diff --git a/.azure-pipelines/ci.yml b/.azure-pipelines/ci.yml deleted file mode 100644 index b5b2765e43844f..00000000000000 --- a/.azure-pipelines/ci.yml +++ /dev/null @@ -1,48 +0,0 @@ -trigger: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7'] - -jobs: -- job: Prebuild - displayName: Pre-build checks - - pool: - vmImage: ubuntu-22.04 - - steps: - - template: ./prebuild-checks.yml - - -- job: Windows_CI_Tests - displayName: Windows CI Tests - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: windows-2022 - - strategy: - matrix: - win32: - arch: win32 - buildOpt: '-p Win32' - testRunTitle: '$(Build.SourceBranchName)-win32' - testRunPlatform: win32 - win64: - arch: amd64 - buildOpt: '-p x64' - testRunTitle: '$(Build.SourceBranchName)-win64' - testRunPlatform: win64 - maxParallel: 4 - - steps: - - template: ./windows-steps.yml - - - template: ./windows-layout-steps.yml - parameters: - kind: nuget - - template: ./windows-layout-steps.yml - parameters: - kind: embed - - template: ./windows-layout-steps.yml - parameters: - kind: appx - fulltest: true diff --git a/.azure-pipelines/find-tools.yml b/.azure-pipelines/find-tools.yml deleted file mode 100644 index 9ad0f5622bb31e..00000000000000 --- a/.azure-pipelines/find-tools.yml +++ /dev/null @@ -1,26 +0,0 @@ -# Locate a set of the tools used for builds - -steps: - - template: windows-release/find-sdk.yml - parameters: - toolname: 'signtool.exe' - - - powershell: | - $vcvarsall = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" ` - -prerelease ` - -latest ` - -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - -find VC\Auxiliary\Build\vcvarsall.bat) - Write-Host "Found vcvarsall at $vcvarsall" - Write-Host "##vso[task.setVariable variable=vcvarsall]$vcvarsall" - displayName: 'Find vcvarsall.bat' - - - powershell: | - $msbuild = (& "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" ` - -prerelease ` - -latest ` - -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` - -find MSBuild\Current\Bin\msbuild.exe) - Write-Host "Found MSBuild at $msbuild" - Write-Host "##vso[task.setVariable variable=msbuild]$msbuild" - displayName: 'Find MSBuild' diff --git a/.azure-pipelines/libffi-build.yml b/.azure-pipelines/libffi-build.yml deleted file mode 100644 index dd26ff215a807a..00000000000000 --- a/.azure-pipelines/libffi-build.yml +++ /dev/null @@ -1,86 +0,0 @@ -name: $(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) - -variables: - IntDir: '$(Build.BinariesDirectory)' - OutDir: '$(Build.ArtifactStagingDirectory)' - - # MUST BE SET AT QUEUE TIME - # SigningCertificate: 'Python Software Foundation' - # SourcesRepo: 'https://github.com/python/cpython-source-deps' - # SourceTag: 'libffi-3.4.2' - -jobs: -- job: Build_LibFFI - displayName: LibFFI - pool: - vmImage: windows-latest - - workspace: - clean: all - - steps: - - checkout: none - - - template: ./find-tools.yml - - - powershell: | - mkdir -Force "$(IntDir)\script" - iwr "https://github.com/python/cpython/raw/main/PCbuild/prepare_libffi.bat" ` - -outfile "$(IntDir)\script\prepare_libffi.bat" - displayName: 'Download build script' - - - powershell: | - git clone $(SourcesRepo) -b $(SourceTag) --depth 1 -c core.autocrlf=false -c core.eol=lf . - displayName: 'Check out LibFFI sources' - - - script: 'prepare_libffi.bat --install-cygwin' - workingDirectory: '$(IntDir)\script' - displayName: 'Install Cygwin and build' - env: - VCVARSALL: '$(vcvarsall)' - LIBFFI_SOURCE: '$(Build.SourcesDirectory)' - LIBFFI_OUT: '$(OutDir)' - - - powershell: | - if ((gci *\*.dll).Count -lt 4) { - Write-Error "Did not generate enough DLL files" - } - if ((gci *\Include\ffi.h).Count -lt 4) { - Write-Error "Did not generate enough include files" - } - failOnStderr: true - workingDirectory: '$(OutDir)' - displayName: 'Verify files were created' - - - publish: '$(OutDir)' - artifact: 'unsigned' - displayName: 'Publish unsigned build' - -- job: Sign_LibFFI - displayName: Sign LibFFI - dependsOn: Build_LibFFI - pool: - name: 'Windows Release' - - workspace: - clean: all - - steps: - - checkout: none - - download: current - artifact: unsigned - - - template: ./find-tools.yml - - - powershell: | - signtool sign /q /a ` - /n "Python Software Foundation" ` - /fd sha256 ` - /tr http://timestamp.digicert.com/ /td sha256 ` - /d "LibFFI for Python" ` - (gci "$(Pipeline.Workspace)\unsigned\*.dll" -r) - displayName: 'Sign files' - - - publish: '$(Pipeline.Workspace)\unsigned' - artifact: 'libffi' - displayName: 'Publish libffi' diff --git a/.azure-pipelines/openssl-build.yml b/.azure-pipelines/openssl-build.yml deleted file mode 100644 index 8aab7ea0b94193..00000000000000 --- a/.azure-pipelines/openssl-build.yml +++ /dev/null @@ -1,110 +0,0 @@ -name: $(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) - -variables: - IntDir: '$(Build.BinariesDirectory)' - OutDir: '$(Build.ArtifactStagingDirectory)' - - # MUST BE SET AT QUEUE TIME - # SigningCertificate: 'Python Software Foundation' - # SourcesRepo: 'https://github.com/python/cpython-source-deps' - # SourceTag: 'openssl-1.1.1k' - -jobs: -- job: Build_SSL - displayName: OpenSSL - pool: - name: 'Windows Release' - #vmImage: windows-latest - - strategy: - matrix: - win32: - Platform: 'win32' - VCPlatform: 'amd64_x86' - OpenSSLPlatform: 'VC-WIN32 no-asm' - amd64: - Platform: 'amd64' - VCPlatform: 'amd64' - OpenSSLPlatform: 'VC-WIN64A-masm' - arm32: - Platform: 'arm32' - VCPlatform: 'amd64_arm' - OpenSSLPlatform: 'VC-WIN32-ARM' - arm64: - Platform: 'arm64' - VCPlatform: 'amd64_arm64' - OpenSSLPlatform: 'VC-WIN64-ARM' - - workspace: - clean: all - - steps: - - checkout: none - - - template: ./find-tools.yml - - - powershell: | - git clone $(SourcesRepo) -b $(SourceTag) --depth 1 . - displayName: 'Check out OpenSSL sources' - - - powershell: | - $f = gi ms\uplink.c - $c1 = gc $f - $c2 = $c1 -replace '\(\(h = GetModuleHandle\(NULL\)\) == NULL\)', '((h = GetModuleHandleA("_ssl.pyd")) == NULL) if ((h = GetModuleHandleA("_ssl_d.pyd")) == NULL) if ((h = GetModuleHandle(NULL)) == NULL /*patched*/)' - if ($c2 -ne $c1) { - $c2 | Out-File $f -Encoding ASCII - } else { - Write-Host '##warning Failed to patch uplink.c' - } - displayName: 'Apply uplink.c patch' - - - script: | - call "$(vcvarsall)" $(VCPlatform) - perl "$(Build.SourcesDirectory)\Configure" $(OpenSSLPlatform) - nmake - workingDirectory: '$(IntDir)' - displayName: 'Build OpenSSL' - - - script: | - call "$(vcvarsall)" $(VCPlatform) - signtool sign /q /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "OpenSSL for Python" *.dll - workingDirectory: '$(IntDir)' - displayName: 'Sign OpenSSL Build' - condition: and(succeeded(), variables['SigningCertificate']) - - - task: CopyFiles@2 - displayName: 'Copy built libraries for upload' - inputs: - SourceFolder: '$(IntDir)' - Contents: | - lib*.dll - lib*.pdb - lib*.lib - include\openssl\*.h - TargetFolder: '$(OutDir)' - - - task: CopyFiles@2 - displayName: 'Copy header files for upload' - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: | - include\openssl\* - TargetFolder: '$(OutDir)' - - - task: CopyFiles@2 - displayName: 'Copy applink files for upload' - inputs: - SourceFolder: '$(Build.SourcesDirectory)\ms' - Contents: applink.c - TargetFolder: '$(OutDir)\include' - - - task: CopyFiles@2 - displayName: 'Copy LICENSE for upload' - inputs: - SourceFolder: '$(Build.SourcesDirectory)' - Contents: LICENSE - TargetFolder: '$(OutDir)' - - - publish: '$(OutDir)' - artifact: '$(Platform)' - displayName: 'Publishing $(Platform)' diff --git a/.azure-pipelines/posix-deps-apt.sh b/.azure-pipelines/posix-deps-apt.sh deleted file mode 100755 index e0f4ca5d8d8e88..00000000000000 --- a/.azure-pipelines/posix-deps-apt.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -apt-get update - -apt-get -yq install \ - build-essential \ - zlib1g-dev \ - libbz2-dev \ - liblzma-dev \ - libncurses5-dev \ - libreadline6-dev \ - libsqlite3-dev \ - libssl-dev \ - libgdbm-dev \ - tk-dev \ - lzma \ - lzma-dev \ - liblzma-dev \ - libffi-dev \ - uuid-dev \ - xvfb - -if [ ! -z "$1" ] -then - echo ##vso[task.prependpath]$PWD/multissl/openssl/$1 - echo ##vso[task.setvariable variable=OPENSSL_DIR]$PWD/multissl/openssl/$1 - python3 Tools/ssl/multissltests.py --steps=library --base-directory $PWD/multissl --openssl $1 --system Linux -fi diff --git a/.azure-pipelines/posix-steps.yml b/.azure-pipelines/posix-steps.yml deleted file mode 100644 index db44255e4422ff..00000000000000 --- a/.azure-pipelines/posix-steps.yml +++ /dev/null @@ -1,27 +0,0 @@ -steps: -- checkout: self - clean: true - fetchDepth: 5 - -# Work around a known issue affecting Ubuntu VMs on Pipelines -- script: sudo setfacl -Rb /home/vsts - displayName: 'Workaround ACL issue' - -- script: sudo ./.azure-pipelines/posix-deps-apt.sh $(openssl_version) - displayName: 'Install dependencies' - -- script: ./configure --with-pydebug - displayName: 'Configure CPython (debug)' - -- script: make -j4 - displayName: 'Build CPython' - -- script: make pythoninfo - displayName: 'Display build info' - -- script: | - git fetch origin - ./python Tools/scripts/patchcheck.py --ci true - displayName: 'Run patchcheck.py' - condition: and(succeeded(), eq(variables['Build.Reason'], 'PullRequest')) - diff --git a/.azure-pipelines/pr.yml b/.azure-pipelines/pr.yml deleted file mode 100644 index 335a4b407cb83c..00000000000000 --- a/.azure-pipelines/pr.yml +++ /dev/null @@ -1,28 +0,0 @@ -pr: ['main', '3.12', '3.11', '3.10', '3.9', '3.8', '3.7'] - -jobs: -- job: Prebuild - displayName: Pre-build checks - - pool: - vmImage: ubuntu-22.04 - - steps: - - template: ./prebuild-checks.yml - - -- job: Ubuntu_Patchcheck - displayName: Ubuntu patchcheck - dependsOn: Prebuild - condition: and(succeeded(), eq(dependencies.Prebuild.outputs['tests.run'], 'true')) - - pool: - vmImage: ubuntu-22.04 - - variables: - testRunTitle: '$(system.pullRequest.TargetBranch)-linux' - testRunPlatform: linux - openssl_version: 1.1.1u - - steps: - - template: ./posix-steps.yml diff --git a/.azure-pipelines/prebuild-checks.yml b/.azure-pipelines/prebuild-checks.yml deleted file mode 100644 index 2c6460d2386735..00000000000000 --- a/.azure-pipelines/prebuild-checks.yml +++ /dev/null @@ -1,24 +0,0 @@ -steps: -- checkout: self - fetchDepth: 5 - -- script: echo "##vso[task.setvariable variable=diffTarget]HEAD~1" - displayName: Set default diff target - -- script: | - git fetch -q origin $(System.PullRequest.TargetBranch) - echo "##vso[task.setvariable variable=diffTarget]HEAD \$(git merge-base HEAD FETCH_HEAD)" - displayName: Fetch comparison tree - condition: and(succeeded(), variables['System.PullRequest.TargetBranch']) - -- script: | - if ! git diff --name-only $(diffTarget) | grep -qvE '(\.rst$|^Doc|^Misc)' - then - echo "Only docs were updated: tests.run=false" - echo "##vso[task.setvariable variable=run;isOutput=true]false" - else - echo "Code was updated: tests.run=true" - echo "##vso[task.setvariable variable=run;isOutput=true]true" - fi - displayName: Detect source changes - name: tests diff --git a/.azure-pipelines/tcltk-build.yml b/.azure-pipelines/tcltk-build.yml deleted file mode 100644 index f9e50d3711a460..00000000000000 --- a/.azure-pipelines/tcltk-build.yml +++ /dev/null @@ -1,71 +0,0 @@ -name: tcl$(TkSourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) - -variables: - IntDir: '$(Build.BinariesDirectory)\obj' - ExternalsDir: '$(Build.BinariesDirectory)\externals' - OutDir: '$(Build.ArtifactStagingDirectory)' - Configuration: 'Release' - - # MUST BE SET AT QUEUE TIME - # SigningCertificate: 'Python Software Foundation' - # SourcesRepo: 'https://github.com/python/cpython-source-deps' - # TclSourceTag: 'tcl-core-8.6.12.0' - # TkSourceTag: 'tk-8.6.12.0' - # TixSourceTag: 'tix-8.4.3.6' - -jobs: -- job: Build_TclTk - displayName: 'Tcl/Tk' - pool: - name: 'Windows Release' - #vmImage: windows-latest - - workspace: - clean: all - - steps: - - template: ./find-tools.yml - - - powershell: | - git clone $(SourcesRepo) -b $(TclSourceTag) --depth 1 "$(ExternalsDir)\$(TclSourceTag)" - displayName: 'Check out Tcl sources' - - - powershell: | - git clone $(SourcesRepo) -b $(TkSourceTag) --depth 1 "$(ExternalsDir)\$(TkSourceTag)" - displayName: 'Check out Tk sources' - - - powershell: | - git clone $(SourcesRepo) -b $(TixSourceTag) --depth 1 "$(ExternalsDir)\$(TixSourceTag)" - displayName: 'Check out Tix sources' - - # This msbuild.rsp file will be used by the build to forcibly override these variables - - powershell: | - del -Force -EA 0 msbuild.rsp - "/p:IntDir=$(IntDir)\" >> msbuild.rsp - "/p:ExternalsDir=$(ExternalsDir)\" >> msbuild.rsp - "/p:tclDir=$(ExternalsDir)\$(TclSourceTag)\" >> msbuild.rsp - "/p:tkDir=$(ExternalsDir)\$(TkSourceTag)\" >> msbuild.rsp - "/p:tixDir=$(ExternalsDir)\$(TixSourceTag)\" >> msbuild.rsp - displayName: 'Generate msbuild.rsp' - - - powershell: | - & "$(msbuild)" PCbuild\tcl.vcxproj "@msbuild.rsp" /p:Platform=Win32 /p:tcltkDir="$(OutDir)\win32" - & "$(msbuild)" PCbuild\tk.vcxproj "@msbuild.rsp" /p:Platform=Win32 /p:tcltkDir="$(OutDir)\win32" - & "$(msbuild)" PCbuild\tix.vcxproj "@msbuild.rsp" /p:Platform=Win32 /p:tcltkDir="$(OutDir)\win32" - displayName: 'Build for win32' - - - powershell: | - & "$(msbuild)" PCbuild\tcl.vcxproj "@msbuild.rsp" /p:Platform=x64 /p:tcltkDir="$(OutDir)\amd64" - & "$(msbuild)" PCbuild\tk.vcxproj "@msbuild.rsp" /p:Platform=x64 /p:tcltkDir="$(OutDir)\amd64" - & "$(msbuild)" PCbuild\tix.vcxproj "@msbuild.rsp" /p:Platform=x64 /p:tcltkDir="$(OutDir)\amd64" - displayName: 'Build for amd64' - - - powershell: | - & "$(msbuild)" PCbuild\tcl.vcxproj "@msbuild.rsp" /p:Platform=ARM64 /p:tcltkDir="$(OutDir)\arm64" - & "$(msbuild)" PCbuild\tk.vcxproj "@msbuild.rsp" /p:Platform=ARM64 /p:tcltkDir="$(OutDir)\arm64" - & "$(msbuild)" PCbuild\tix.vcxproj "@msbuild.rsp" /p:Platform=ARM64 /p:tcltkDir="$(OutDir)\arm64" - displayName: 'Build for arm64' - - - publish: '$(OutDir)' - artifact: 'tcltk' - displayName: 'Publishing tcltk' diff --git a/.azure-pipelines/windows-layout-steps.yml b/.azure-pipelines/windows-layout-steps.yml deleted file mode 100644 index afd89781790494..00000000000000 --- a/.azure-pipelines/windows-layout-steps.yml +++ /dev/null @@ -1,28 +0,0 @@ -parameters: - kind: nuget - extraOpts: --precompile - fulltest: false - -steps: -- script: .\python.bat PC\layout -vv -s "$(Build.SourcesDirectory)" -b "$(Py_OutDir)\$(arch)" -t "$(Build.BinariesDirectory)\layout-tmp-${{ parameters.kind }}-$(arch)" --copy "$(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch)" ${{ parameters.extraOpts }} --preset-${{ parameters.kind }} --include-tests - displayName: Create ${{ parameters.kind }} layout - -- script: .\python.exe -m test.pythoninfo - workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch) - displayName: Show layout info (${{ parameters.kind }}) - -- ${{ if eq(parameters.fulltest, 'true') }}: - - script: .\python.exe -m test -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml" --tempdir "$(Build.BinariesDirectory)\tmp-${{ parameters.kind }}-$(arch)" -i test_launcher - workingDirectory: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch) - displayName: ${{ parameters.kind }} Tests - env: - PREFIX: $(Build.BinariesDirectory)\layout-${{ parameters.kind }}-$(arch) - - - task: PublishTestResults@2 - displayName: Publish ${{ parameters.kind }} Test Results - inputs: - testResultsFiles: $(Build.BinariesDirectory)\test-results-${{ parameters.kind }}.xml - mergeTestResults: true - testRunTitle: ${{ parameters.kind }}-$(testRunTitle) - platform: $(testRunPlatform) - condition: succeededOrFailed() diff --git a/.azure-pipelines/windows-release.yml b/.azure-pipelines/windows-release.yml deleted file mode 100644 index 581f48ba22882b..00000000000000 --- a/.azure-pipelines/windows-release.yml +++ /dev/null @@ -1,220 +0,0 @@ -name: Release_$(Build.SourceBranchName)_$(SourceTag)_$(Date:yyyyMMdd)$(Rev:.rr) - -parameters: -- name: GitRemote - displayName: "Git remote" - type: string - default: python - values: - - 'python' - - 'pablogsal' - - 'ambv' - - '(Other)' -- name: GitRemote_Other - displayName: "If Other, specify Git remote" - type: string - default: 'python' -- name: SourceTag - displayName: "Git tag" - type: string - default: main -- name: DoPublish - displayName: "Publish release" - type: boolean - default: false -- name: SigningCertificate - displayName: "Code signing certificate" - type: string - default: 'Python Software Foundation' - values: - - 'Python Software Foundation' - - 'TestSign' - - 'Unsigned' -- name: SigningDescription - displayName: "Signature description" - type: string - default: 'Built: $(Build.BuildNumber)' -- name: DoARM64 - displayName: "Publish ARM64 build" - type: boolean - default: true -# Because there is no ARM64 Tcl/Tk pre-3.11, we need a separate option -# to keep those builds working when the files are going to be absent. -# Eventually when we stop releasing anything that old, we can drop this -# argument (and make it implicitly always 'true') -- name: ARM64TclTk - displayName: "Use Tcl/Tk for ARM64 (3.11 and later)" - type: boolean - default: true -- name: DoPGO - displayName: "Run PGO" - type: boolean - default: true -- name: DoCHM - displayName: "Produce compiled help document (pre-3.11)" - type: boolean - default: false -- name: DoLayout - displayName: "Produce full layout artifact" - type: boolean - default: true -- name: DoMSIX - displayName: "Produce Store packages" - type: boolean - default: true -- name: DoNuget - displayName: "Produce Nuget packages" - type: boolean - default: true -- name: DoEmbed - displayName: "Produce embeddable package" - type: boolean - default: true -- name: DoMSI - displayName: "Produce EXE/MSI installer" - type: boolean - default: true -- name: BuildToPublish - displayName: "Build number to publish (0 to skip)" - type: number - default: '0' - -variables: - __RealSigningCertificate: 'Python Software Foundation' - ${{ if ne(parameters.GitRemote, '(Other)') }}: - GitRemote: ${{ parameters.GitRemote }} - ${{ else }}: - GitRemote: ${{ parameters.GitRemote_Other }} - SourceTag: ${{ parameters.SourceTag }} - DoPGO: ${{ parameters.DoPGO }} - ${{ if ne(parameters.SigningCertificate, 'Unsigned') }}: - SigningCertificate: ${{ parameters.SigningCertificate }} - SigningDescription: ${{ parameters.SigningDescription }} - DoCHM: ${{ parameters.DoCHM }} - DoLayout: ${{ parameters.DoLayout }} - DoMSIX: ${{ parameters.DoMSIX }} - DoNuget: ${{ parameters.DoNuget }} - DoEmbed: ${{ parameters.DoEmbed }} - DoMSI: ${{ parameters.DoMSI }} - DoPublish: ${{ parameters.DoPublish }} - PublishARM64: ${{ parameters.DoARM64 }} -# QUEUE TIME VARIABLES -# PyDotOrgUsername: '' -# PyDotOrgServer: '' - -trigger: none -pr: none - -stages: -- ${{ if eq(parameters.BuildToPublish, '0') }}: - - stage: Build - displayName: Build binaries - jobs: - - template: windows-release/stage-build.yml - parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} - - - stage: Sign - displayName: Sign binaries - dependsOn: Build - jobs: - - template: windows-release/stage-sign.yml - - - stage: Layout - displayName: Generate layouts - dependsOn: Sign - jobs: - - template: windows-release/stage-layout-full.yml - parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} - - template: windows-release/stage-layout-embed.yml - - template: windows-release/stage-layout-nuget.yml - - - stage: Pack - dependsOn: Layout - jobs: - - template: windows-release/stage-pack-nuget.yml - - - stage: Test - dependsOn: Pack - jobs: - - template: windows-release/stage-test-embed.yml - - template: windows-release/stage-test-nuget.yml - - - ${{ if eq(parameters.DoMSIX, 'true') }}: - - stage: Layout_MSIX - displayName: Generate MSIX layouts - dependsOn: Sign - jobs: - - template: windows-release/stage-layout-msix.yml - parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} - - - stage: Pack_MSIX - displayName: Package MSIX - dependsOn: Layout_MSIX - jobs: - - template: windows-release/stage-pack-msix.yml - - - ${{ if eq(parameters.DoMSI, 'true') }}: - - stage: Build_MSI - displayName: Build MSI installer - dependsOn: Sign - jobs: - - template: windows-release/stage-msi.yml - parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} - - - stage: Test_MSI - displayName: Test MSI installer - dependsOn: Build_MSI - jobs: - - template: windows-release/stage-test-msi.yml - - - ${{ if eq(parameters.DoPublish, 'true') }}: - - ${{ if eq(parameters.DoMSI, 'true') }}: - - stage: PublishPyDotOrg - displayName: Publish to python.org - dependsOn: ['Test_MSI', 'Test'] - jobs: - - template: windows-release/stage-publish-pythonorg.yml - - - ${{ if eq(parameters.DoNuget, 'true') }}: - - stage: PublishNuget - displayName: Publish to nuget.org - ${{ if eq(parameters.DoMSI, 'true') }}: - dependsOn: ['Test_MSI', 'Test'] - ${{ else }}: - dependsOn: 'Test' - jobs: - - template: windows-release/stage-publish-nugetorg.yml - - - ${{ if eq(parameters.DoMSIX, 'true') }}: - - stage: PublishStore - displayName: Publish to Store - ${{ if eq(parameters.DoMSI, 'true') }}: - dependsOn: ['Test_MSI', 'Pack_MSIX'] - ${{ else }}: - dependsOn: 'Pack_MSIX' - jobs: - - template: windows-release/stage-publish-store.yml - -- ${{ else }}: - - stage: PublishExisting - displayName: Publish existing build - dependsOn: [] - jobs: - - ${{ if eq(parameters.DoMSI, 'true') }}: - - template: windows-release/stage-publish-pythonorg.yml - parameters: - BuildToPublish: ${{ parameters.BuildToPublish }} - - - ${{ if eq(parameters.DoNuget, 'true') }}: - - template: windows-release/stage-publish-nugetorg.yml - parameters: - BuildToPublish: ${{ parameters.BuildToPublish }} - - - ${{ if eq(parameters.DoMSIX, 'true') }}: - - template: windows-release/stage-publish-store.yml - parameters: - BuildToPublish: ${{ parameters.BuildToPublish }} diff --git a/.azure-pipelines/windows-release/build-steps.yml b/.azure-pipelines/windows-release/build-steps.yml deleted file mode 100644 index 5ca2016d65f9e1..00000000000000 --- a/.azure-pipelines/windows-release/build-steps.yml +++ /dev/null @@ -1,84 +0,0 @@ -parameters: - ShouldPGO: false - -steps: -- template: ./checkout.yml - -- powershell: | - $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; - Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" - Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" - Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" - Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" - Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)" - Write-Host "##vso[build.addbuildtag]$($d.PythonVersion)-$(Name)" - displayName: 'Extract version numbers' - -- ${{ if eq(parameters.ShouldPGO, 'false') }}: - - powershell: | - $env:SigningCertificate = $null - .\PCbuild\build.bat -v -p $(Platform) -c $(Configuration) - displayName: 'Run build' - env: - IncludeUwp: true - Py_OutDir: '$(Build.BinariesDirectory)\bin' - -- ${{ if eq(parameters.ShouldPGO, 'true') }}: - - powershell: | - $env:SigningCertificate = $null - .\PCbuild\build.bat -v -p $(Platform) --pgo - displayName: 'Run build with PGO' - env: - IncludeUwp: true - Py_OutDir: '$(Build.BinariesDirectory)\bin' - -- powershell: | - $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 - $tool = (gci -r "$kitroot\Bin\*\x64\signtool.exe" | sort FullName -Desc | select -First 1) - if (-not $tool) { - throw "SDK is not available" - } - Write-Host "##vso[task.prependpath]$($tool.Directory)" - displayName: 'Add WinSDK tools to path' - -- powershell: | - $env:SigningCertificate = $null - $(_HostPython) PC\layout -vv -b "$(Build.BinariesDirectory)\bin" -t "$(Build.BinariesDirectory)\catalog" --catalog "${env:CAT}.cdf" --preset-default --arch $(Arch) - makecat "${env:CAT}.cdf" - del "${env:CAT}.cdf" - if (-not (Test-Path "${env:CAT}.cat")) { - throw "Failed to build catalog file" - } - displayName: 'Generate catalog' - env: - CAT: $(Build.BinariesDirectory)\bin\$(Arch)\python - PYTHON_HEXVERSION: $(VersionHex) - -- task: PublishPipelineArtifact@0 - displayName: 'Publish binaries' - condition: and(succeeded(), not(and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate']))) - inputs: - targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' - artifactName: bin_$(Name) - -- task: PublishPipelineArtifact@0 - displayName: 'Publish binaries for signing' - condition: and(succeeded(), and(eq(variables['Configuration'], 'Release'), variables['SigningCertificate'])) - inputs: - targetPath: '$(Build.BinariesDirectory)\bin\$(Arch)' - artifactName: unsigned_bin_$(Name) - -- task: CopyFiles@2 - displayName: 'Layout Artifact: symbols' - inputs: - sourceFolder: $(Build.BinariesDirectory)\bin\$(Arch) - targetFolder: $(Build.ArtifactStagingDirectory)\symbols\$(Name) - flatten: true - contents: | - **\*.pdb - -- task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: symbols' - inputs: - PathToPublish: '$(Build.ArtifactStagingDirectory)\symbols' - ArtifactName: symbols diff --git a/.azure-pipelines/windows-release/checkout.yml b/.azure-pipelines/windows-release/checkout.yml deleted file mode 100644 index d42d55fff08dda..00000000000000 --- a/.azure-pipelines/windows-release/checkout.yml +++ /dev/null @@ -1,21 +0,0 @@ -parameters: - depth: 3 - -steps: -- checkout: none - -- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch https://github.com/$(GitRemote)/cpython.git . - displayName: 'git clone ($(GitRemote)/$(SourceTag))' - condition: and(succeeded(), and(variables['GitRemote'], variables['SourceTag'])) - -- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(SourceTag) --single-branch $(Build.Repository.Uri) . - displayName: 'git clone (/$(SourceTag))' - condition: and(succeeded(), and(not(variables['GitRemote']), variables['SourceTag'])) - -- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch https://github.com/$(GitRemote)/cpython.git . - displayName: 'git clone ($(GitRemote)/)' - condition: and(succeeded(), and(variables['GitRemote'], not(variables['SourceTag']))) - -- script: git clone --progress -v --depth ${{ parameters.depth }} --branch $(Build.SourceBranchName) --single-branch $(Build.Repository.Uri) . - displayName: 'git clone' - condition: and(succeeded(), and(not(variables['GitRemote']), not(variables['SourceTag']))) diff --git a/.azure-pipelines/windows-release/find-sdk.yml b/.azure-pipelines/windows-release/find-sdk.yml deleted file mode 100644 index e4de78555b3f66..00000000000000 --- a/.azure-pipelines/windows-release/find-sdk.yml +++ /dev/null @@ -1,17 +0,0 @@ -# Locate the Windows SDK and add its binaries directory to PATH -# -# `toolname` can be overridden to use a different marker file. - -parameters: - toolname: signtool.exe - -steps: - - powershell: | - $kitroot = (gp 'HKLM:\SOFTWARE\Microsoft\Windows Kits\Installed Roots\').KitsRoot10 - $tool = (gci -r "$kitroot\Bin\*\${{ parameters.toolname }}" | sort FullName -Desc | select -First 1) - if (-not $tool) { - throw "SDK is not available" - } - Write-Host "##vso[task.prependpath]$($tool.Directory)" - Write-Host "Adding $($tool.Directory) to PATH" - displayName: 'Add WinSDK tools to path' diff --git a/.azure-pipelines/windows-release/layout-command.yml b/.azure-pipelines/windows-release/layout-command.yml deleted file mode 100644 index 406ccd859faa6a..00000000000000 --- a/.azure-pipelines/windows-release/layout-command.yml +++ /dev/null @@ -1,23 +0,0 @@ -steps: -- task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_$(HostArch)' - condition: and(succeeded(), variables['HostArch']) - inputs: - artifactName: bin_$(HostArch) - targetPath: $(Build.BinariesDirectory)\bin_$(HostArch) - -- powershell: > - Write-Host ( - '##vso[task.setvariable variable=LayoutCmd]& - "$(Python)" - "{1}\PC\layout" - -vv - --source "{1}" - --build "{0}\bin" - --arch "$(Name)" - --temp "{0}\layout-temp" - --include-cat "{0}\bin\python.cat" - --doc-build "{0}\doc"' - -f ("$(Build.BinariesDirectory)", "$(Build.SourcesDirectory)") - ) - displayName: 'Set LayoutCmd' diff --git a/.azure-pipelines/windows-release/mingw-lib.yml b/.azure-pipelines/windows-release/mingw-lib.yml deleted file mode 100644 index 30f7d34fa61d23..00000000000000 --- a/.azure-pipelines/windows-release/mingw-lib.yml +++ /dev/null @@ -1,13 +0,0 @@ -parameters: - DllToolOpt: -m i386:x86-64 - #DllToolOpt: -m i386 --as-flags=--32 - -steps: -- powershell: | - git clone https://github.com/python/cpython-bin-deps --branch binutils --single-branch --depth 1 --progress -v "binutils" - gci "bin\$(Arch)\python*.dll" | %{ - & "binutils\gendef.exe" $_ | Out-File -Encoding ascii tmp.def - & "binutils\dlltool.exe" --dllname $($_.BaseName).dll --def tmp.def --output-lib "$($_.Directory)\lib$($_.BaseName).a" ${{ parameters.DllToolOpt }} - } - displayName: 'Generate MinGW import library' - workingDirectory: $(Build.BinariesDirectory) diff --git a/.azure-pipelines/windows-release/msi-steps.yml b/.azure-pipelines/windows-release/msi-steps.yml deleted file mode 100644 index 79fc6f5ed292d4..00000000000000 --- a/.azure-pipelines/windows-release/msi-steps.yml +++ /dev/null @@ -1,181 +0,0 @@ -parameters: - ARM64TclTk: true - -steps: - - template: ./checkout.yml - - - powershell: | - $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; - Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" - displayName: 'Update signing description' - condition: and(succeeded(), not(variables['SigningDescription'])) - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: doc' - inputs: - artifactName: doc - targetPath: $(Build.BinariesDirectory)\doc - - - task: CopyFiles@2 - displayName: 'Merge documentation files' - inputs: - sourceFolder: $(Build.BinariesDirectory)\doc - targetFolder: $(Build.SourcesDirectory)\Doc\build - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_win32' - inputs: - artifactName: bin_win32 - targetPath: $(Build.BinariesDirectory)\win32 - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_win32_d' - inputs: - artifactName: bin_win32_d - targetPath: $(Build.BinariesDirectory)\win32 - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_amd64' - inputs: - artifactName: bin_amd64 - targetPath: $(Build.BinariesDirectory)\amd64 - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_amd64_d' - inputs: - artifactName: bin_amd64_d - targetPath: $(Build.BinariesDirectory)\amd64 - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_arm64' - condition: and(succeeded(), eq(variables['PublishARM64'], 'true')) - inputs: - artifactName: bin_arm64 - targetPath: $(Build.BinariesDirectory)\arm64 - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_arm64_d' - condition: and(succeeded(), eq(variables['PublishARM64'], 'true')) - inputs: - artifactName: bin_arm64_d - targetPath: $(Build.BinariesDirectory)\arm64 - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: tcltk_lib_win32' - inputs: - artifactName: tcltk_lib_win32 - targetPath: $(Build.BinariesDirectory)\tcltk_lib_win32 - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: tcltk_lib_amd64' - inputs: - artifactName: tcltk_lib_amd64 - targetPath: $(Build.BinariesDirectory)\tcltk_lib_amd64 - - - ${{ if eq(parameters.ARM64TclTk, true) }}: - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: tcltk_lib_arm64' - condition: and(succeeded(), eq(variables['PublishARM64'], 'true')) - inputs: - artifactName: tcltk_lib_arm64 - targetPath: $(Build.BinariesDirectory)\tcltk_lib_arm64 - - - powershell: | - copy $(Build.BinariesDirectory)\amd64\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force - displayName: 'Copy signed files into sources' - condition: and(succeeded(), variables['SigningCertificate']) - - - script: | - call Tools\msi\get_externals.bat - call PCbuild\find_python.bat - echo ##vso[task.setvariable variable=PYTHON]%PYTHON% - call PCbuild/find_msbuild.bat - echo ##vso[task.setvariable variable=MSBUILD]%MSBUILD% - displayName: 'Get external dependencies' - - - script: | - %PYTHON% -m pip install blurb - %PYTHON% -m blurb merge -f Misc\NEWS - displayName: 'Merge NEWS file' - - - script: | - %MSBUILD% Tools\msi\launcher\launcher.wixproj - displayName: 'Build launcher installer' - env: - Platform: x86 - Py_OutDir: $(Build.BinariesDirectory) - - - script: | - %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true - displayName: 'Build win32 installer' - env: - Platform: x86 - Py_OutDir: $(Build.BinariesDirectory) - PYTHON: $(Build.BinariesDirectory)\win32\python.exe - PythonForBuild: $(Build.BinariesDirectory)\win32\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_win32 - BuildForRelease: true - - - script: | - %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true - displayName: 'Build amd64 installer' - env: - Platform: x64 - Py_OutDir: $(Build.BinariesDirectory) - PYTHON: $(Build.BinariesDirectory)\amd64\python.exe - PythonForBuild: $(Build.BinariesDirectory)\amd64\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_amd64 - BuildForRelease: true - - - script: | - %MSBUILD% Tools\msi\bundle\releaselocal.wixproj /t:Rebuild /p:RebuildAll=true - displayName: 'Build arm64 installer' - condition: and(succeeded(), eq(variables['PublishARM64'], 'true')) - env: - Platform: ARM64 - Py_OutDir: $(Build.BinariesDirectory) - PYTHON: $(Build.BinariesDirectory)\win32\python.exe - PythonForBuild: $(Build.BinariesDirectory)\win32\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - BuildForRelease: true - ${{ if eq(parameters.ARM64TclTk, true) }}: - TclTkLibraryDir: $(Build.BinariesDirectory)\tcltk_lib_arm64 - - - task: CopyFiles@2 - displayName: 'Assemble artifact: msi (win32)' - inputs: - sourceFolder: $(Build.BinariesDirectory)\win32\en-us - targetFolder: $(Build.ArtifactStagingDirectory)\msi\win32 - contents: | - *.msi - *.cab - *.exe - - - task: CopyFiles@2 - displayName: 'Assemble artifact: msi (amd64)' - inputs: - sourceFolder: $(Build.BinariesDirectory)\amd64\en-us - targetFolder: $(Build.ArtifactStagingDirectory)\msi\amd64 - contents: | - *.msi - *.cab - *.exe - - - task: CopyFiles@2 - displayName: 'Assemble artifact: msi (arm64)' - condition: and(succeeded(), eq(variables['PublishARM64'], 'true')) - inputs: - sourceFolder: $(Build.BinariesDirectory)\arm64\en-us - targetFolder: $(Build.ArtifactStagingDirectory)\msi\arm64 - contents: | - *.msi - *.cab - *.exe - - - task: PublishPipelineArtifact@0 - displayName: 'Publish MSI' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\msi' - artifactName: msi diff --git a/.azure-pipelines/windows-release/stage-build.yml b/.azure-pipelines/windows-release/stage-build.yml deleted file mode 100644 index 26f43177504362..00000000000000 --- a/.azure-pipelines/windows-release/stage-build.yml +++ /dev/null @@ -1,193 +0,0 @@ -parameters: - ARM64TclTk: true - -jobs: -- job: Build_Docs - displayName: Docs build - pool: - vmImage: windows-2022 - - workspace: - clean: all - - steps: - - template: ./checkout.yml - - - script: Doc\make.bat html - displayName: 'Build HTML docs' - env: - BUILDDIR: $(Build.BinariesDirectory)\Doc - - - script: Doc\make.bat htmlhelp - displayName: 'Build CHM docs' - condition: and(succeeded(), eq(variables['DoCHM'], 'true')) - env: - BUILDDIR: $(Build.BinariesDirectory)\Doc - - - task: CopyFiles@2 - displayName: 'Assemble artifact: Doc' - inputs: - sourceFolder: $(Build.BinariesDirectory)\Doc - targetFolder: $(Build.ArtifactStagingDirectory)\Doc - contents: | - html\**\* - htmlhelp\*.chm - - - task: PublishPipelineArtifact@0 - displayName: 'Publish artifact: doc' - inputs: - targetPath: $(Build.ArtifactStagingDirectory)\Doc - artifactName: doc - - -- job: Build_Python - displayName: Python build - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - win32: - Name: win32 - Arch: win32 - Platform: x86 - Configuration: Release - _HostPython: .\python - win32_d: - Name: win32_d - Arch: win32 - Platform: x86 - Configuration: Debug - _HostPython: .\python - amd64_d: - Name: amd64_d - Arch: amd64 - Platform: x64 - Configuration: Debug - _HostPython: .\python - arm64: - Name: arm64 - Arch: arm64 - Platform: ARM64 - Configuration: Release - _HostPython: python - arm64_d: - Name: arm64_d - Arch: arm64 - Platform: ARM64 - Configuration: Debug - _HostPython: python - - steps: - - template: ./build-steps.yml - -- job: Build_Python_NonPGO - displayName: Python non-PGO build - condition: and(succeeded(), ne(variables['DoPGO'], 'true')) - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - amd64: - Name: amd64 - Arch: amd64 - Platform: x64 - Configuration: Release - _HostPython: .\python - - steps: - - template: ./build-steps.yml - - -- job: Build_Python_PGO - displayName: Python PGO build - condition: and(succeeded(), eq(variables['DoPGO'], 'true')) - - # Allow up to five hours for PGO - timeoutInMinutes: 300 - - pool: - name: 'Windows Release' - - workspace: - clean: all - - strategy: - matrix: - amd64: - Name: amd64 - Arch: amd64 - Platform: x64 - Configuration: Release - _HostPython: .\python - - steps: - - template: ./build-steps.yml - parameters: - ShouldPGO: true - - -- job: TclTk_Lib - displayName: Publish Tcl/Tk Library - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - steps: - - template: ./checkout.yml - - - script: PCbuild\get_externals.bat --no-openssl --no-libffi - displayName: 'Get external dependencies' - - - task: MSBuild@1 - displayName: 'Copy Tcl/Tk lib for publish' - inputs: - solution: PCbuild\tcltk.props - platform: x86 - msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_win32" - - - task: MSBuild@1 - displayName: 'Copy Tcl/Tk lib for publish' - inputs: - solution: PCbuild\tcltk.props - platform: x64 - msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_amd64" - - - ${{ if eq(parameters.ARM64TclTk, true) }}: - - task: MSBuild@1 - displayName: 'Copy Tcl/Tk lib for publish' - inputs: - solution: PCbuild\tcltk.props - platform: ARM64 - msbuildArguments: /t:CopyTclTkLib /p:OutDir="$(Build.ArtifactStagingDirectory)\tcl_arm64" - - - task: PublishPipelineArtifact@0 - displayName: 'Publish artifact: tcltk_lib_win32' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\tcl_win32' - artifactName: tcltk_lib_win32 - - - task: PublishPipelineArtifact@0 - displayName: 'Publish artifact: tcltk_lib_amd64' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\tcl_amd64' - artifactName: tcltk_lib_amd64 - - - ${{ if eq(parameters.ARM64TclTk, true) }}: - - task: PublishPipelineArtifact@0 - displayName: 'Publish artifact: tcltk_lib_arm64' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\tcl_arm64' - artifactName: tcltk_lib_arm64 diff --git a/.azure-pipelines/windows-release/stage-layout-embed.yml b/.azure-pipelines/windows-release/stage-layout-embed.yml deleted file mode 100644 index c8b23d308d81e9..00000000000000 --- a/.azure-pipelines/windows-release/stage-layout-embed.yml +++ /dev/null @@ -1,61 +0,0 @@ -jobs: -- job: Make_Embed_Layout - displayName: Make embeddable layout - condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - win32: - Name: win32 - Python: $(Build.BinariesDirectory)\bin\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - amd64: - Name: amd64 - Python: $(Build.BinariesDirectory)\bin\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - arm64: - Name: arm64 - HostArch: amd64 - Python: $(Build.BinariesDirectory)\bin_amd64\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - - steps: - - template: ./checkout.yml - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_$(Name)' - inputs: - artifactName: bin_$(Name) - targetPath: $(Build.BinariesDirectory)\bin - - - template: ./layout-command.yml - - - powershell: | - $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; - Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" - displayName: 'Extract version numbers' - - - powershell: > - $(LayoutCmd) - --copy "$(Build.ArtifactStagingDirectory)\layout" - --zip "$(Build.ArtifactStagingDirectory)\embed\python-$(VersionText)-embed-$(Name).zip" - --preset-embed - displayName: 'Generate embeddable layout' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: layout_embed_$(Name)' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\layout' - artifactName: layout_embed_$(Name) - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: embed' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\embed' - ArtifactName: embed diff --git a/.azure-pipelines/windows-release/stage-layout-full.yml b/.azure-pipelines/windows-release/stage-layout-full.yml deleted file mode 100644 index 343ee1f73c215b..00000000000000 --- a/.azure-pipelines/windows-release/stage-layout-full.yml +++ /dev/null @@ -1,80 +0,0 @@ -parameters: - ARM64TclTk: true - -jobs: -- job: Make_Layouts - displayName: Make layouts - condition: and(succeeded(), eq(variables['DoLayout'], 'true')) - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - win32: - Name: win32 - Python: $(Build.BinariesDirectory)\bin\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 - amd64: - Name: amd64 - Python: $(Build.BinariesDirectory)\bin\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 - arm64: - Name: arm64 - HostArch: amd64 - Python: $(Build.BinariesDirectory)\bin_amd64\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - ${{ if eq(parameters.ARM64TclTk, true) }}: - TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 - - steps: - - template: ./checkout.yml - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_$(Name)' - inputs: - artifactName: bin_$(Name) - targetPath: $(Build.BinariesDirectory)\bin - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_$(Name)_d' - inputs: - artifactName: bin_$(Name)_d - targetPath: $(Build.BinariesDirectory)\bin - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: doc' - inputs: - artifactName: doc - targetPath: $(Build.BinariesDirectory)\doc - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: tcltk_lib_$(Name)' - condition: and(succeeded(), variables['TclLibrary']) - inputs: - artifactName: tcltk_lib_$(Name) - targetPath: $(Build.BinariesDirectory)\tcltk_lib - - - powershell: | - copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force - displayName: 'Copy signed files into sources' - condition: and(succeeded(), variables['SigningCertificate']) - - - template: ./layout-command.yml - - - powershell: | - $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\layout" --preset-default - displayName: 'Generate full layout' - env: - TCL_LIBRARY: $(TclLibrary) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: layout_full_$(Name)' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\layout' - artifactName: layout_full_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-msix.yml b/.azure-pipelines/windows-release/stage-layout-msix.yml deleted file mode 100644 index a44e1ede20326c..00000000000000 --- a/.azure-pipelines/windows-release/stage-layout-msix.yml +++ /dev/null @@ -1,102 +0,0 @@ -parameters: - ARM64TclTk: true - -jobs: -- job: Make_MSIX_Layout - displayName: Make MSIX layout - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - #win32: - # Name: win32 - # Python: $(Build.BinariesDirectory)\bin\python.exe - # PYTHONHOME: $(Build.SourcesDirectory) - # TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 - amd64: - Name: amd64 - Python: $(Build.BinariesDirectory)\bin\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 - arm64: - Name: arm64 - HostArch: amd64 - Python: $(Build.BinariesDirectory)\bin_amd64\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - ${{ if eq(parameters.ARM64TclTk, true) }}: - TclLibrary: $(Build.BinariesDirectory)\tcltk_lib\tcl8 - - steps: - - template: ./checkout.yml - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_$(Name)' - inputs: - artifactName: bin_$(Name) - targetPath: $(Build.BinariesDirectory)\bin - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_$(Name)_d' - inputs: - artifactName: bin_$(Name)_d - targetPath: $(Build.BinariesDirectory)\bin - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: tcltk_lib_$(Name)' - condition: and(succeeded(), variables['TclLibrary']) - inputs: - artifactName: tcltk_lib_$(Name) - targetPath: $(Build.BinariesDirectory)\tcltk_lib - - - powershell: | - copy "$(Build.BinariesDirectory)\bin\Activate.ps1" Lib\venv\scripts\common\Activate.ps1 -Force - displayName: 'Copy signed files into sources' - condition: and(succeeded(), variables['SigningCertificate']) - - - template: ./layout-command.yml - - - powershell: | - Remove-Item "$(Build.ArtifactStagingDirectory)\appx-store" -Recurse -Force -EA 0 - $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx-store" --preset-appx --precompile - displayName: 'Generate store APPX layout' - env: - TCL_LIBRARY: $(TclLibrary) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: layout_appxstore_$(Name)' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\appx-store' - artifactName: layout_appxstore_$(Name) - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: cert' - condition: and(succeeded(), variables['SigningCertificate']) - inputs: - artifactName: cert - targetPath: $(Build.BinariesDirectory)\cert - - - powershell: | - $info = (gc "$(Build.BinariesDirectory)\cert\certinfo.json" | ConvertFrom-JSON) - Write-Host "Side-loadable APPX must be signed with '$($info.Subject)'" - Write-Host "##vso[task.setvariable variable=APPX_DATA_PUBLISHER]$($info.Subject)" - Write-Host "##vso[task.setvariable variable=APPX_DATA_SHA256]$($info.SHA256)" - displayName: 'Override signing parameters' - condition: and(succeeded(), variables['SigningCertificate']) - - - powershell: | - Remove-Item "$(Build.ArtifactStagingDirectory)\appx" -Recurse -Force -EA 0 - $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\appx" --preset-appx --precompile --include-symbols --include-tests - displayName: 'Generate sideloading APPX layout' - env: - TCL_LIBRARY: $(TclLibrary) - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: layout_appx_$(Name)' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\appx' - artifactName: layout_appx_$(Name) diff --git a/.azure-pipelines/windows-release/stage-layout-nuget.yml b/.azure-pipelines/windows-release/stage-layout-nuget.yml deleted file mode 100644 index b60a324dd90e3d..00000000000000 --- a/.azure-pipelines/windows-release/stage-layout-nuget.yml +++ /dev/null @@ -1,52 +0,0 @@ -jobs: -- job: Make_Nuget_Layout - displayName: Make Nuget layout - condition: and(succeeded(), eq(variables['DoNuget'], 'true')) - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - win32: - Name: win32 - Python: $(Build.BinariesDirectory)\bin\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - amd64: - Name: amd64 - Python: $(Build.BinariesDirectory)\bin\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - arm64: - Name: arm64 - HostArch: amd64 - Python: $(Build.BinariesDirectory)\bin_amd64\python.exe - PYTHONHOME: $(Build.SourcesDirectory) - - steps: - - template: ./checkout.yml - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: bin_$(Name)' - inputs: - artifactName: bin_$(Name) - targetPath: $(Build.BinariesDirectory)\bin - - - powershell: | - copy $(Build.BinariesDirectory)\bin\Activate.ps1 Lib\venv\scripts\common\Activate.ps1 -Force - displayName: 'Copy signed files into sources' - condition: and(succeeded(), variables['SigningCertificate']) - - - template: ./layout-command.yml - - - powershell: | - $(LayoutCmd) --copy "$(Build.ArtifactStagingDirectory)\nuget" --preset-nuget - displayName: 'Generate nuget layout' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: layout_nuget_$(Name)' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\nuget' - artifactName: layout_nuget_$(Name) diff --git a/.azure-pipelines/windows-release/stage-msi.yml b/.azure-pipelines/windows-release/stage-msi.yml deleted file mode 100644 index 0566544a6eb63e..00000000000000 --- a/.azure-pipelines/windows-release/stage-msi.yml +++ /dev/null @@ -1,43 +0,0 @@ -parameters: - ARM64TclTk: true - -jobs: -- job: Make_MSI - displayName: Make MSI - condition: and(succeeded(), not(variables['SigningCertificate'])) - - pool: - vmImage: windows-2022 - - variables: - ReleaseUri: http://www.python.org/{arch} - DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} - Py_OutDir: $(Build.BinariesDirectory) - - workspace: - clean: all - - steps: - - template: msi-steps.yml - parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} - -- job: Make_Signed_MSI - displayName: Make signed MSI - condition: and(succeeded(), variables['SigningCertificate']) - - pool: - name: 'Windows Release' - - variables: - ReleaseUri: http://www.python.org/{arch} - DownloadUrl: https://www.python.org/ftp/python/{version}/{arch}{releasename}/{msi} - Py_OutDir: $(Build.BinariesDirectory) - - workspace: - clean: all - - steps: - - template: msi-steps.yml - parameters: - ARM64TclTk: ${{ parameters.ARM64TclTk }} diff --git a/.azure-pipelines/windows-release/stage-pack-msix.yml b/.azure-pipelines/windows-release/stage-pack-msix.yml deleted file mode 100644 index 95988151a03db7..00000000000000 --- a/.azure-pipelines/windows-release/stage-pack-msix.yml +++ /dev/null @@ -1,148 +0,0 @@ -jobs: -- job: Pack_MSIX - displayName: Pack MSIX bundles - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - amd64: - Name: amd64 - Artifact: appx - Suffix: - ShouldSign: true - amd64_store: - Name: amd64 - Artifact: appxstore - Suffix: -store - Upload: true - arm64: - Name: arm64 - Artifact: appx - Suffix: - ShouldSign: true - arm64_store: - Name: arm64 - Artifact: appxstore - Suffix: -store - Upload: true - - steps: - - template: ./checkout.yml - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: layout_$(Artifact)_$(Name)' - inputs: - artifactName: layout_$(Artifact)_$(Name) - targetPath: $(Build.BinariesDirectory)\layout - - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: symbols' - inputs: - artifactName: symbols - downloadPath: $(Build.BinariesDirectory) - - - powershell: | - $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; - Write-Host "##vso[task.setvariable variable=VersionText]$($d.PythonVersion)" - Write-Host "##vso[task.setvariable variable=VersionNumber]$($d.PythonVersionNumber)" - Write-Host "##vso[task.setvariable variable=VersionHex]$($d.PythonVersionHex)" - Write-Host "##vso[task.setvariable variable=VersionUnique]$($d.PythonVersionUnique)" - Write-Host "##vso[task.setvariable variable=Filename]python-$($d.PythonVersion)-$(Name)$(Suffix)" - displayName: 'Extract version numbers' - - - powershell: | - ./Tools/msi/make_appx.ps1 -layout "$(Build.BinariesDirectory)\layout" -msix "$(Build.ArtifactStagingDirectory)\msix\$(Filename).msix" - displayName: 'Build msix' - - - powershell: | - 7z a -tzip "$(Build.ArtifactStagingDirectory)\msix\$(Filename).appxsym" *.pdb - displayName: 'Build appxsym' - workingDirectory: $(Build.BinariesDirectory)\symbols\$(Name) - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: MSIX' - condition: and(succeeded(), or(ne(variables['ShouldSign'], 'true'), not(variables['SigningCertificate']))) - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' - ArtifactName: msix - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: MSIX' - condition: and(succeeded(), and(eq(variables['ShouldSign'], 'true'), variables['SigningCertificate'])) - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\msix' - ArtifactName: unsigned_msix - - - powershell: | - 7z a -tzip "$(Build.ArtifactStagingDirectory)\msixupload\$(Filename).msixupload" * - displayName: 'Build msixupload' - condition: and(succeeded(), eq(variables['Upload'], 'true')) - workingDirectory: $(Build.ArtifactStagingDirectory)\msix - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: MSIXUpload' - condition: and(succeeded(), eq(variables['Upload'], 'true')) - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\msixupload' - ArtifactName: msixupload - - -- job: Sign_MSIX - displayName: Sign side-loadable MSIX bundles - dependsOn: - - Pack_MSIX - condition: and(succeeded(), variables['SigningCertificate']) - - pool: - name: 'Windows Release' - - workspace: - clean: all - - steps: - - template: ./checkout.yml - - template: ./find-sdk.yml - - - powershell: | - $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; - Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" - displayName: 'Update signing description' - condition: and(succeeded(), not(variables['SigningDescription'])) - - - task: DownloadBuildArtifacts@0 - displayName: 'Download Artifact: unsigned_msix' - inputs: - artifactName: unsigned_msix - downloadPath: $(Build.BinariesDirectory) - - # MSIX must be signed and timestamped simultaneously - # - # Getting "Error: SignerSign() failed." (-2147024885/0x8007000b)"? - # It may be that the certificate info collected in stage-sign.yml is wrong. Check that - # you do not have multiple matches for the certificate name you have specified. - - powershell: | - $failed = $true - foreach ($retry in 1..3) { - signtool sign /a /n "$(SigningCertificate)" /fd sha256 /tr http://timestamp.digicert.com/ /td sha256 /d "$(SigningDescription)" (gi *.msix) - if ($?) { - $failed = $false - break - } - sleep 1 - } - if ($failed) { - throw "Failed to sign MSIX" - } - displayName: 'Sign MSIX' - workingDirectory: $(Build.BinariesDirectory)\unsigned_msix - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: MSIX' - inputs: - PathtoPublish: '$(Build.BinariesDirectory)\unsigned_msix' - ArtifactName: msix diff --git a/.azure-pipelines/windows-release/stage-pack-nuget.yml b/.azure-pipelines/windows-release/stage-pack-nuget.yml deleted file mode 100644 index 85b44e389ab5d1..00000000000000 --- a/.azure-pipelines/windows-release/stage-pack-nuget.yml +++ /dev/null @@ -1,66 +0,0 @@ -jobs: -- job: Pack_Nuget - displayName: Pack Nuget bundles - condition: and(succeeded(), eq(variables['DoNuget'], 'true')) - - pool: - name: 'Windows Release' - - workspace: - clean: all - - strategy: - matrix: - amd64: - Name: amd64 - win32: - Name: win32 - arm64: - Name: arm64 - - steps: - - checkout: none - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: layout_nuget_$(Name)' - inputs: - artifactName: layout_nuget_$(Name) - targetPath: $(Build.BinariesDirectory)\layout - - - task: NugetToolInstaller@0 - displayName: 'Install Nuget' - inputs: - versionSpec: '>=5.0' - - - powershell: > - nuget pack - "$(Build.BinariesDirectory)\layout\python.nuspec" - -OutputDirectory $(Build.ArtifactStagingDirectory) - -NoPackageAnalysis - -NonInteractive - condition: and(succeeded(), not(variables['OverrideNugetVersion'])) - displayName: 'Create nuget package' - - - powershell: > - nuget pack - "$(Build.BinariesDirectory)\layout\python.nuspec" - -OutputDirectory $(Build.ArtifactStagingDirectory) - -NoPackageAnalysis - -NonInteractive - -Version "$(OverrideNugetVersion)" - condition: and(succeeded(), variables['OverrideNugetVersion']) - displayName: 'Create nuget package' - - - powershell: | - gci *.nupkg | %{ - nuget sign "$_" -CertificateSubjectName "$(SigningCertificate)" -Timestamper http://timestamp.digicert.com/ -Overwrite - } - displayName: 'Sign nuget package' - workingDirectory: $(Build.ArtifactStagingDirectory) - condition: and(succeeded(), variables['SigningCertificate']) - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: nuget' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)' - ArtifactName: nuget diff --git a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml b/.azure-pipelines/windows-release/stage-publish-nugetorg.yml deleted file mode 100644 index abb9d0f0fd485a..00000000000000 --- a/.azure-pipelines/windows-release/stage-publish-nugetorg.yml +++ /dev/null @@ -1,50 +0,0 @@ -parameters: - BuildToPublish: '' - -jobs: -- job: Publish_Nuget - displayName: Publish Nuget packages - condition: and(succeeded(), eq(variables['DoNuget'], 'true'), ne(variables['SkipNugetPublish'], 'true')) - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - steps: - - checkout: none - - - ${{ if parameters.BuildToPublish }}: - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact from ${{ parameters.BuildToPublish }}' - inputs: - artifactName: nuget - downloadPath: $(Build.BinariesDirectory) - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - ${{ else }}: - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: nuget' - inputs: - artifactName: nuget - downloadPath: $(Build.BinariesDirectory) - - - - powershell: 'gci pythonarm*.nupkg | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' - displayName: 'Prevent publishing ARM64 packages' - workingDirectory: '$(Build.BinariesDirectory)\nuget' - condition: and(succeeded(), ne(variables['PublishARM64'], 'true')) - - - task: NuGetCommand@2 - displayName: Push packages - condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) - inputs: - command: push - packagesToPush: '$(Build.BinariesDirectory)\nuget\*.nupkg' - nuGetFeedType: external - publishFeedCredentials: 'Python on Nuget' diff --git a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml b/.azure-pipelines/windows-release/stage-publish-pythonorg.yml deleted file mode 100644 index 084134e009902e..00000000000000 --- a/.azure-pipelines/windows-release/stage-publish-pythonorg.yml +++ /dev/null @@ -1,192 +0,0 @@ -parameters: - BuildToPublish: '' - -jobs: -- job: Publish_Python - displayName: Publish python.org packages - condition: and(succeeded(), eq(variables['DoMSI'], 'true'), eq(variables['DoEmbed'], 'true'), ne(variables['SkipPythonOrgPublish'], 'true')) - - pool: - #vmImage: windows-2022 - name: 'Windows Release' - - workspace: - clean: all - - steps: - - template: ./checkout.yml - - - task: UsePythonVersion@0 - displayName: 'Use Python 3.6 or later' - inputs: - versionSpec: '>=3.6' - - - ${{ if parameters.BuildToPublish }}: - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact from ${{ parameters.BuildToPublish }}: Doc' - inputs: - artifactName: Doc - targetPath: $(Build.BinariesDirectory)\Doc - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact from ${{ parameters.BuildToPublish }}: msi' - inputs: - artifactName: msi - targetPath: $(Build.BinariesDirectory)\msi - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - # Note that embed is a 'build' artifact, not a 'pipeline' artifact - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact from ${{ parameters.BuildToPublish }}: embed' - inputs: - artifactName: embed - downloadPath: $(Build.BinariesDirectory) - buildType: specific - project: $(System.TeamProject) - pipeline: $(Build.DefinitionName) - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - ${{ else }}: - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: Doc' - inputs: - artifactName: Doc - targetPath: $(Build.BinariesDirectory)\Doc - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: msi' - inputs: - artifactName: msi - targetPath: $(Build.BinariesDirectory)\msi - - # Note that embed is a 'build' artifact, not a 'pipeline' artifact - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: embed' - inputs: - artifactName: embed - downloadPath: $(Build.BinariesDirectory) - - - # Note that ARM64 MSIs are skipped at build when this option is specified - - powershell: 'gci *embed-arm*.zip | %{ Write-Host "Not publishing: $($_.Name)"; gi $_ } | del' - displayName: 'Prevent publishing ARM64 packages' - workingDirectory: '$(Build.BinariesDirectory)\embed' - condition: and(succeeded(), ne(variables['PublishARM64'], 'true')) - - - - task: DownloadSecureFile@1 - name: gpgkey - inputs: - secureFile: 'python-signing.key' - displayName: 'Download GPG key' - - - powershell: | - git clone https://github.com/python/cpython-bin-deps --branch gpg --single-branch --depth 1 --progress -v "gpg" - gpg/gpg2.exe --import "$(gpgkey.secureFilePath)" - $files = gci -File "msi\*\*", "embed\*.zip" - if ("$(DoCHM)" -ieq "true") { - $files = $files + (gci -File "doc\htmlhelp\*.chm") - } - $files.FullName | %{ - gpg/gpg2.exe -ba --batch --passphrase $(GPGPassphrase) $_ - "Made signature for $_" - } - displayName: 'Generate GPG signatures' - workingDirectory: $(Build.BinariesDirectory) - - - powershell: | - $p = gps "gpg-agent" -EA 0 - if ($p) { $p.Kill() } - displayName: 'Kill GPG agent' - condition: true - - - - powershell: > - $(Build.SourcesDirectory)\Tools\msi\uploadrelease.ps1 - -build msi - -user $(PyDotOrgUsername) - -server $(PyDotOrgServer) - -doc_htmlhelp doc\htmlhelp - -embed embed - -skippurge - -skiptest - -skiphash - condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) - workingDirectory: $(Build.BinariesDirectory) - displayName: 'Upload files to python.org' - - - powershell: > - python - "$(Build.SourcesDirectory)\Tools\msi\purge.py" - (gci msi\*\python-*.exe | %{ $_.Name -replace 'python-(.+?)(-|\.exe).+', '$1' } | select -First 1) - workingDirectory: $(Build.BinariesDirectory) - condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) - displayName: 'Purge CDN' - - - powershell: | - $failures = 0 - gci "msi\*\*.exe" -File | %{ - $d = mkdir "tests\$($_.BaseName)" -Force - gci $d -r -File | del - $ic = copy $_ $d -PassThru - "Checking layout for $($ic.Name)" - Start-Process -wait $ic "/passive", "/layout", "$d\layout", "/log", "$d\log\install.log" - if (-not $?) { - Write-Error "Failed to validate layout of $($inst.Name)" - $failures += 1 - } - } - if ($failures) { - Write-Error "Failed to validate $failures installers" - exit 1 - } - condition: and(succeeded(), eq(variables['SigningCertificate'], variables['__RealSigningCertificate'])) - workingDirectory: $(Build.BinariesDirectory) - displayName: 'Test layouts' - - - powershell: | - $files = gci -File "msi\*\*.exe", "embed\*.zip" - if ("$(DoCHM)" -ieq "true") { - $files = $files + (gci -File "doc\htmlhelp\python*.chm") - } - $hashes = $files | ` - Sort-Object Name | ` - Format-Table Name, @{ - Label="MD5"; - Expression={(Get-FileHash $_ -Algorithm MD5).Hash} - }, Length -AutoSize | ` - Out-String -Width 4096 - $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force - $hashes | Out-File "$d\hashes.txt" -Encoding ascii - $hashes - workingDirectory: $(Build.BinariesDirectory) - displayName: 'Generate hashes' - - - powershell: | - "Copying:" - $files = gci -File "msi\*\python*.asc", "embed\*.asc" - if ("$(DoCHM)" -ieq "true") { - $files = $files + (gci -File "doc\htmlhelp\*.asc") - } - $files.FullName - $d = mkdir "$(Build.ArtifactStagingDirectory)\hashes" -Force - move $files $d -Force - gci msi -Directory | %{ move "msi\$_\*.asc" (mkdir "$d\$_" -Force) } - workingDirectory: $(Build.BinariesDirectory) - displayName: 'Copy GPG signatures for build' - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Artifact: hashes' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)\hashes' - artifactName: hashes diff --git a/.azure-pipelines/windows-release/stage-publish-store.yml b/.azure-pipelines/windows-release/stage-publish-store.yml deleted file mode 100644 index 0eae21edaa7f8e..00000000000000 --- a/.azure-pipelines/windows-release/stage-publish-store.yml +++ /dev/null @@ -1,38 +0,0 @@ -parameters: - BuildToPublish: '' - -jobs: -- job: Publish_Store - displayName: Publish Store packages - condition: and(succeeded(), eq(variables['DoMSIX'], 'true'), ne(variables['SkipMSIXPublish'], 'true')) - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - steps: - - checkout: none - - - ${{ if parameters.BuildToPublish }}: - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: msixupload' - inputs: - artifactName: msixupload - downloadPath: $(Build.BinariesDirectory) - buildType: specific - project: cpython - pipeline: Windows-Release - buildVersionToDownload: specific - buildId: ${{ parameters.BuildToPublish }} - - - ${{ else }}: - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: msixupload' - inputs: - artifactName: msixupload - downloadPath: $(Build.BinariesDirectory) - - # TODO: eq(variables['SigningCertificate'], variables['__RealSigningCertificate']) - # If we are not real-signed, DO NOT PUBLISH diff --git a/.azure-pipelines/windows-release/stage-sign.yml b/.azure-pipelines/windows-release/stage-sign.yml deleted file mode 100644 index 4481aa86edc2cc..00000000000000 --- a/.azure-pipelines/windows-release/stage-sign.yml +++ /dev/null @@ -1,130 +0,0 @@ -parameters: - Include: '*.exe, *.dll, *.pyd, *.cat, *.ps1' - Exclude: 'vcruntime*, libffi*, libcrypto*, libssl*' - -jobs: -- job: Sign_Python - displayName: Sign Python binaries - condition: and(succeeded(), variables['SigningCertificate']) - - pool: - name: 'Windows Release' - - workspace: - clean: all - - strategy: - matrix: - win32: - Name: win32 - amd64: - Name: amd64 - arm64: - Name: arm64 - - steps: - - template: ./checkout.yml - - template: ./find-sdk.yml - - - powershell: | - $d = (.\PCbuild\build.bat -V) | %{ if($_ -match '\s+(\w+):\s*(.+)\s*$') { @{$Matches[1] = $Matches[2];} }}; - Write-Host "##vso[task.setvariable variable=SigningDescription]Python $($d.PythonVersion)" - displayName: 'Update signing description' - condition: and(succeeded(), not(variables['SigningDescription'])) - - - powershell: | - Write-Host "##vso[build.addbuildtag]signed" - displayName: 'Add build tags' - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: unsigned_bin_$(Name)' - inputs: - artifactName: unsigned_bin_$(Name) - targetPath: $(Build.BinariesDirectory)\bin - - - powershell: | - copy "$(Build.SourcesDirectory)\Lib\venv\scripts\common\Activate.ps1" . - displayName: 'Copy files from source' - workingDirectory: $(Build.BinariesDirectory)\bin - - - powershell: | - $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) - signtool sign /a /n "$(SigningCertificate)" /fd sha256 /d "$(SigningDescription)" $files - displayName: 'Sign binaries' - workingDirectory: $(Build.BinariesDirectory)\bin - - - powershell: | - $files = (gi ${{ parameters.Include }} -Exclude ${{ parameters.Exclude }}) - $failed = $true - foreach ($retry in 1..10) { - signtool timestamp /tr http://timestamp.digicert.com/ /td sha256 $files - if ($?) { - $failed = $false - break - } - sleep 5 - } - if ($failed) { - Write-Host "##vso[task.logissue type=error]Failed to timestamp files" - } - displayName: 'Timestamp binaries' - workingDirectory: $(Build.BinariesDirectory)\bin - continueOnError: true - - - task: PublishPipelineArtifact@0 - displayName: 'Publish artifact: bin_$(Name)' - inputs: - targetPath: '$(Build.BinariesDirectory)\bin' - artifactName: bin_$(Name) - - -- job: Dump_CertInfo - displayName: Capture certificate info - condition: and(succeeded(), variables['SigningCertificate']) - - pool: - name: 'Windows Release' - - steps: - - checkout: none - - - powershell: | - $m = 'CN=$(SigningCertificate)' - $c = ((gci Cert:\CurrentUser\My), (gci Cert:\LocalMachine\My)) | %{ $_ } | ` - ?{ $_.Subject -match $m -and $_.NotBefore -lt (Get-Date) -and $_.NotAfter -gt (Get-Date) } | ` - select -First 1 - if (-not $c) { - Write-Host "Failed to find certificate for $(SigningCertificate)" - exit - } - $d = mkdir "$(Build.BinariesDirectory)\tmp" -Force - $cf = "$d\cert.cer" - [IO.File]::WriteAllBytes($cf, $c.Export("Cer")) - $csha = (certutil -dump $cf | sls "Cert Hash\(sha256\): (.+)").Matches.Groups[1].Value - - $info = @{ Subject=$c.Subject; SHA256=$csha; } - - $d = mkdir "$(Build.BinariesDirectory)\cert" -Force - $info | ConvertTo-JSON -Compress | Out-File -Encoding utf8 "$d\certinfo.json" - displayName: "Extract certificate info" - - - task: PublishPipelineArtifact@0 - displayName: 'Publish artifact: cert' - inputs: - targetPath: '$(Build.BinariesDirectory)\cert' - artifactName: cert - - -- job: Mark_Unsigned - displayName: Tag unsigned build - condition: and(succeeded(), not(variables['SigningCertificate'])) - - pool: - vmImage: windows-2022 - - steps: - - checkout: none - - - powershell: | - Write-Host "##vso[build.addbuildtag]unsigned" - displayName: 'Add build tag' diff --git a/.azure-pipelines/windows-release/stage-test-embed.yml b/.azure-pipelines/windows-release/stage-test-embed.yml deleted file mode 100644 index 252db959930f2f..00000000000000 --- a/.azure-pipelines/windows-release/stage-test-embed.yml +++ /dev/null @@ -1,41 +0,0 @@ -jobs: -- job: Test_Embed - displayName: Test Embed - condition: and(succeeded(), eq(variables['DoEmbed'], 'true')) - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - win32: - Name: win32 - amd64: - Name: amd64 - - steps: - - checkout: none - - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: embed' - inputs: - artifactName: embed - downloadPath: $(Build.BinariesDirectory) - - - powershell: | - $p = gi "$(Build.BinariesDirectory)\embed\python*embed-$(Name).zip" - Expand-Archive -Path $p -DestinationPath "$(Build.BinariesDirectory)\Python" - $p = gi "$(Build.BinariesDirectory)\Python\python.exe" - Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" - displayName: 'Install Python and add to PATH' - - - script: | - python -c "import sys; print(sys.version)" - displayName: 'Collect version number' - - - script: | - python -m site - displayName: 'Collect site' diff --git a/.azure-pipelines/windows-release/stage-test-msi.yml b/.azure-pipelines/windows-release/stage-test-msi.yml deleted file mode 100644 index a471d05bc6a4a2..00000000000000 --- a/.azure-pipelines/windows-release/stage-test-msi.yml +++ /dev/null @@ -1,108 +0,0 @@ -jobs: -- job: Test_MSI - displayName: Test MSI - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - win32_User: - ExeMatch: 'python-[\dabrc.]+\.exe' - Logs: $(Build.ArtifactStagingDirectory)\logs\win32_User - InstallAllUsers: 0 - win32_Machine: - ExeMatch: 'python-[\dabrc.]+\.exe' - Logs: $(Build.ArtifactStagingDirectory)\logs\win32_Machine - InstallAllUsers: 1 - amd64_User: - ExeMatch: 'python-[\dabrc.]+-amd64\.exe' - Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_User - InstallAllUsers: 0 - amd64_Machine: - ExeMatch: 'python-[\dabrc.]+-amd64\.exe' - Logs: $(Build.ArtifactStagingDirectory)\logs\amd64_Machine - InstallAllUsers: 1 - - steps: - - checkout: none - - - task: DownloadPipelineArtifact@1 - displayName: 'Download artifact: msi' - inputs: - artifactName: msi - targetPath: $(Build.BinariesDirectory)\msi - - - powershell: | - $p = (gci -r *.exe | ?{ $_.Name -match '$(ExeMatch)' } | select -First 1) - Write-Host "##vso[task.setvariable variable=SetupExe]$($p.FullName)" - Write-Host "##vso[task.setvariable variable=SetupExeName]$($p.Name)" - displayName: 'Find installer executable' - workingDirectory: $(Build.BinariesDirectory)\msi - - - script: > - "$(SetupExe)" - /passive - /log "$(Logs)\install\log.txt" - TargetDir="$(Build.BinariesDirectory)\Python" - Include_debug=1 - Include_symbols=1 - InstallAllUsers=$(InstallAllUsers) - displayName: 'Install Python' - - - powershell: | - $p = gi "$(Build.BinariesDirectory)\Python\python.exe" - Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" - displayName: 'Add test Python to PATH' - - - script: | - python -c "import sys; print(sys.version)" - displayName: 'Collect version number' - - - script: | - python -m site - displayName: 'Collect site' - - - powershell: | - gci -r "${env:PROGRAMDATA}\Microsoft\Windows\Start Menu\Programs\Python*" - displayName: 'Capture per-machine Start Menu items' - - powershell: | - gci -r "${env:APPDATA}\Microsoft\Windows\Start Menu\Programs\Python*" - displayName: 'Capture per-user Start Menu items' - - - powershell: | - gci -r "HKLM:\Software\WOW6432Node\Python" - displayName: 'Capture per-machine 32-bit registry' - - powershell: | - gci -r "HKLM:\Software\Python" - displayName: 'Capture per-machine native registry' - - powershell: | - gci -r "HKCU:\Software\Python" - displayName: 'Capture current-user registry' - - - script: | - python -m pip install "azure<0.10" - python -m pip uninstall -y azure python-dateutil six - displayName: 'Test (un)install package' - - - script: | - python -m test -uall -v test_ttk_guionly test_tk test_idle - displayName: 'Test Tkinter and Idle' - - - script: > - "$(SetupExe)" - /passive - /uninstall - /log "$(Logs)\uninstall\log.txt" - displayName: 'Uninstall Python' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: logs' - condition: true - continueOnError: true - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)\logs' - ArtifactName: msi_testlogs diff --git a/.azure-pipelines/windows-release/stage-test-nuget.yml b/.azure-pipelines/windows-release/stage-test-nuget.yml deleted file mode 100644 index c500baf29b4571..00000000000000 --- a/.azure-pipelines/windows-release/stage-test-nuget.yml +++ /dev/null @@ -1,58 +0,0 @@ -jobs: -- job: Test_Nuget - displayName: Test Nuget - condition: and(succeeded(), eq(variables['DoNuget'], 'true')) - - pool: - vmImage: windows-2022 - - workspace: - clean: all - - strategy: - matrix: - win32: - Package: pythonx86 - amd64: - Package: python - - steps: - - checkout: none - - - task: DownloadBuildArtifacts@0 - displayName: 'Download artifact: nuget' - inputs: - artifactName: nuget - downloadPath: $(Build.BinariesDirectory) - - - task: NugetToolInstaller@0 - inputs: - versionSpec: '>= 5' - - - powershell: > - nuget install - $(Package) - -Source "$(Build.BinariesDirectory)\nuget" - -OutputDirectory "$(Build.BinariesDirectory)\install" - -Prerelease - -ExcludeVersion - -NonInteractive - displayName: 'Install Python' - - - powershell: | - $p = gi "$(Build.BinariesDirectory)\install\$(Package)\tools\python.exe" - Write-Host "##vso[task.prependpath]$(Split-Path -Parent $p)" - displayName: 'Add test Python to PATH' - - - script: | - python -c "import sys; print(sys.version)" - displayName: 'Collect version number' - - - script: | - python -m site - displayName: 'Collect site' - - - script: | - python -m pip install "azure<0.10" - python -m pip uninstall -y azure python-dateutil six - displayName: 'Test (un)install package' diff --git a/.azure-pipelines/windows-steps.yml b/.azure-pipelines/windows-steps.yml deleted file mode 100644 index f502c40637c310..00000000000000 --- a/.azure-pipelines/windows-steps.yml +++ /dev/null @@ -1,37 +0,0 @@ -steps: -- checkout: self - clean: false - fetchDepth: 5 - -- powershell: | - # Relocate build outputs outside of source directory to make cleaning faster - Write-Host '##vso[task.setvariable variable=Py_IntDir]$(Build.BinariesDirectory)\obj' - # UNDONE: Do not build to a different directory because of broken tests - Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.SourcesDirectory)\PCbuild' - #Write-Host '##vso[task.setvariable variable=Py_OutDir]$(Build.BinariesDirectory)\bin' - Write-Host '##vso[task.setvariable variable=EXTERNALS_DIR]$(Build.BinariesDirectory)\externals' - displayName: Update build locations - -- script: PCbuild\build.bat -e $(buildOpt) - displayName: 'Build CPython' - env: - IncludeUwp: true - -- script: python.bat -m test.pythoninfo - displayName: 'Display build info' - condition: and(succeeded(), variables['testRunPlatform']) - -- script: PCbuild\rt.bat -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 --junit-xml="$(Build.BinariesDirectory)\test-results.xml" --tempdir="$(Build.BinariesDirectory)\test" - displayName: 'Tests' - condition: and(succeeded(), variables['testRunPlatform']) - env: - PREFIX: $(Py_OutDir)\$(arch) - -- task: PublishTestResults@2 - displayName: 'Publish Test Results' - inputs: - testResultsFiles: '$(Build.BinariesDirectory)\test-results.xml' - mergeTestResults: true - testRunTitle: $(testRunTitle) - platform: $(testRunPlatform) - condition: and(succeededOrFailed(), variables['testRunPlatform']) diff --git a/.github/CONTRIBUTING.rst b/.github/CONTRIBUTING.rst deleted file mode 100644 index f4affee76e1d44..00000000000000 --- a/.github/CONTRIBUTING.rst +++ /dev/null @@ -1,67 +0,0 @@ -Contributing to Python -====================== - -Build Status ------------- - -- main - - + `Stable buildbots `_ - -- 3.9 - - + `Stable buildbots `_ - -- 3.8 - - + `Stable buildbots `_ - -- 3.7 - - + `Stable buildbots `_ - - -Thank You ---------- -First off, thanks for contributing to the maintenance of the Python programming -language and the CPython interpreter! Even if your contribution is not -ultimately accepted, the fact you put time and effort into helping out is -greatly appreciated. - - -Contribution Guidelines ------------------------ -Please read the `devguide `_ for -guidance on how to contribute to this project. The documentation covers -everything from how to build the code to submitting a pull request. There are -also suggestions on how you can most effectively help the project. - -Please be aware that our workflow does deviate slightly from the typical GitHub -project. Details on how to properly submit a pull request are covered in -`Lifecycle of a Pull Request `_. -We utilize various bots and status checks to help with this, so do follow the -comments they leave and their "Details" links, respectively. The key points of -our workflow that are not covered by a bot or status check are: - -- All discussions that are not directly related to the code in the pull request - should happen on `GitHub Issues `_. -- Upon your first non-trivial pull request (which includes documentation changes), - feel free to add yourself to ``Misc/ACKS`` - - -Setting Expectations --------------------- -Due to the fact that this project is entirely volunteer-run (i.e. no one is paid -to work on Python full-time), we unfortunately can make no guarantees as to if -or when a core developer will get around to reviewing your pull request. -If no core developer has done a review or responded to changes made because of a -"changes requested" review, please feel free to email python-dev to ask if -someone could take a look at your pull request. - - -Code of Conduct ---------------- -All interactions for this project are covered by the -`PSF Code of Conduct `_. Everyone is -expected to be open, considerate, and respectful of others no matter their -position within the project. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 71901d4cab2ae5..00000000000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,507 +0,0 @@ -name: Tests - -# gh-84728: "paths-ignore" is not used to skip documentation-only PRs, because -# it prevents to mark a job as mandatory. A PR cannot be merged if a job is -# mandatory but not scheduled because of "paths-ignore". -on: - workflow_dispatch: - push: - branches: - - 'main' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - pull_request: - branches: - - 'main' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}-reusable - cancel-in-progress: true - -jobs: - check_source: - name: 'Check for source changes' - runs-on: ubuntu-latest - timeout-minutes: 10 - outputs: - run-docs: ${{ steps.docs-changes.outputs.run-docs || false }} - run_tests: ${{ steps.check.outputs.run_tests }} - run_ssl_tests: ${{ steps.check.outputs.run_ssl_tests }} - config_hash: ${{ steps.config_hash.outputs.hash }} - steps: - - uses: actions/checkout@v4 - - name: Check for source changes - id: check - run: | - if [ -z "$GITHUB_BASE_REF" ]; then - echo "run_tests=true" >> $GITHUB_OUTPUT - echo "run_ssl_tests=true" >> $GITHUB_OUTPUT - else - git fetch origin $GITHUB_BASE_REF --depth=1 - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) may be more - # reliable than git diff "origin/$GITHUB_BASE_REF.." (2 dots), - # but it requires to download more commits (this job uses - # "git fetch --depth=1"). - # - # git diff "origin/$GITHUB_BASE_REF..." (3 dots) works with Git - # 2.26, but Git 2.28 is stricter and fails with "no merge base". - # - # git diff "origin/$GITHUB_BASE_REF.." (2 dots) should be enough on - # GitHub, since GitHub starts by merging origin/$GITHUB_BASE_REF - # into the PR branch anyway. - # - # https://github.com/python/core-workflow/issues/373 - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qvE '(\.rst$|^Doc|^Misc|^\.pre-commit-config\.yaml$|\.ruff\.toml$)' && echo "run_tests=true" >> $GITHUB_OUTPUT || true - git diff --name-only origin/$GITHUB_BASE_REF.. | grep -qE '(ssl|hashlib|hmac|^.github)' && echo "run_ssl_tests=true" >> $GITHUB_OUTPUT || true - fi - - name: Compute hash for config cache key - id: config_hash - run: | - echo "hash=${{ hashFiles('configure', 'configure.ac', '.github/workflows/build.yml') }}" >> $GITHUB_OUTPUT - - name: Get a list of the changed documentation-related files - if: github.event_name == 'pull_request' - id: changed-docs-files - uses: Ana06/get-changed-files@v2.2.0 - with: - filter: | - Doc/** - Misc/** - .github/workflows/reusable-docs.yml - format: csv # works for paths with spaces - - name: Check for docs changes - if: >- - github.event_name == 'pull_request' - && steps.changed-docs-files.outputs.added_modified_renamed != '' - id: docs-changes - run: | - echo "run-docs=true" >> "${GITHUB_OUTPUT}" - - check-docs: - name: Docs - needs: check_source - if: fromJSON(needs.check_source.outputs.run-docs) - uses: ./.github/workflows/reusable-docs.yml - - check_abi: - name: 'Check if the ABI has changed' - runs-on: ubuntu-20.04 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - - name: Install Dependencies - run: | - sudo ./.github/workflows/posix-deps-apt.sh - sudo apt-get install -yq abigail-tools - - name: Build CPython - env: - CFLAGS: -g3 -O0 - run: | - # Build Python with the libpython dynamic library - ./configure --enable-shared - make -j4 - - name: Check for changes in the ABI - id: check - run: | - if ! make check-abidump; then - echo "Generated ABI file is not up to date." - echo "Please, add the release manager of this branch as a reviewer of this PR." - echo "" - echo "The up to date ABI file should be attached to this build as an artifact." - echo "" - echo "To learn more about this check: https://devguide.python.org/setup/#regenerate-the-abi-dump" - echo "" - exit 1 - fi - - name: Generate updated ABI files - if: ${{ failure() && steps.check.conclusion == 'failure' }} - run: | - make regen-abidump - - uses: actions/upload-artifact@v3 - name: Publish updated ABI files - if: ${{ failure() && steps.check.conclusion == 'failure' }} - with: - name: abi-data - path: ./Doc/data/*.abi - - check_generated_files: - name: 'Check if generated files are up to date' - runs-on: ubuntu-latest - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 - with: - python-version: '3.x' - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }}-${{ env.pythonLocation }} - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Add ccache to PATH - run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Check Autoconf version 2.69 and aclocal 1.16.3 - run: | - grep "Generated by GNU Autoconf 2.69" configure - grep "aclocal 1.16.3" aclocal.m4 - grep -q "runstatedir" configure - grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4 - - name: Configure CPython - run: | - # Build Python with the libpython dynamic library - ./configure --config-cache --with-pydebug --enable-shared - - name: Regenerate autoconf files with container image - run: make regen-configure - - name: Build CPython - run: | - # Deepfreeze will usually cause global objects to be added or removed, - # so we run it before regen-global-objects gets rum (in regen-all). - make regen-deepfreeze - make -j4 regen-all - make regen-stdlib-module-names - - name: Check for changes - run: | - git add -u - changes=$(git status --porcelain) - # Check for changes in regenerated files - if test -n "$changes"; then - echo "Generated files not up to date." - echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)" - echo "configure files must be regenerated with a specific version of autoconf." - echo "$changes" - echo "" - git diff --staged || true - exit 1 - fi - - name: Check exported libpython symbols - run: make smelly - - name: Check limited ABI symbols - run: make check-limited-abi - - build_win32: - name: 'Windows (x86)' - runs-on: windows-latest - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p Win32 - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p Win32 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 - - build_win_amd64: - name: 'Windows (x64)' - runs-on: windows-latest - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Register MSVC problem matcher - run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p x64 - - name: Display build info - run: .\python.bat -m test.pythoninfo - - name: Tests - run: .\PCbuild\rt.bat -p x64 -d -q -uall -u-cpu -rwW --slowest --timeout=1200 -j0 - - build_win_arm64: - name: 'Windows (arm64)' - runs-on: windows-latest - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - IncludeUwp: 'true' - steps: - - uses: actions/checkout@v4 - - name: Register MSVC problem matcher - run: echo "::add-matcher::.github/problem-matchers/msvc.json" - - name: Build CPython - run: .\PCbuild\build.bat -e -d -p arm64 - - build_macos: - name: 'macOS' - runs-on: macos-latest - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - HOMEBREW_NO_ANALYTICS: 1 - HOMEBREW_NO_AUTO_UPDATE: 1 - HOMEBREW_NO_INSTALL_CLEANUP: 1 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v4 - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Install Homebrew dependencies - run: brew install pkg-config openssl@3.0 xz gdbm tcl-tk - - name: Configure CPython - run: | - GDBM_CFLAGS="-I$(brew --prefix gdbm)/include" \ - GDBM_LIBS="-L$(brew --prefix gdbm)/lib -lgdbm" \ - ./configure \ - --config-cache \ - --with-pydebug \ - --prefix=/opt/python-dev \ - --with-openssl="$(brew --prefix openssl@3.0)" - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: make buildbottest TESTOPTS="-j4 -uall,-cpu" - - build_ubuntu: - name: 'Ubuntu' - runs-on: ubuntu-20.04 - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - OPENSSL_VER: 3.0.11 - PYTHONSTRICTEXTENSIONBUILD: 1 - steps: - - uses: actions/checkout@v4 - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Setup directory envs for out-of-tree builds - run: | - echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV - echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV - - name: Create directories for read-only out-of-tree builds - run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR - - name: Bind mount sources read-only - run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: ${{ env.CPYTHON_BUILDDIR }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Configure CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: | - ../cpython-ro-srcdir/configure \ - --config-cache \ - --with-pydebug \ - --with-openssl=$OPENSSL_DIR - - name: Build CPython out-of-tree - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make -j4 - - name: Display build info - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: make pythoninfo - - name: Remount sources writable for tests - # some tests write to srcdir, lack of pyc files slows down testing - run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw - - name: Tests - working-directory: ${{ env.CPYTHON_BUILDDIR }} - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" - - build_ubuntu_ssltests: - name: 'Ubuntu SSL tests with OpenSSL' - runs-on: ubuntu-20.04 - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' && needs.check_source.outputs.run_ssl_tests == 'true' - strategy: - fail-fast: false - matrix: - openssl_ver: [1.1.1w, 3.0.11, 3.1.3] - env: - OPENSSL_VER: ${{ matrix.openssl_ver }} - MULTISSL_DIR: ${{ github.workspace }}/multissl - OPENSSL_DIR: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }} - LD_LIBRARY_PATH: ${{ github.workspace }}/multissl/openssl/${{ matrix.openssl_ver }}/lib - steps: - - uses: actions/checkout@v4 - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Configure CPython - run: ./configure --config-cache --with-pydebug --with-openssl=$OPENSSL_DIR - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: SSL tests - run: ./python Lib/test/ssltests.py - - build_asan: - name: 'Address sanitizer' - runs-on: ubuntu-20.04 - timeout-minutes: 60 - needs: check_source - if: needs.check_source.outputs.run_tests == 'true' - env: - OPENSSL_VER: 3.0.11 - PYTHONSTRICTEXTENSIONBUILD: 1 - ASAN_OPTIONS: detect_leaks=0:allocator_may_return_null=1:handle_segv=0 - steps: - - uses: actions/checkout@v4 - - name: Restore config.cache - uses: actions/cache@v3 - with: - path: config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ needs.check_source.outputs.config_hash }} - - name: Register gcc problem matcher - run: echo "::add-matcher::.github/problem-matchers/gcc.json" - - name: Install Dependencies - run: sudo ./.github/workflows/posix-deps-apt.sh - - name: Set up GCC-10 for ASAN - uses: egor-tensin/setup-gcc@v1 - with: - version: 10 - - name: Configure OpenSSL env vars - run: | - echo "MULTISSL_DIR=${GITHUB_WORKSPACE}/multissl" >> $GITHUB_ENV - echo "OPENSSL_DIR=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}" >> $GITHUB_ENV - echo "LD_LIBRARY_PATH=${GITHUB_WORKSPACE}/multissl/openssl/${OPENSSL_VER}/lib" >> $GITHUB_ENV - - name: 'Restore OpenSSL build' - id: cache-openssl - uses: actions/cache@v3 - with: - path: ./multissl/openssl/${{ env.OPENSSL_VER }} - key: ${{ runner.os }}-multissl-openssl-${{ env.OPENSSL_VER }} - - name: Install OpenSSL - if: steps.cache-openssl.outputs.cache-hit != 'true' - run: python3 Tools/ssl/multissltests.py --steps=library --base-directory $MULTISSL_DIR --openssl $OPENSSL_VER --system Linux - - name: Add ccache to PATH - run: | - echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV - - name: Configure ccache action - uses: hendrikmuhs/ccache-action@v1.2 - - name: Configure CPython - run: ./configure --config-cache --with-address-sanitizer --without-pymalloc - - name: Build CPython - run: make -j4 - - name: Display build info - run: make pythoninfo - - name: Tests - run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu" - - all-required-green: # This job does nothing and is only used for the branch protection - name: All required checks pass - if: always() - - needs: - - check_source # Transitive dependency, needed to access `run_tests` value - - check-docs - - check_generated_files - - build_win32 - - build_win_amd64 - - build_win_arm64 - - build_macos - - build_ubuntu - - build_ubuntu_ssltests - - build_asan - - runs-on: ubuntu-latest - - steps: - - name: Check whether the needed jobs succeeded or failed - uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe - with: - allowed-failures: >- - build_macos, - build_ubuntu_ssltests, - build_win32, - build_win_arm64, - allowed-skips: >- - ${{ - !fromJSON(needs.check_source.outputs.run-docs) - && ' - check-docs, - ' - || '' - }} - ${{ - needs.check_source.outputs.run_tests != 'true' - && ' - check_generated_files, - build_win32, - build_win_amd64, - build_win_arm64, - build_macos, - build_ubuntu, - build_ubuntu_ssltests, - build_asan, - ' - || '' - }} - jobs: ${{ toJSON(needs) }} diff --git a/.github/workflows/build_msi.yml b/.github/workflows/build_msi.yml deleted file mode 100644 index 4c6e2678f716b5..00000000000000 --- a/.github/workflows/build_msi.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: TestsMSI - -on: - workflow_dispatch: - push: - branches: - - 'main' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - - '3.7' - paths: - - 'Tools/msi/**' - pull_request: - branches: - - 'main' - - '3.11' - - '3.10' - - '3.9' - - '3.8' - - '3.7' - paths: - - 'Tools/msi/**' - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} - cancel-in-progress: true - -jobs: - build_win32: - name: 'Windows (x86) Installer' - runs-on: windows-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - name: Build CPython installer - run: .\Tools\msi\build.bat -x86 - - build_win_amd64: - name: 'Windows (x64) Installer' - runs-on: windows-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - name: Build CPython installer - run: .\Tools\msi\build.bat -x64 - - build_win_arm64: - name: 'Windows (ARM64) Installer' - runs-on: windows-latest - timeout-minutes: 60 - steps: - - uses: actions/checkout@v4 - - name: Build CPython installer - run: .\Tools\msi\build.bat -arm64 diff --git a/.gitignore b/.gitignore index d42e111666f1e9..3e46cec91c5eda 100644 --- a/.gitignore +++ b/.gitignore @@ -151,3 +151,10 @@ Python/frozen_modules/MANIFEST # Ignore ./python binary on Unix but still look into ./Python/ directory. /python !/Python/ + +# Nuitka-Python specifics here: +nuget-result* +output +output32 + +dep-build/ diff --git a/Doc/Makefile b/Doc/Makefile deleted file mode 100644 index ca82353eb454f3..00000000000000 --- a/Doc/Makefile +++ /dev/null @@ -1,252 +0,0 @@ -# -# Makefile for Python documentation -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -# - -# You can set these variables from the command line. -PYTHON = python3 -VENVDIR = ./venv -SPHINXBUILD = PATH=$(VENVDIR)/bin:$$PATH sphinx-build -BLURB = PATH=$(VENVDIR)/bin:$$PATH blurb -JOBS = auto -PAPER = -SOURCES = -DISTVERSION = $(shell $(PYTHON) tools/extensions/patchlevel.py) -REQUIREMENTS = requirements.txt -SPHINXERRORHANDLING = -W - -# Internal variables. -PAPEROPT_a4 = -D latex_elements.papersize=a4paper -PAPEROPT_letter = -D latex_elements.papersize=letterpaper - -ALLSPHINXOPTS = -b $(BUILDER) -d build/doctrees $(PAPEROPT_$(PAPER)) -j $(JOBS) \ - $(SPHINXOPTS) $(SPHINXERRORHANDLING) . build/$(BUILDER) $(SOURCES) - -.PHONY: help build html htmlhelp latex text texinfo changes linkcheck \ - suspicious coverage doctest pydoc-topics htmlview clean dist check serve \ - autobuild-dev autobuild-stable venv - -help: - @echo "Please use \`make ' where is one of" - @echo " clean to remove build files" - @echo " venv to create a venv with necessary tools" - @echo " html to make standalone HTML files" - @echo " htmlview to open the index page built by the html target in your browser" - @echo " htmlhelp to make HTML files and a HTML help project" - @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" - @echo " text to make plain text files" - @echo " texinfo to make Texinfo file" - @echo " epub to make EPUB files" - @echo " changes to make an overview over all changed/added/deprecated items" - @echo " linkcheck to check all external links for integrity" - @echo " coverage to check documentation coverage for library and C API" - @echo " doctest to run doctests in the documentation" - @echo " pydoc-topics to regenerate the pydoc topics file" - @echo " dist to create a \"dist\" directory with archived docs for download" - @echo " suspicious to check for suspicious markup in output text" - @echo " check to run a check for frequent markup errors" - -build: - -mkdir -p build -# Look first for a Misc/NEWS file (building from a source release tarball -# or old repo) and use that, otherwise look for a Misc/NEWS.d directory -# (building from a newer repo) and use blurb to generate the NEWS file. - @if [ -f ../Misc/NEWS ] ; then \ - echo "Using existing Misc/NEWS file"; \ - cp ../Misc/NEWS build/NEWS; \ - elif $(BLURB) help >/dev/null 2>&1 && $(SPHINXBUILD) --version >/dev/null 2>&1; then \ - if [ -d ../Misc/NEWS.d ]; then \ - echo "Building NEWS from Misc/NEWS.d with blurb"; \ - $(BLURB) merge -f build/NEWS; \ - else \ - echo "Neither Misc/NEWS.d nor Misc/NEWS found; cannot build docs"; \ - exit 1; \ - fi \ - else \ - echo ""; \ - echo "Missing the required blurb or sphinx-build tools."; \ - echo "Please run 'make venv' to install local copies."; \ - echo ""; \ - exit 1; \ - fi - $(SPHINXBUILD) $(ALLSPHINXOPTS) - @echo - -html: BUILDER = html -html: build - @echo "Build finished. The HTML pages are in build/html." - -htmlhelp: BUILDER = htmlhelp -htmlhelp: build - @echo "Build finished; now you can run HTML Help Workshop with the" \ - "build/htmlhelp/pydoc.hhp project file." - -latex: BUILDER = latex -latex: build - @echo "Build finished; the LaTeX files are in build/latex." - @echo "Run \`make all-pdf' or \`make all-ps' in that directory to" \ - "run these through (pdf)latex." - -text: BUILDER = text -text: build - @echo "Build finished; the text files are in build/text." - -texinfo: BUILDER = texinfo -texinfo: build - @echo "Build finished; the python.texi file is in build/texinfo." - @echo "Run \`make info' in that directory to run it through makeinfo." - -epub: BUILDER = epub -epub: build - @echo "Build finished; the epub files are in build/epub." - -changes: BUILDER = changes -changes: build - @echo "The overview file is in build/changes." - -linkcheck: BUILDER = linkcheck -linkcheck: - @$(MAKE) build BUILDER=$(BUILDER) || { \ - echo "Link check complete; look for any errors in the above output" \ - "or in build/$(BUILDER)/output.txt"; \ - false; } - -suspicious: BUILDER = suspicious -suspicious: - @$(MAKE) build BUILDER=$(BUILDER) || { \ - echo "Suspicious check complete; look for any errors in the above output" \ - "or in build/$(BUILDER)/suspicious.csv. If all issues are false" \ - "positives, append that file to tools/susp-ignored.csv."; \ - false; } - @echo "⚠ make suspicious is deprecated and will be removed soon." - @echo "⚠ Use:" - @echo "⚠ make check" - @echo "⚠ instead." - -coverage: BUILDER = coverage -coverage: build - @echo "Coverage finished; see c.txt and python.txt in build/coverage" - -doctest: BUILDER = doctest -doctest: - @$(MAKE) build BUILDER=$(BUILDER) || { \ - echo "Testing of doctests in the sources finished, look at the" \ - "results in build/doctest/output.txt"; \ - false; } - -pydoc-topics: BUILDER = pydoc-topics -pydoc-topics: build - @echo "Building finished; now run this:" \ - "cp build/pydoc-topics/topics.py ../Lib/pydoc_data/topics.py" - -htmlview: html - $(PYTHON) -c "import os, webbrowser; webbrowser.open('file://' + os.path.realpath('build/html/index.html'))" - -clean: clean-venv - -rm -rf build/* - -clean-venv: - rm -rf $(VENVDIR) - -venv: - @if [ -d $(VENVDIR) ] ; then \ - echo "venv already exists."; \ - echo "To recreate it, remove it first with \`make clean-venv'."; \ - else \ - $(PYTHON) -m venv $(VENVDIR); \ - $(VENVDIR)/bin/python3 -m pip install --upgrade pip; \ - $(VENVDIR)/bin/python3 -m pip install -r $(REQUIREMENTS); \ - echo "The venv has been created in the $(VENVDIR) directory"; \ - fi - -dist: - rm -rf dist - mkdir -p dist - - # archive the HTML - make html - cp -pPR build/html dist/python-$(DISTVERSION)-docs-html - tar -C dist -cf dist/python-$(DISTVERSION)-docs-html.tar python-$(DISTVERSION)-docs-html - bzip2 -9 -k dist/python-$(DISTVERSION)-docs-html.tar - (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-html.zip python-$(DISTVERSION)-docs-html) - rm -r dist/python-$(DISTVERSION)-docs-html - rm dist/python-$(DISTVERSION)-docs-html.tar - - # archive the text build - make text - cp -pPR build/text dist/python-$(DISTVERSION)-docs-text - tar -C dist -cf dist/python-$(DISTVERSION)-docs-text.tar python-$(DISTVERSION)-docs-text - bzip2 -9 -k dist/python-$(DISTVERSION)-docs-text.tar - (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-text.zip python-$(DISTVERSION)-docs-text) - rm -r dist/python-$(DISTVERSION)-docs-text - rm dist/python-$(DISTVERSION)-docs-text.tar - - # archive the A4 latex - rm -rf build/latex - make latex PAPER=a4 - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) - cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-a4.zip - cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-a4.tar.bz2 - - # archive the letter latex - rm -rf build/latex - make latex PAPER=letter - -sed -i 's/makeindex/makeindex -q/' build/latex/Makefile - (cd build/latex; make clean && make all-pdf && make FMT=pdf zip bz2) - cp build/latex/docs-pdf.zip dist/python-$(DISTVERSION)-docs-pdf-letter.zip - cp build/latex/docs-pdf.tar.bz2 dist/python-$(DISTVERSION)-docs-pdf-letter.tar.bz2 - - # copy the epub build - rm -rf build/epub - make epub - cp -pPR build/epub/Python.epub dist/python-$(DISTVERSION)-docs.epub - - # archive the texinfo build - rm -rf build/texinfo - make texinfo - make info --directory=build/texinfo - cp -pPR build/texinfo dist/python-$(DISTVERSION)-docs-texinfo - tar -C dist -cf dist/python-$(DISTVERSION)-docs-texinfo.tar python-$(DISTVERSION)-docs-texinfo - bzip2 -9 -k dist/python-$(DISTVERSION)-docs-texinfo.tar - (cd dist; zip -q -r -9 python-$(DISTVERSION)-docs-texinfo.zip python-$(DISTVERSION)-docs-texinfo) - rm -r dist/python-$(DISTVERSION)-docs-texinfo - rm dist/python-$(DISTVERSION)-docs-texinfo.tar - -check: venv - $(VENVDIR)/bin/python3 -m pre_commit --version > /dev/null || $(VENVDIR)/bin/python3 -m pip install pre-commit - $(VENVDIR)/bin/python3 -m pre_commit run --all-files - -serve: - @echo "The serve target was removed, use htmlview instead (see bpo-36329)" - -# Targets for daily automated doc build -# By default, Sphinx only rebuilds pages where the page content has changed. -# This means it doesn't always pick up changes to preferred link targets, etc -# To ensure such changes are picked up, we build the published docs with -# `-E` (to ignore the cached environment) and `-a` (to ignore already existing -# output files) - -# for development releases: always build -autobuild-dev: - make dist SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' - -# for quick rebuilds (HTML only) -autobuild-dev-html: - make html SPHINXOPTS='$(SPHINXOPTS) -Ea -A daily=1' - -# for stable releases: only build if not in pre-release stage (alpha, beta) -# release candidate downloads are okay, since the stable tree can be in that stage -autobuild-stable: - @case $(DISTVERSION) in *[ab]*) \ - echo "Not building; $(DISTVERSION) is not a release version."; \ - exit 1;; \ - esac - @make autobuild-dev - -autobuild-stable-html: - @case $(DISTVERSION) in *[ab]*) \ - echo "Not building; $(DISTVERSION) is not a release version."; \ - exit 1;; \ - esac - @make autobuild-dev-html diff --git a/Doc/README.rst b/Doc/README.rst deleted file mode 100644 index d67cad79916b38..00000000000000 --- a/Doc/README.rst +++ /dev/null @@ -1,141 +0,0 @@ -Python Documentation README -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This directory contains the reStructuredText (reST) sources to the Python -documentation. You don't need to build them yourself, `prebuilt versions are -available `_. - -Documentation on authoring Python documentation, including information about -both style and markup, is available in the "`Documenting Python -`_" chapter of the -developers guide. - - -Building the docs -================= - -The documentation is built with several tools which are not included in this -tree but are maintained separately and are available from -`PyPI `_. - -* `Sphinx `_ -* `blurb `_ -* `python-docs-theme `_ - -The easiest way to install these tools is to create a virtual environment and -install the tools into there. - -Using make ----------- - -To get started on UNIX, you can create a virtual environment and build -documentation with the commands:: - - make venv - make html - -The virtual environment in the ``venv`` directory will contain all the tools -necessary to build the documentation downloaded and installed from PyPI. -If you'd like to create the virtual environment in a different location, -you can specify it using the ``VENVDIR`` variable. - -You can also skip creating the virtual environment altogether, in which case -the Makefile will look for instances of ``sphinx-build`` and ``blurb`` -installed on your process ``PATH`` (configurable with the ``SPHINXBUILD`` and -``BLURB`` variables). - -On Windows, we try to emulate the Makefile as closely as possible with a -``make.bat`` file. If you need to specify the Python interpreter to use, -set the PYTHON environment variable. - -Available make targets are: - -* "clean", which removes all build files and the virtual environment. - -* "clean-venv", which removes the virtual environment directory. - -* "venv", which creates a virtual environment with all necessary tools - installed. - -* "html", which builds standalone HTML files for offline viewing. - -* "htmlview", which re-uses the "html" builder, but then opens the main page - in your default web browser. - -* "htmlhelp", which builds HTML files and a HTML Help project file usable to - convert them into a single Compiled HTML (.chm) file -- these are popular - under Microsoft Windows, but very handy on every platform. - - To create the CHM file, you need to run the Microsoft HTML Help Workshop - over the generated project (.hhp) file. The make.bat script does this for - you on Windows. - -* "latex", which builds LaTeX source files as input to "pdflatex" to produce - PDF documents. - -* "text", which builds a plain text file for each source file. - -* "epub", which builds an EPUB document, suitable to be viewed on e-book - readers. - -* "linkcheck", which checks all external references to see whether they are - broken, redirected or malformed, and outputs this information to stdout as - well as a plain-text (.txt) file. - -* "changes", which builds an overview over all versionadded/versionchanged/ - deprecated items in the current version. This is meant as a help for the - writer of the "What's New" document. - -* "coverage", which builds a coverage overview for standard library modules and - C API. - -* "pydoc-topics", which builds a Python module containing a dictionary with - plain text documentation for the labels defined in - ``tools/pyspecific.py`` -- pydoc needs these to show topic and keyword help. - -* "suspicious", which checks the parsed markup for text that looks like - malformed and thus unconverted reST. - -* "check", which checks for frequent markup errors. - -* "serve", which serves the build/html directory on port 8000. - -* "dist", (Unix only) which creates distributable archives of HTML, text, - PDF, and EPUB builds. - - -Without make ------------- - -First, install the tool dependencies from PyPI. - -Then, from the ``Doc`` directory, run :: - - sphinx-build -b . build/ - -where ```` is one of html, text, latex, or htmlhelp (for explanations -see the make targets above). - -Deprecation header -================== - -You can define the ``outdated`` variable in ``html_context`` to show a -red banner on each page redirecting to the "latest" version. - -The link points to the same page on ``/3/``, sadly for the moment the -language is lost during the process. - - -Contributing -============ - -Bugs in the content should be reported to the -`Python bug tracker `_. - -Bugs in the toolset should be reported to the tools themselves. - -You can also send a mail to the Python Documentation Team at docs@python.org, -and we will process your request as soon as possible. - -If you want to help the Documentation Team, you are always welcome. Just send -a mail to docs@python.org. diff --git a/Doc/about.rst b/Doc/about.rst deleted file mode 100644 index 5e6160ff2700ed..00000000000000 --- a/Doc/about.rst +++ /dev/null @@ -1,38 +0,0 @@ -===================== -About these documents -===================== - - -These documents are generated from `reStructuredText`_ sources by `Sphinx`_, a -document processor specifically written for the Python documentation. - -.. _reStructuredText: https://docutils.sourceforge.io/rst.html -.. _Sphinx: https://www.sphinx-doc.org/ - -.. In the online version of these documents, you can submit comments and suggest - changes directly on the documentation pages. - -Development of the documentation and its toolchain is an entirely volunteer -effort, just like Python itself. If you want to contribute, please take a -look at the :ref:`reporting-bugs` page for information on how to do so. New -volunteers are always welcome! - -Many thanks go to: - -* Fred L. Drake, Jr., the creator of the original Python documentation toolset - and writer of much of the content; -* the `Docutils `_ project for creating - reStructuredText and the Docutils suite; -* Fredrik Lundh for his Alternative Python Reference project from which Sphinx - got many good ideas. - - -Contributors to the Python Documentation ----------------------------------------- - -Many people have contributed to the Python language, the Python standard -library, and the Python documentation. See :source:`Misc/ACKS` in the Python -source distribution for a partial list of contributors. - -It is only with the input and contributions of the Python community -that Python has such wonderful documentation -- Thank You! diff --git a/Doc/bugs.rst b/Doc/bugs.rst deleted file mode 100644 index d98192b369603e..00000000000000 --- a/Doc/bugs.rst +++ /dev/null @@ -1,108 +0,0 @@ -.. _reporting-bugs: - -***************** -Dealing with Bugs -***************** - -Python is a mature programming language which has established a reputation for -stability. In order to maintain this reputation, the developers would like to -know of any deficiencies you find in Python. - -It can be sometimes faster to fix bugs yourself and contribute patches to -Python as it streamlines the process and involves less people. Learn how to -:ref:`contribute `. - -Documentation bugs -================== - -If you find a bug in this documentation or would like to propose an improvement, -please submit a bug report on the :ref:`tracker `. If you -have a suggestion on how to fix it, include that as well. - -You can also open a discussion item on our -`Documentation Discourse forum `_. - -If you're short on time, you can also email documentation bug reports to -docs@python.org (behavioral bugs can be sent to python-list@python.org). -'docs@' is a mailing list run by volunteers; your request will be noticed, -though it may take a while to be processed. - -.. seealso:: - - `Documentation bugs`_ - A list of documentation bugs that have been submitted to the Python issue tracker. - - `Issue Tracking `_ - Overview of the process involved in reporting an improvement on the tracker. - - `Helping with Documentation `_ - Comprehensive guide for individuals that are interested in contributing to Python documentation. - - `Documentation Translations `_ - A list of GitHub pages for documentation translation and their primary contacts. - - -.. _using-the-tracker: - -Using the Python issue tracker -============================== - -Issue reports for Python itself should be submitted via the GitHub issues -tracker (https://github.com/python/cpython/issues). -The GitHub issues tracker offers a web form which allows pertinent information -to be entered and submitted to the developers. - -The first step in filing a report is to determine whether the problem has -already been reported. The advantage in doing so, aside from saving the -developers' time, is that you learn what has been done to fix it; it may be that -the problem has already been fixed for the next release, or additional -information is needed (in which case you are welcome to provide it if you can!). -To do this, search the tracker using the search box at the top of the page. - -If the problem you're reporting is not already in the list, log in to GitHub. -If you don't already have a GitHub account, create a new account using the -"Sign up" link. -It is not possible to submit a bug report anonymously. - -Being now logged in, you can submit an issue. -Click on the "New issue" button in the top bar to report a new issue. - -The submission form has two fields, "Title" and "Comment". - -For the "Title" field, enter a *very* short description of the problem; -fewer than ten words is good. - -In the "Comment" field, describe the problem in detail, including what you -expected to happen and what did happen. Be sure to include whether any -extension modules were involved, and what hardware and software platform you -were using (including version information as appropriate). - -Each issue report will be reviewed by a developer who will determine what needs to -be done to correct the problem. You will receive an update each time an action is -taken on the issue. - - -.. seealso:: - - `How to Report Bugs Effectively `_ - Article which goes into some detail about how to create a useful bug report. - This describes what kind of information is useful and why it is useful. - - `Bug Writing Guidelines `_ - Information about writing a good bug report. Some of this is specific to the - Mozilla project, but describes general good practices. - -.. _contributing-to-python: - -Getting started contributing to Python yourself -=============================================== - -Beyond just reporting bugs that you find, you are also welcome to submit -patches to fix them. You can find more information on how to get started -patching Python in the `Python Developer's Guide`_. If you have questions, -the `core-mentorship mailing list`_ is a friendly place to get answers to -any and all questions pertaining to the process of fixing issues in Python. - -.. _Documentation bugs: https://github.com/python/cpython/issues?q=is%3Aissue+is%3Aopen+label%3Adocs -.. _Python Developer's Guide: https://devguide.python.org/ -.. _core-mentorship mailing list: https://mail.python.org/mailman3/lists/core-mentorship.python.org/ diff --git a/Doc/c-api/abstract.rst b/Doc/c-api/abstract.rst deleted file mode 100644 index 1823f9d70c79f3..00000000000000 --- a/Doc/c-api/abstract.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. highlight:: c - -.. _abstract: - -********************** -Abstract Objects Layer -********************** - -The functions in this chapter interact with Python objects regardless of their -type, or with wide classes of object types (e.g. all numerical types, or all -sequence types). When used on object types for which they do not apply, they -will raise a Python exception. - -It is not possible to use these functions on objects that are not properly -initialized, such as a list object that has been created by :c:func:`PyList_New`, -but whose items have not been set to some non-\ ``NULL`` value yet. - -.. toctree:: - - object.rst - call.rst - number.rst - sequence.rst - mapping.rst - iter.rst - buffer.rst - objbuffer.rst diff --git a/Doc/c-api/allocation.rst b/Doc/c-api/allocation.rst deleted file mode 100644 index b3609c233156b6..00000000000000 --- a/Doc/c-api/allocation.rst +++ /dev/null @@ -1,75 +0,0 @@ -.. highlight:: c - -.. _allocating-objects: - -Allocating Objects on the Heap -============================== - - -.. c:function:: PyObject* _PyObject_New(PyTypeObject *type) - - -.. c:function:: PyVarObject* _PyObject_NewVar(PyTypeObject *type, Py_ssize_t size) - - -.. c:function:: PyObject* PyObject_Init(PyObject *op, PyTypeObject *type) - - Initialize a newly allocated object *op* with its type and initial - reference. Returns the initialized object. If *type* indicates that the - object participates in the cyclic garbage detector, it is added to the - detector's set of observed objects. Other fields of the object are not - affected. - - -.. c:function:: PyVarObject* PyObject_InitVar(PyVarObject *op, PyTypeObject *type, Py_ssize_t size) - - This does everything :c:func:`PyObject_Init` does, and also initializes the - length information for a variable-size object. - - -.. c:macro:: PyObject_New(TYPE, typeobj) - - Allocate a new Python object using the C structure type *TYPE* - and the Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header are not initialized. - The caller will own the only reference to the object - (i.e. its reference count will be one). - The size of the memory allocation is determined from the - :c:member:`~PyTypeObject.tp_basicsize` field of the type object. - - -.. c:macro:: PyObject_NewVar(TYPE, typeobj, size) - - Allocate a new Python object using the C structure type *TYPE* and the - Python type object *typeobj* (``PyTypeObject*``). - Fields not defined by the Python object header - are not initialized. The allocated memory allows for the *TYPE* structure - plus *size* (``Py_ssize_t``) fields of the size - given by the :c:member:`~PyTypeObject.tp_itemsize` field of - *typeobj*. This is useful for implementing objects like tuples, which are - able to determine their size at construction time. Embedding the array of - fields into the same allocation decreases the number of allocations, - improving the memory management efficiency. - - -.. c:function:: void PyObject_Del(void *op) - - Releases memory allocated to an object using :c:macro:`PyObject_New` or - :c:macro:`PyObject_NewVar`. This is normally called from the - :c:member:`~PyTypeObject.tp_dealloc` handler specified in the object's type. The fields of - the object should not be accessed after this call as the memory is no - longer a valid Python object. - - -.. c:var:: PyObject _Py_NoneStruct - - Object which is visible in Python as ``None``. This should only be accessed - using the :c:macro:`Py_None` macro, which evaluates to a pointer to this - object. - - -.. seealso:: - - :c:func:`PyModule_Create` - To allocate and create extension modules. - diff --git a/Doc/c-api/apiabiversion.rst b/Doc/c-api/apiabiversion.rst deleted file mode 100644 index f6c8284daeacb0..00000000000000 --- a/Doc/c-api/apiabiversion.rst +++ /dev/null @@ -1,73 +0,0 @@ -.. highlight:: c - -.. _apiabiversion: - -*********************** -API and ABI Versioning -*********************** - -CPython exposes its version number in the following macros. -Note that these correspond to the version code is **built** with, -not necessarily the version used at **run time**. - -See :ref:`stable` for a discussion of API and ABI stability across versions. - -.. c:macro:: PY_MAJOR_VERSION - - The ``3`` in ``3.4.1a2``. - -.. c:macro:: PY_MINOR_VERSION - - The ``4`` in ``3.4.1a2``. - -.. c:macro:: PY_MICRO_VERSION - - The ``1`` in ``3.4.1a2``. - -.. c:macro:: PY_RELEASE_LEVEL - - The ``a`` in ``3.4.1a2``. - This can be ``0xA`` for alpha, ``0xB`` for beta, ``0xC`` for release - candidate or ``0xF`` for final. - -.. c:macro:: PY_RELEASE_SERIAL - - The ``2`` in ``3.4.1a2``. Zero for final releases. - -.. c:macro:: PY_VERSION_HEX - - The Python version number encoded in a single integer. - - The underlying version information can be found by treating it as a 32 bit - number in the following manner: - - +-------+-------------------------+-------------------------+--------------------------+ - | Bytes | Bits (big endian order) | Meaning | Value for ``3.4.1a2`` | - +=======+=========================+=========================+==========================+ - | 1 | 1-8 | ``PY_MAJOR_VERSION`` | ``0x03`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 2 | 9-16 | ``PY_MINOR_VERSION`` | ``0x04`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 3 | 17-24 | ``PY_MICRO_VERSION`` | ``0x01`` | - +-------+-------------------------+-------------------------+--------------------------+ - | 4 | 25-28 | ``PY_RELEASE_LEVEL`` | ``0xA`` | - + +-------------------------+-------------------------+--------------------------+ - | | 29-32 | ``PY_RELEASE_SERIAL`` | ``0x2`` | - +-------+-------------------------+-------------------------+--------------------------+ - - Thus ``3.4.1a2`` is hexversion ``0x030401a2`` and ``3.10.0`` is - hexversion ``0x030a00f0``. - - Use this for numeric comparisons, e.g. ``#if PY_VERSION_HEX >= ...``. - - This version is also available via the symbol :c:var:`Py_Version`. - -.. c:var:: const unsigned long Py_Version - - The Python runtime version number encoded in a single constant integer, with - the same format as the :c:macro:`PY_VERSION_HEX` macro. - This contains the Python version used at run time. - - .. versionadded:: 3.11 - -All the given macros are defined in :source:`Include/patchlevel.h`. diff --git a/Doc/c-api/arg.rst b/Doc/c-api/arg.rst deleted file mode 100644 index 08d6cf788e299e..00000000000000 --- a/Doc/c-api/arg.rst +++ /dev/null @@ -1,696 +0,0 @@ -.. highlight:: c - -.. _arg-parsing: - -Parsing arguments and building values -===================================== - -These functions are useful when creating your own extensions functions and -methods. Additional information and examples are available in -:ref:`extending-index`. - -The first three of these functions described, :c:func:`PyArg_ParseTuple`, -:c:func:`PyArg_ParseTupleAndKeywords`, and :c:func:`PyArg_Parse`, all use *format -strings* which are used to tell the function about the expected arguments. The -format strings use the same syntax for each of these functions. - ------------------ -Parsing arguments ------------------ - -A format string consists of zero or more "format units." A format unit -describes one Python object; it is usually a single character or a parenthesized -sequence of format units. With a few exceptions, a format unit that is not a -parenthesized sequence normally corresponds to a single address argument to -these functions. In the following description, the quoted form is the format -unit; the entry in (round) parentheses is the Python object type that matches -the format unit; and the entry in [square] brackets is the type of the C -variable(s) whose address should be passed. - -Strings and buffers -------------------- - -These formats allow accessing an object as a contiguous chunk of memory. -You don't have to provide raw storage for the returned unicode or bytes -area. - -Unless otherwise stated, buffers are not NUL-terminated. - -There are three ways strings and buffers can be converted to C: - -* Formats such as ``y*`` and ``s*`` fill a :c:type:`Py_buffer` structure. - This locks the underlying buffer so that the caller can subsequently use - the buffer even inside a :c:type:`Py_BEGIN_ALLOW_THREADS` - block without the risk of mutable data being resized or destroyed. - As a result, **you have to call** :c:func:`PyBuffer_Release` after you have - finished processing the data (or in any early abort case). - -* The ``es``, ``es#``, ``et`` and ``et#`` formats allocate the result buffer. - **You have to call** :c:func:`PyMem_Free` after you have finished - processing the data (or in any early abort case). - -* .. _c-arg-borrowed-buffer: - - Other formats take a :class:`str` or a read-only :term:`bytes-like object`, - such as :class:`bytes`, and provide a ``const char *`` pointer to - its buffer. - In this case the buffer is "borrowed": it is managed by the corresponding - Python object, and shares the lifetime of this object. - You won't have to release any memory yourself. - - To ensure that the underlying buffer may be safely borrowed, the object's - :c:member:`PyBufferProcs.bf_releasebuffer` field must be ``NULL``. - This disallows common mutable objects such as :class:`bytearray`, - but also some read-only objects such as :class:`memoryview` of - :class:`bytes`. - - Besides this ``bf_releasebuffer`` requirement, there is no check to verify - whether the input object is immutable (e.g. whether it would honor a request - for a writable buffer, or whether another thread can mutate the data). - -.. note:: - - For all ``#`` variants of formats (``s#``, ``y#``, etc.), the macro - :c:macro:`PY_SSIZE_T_CLEAN` must be defined before including - :file:`Python.h`. On Python 3.9 and older, the type of the length argument - is :c:type:`Py_ssize_t` if the :c:macro:`PY_SSIZE_T_CLEAN` macro is defined, - or int otherwise. - - -``s`` (:class:`str`) [const char \*] - Convert a Unicode object to a C pointer to a character string. - A pointer to an existing string is stored in the character pointer - variable whose address you pass. The C string is NUL-terminated. - The Python string must not contain embedded null code points; if it does, - a :exc:`ValueError` exception is raised. Unicode objects are converted - to C strings using ``'utf-8'`` encoding. If this conversion fails, a - :exc:`UnicodeError` is raised. - - .. note:: - This format does not accept :term:`bytes-like objects - `. If you want to accept - filesystem paths and convert them to C character strings, it is - preferable to use the ``O&`` format with :c:func:`PyUnicode_FSConverter` - as *converter*. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when embedded null code points - were encountered in the Python string. - -``s*`` (:class:`str` or :term:`bytes-like object`) [Py_buffer] - This format accepts Unicode objects as well as bytes-like objects. - It fills a :c:type:`Py_buffer` structure provided by the caller. - In this case the resulting C string may contain embedded NUL bytes. - Unicode objects are converted to C strings using ``'utf-8'`` encoding. - -``s#`` (:class:`str`, read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] - Like ``s*``, except that it provides a :ref:`borrowed buffer `. - The result is stored into two C variables, - the first one a pointer to a C string, the second one its length. - The string may contain embedded null bytes. Unicode objects are converted - to C strings using ``'utf-8'`` encoding. - -``z`` (:class:`str` or ``None``) [const char \*] - Like ``s``, but the Python object may also be ``None``, in which case the C - pointer is set to ``NULL``. - -``z*`` (:class:`str`, :term:`bytes-like object` or ``None``) [Py_buffer] - Like ``s*``, but the Python object may also be ``None``, in which case the - ``buf`` member of the :c:type:`Py_buffer` structure is set to ``NULL``. - -``z#`` (:class:`str`, read-only :term:`bytes-like object` or ``None``) [const char \*, :c:type:`Py_ssize_t`] - Like ``s#``, but the Python object may also be ``None``, in which case the C - pointer is set to ``NULL``. - -``y`` (read-only :term:`bytes-like object`) [const char \*] - This format converts a bytes-like object to a C pointer to a - :ref:`borrowed ` character string; - it does not accept Unicode objects. The bytes buffer must not - contain embedded null bytes; if it does, a :exc:`ValueError` - exception is raised. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when embedded null bytes were - encountered in the bytes buffer. - -``y*`` (:term:`bytes-like object`) [Py_buffer] - This variant on ``s*`` doesn't accept Unicode objects, only - bytes-like objects. **This is the recommended way to accept - binary data.** - -``y#`` (read-only :term:`bytes-like object`) [const char \*, :c:type:`Py_ssize_t`] - This variant on ``s#`` doesn't accept Unicode objects, only bytes-like - objects. - -``S`` (:class:`bytes`) [PyBytesObject \*] - Requires that the Python object is a :class:`bytes` object, without - attempting any conversion. Raises :exc:`TypeError` if the object is not - a bytes object. The C variable may also be declared as :c:expr:`PyObject*`. - -``Y`` (:class:`bytearray`) [PyByteArrayObject \*] - Requires that the Python object is a :class:`bytearray` object, without - attempting any conversion. Raises :exc:`TypeError` if the object is not - a :class:`bytearray` object. The C variable may also be declared as :c:expr:`PyObject*`. - -``u`` (:class:`str`) [const Py_UNICODE \*] - Convert a Python Unicode object to a C pointer to a NUL-terminated buffer of - Unicode characters. You must pass the address of a :c:type:`Py_UNICODE` - pointer variable, which will be filled with the pointer to an existing - Unicode buffer. Please note that the width of a :c:type:`Py_UNICODE` - character depends on compilation options (it is either 16 or 32 bits). - The Python string must not contain embedded null code points; if it does, - a :exc:`ValueError` exception is raised. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when embedded null code points - were encountered in the Python string. - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsWideCharString`. - -``u#`` (:class:`str`) [const Py_UNICODE \*, :c:type:`Py_ssize_t`] - This variant on ``u`` stores into two C variables, the first one a pointer to a - Unicode data buffer, the second one its length. This variant allows - null code points. - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsWideCharString`. - -``Z`` (:class:`str` or ``None``) [const Py_UNICODE \*] - Like ``u``, but the Python object may also be ``None``, in which case the - :c:type:`Py_UNICODE` pointer is set to ``NULL``. - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsWideCharString`. - -``Z#`` (:class:`str` or ``None``) [const Py_UNICODE \*, :c:type:`Py_ssize_t`] - Like ``u#``, but the Python object may also be ``None``, in which case the - :c:type:`Py_UNICODE` pointer is set to ``NULL``. - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style :c:type:`Py_UNICODE` API; please migrate to using - :c:func:`PyUnicode_AsWideCharString`. - -``U`` (:class:`str`) [PyObject \*] - Requires that the Python object is a Unicode object, without attempting - any conversion. Raises :exc:`TypeError` if the object is not a Unicode - object. The C variable may also be declared as :c:expr:`PyObject*`. - -``w*`` (read-write :term:`bytes-like object`) [Py_buffer] - This format accepts any object which implements the read-write buffer - interface. It fills a :c:type:`Py_buffer` structure provided by the caller. - The buffer may contain embedded null bytes. The caller have to call - :c:func:`PyBuffer_Release` when it is done with the buffer. - -``es`` (:class:`str`) [const char \*encoding, char \*\*buffer] - This variant on ``s`` is used for encoding Unicode into a character buffer. - It only works for encoded data without embedded NUL bytes. - - This format requires two arguments. The first is only used as input, and - must be a :c:expr:`const char*` which points to the name of an encoding as a - NUL-terminated string, or ``NULL``, in which case ``'utf-8'`` encoding is used. - An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:expr:`char**`; the value of the pointer it - references will be set to a buffer with the contents of the argument text. - The text will be encoded in the encoding specified by the first argument. - - :c:func:`PyArg_ParseTuple` will allocate a buffer of the needed size, copy the - encoded data into this buffer and adjust *\*buffer* to reference the newly - allocated storage. The caller is responsible for calling :c:func:`PyMem_Free` to - free the allocated buffer after use. - -``et`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer] - Same as ``es`` except that byte string objects are passed through without - recoding them. Instead, the implementation assumes that the byte string object uses - the encoding passed in as parameter. - -``es#`` (:class:`str`) [const char \*encoding, char \*\*buffer, :c:type:`Py_ssize_t` \*buffer_length] - This variant on ``s#`` is used for encoding Unicode into a character buffer. - Unlike the ``es`` format, this variant allows input data which contains NUL - characters. - - It requires three arguments. The first is only used as input, and must be a - :c:expr:`const char*` which points to the name of an encoding as a - NUL-terminated string, or ``NULL``, in which case ``'utf-8'`` encoding is used. - An exception is raised if the named encoding is not known to Python. The - second argument must be a :c:expr:`char**`; the value of the pointer it - references will be set to a buffer with the contents of the argument text. - The text will be encoded in the encoding specified by the first argument. - The third argument must be a pointer to an integer; the referenced integer - will be set to the number of bytes in the output buffer. - - There are two modes of operation: - - If *\*buffer* points a ``NULL`` pointer, the function will allocate a buffer of - the needed size, copy the encoded data into this buffer and set *\*buffer* to - reference the newly allocated storage. The caller is responsible for calling - :c:func:`PyMem_Free` to free the allocated buffer after usage. - - If *\*buffer* points to a non-``NULL`` pointer (an already allocated buffer), - :c:func:`PyArg_ParseTuple` will use this location as the buffer and interpret the - initial value of *\*buffer_length* as the buffer size. It will then copy the - encoded data into the buffer and NUL-terminate it. If the buffer is not large - enough, a :exc:`ValueError` will be set. - - In both cases, *\*buffer_length* is set to the length of the encoded data - without the trailing NUL byte. - -``et#`` (:class:`str`, :class:`bytes` or :class:`bytearray`) [const char \*encoding, char \*\*buffer, :c:type:`Py_ssize_t` \*buffer_length] - Same as ``es#`` except that byte string objects are passed through without recoding - them. Instead, the implementation assumes that the byte string object uses the - encoding passed in as parameter. - -Numbers -------- - -``b`` (:class:`int`) [unsigned char] - Convert a nonnegative Python integer to an unsigned tiny int, stored in a C - :c:expr:`unsigned char`. - -``B`` (:class:`int`) [unsigned char] - Convert a Python integer to a tiny int without overflow checking, stored in a C - :c:expr:`unsigned char`. - -``h`` (:class:`int`) [short int] - Convert a Python integer to a C :c:expr:`short int`. - -``H`` (:class:`int`) [unsigned short int] - Convert a Python integer to a C :c:expr:`unsigned short int`, without overflow - checking. - -``i`` (:class:`int`) [int] - Convert a Python integer to a plain C :c:expr:`int`. - -``I`` (:class:`int`) [unsigned int] - Convert a Python integer to a C :c:expr:`unsigned int`, without overflow - checking. - -``l`` (:class:`int`) [long int] - Convert a Python integer to a C :c:expr:`long int`. - -``k`` (:class:`int`) [unsigned long] - Convert a Python integer to a C :c:expr:`unsigned long` without - overflow checking. - -``L`` (:class:`int`) [long long] - Convert a Python integer to a C :c:expr:`long long`. - -``K`` (:class:`int`) [unsigned long long] - Convert a Python integer to a C :c:expr:`unsigned long long` - without overflow checking. - -``n`` (:class:`int`) [:c:type:`Py_ssize_t`] - Convert a Python integer to a C :c:type:`Py_ssize_t`. - -``c`` (:class:`bytes` or :class:`bytearray` of length 1) [char] - Convert a Python byte, represented as a :class:`bytes` or - :class:`bytearray` object of length 1, to a C :c:expr:`char`. - - .. versionchanged:: 3.3 - Allow :class:`bytearray` objects. - -``C`` (:class:`str` of length 1) [int] - Convert a Python character, represented as a :class:`str` object of - length 1, to a C :c:expr:`int`. - -``f`` (:class:`float`) [float] - Convert a Python floating point number to a C :c:expr:`float`. - -``d`` (:class:`float`) [double] - Convert a Python floating point number to a C :c:expr:`double`. - -``D`` (:class:`complex`) [Py_complex] - Convert a Python complex number to a C :c:type:`Py_complex` structure. - -Other objects -------------- - -``O`` (object) [PyObject \*] - Store a Python object (without any conversion) in a C object pointer. The C - program thus receives the actual object that was passed. A new - :term:`strong reference` to the object is not created - (i.e. its reference count is not increased). - The pointer stored is not ``NULL``. - -``O!`` (object) [*typeobject*, PyObject \*] - Store a Python object in a C object pointer. This is similar to ``O``, but - takes two C arguments: the first is the address of a Python type object, the - second is the address of the C variable (of type :c:expr:`PyObject*`) into which - the object pointer is stored. If the Python object does not have the required - type, :exc:`TypeError` is raised. - -.. _o_ampersand: - -``O&`` (object) [*converter*, *anything*] - Convert a Python object to a C variable through a *converter* function. This - takes two arguments: the first is a function, the second is the address of a C - variable (of arbitrary type), converted to :c:expr:`void *`. The *converter* - function in turn is called as follows:: - - status = converter(object, address); - - where *object* is the Python object to be converted and *address* is the - :c:expr:`void*` argument that was passed to the ``PyArg_Parse*`` function. - The returned *status* should be ``1`` for a successful conversion and ``0`` if - the conversion has failed. When the conversion fails, the *converter* function - should raise an exception and leave the content of *address* unmodified. - - If the *converter* returns ``Py_CLEANUP_SUPPORTED``, it may get called a - second time if the argument parsing eventually fails, giving the converter a - chance to release any memory that it had already allocated. In this second - call, the *object* parameter will be ``NULL``; *address* will have the same value - as in the original call. - - .. versionchanged:: 3.1 - ``Py_CLEANUP_SUPPORTED`` was added. - -``p`` (:class:`bool`) [int] - Tests the value passed in for truth (a boolean **p**\ redicate) and converts - the result to its equivalent C true/false integer value. - Sets the int to ``1`` if the expression was true and ``0`` if it was false. - This accepts any valid Python value. See :ref:`truth` for more - information about how Python tests values for truth. - - .. versionadded:: 3.3 - -``(items)`` (:class:`tuple`) [*matching-items*] - The object must be a Python sequence whose length is the number of format units - in *items*. The C arguments must correspond to the individual format units in - *items*. Format units for sequences may be nested. - -It is possible to pass "long" integers (integers whose value exceeds the -platform's :c:macro:`LONG_MAX`) however no proper range checking is done --- the -most significant bits are silently truncated when the receiving field is too -small to receive the value (actually, the semantics are inherited from downcasts -in C --- your mileage may vary). - -A few other characters have a meaning in a format string. These may not occur -inside nested parentheses. They are: - -``|`` - Indicates that the remaining arguments in the Python argument list are optional. - The C variables corresponding to optional arguments should be initialized to - their default value --- when an optional argument is not specified, - :c:func:`PyArg_ParseTuple` does not touch the contents of the corresponding C - variable(s). - -``$`` - :c:func:`PyArg_ParseTupleAndKeywords` only: - Indicates that the remaining arguments in the Python argument list are - keyword-only. Currently, all keyword-only arguments must also be optional - arguments, so ``|`` must always be specified before ``$`` in the format - string. - - .. versionadded:: 3.3 - -``:`` - The list of format units ends here; the string after the colon is used as the - function name in error messages (the "associated value" of the exception that - :c:func:`PyArg_ParseTuple` raises). - -``;`` - The list of format units ends here; the string after the semicolon is used as - the error message *instead* of the default error message. ``:`` and ``;`` - mutually exclude each other. - -Note that any Python object references which are provided to the caller are -*borrowed* references; do not release them -(i.e. do not decrement their reference count)! - -Additional arguments passed to these functions must be addresses of variables -whose type is determined by the format string; these are used to store values -from the input tuple. There are a few cases, as described in the list of format -units above, where these parameters are used as input values; they should match -what is specified for the corresponding format unit in that case. - -For the conversion to succeed, the *arg* object must match the format -and the format must be exhausted. On success, the -``PyArg_Parse*`` functions return true, otherwise they return -false and raise an appropriate exception. When the -``PyArg_Parse*`` functions fail due to conversion failure in one -of the format units, the variables at the addresses corresponding to that -and the following format units are left untouched. - -API Functions -------------- - -.. c:function:: int PyArg_ParseTuple(PyObject *args, const char *format, ...) - - Parse the parameters of a function that takes only positional parameters into - local variables. Returns true on success; on failure, it returns false and - raises the appropriate exception. - - -.. c:function:: int PyArg_VaParse(PyObject *args, const char *format, va_list vargs) - - Identical to :c:func:`PyArg_ParseTuple`, except that it accepts a va_list rather - than a variable number of arguments. - - -.. c:function:: int PyArg_ParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], ...) - - Parse the parameters of a function that takes both positional and keyword - parameters into local variables. The *keywords* argument is a - ``NULL``-terminated array of keyword parameter names. Empty names denote - :ref:`positional-only parameters `. - Returns true on success; on failure, it returns false and raises the - appropriate exception. - - .. versionchanged:: 3.6 - Added support for :ref:`positional-only parameters - `. - - -.. c:function:: int PyArg_VaParseTupleAndKeywords(PyObject *args, PyObject *kw, const char *format, char *keywords[], va_list vargs) - - Identical to :c:func:`PyArg_ParseTupleAndKeywords`, except that it accepts a - va_list rather than a variable number of arguments. - - -.. c:function:: int PyArg_ValidateKeywordArguments(PyObject *) - - Ensure that the keys in the keywords argument dictionary are strings. This - is only needed if :c:func:`PyArg_ParseTupleAndKeywords` is not used, since the - latter already does this check. - - .. versionadded:: 3.2 - - -.. XXX deprecated, will be removed -.. c:function:: int PyArg_Parse(PyObject *args, const char *format, ...) - - Function used to deconstruct the argument lists of "old-style" functions --- - these are functions which use the :const:`METH_OLDARGS` parameter parsing - method, which has been removed in Python 3. This is not recommended for use - in parameter parsing in new code, and most code in the standard interpreter - has been modified to no longer use this for that purpose. It does remain a - convenient way to decompose other tuples, however, and may continue to be - used for that purpose. - - -.. c:function:: int PyArg_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, ...) - - A simpler form of parameter retrieval which does not use a format string to - specify the types of the arguments. Functions which use this method to retrieve - their parameters should be declared as :c:macro:`METH_VARARGS` in function or - method tables. The tuple containing the actual parameters should be passed as - *args*; it must actually be a tuple. The length of the tuple must be at least - *min* and no more than *max*; *min* and *max* may be equal. Additional - arguments must be passed to the function, each of which should be a pointer to a - :c:expr:`PyObject*` variable; these will be filled in with the values from - *args*; they will contain :term:`borrowed references `. - The variables which correspond - to optional parameters not given by *args* will not be filled in; these should - be initialized by the caller. This function returns true on success and false if - *args* is not a tuple or contains the wrong number of elements; an exception - will be set if there was a failure. - - This is an example of the use of this function, taken from the sources for the - :mod:`!_weakref` helper module for weak references:: - - static PyObject * - weakref_ref(PyObject *self, PyObject *args) - { - PyObject *object; - PyObject *callback = NULL; - PyObject *result = NULL; - - if (PyArg_UnpackTuple(args, "ref", 1, 2, &object, &callback)) { - result = PyWeakref_NewRef(object, callback); - } - return result; - } - - The call to :c:func:`PyArg_UnpackTuple` in this example is entirely equivalent to - this call to :c:func:`PyArg_ParseTuple`:: - - PyArg_ParseTuple(args, "O|O:ref", &object, &callback) - - ---------------- -Building values ---------------- - -.. c:function:: PyObject* Py_BuildValue(const char *format, ...) - - Create a new value based on a format string similar to those accepted by the - ``PyArg_Parse*`` family of functions and a sequence of values. Returns - the value or ``NULL`` in the case of an error; an exception will be raised if - ``NULL`` is returned. - - :c:func:`Py_BuildValue` does not always build a tuple. It builds a tuple only if - its format string contains two or more format units. If the format string is - empty, it returns ``None``; if it contains exactly one format unit, it returns - whatever object is described by that format unit. To force it to return a tuple - of size 0 or one, parenthesize the format string. - - When memory buffers are passed as parameters to supply data to build objects, as - for the ``s`` and ``s#`` formats, the required data is copied. Buffers provided - by the caller are never referenced by the objects created by - :c:func:`Py_BuildValue`. In other words, if your code invokes :c:func:`malloc` - and passes the allocated memory to :c:func:`Py_BuildValue`, your code is - responsible for calling :c:func:`free` for that memory once - :c:func:`Py_BuildValue` returns. - - In the following description, the quoted form is the format unit; the entry in - (round) parentheses is the Python object type that the format unit will return; - and the entry in [square] brackets is the type of the C value(s) to be passed. - - The characters space, tab, colon and comma are ignored in format strings (but - not within format units such as ``s#``). This can be used to make long format - strings a tad more readable. - - ``s`` (:class:`str` or ``None``) [const char \*] - Convert a null-terminated C string to a Python :class:`str` object using ``'utf-8'`` - encoding. If the C string pointer is ``NULL``, ``None`` is used. - - ``s#`` (:class:`str` or ``None``) [const char \*, :c:type:`Py_ssize_t`] - Convert a C string and its length to a Python :class:`str` object using ``'utf-8'`` - encoding. If the C string pointer is ``NULL``, the length is ignored and - ``None`` is returned. - - ``y`` (:class:`bytes`) [const char \*] - This converts a C string to a Python :class:`bytes` object. If the C - string pointer is ``NULL``, ``None`` is returned. - - ``y#`` (:class:`bytes`) [const char \*, :c:type:`Py_ssize_t`] - This converts a C string and its lengths to a Python object. If the C - string pointer is ``NULL``, ``None`` is returned. - - ``z`` (:class:`str` or ``None``) [const char \*] - Same as ``s``. - - ``z#`` (:class:`str` or ``None``) [const char \*, :c:type:`Py_ssize_t`] - Same as ``s#``. - - ``u`` (:class:`str`) [const wchar_t \*] - Convert a null-terminated :c:type:`wchar_t` buffer of Unicode (UTF-16 or UCS-4) - data to a Python Unicode object. If the Unicode buffer pointer is ``NULL``, - ``None`` is returned. - - ``u#`` (:class:`str`) [const wchar_t \*, :c:type:`Py_ssize_t`] - Convert a Unicode (UTF-16 or UCS-4) data buffer and its length to a Python - Unicode object. If the Unicode buffer pointer is ``NULL``, the length is ignored - and ``None`` is returned. - - ``U`` (:class:`str` or ``None``) [const char \*] - Same as ``s``. - - ``U#`` (:class:`str` or ``None``) [const char \*, :c:type:`Py_ssize_t`] - Same as ``s#``. - - ``i`` (:class:`int`) [int] - Convert a plain C :c:expr:`int` to a Python integer object. - - ``b`` (:class:`int`) [char] - Convert a plain C :c:expr:`char` to a Python integer object. - - ``h`` (:class:`int`) [short int] - Convert a plain C :c:expr:`short int` to a Python integer object. - - ``l`` (:class:`int`) [long int] - Convert a C :c:expr:`long int` to a Python integer object. - - ``B`` (:class:`int`) [unsigned char] - Convert a C :c:expr:`unsigned char` to a Python integer object. - - ``H`` (:class:`int`) [unsigned short int] - Convert a C :c:expr:`unsigned short int` to a Python integer object. - - ``I`` (:class:`int`) [unsigned int] - Convert a C :c:expr:`unsigned int` to a Python integer object. - - ``k`` (:class:`int`) [unsigned long] - Convert a C :c:expr:`unsigned long` to a Python integer object. - - ``L`` (:class:`int`) [long long] - Convert a C :c:expr:`long long` to a Python integer object. - - ``K`` (:class:`int`) [unsigned long long] - Convert a C :c:expr:`unsigned long long` to a Python integer object. - - ``n`` (:class:`int`) [:c:type:`Py_ssize_t`] - Convert a C :c:type:`Py_ssize_t` to a Python integer. - - ``c`` (:class:`bytes` of length 1) [char] - Convert a C :c:expr:`int` representing a byte to a Python :class:`bytes` object of - length 1. - - ``C`` (:class:`str` of length 1) [int] - Convert a C :c:expr:`int` representing a character to Python :class:`str` - object of length 1. - - ``d`` (:class:`float`) [double] - Convert a C :c:expr:`double` to a Python floating point number. - - ``f`` (:class:`float`) [float] - Convert a C :c:expr:`float` to a Python floating point number. - - ``D`` (:class:`complex`) [Py_complex \*] - Convert a C :c:type:`Py_complex` structure to a Python complex number. - - ``O`` (object) [PyObject \*] - Pass a Python object untouched but create a new - :term:`strong reference` to it - (i.e. its reference count is incremented by one). - If the object passed in is a ``NULL`` pointer, it is assumed - that this was caused because the call producing the argument found an error and - set an exception. Therefore, :c:func:`Py_BuildValue` will return ``NULL`` but won't - raise an exception. If no exception has been raised yet, :exc:`SystemError` is - set. - - ``S`` (object) [PyObject \*] - Same as ``O``. - - ``N`` (object) [PyObject \*] - Same as ``O``, except it doesn't create a new :term:`strong reference`. - Useful when the object is created by a call to an object constructor in the - argument list. - - ``O&`` (object) [*converter*, *anything*] - Convert *anything* to a Python object through a *converter* function. The - function is called with *anything* (which should be compatible with :c:expr:`void*`) - as its argument and should return a "new" Python object, or ``NULL`` if an - error occurred. - - ``(items)`` (:class:`tuple`) [*matching-items*] - Convert a sequence of C values to a Python tuple with the same number of items. - - ``[items]`` (:class:`list`) [*matching-items*] - Convert a sequence of C values to a Python list with the same number of items. - - ``{items}`` (:class:`dict`) [*matching-items*] - Convert a sequence of C values to a Python dictionary. Each pair of consecutive - C values adds one item to the dictionary, serving as key and value, - respectively. - - If there is an error in the format string, the :exc:`SystemError` exception is - set and ``NULL`` returned. - -.. c:function:: PyObject* Py_VaBuildValue(const char *format, va_list vargs) - - Identical to :c:func:`Py_BuildValue`, except that it accepts a va_list - rather than a variable number of arguments. diff --git a/Doc/c-api/bool.rst b/Doc/c-api/bool.rst deleted file mode 100644 index b15ee156c52c21..00000000000000 --- a/Doc/c-api/bool.rst +++ /dev/null @@ -1,53 +0,0 @@ -.. highlight:: c - -.. _boolobjects: - -Boolean Objects ---------------- - -Booleans in Python are implemented as a subclass of integers. There are only -two booleans, :const:`Py_False` and :const:`Py_True`. As such, the normal -creation and deletion functions don't apply to booleans. The following macros -are available, however. - - -.. c:var:: PyTypeObject PyBool_Type - - This instance of :c:type:`PyTypeObject` represents the Python boolean type; it - is the same object as :class:`bool` in the Python layer. - - -.. c:function:: int PyBool_Check(PyObject *o) - - Return true if *o* is of type :c:data:`PyBool_Type`. This function always - succeeds. - - -.. c:var:: PyObject* Py_False - - The Python ``False`` object. This object has no methods. It needs to be - treated just like any other object with respect to reference counts. - - -.. c:var:: PyObject* Py_True - - The Python ``True`` object. This object has no methods. It needs to be treated - just like any other object with respect to reference counts. - - -.. c:macro:: Py_RETURN_FALSE - - Return :const:`Py_False` from a function, properly incrementing its reference - count. - - -.. c:macro:: Py_RETURN_TRUE - - Return :const:`Py_True` from a function, properly incrementing its reference - count. - - -.. c:function:: PyObject* PyBool_FromLong(long v) - - Return a new reference to :const:`Py_True` or :const:`Py_False` depending on the - truth value of *v*. diff --git a/Doc/c-api/buffer.rst b/Doc/c-api/buffer.rst deleted file mode 100644 index e572815ffd6259..00000000000000 --- a/Doc/c-api/buffer.rst +++ /dev/null @@ -1,543 +0,0 @@ -.. highlight:: c - -.. index:: - single: buffer protocol - single: buffer interface; (see buffer protocol) - single: buffer object; (see buffer protocol) - -.. _bufferobjects: - -Buffer Protocol ---------------- - -.. sectionauthor:: Greg Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - - -Certain objects available in Python wrap access to an underlying memory -array or *buffer*. Such objects include the built-in :class:`bytes` and -:class:`bytearray`, and some extension types like :class:`array.array`. -Third-party libraries may define their own types for special purposes, such -as image processing or numeric analysis. - -While each of these types have their own semantics, they share the common -characteristic of being backed by a possibly large memory buffer. It is -then desirable, in some situations, to access that buffer directly and -without intermediate copying. - -Python provides such a facility at the C level in the form of the :ref:`buffer -protocol `. This protocol has two sides: - -.. index:: single: PyBufferProcs - -- on the producer side, a type can export a "buffer interface" which allows - objects of that type to expose information about their underlying buffer. - This interface is described in the section :ref:`buffer-structs`; - -- on the consumer side, several means are available to obtain a pointer to - the raw underlying data of an object (for example a method parameter). - -Simple objects such as :class:`bytes` and :class:`bytearray` expose their -underlying buffer in byte-oriented form. Other forms are possible; for example, -the elements exposed by an :class:`array.array` can be multi-byte values. - -An example consumer of the buffer interface is the :meth:`~io.BufferedIOBase.write` -method of file objects: any object that can export a series of bytes through -the buffer interface can be written to a file. While :meth:`!write` only -needs read-only access to the internal contents of the object passed to it, -other methods such as :meth:`~io.BufferedIOBase.readinto` need write access -to the contents of their argument. The buffer interface allows objects to -selectively allow or reject exporting of read-write and read-only buffers. - -There are two ways for a consumer of the buffer interface to acquire a buffer -over a target object: - -* call :c:func:`PyObject_GetBuffer` with the right parameters; - -* call :c:func:`PyArg_ParseTuple` (or one of its siblings) with one of the - ``y*``, ``w*`` or ``s*`` :ref:`format codes `. - -In both cases, :c:func:`PyBuffer_Release` must be called when the buffer -isn't needed anymore. Failure to do so could lead to various issues such as -resource leaks. - - -.. _buffer-structure: - -Buffer structure -================ - -Buffer structures (or simply "buffers") are useful as a way to expose the -binary data from another object to the Python programmer. They can also be -used as a zero-copy slicing mechanism. Using their ability to reference a -block of memory, it is possible to expose any data to the Python programmer -quite easily. The memory could be a large, constant array in a C extension, -it could be a raw block of memory for manipulation before passing to an -operating system library, or it could be used to pass around structured data -in its native, in-memory format. - -Contrary to most data types exposed by the Python interpreter, buffers -are not :c:type:`PyObject` pointers but rather simple C structures. This -allows them to be created and copied very simply. When a generic wrapper -around a buffer is needed, a :ref:`memoryview ` object -can be created. - -For short instructions how to write an exporting object, see -:ref:`Buffer Object Structures `. For obtaining -a buffer, see :c:func:`PyObject_GetBuffer`. - -.. c:type:: Py_buffer - - .. c:member:: void *buf - - A pointer to the start of the logical structure described by the buffer - fields. This can be any location within the underlying physical memory - block of the exporter. For example, with negative :c:member:`~Py_buffer.strides` - the value may point to the end of the memory block. - - For :term:`contiguous` arrays, the value points to the beginning of - the memory block. - - .. c:member:: PyObject *obj - - A new reference to the exporting object. The reference is owned by - the consumer and automatically released - (i.e. reference count decremented) - and set to ``NULL`` by - :c:func:`PyBuffer_Release`. The field is the equivalent of the return - value of any standard C-API function. - - As a special case, for *temporary* buffers that are wrapped by - :c:func:`PyMemoryView_FromBuffer` or :c:func:`PyBuffer_FillInfo` - this field is ``NULL``. In general, exporting objects MUST NOT - use this scheme. - - .. c:member:: Py_ssize_t len - - ``product(shape) * itemsize``. For contiguous arrays, this is the length - of the underlying memory block. For non-contiguous arrays, it is the length - that the logical structure would have if it were copied to a contiguous - representation. - - Accessing ``((char *)buf)[0] up to ((char *)buf)[len-1]`` is only valid - if the buffer has been obtained by a request that guarantees contiguity. In - most cases such a request will be :c:macro:`PyBUF_SIMPLE` or :c:macro:`PyBUF_WRITABLE`. - - .. c:member:: int readonly - - An indicator of whether the buffer is read-only. This field is controlled - by the :c:macro:`PyBUF_WRITABLE` flag. - - .. c:member:: Py_ssize_t itemsize - - Item size in bytes of a single element. Same as the value of :func:`struct.calcsize` - called on non-``NULL`` :c:member:`~Py_buffer.format` values. - - Important exception: If a consumer requests a buffer without the - :c:macro:`PyBUF_FORMAT` flag, :c:member:`~Py_buffer.format` will - be set to ``NULL``, but :c:member:`~Py_buffer.itemsize` still has - the value for the original format. - - If :c:member:`~Py_buffer.shape` is present, the equality - ``product(shape) * itemsize == len`` still holds and the consumer - can use :c:member:`~Py_buffer.itemsize` to navigate the buffer. - - If :c:member:`~Py_buffer.shape` is ``NULL`` as a result of a :c:macro:`PyBUF_SIMPLE` - or a :c:macro:`PyBUF_WRITABLE` request, the consumer must disregard - :c:member:`~Py_buffer.itemsize` and assume ``itemsize == 1``. - - .. c:member:: const char *format - - A *NUL* terminated string in :mod:`struct` module style syntax describing - the contents of a single item. If this is ``NULL``, ``"B"`` (unsigned bytes) - is assumed. - - This field is controlled by the :c:macro:`PyBUF_FORMAT` flag. - - .. c:member:: int ndim - - The number of dimensions the memory represents as an n-dimensional array. - If it is ``0``, :c:member:`~Py_buffer.buf` points to a single item representing - a scalar. In this case, :c:member:`~Py_buffer.shape`, :c:member:`~Py_buffer.strides` - and :c:member:`~Py_buffer.suboffsets` MUST be ``NULL``. - The maximum number of dimensions is given by :c:macro:`PyBUF_MAX_NDIM`. - - .. c:member:: Py_ssize_t *shape - - An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` - indicating the shape of the memory as an n-dimensional array. Note that - ``shape[0] * ... * shape[ndim-1] * itemsize`` MUST be equal to - :c:member:`~Py_buffer.len`. - - Shape values are restricted to ``shape[n] >= 0``. The case - ``shape[n] == 0`` requires special attention. See `complex arrays`_ - for further information. - - The shape array is read-only for the consumer. - - .. c:member:: Py_ssize_t *strides - - An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim` - giving the number of bytes to skip to get to a new element in each - dimension. - - Stride values can be any integer. For regular arrays, strides are - usually positive, but a consumer MUST be able to handle the case - ``strides[n] <= 0``. See `complex arrays`_ for further information. - - The strides array is read-only for the consumer. - - .. c:member:: Py_ssize_t *suboffsets - - An array of :c:type:`Py_ssize_t` of length :c:member:`~Py_buffer.ndim`. - If ``suboffsets[n] >= 0``, the values stored along the nth dimension are - pointers and the suboffset value dictates how many bytes to add to each - pointer after de-referencing. A suboffset value that is negative - indicates that no de-referencing should occur (striding in a contiguous - memory block). - - If all suboffsets are negative (i.e. no de-referencing is needed), then - this field must be ``NULL`` (the default value). - - This type of array representation is used by the Python Imaging Library - (PIL). See `complex arrays`_ for further information how to access elements - of such an array. - - The suboffsets array is read-only for the consumer. - - .. c:member:: void *internal - - This is for use internally by the exporting object. For example, this - might be re-cast as an integer by the exporter and used to store flags - about whether or not the shape, strides, and suboffsets arrays must be - freed when the buffer is released. The consumer MUST NOT alter this - value. - - -Constants: - -.. c:macro:: PyBUF_MAX_NDIM - - The maximum number of dimensions the memory represents. - Exporters MUST respect this limit, consumers of multi-dimensional - buffers SHOULD be able to handle up to :c:macro:`!PyBUF_MAX_NDIM` dimensions. - Currently set to 64. - - -.. _buffer-request-types: - -Buffer request types -==================== - -Buffers are usually obtained by sending a buffer request to an exporting -object via :c:func:`PyObject_GetBuffer`. Since the complexity of the logical -structure of the memory can vary drastically, the consumer uses the *flags* -argument to specify the exact buffer type it can handle. - -All :c:type:`Py_buffer` fields are unambiguously defined by the request -type. - -request-independent fields -~~~~~~~~~~~~~~~~~~~~~~~~~~ -The following fields are not influenced by *flags* and must always be filled in -with the correct values: :c:member:`~Py_buffer.obj`, :c:member:`~Py_buffer.buf`, -:c:member:`~Py_buffer.len`, :c:member:`~Py_buffer.itemsize`, :c:member:`~Py_buffer.ndim`. - - -readonly, format -~~~~~~~~~~~~~~~~ - - .. c:macro:: PyBUF_WRITABLE - - Controls the :c:member:`~Py_buffer.readonly` field. If set, the exporter - MUST provide a writable buffer or else report failure. Otherwise, the - exporter MAY provide either a read-only or writable buffer, but the choice - MUST be consistent for all consumers. - - .. c:macro:: PyBUF_FORMAT - - Controls the :c:member:`~Py_buffer.format` field. If set, this field MUST - be filled in correctly. Otherwise, this field MUST be ``NULL``. - - -:c:macro:`PyBUF_WRITABLE` can be \|'d to any of the flags in the next section. -Since :c:macro:`PyBUF_SIMPLE` is defined as 0, :c:macro:`PyBUF_WRITABLE` -can be used as a stand-alone flag to request a simple writable buffer. - -:c:macro:`PyBUF_FORMAT` can be \|'d to any of the flags except :c:macro:`PyBUF_SIMPLE`. -The latter already implies format ``B`` (unsigned bytes). - - -shape, strides, suboffsets -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The flags that control the logical structure of the memory are listed -in decreasing order of complexity. Note that each flag contains all bits -of the flags below it. - -.. tabularcolumns:: |p{0.35\linewidth}|l|l|l| - -+-----------------------------+-------+---------+------------+ -| Request | shape | strides | suboffsets | -+=============================+=======+=========+============+ -| .. c:macro:: PyBUF_INDIRECT | yes | yes | if needed | -+-----------------------------+-------+---------+------------+ -| .. c:macro:: PyBUF_STRIDES | yes | yes | NULL | -+-----------------------------+-------+---------+------------+ -| .. c:macro:: PyBUF_ND | yes | NULL | NULL | -+-----------------------------+-------+---------+------------+ -| .. c:macro:: PyBUF_SIMPLE | NULL | NULL | NULL | -+-----------------------------+-------+---------+------------+ - - -.. index:: contiguous, C-contiguous, Fortran contiguous - -contiguity requests -~~~~~~~~~~~~~~~~~~~ - -C or Fortran :term:`contiguity ` can be explicitly requested, -with and without stride information. Without stride information, the buffer -must be C-contiguous. - -.. tabularcolumns:: |p{0.35\linewidth}|l|l|l|l| - -+-----------------------------------+-------+---------+------------+--------+ -| Request | shape | strides | suboffsets | contig | -+===================================+=======+=========+============+========+ -| .. c:macro:: PyBUF_C_CONTIGUOUS | yes | yes | NULL | C | -+-----------------------------------+-------+---------+------------+--------+ -| .. c:macro:: PyBUF_F_CONTIGUOUS | yes | yes | NULL | F | -+-----------------------------------+-------+---------+------------+--------+ -| .. c:macro:: PyBUF_ANY_CONTIGUOUS | yes | yes | NULL | C or F | -+-----------------------------------+-------+---------+------------+--------+ -| :c:macro:`PyBUF_ND` | yes | NULL | NULL | C | -+-----------------------------------+-------+---------+------------+--------+ - - -compound requests -~~~~~~~~~~~~~~~~~ - -All possible requests are fully defined by some combination of the flags in -the previous section. For convenience, the buffer protocol provides frequently -used combinations as single flags. - -In the following table *U* stands for undefined contiguity. The consumer would -have to call :c:func:`PyBuffer_IsContiguous` to determine contiguity. - -.. tabularcolumns:: |p{0.35\linewidth}|l|l|l|l|l|l| - -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| Request | shape | strides | suboffsets | contig | readonly | format | -+===============================+=======+=========+============+========+==========+========+ -| .. c:macro:: PyBUF_FULL | yes | yes | if needed | U | 0 | yes | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_FULL_RO | yes | yes | if needed | U | 1 or 0 | yes | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_RECORDS | yes | yes | NULL | U | 0 | yes | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_RECORDS_RO | yes | yes | NULL | U | 1 or 0 | yes | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_STRIDED | yes | yes | NULL | U | 0 | NULL | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_STRIDED_RO | yes | yes | NULL | U | 1 or 0 | NULL | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_CONTIG | yes | NULL | NULL | C | 0 | NULL | -+-------------------------------+-------+---------+------------+--------+----------+--------+ -| .. c:macro:: PyBUF_CONTIG_RO | yes | NULL | NULL | C | 1 or 0 | NULL | -+-------------------------------+-------+---------+------------+--------+----------+--------+ - - -Complex arrays -============== - -NumPy-style: shape and strides -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The logical structure of NumPy-style arrays is defined by :c:member:`~Py_buffer.itemsize`, -:c:member:`~Py_buffer.ndim`, :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides`. - -If ``ndim == 0``, the memory location pointed to by :c:member:`~Py_buffer.buf` is -interpreted as a scalar of size :c:member:`~Py_buffer.itemsize`. In that case, -both :c:member:`~Py_buffer.shape` and :c:member:`~Py_buffer.strides` are ``NULL``. - -If :c:member:`~Py_buffer.strides` is ``NULL``, the array is interpreted as -a standard n-dimensional C-array. Otherwise, the consumer must access an -n-dimensional array as follows: - -.. code-block:: c - - ptr = (char *)buf + indices[0] * strides[0] + ... + indices[n-1] * strides[n-1]; - item = *((typeof(item) *)ptr); - - -As noted above, :c:member:`~Py_buffer.buf` can point to any location within -the actual memory block. An exporter can check the validity of a buffer with -this function: - -.. code-block:: python - - def verify_structure(memlen, itemsize, ndim, shape, strides, offset): - """Verify that the parameters represent a valid array within - the bounds of the allocated memory: - char *mem: start of the physical memory block - memlen: length of the physical memory block - offset: (char *)buf - mem - """ - if offset % itemsize: - return False - if offset < 0 or offset+itemsize > memlen: - return False - if any(v % itemsize for v in strides): - return False - - if ndim <= 0: - return ndim == 0 and not shape and not strides - if 0 in shape: - return True - - imin = sum(strides[j]*(shape[j]-1) for j in range(ndim) - if strides[j] <= 0) - imax = sum(strides[j]*(shape[j]-1) for j in range(ndim) - if strides[j] > 0) - - return 0 <= offset+imin and offset+imax+itemsize <= memlen - - -PIL-style: shape, strides and suboffsets -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In addition to the regular items, PIL-style arrays can contain pointers -that must be followed in order to get to the next element in a dimension. -For example, the regular three-dimensional C-array ``char v[2][2][3]`` can -also be viewed as an array of 2 pointers to 2 two-dimensional arrays: -``char (*v[2])[2][3]``. In suboffsets representation, those two pointers -can be embedded at the start of :c:member:`~Py_buffer.buf`, pointing -to two ``char x[2][3]`` arrays that can be located anywhere in memory. - - -Here is a function that returns a pointer to the element in an N-D array -pointed to by an N-dimensional index when there are both non-``NULL`` strides -and suboffsets:: - - void *get_item_pointer(int ndim, void *buf, Py_ssize_t *strides, - Py_ssize_t *suboffsets, Py_ssize_t *indices) { - char *pointer = (char*)buf; - int i; - for (i = 0; i < ndim; i++) { - pointer += strides[i] * indices[i]; - if (suboffsets[i] >=0 ) { - pointer = *((char**)pointer) + suboffsets[i]; - } - } - return (void*)pointer; - } - - -Buffer-related functions -======================== - -.. c:function:: int PyObject_CheckBuffer(PyObject *obj) - - Return ``1`` if *obj* supports the buffer interface otherwise ``0``. When ``1`` is - returned, it doesn't guarantee that :c:func:`PyObject_GetBuffer` will - succeed. This function always succeeds. - - -.. c:function:: int PyObject_GetBuffer(PyObject *exporter, Py_buffer *view, int flags) - - Send a request to *exporter* to fill in *view* as specified by *flags*. - If the exporter cannot provide a buffer of the exact type, it MUST raise - :exc:`BufferError`, set ``view->obj`` to ``NULL`` and - return ``-1``. - - On success, fill in *view*, set ``view->obj`` to a new reference - to *exporter* and return 0. In the case of chained buffer providers - that redirect requests to a single object, ``view->obj`` MAY - refer to this object instead of *exporter* (See :ref:`Buffer Object Structures `). - - Successful calls to :c:func:`PyObject_GetBuffer` must be paired with calls - to :c:func:`PyBuffer_Release`, similar to :c:func:`malloc` and :c:func:`free`. - Thus, after the consumer is done with the buffer, :c:func:`PyBuffer_Release` - must be called exactly once. - - -.. c:function:: void PyBuffer_Release(Py_buffer *view) - - Release the buffer *view* and release the :term:`strong reference` - (i.e. decrement the reference count) to the view's supporting object, - ``view->obj``. This function MUST be called when the buffer - is no longer being used, otherwise reference leaks may occur. - - It is an error to call this function on a buffer that was not obtained via - :c:func:`PyObject_GetBuffer`. - - -.. c:function:: Py_ssize_t PyBuffer_SizeFromFormat(const char *format) - - Return the implied :c:member:`~Py_buffer.itemsize` from :c:member:`~Py_buffer.format`. - On error, raise an exception and return -1. - - .. versionadded:: 3.9 - - -.. c:function:: int PyBuffer_IsContiguous(const Py_buffer *view, char order) - - Return ``1`` if the memory defined by the *view* is C-style (*order* is - ``'C'``) or Fortran-style (*order* is ``'F'``) :term:`contiguous` or either one - (*order* is ``'A'``). Return ``0`` otherwise. This function always succeeds. - - -.. c:function:: void* PyBuffer_GetPointer(const Py_buffer *view, const Py_ssize_t *indices) - - Get the memory area pointed to by the *indices* inside the given *view*. - *indices* must point to an array of ``view->ndim`` indices. - - -.. c:function:: int PyBuffer_FromContiguous(const Py_buffer *view, const void *buf, Py_ssize_t len, char fort) - - Copy contiguous *len* bytes from *buf* to *view*. - *fort* can be ``'C'`` or ``'F'`` (for C-style or Fortran-style ordering). - ``0`` is returned on success, ``-1`` on error. - - -.. c:function:: int PyBuffer_ToContiguous(void *buf, const Py_buffer *src, Py_ssize_t len, char order) - - Copy *len* bytes from *src* to its contiguous representation in *buf*. - *order* can be ``'C'`` or ``'F'`` or ``'A'`` (for C-style or Fortran-style - ordering or either one). ``0`` is returned on success, ``-1`` on error. - - This function fails if *len* != *src->len*. - - -.. c:function:: int PyObject_CopyData(PyObject *dest, PyObject *src) - - Copy data from *src* to *dest* buffer. Can convert between C-style and - or Fortran-style buffers. - - ``0`` is returned on success, ``-1`` on error. - -.. c:function:: void PyBuffer_FillContiguousStrides(int ndims, Py_ssize_t *shape, Py_ssize_t *strides, int itemsize, char order) - - Fill the *strides* array with byte-strides of a :term:`contiguous` (C-style if - *order* is ``'C'`` or Fortran-style if *order* is ``'F'``) array of the - given shape with the given number of bytes per element. - - -.. c:function:: int PyBuffer_FillInfo(Py_buffer *view, PyObject *exporter, void *buf, Py_ssize_t len, int readonly, int flags) - - Handle buffer requests for an exporter that wants to expose *buf* of size *len* - with writability set according to *readonly*. *buf* is interpreted as a sequence - of unsigned bytes. - - The *flags* argument indicates the request type. This function always fills in - *view* as specified by flags, unless *buf* has been designated as read-only - and :c:macro:`PyBUF_WRITABLE` is set in *flags*. - - On success, set ``view->obj`` to a new reference to *exporter* and - return 0. Otherwise, raise :exc:`BufferError`, set - ``view->obj`` to ``NULL`` and return ``-1``; - - If this function is used as part of a :ref:`getbufferproc `, - *exporter* MUST be set to the exporting object and *flags* must be passed - unmodified. Otherwise, *exporter* MUST be ``NULL``. diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst deleted file mode 100644 index 456f7d89bca03c..00000000000000 --- a/Doc/c-api/bytearray.rst +++ /dev/null @@ -1,85 +0,0 @@ -.. highlight:: c - -.. _bytearrayobjects: - -Byte Array Objects ------------------- - -.. index:: pair: object; bytearray - - -.. c:type:: PyByteArrayObject - - This subtype of :c:type:`PyObject` represents a Python bytearray object. - - -.. c:var:: PyTypeObject PyByteArray_Type - - This instance of :c:type:`PyTypeObject` represents the Python bytearray type; - it is the same object as :class:`bytearray` in the Python layer. - - -Type check macros -^^^^^^^^^^^^^^^^^ - -.. c:function:: int PyByteArray_Check(PyObject *o) - - Return true if the object *o* is a bytearray object or an instance of a - subtype of the bytearray type. This function always succeeds. - - -.. c:function:: int PyByteArray_CheckExact(PyObject *o) - - Return true if the object *o* is a bytearray object, but not an instance of a - subtype of the bytearray type. This function always succeeds. - - -Direct API functions -^^^^^^^^^^^^^^^^^^^^ - -.. c:function:: PyObject* PyByteArray_FromObject(PyObject *o) - - Return a new bytearray object from any object, *o*, that implements the - :ref:`buffer protocol `. - - -.. c:function:: PyObject* PyByteArray_FromStringAndSize(const char *string, Py_ssize_t len) - - Create a new bytearray object from *string* and its length, *len*. On - failure, ``NULL`` is returned. - - -.. c:function:: PyObject* PyByteArray_Concat(PyObject *a, PyObject *b) - - Concat bytearrays *a* and *b* and return a new bytearray with the result. - - -.. c:function:: Py_ssize_t PyByteArray_Size(PyObject *bytearray) - - Return the size of *bytearray* after checking for a ``NULL`` pointer. - - -.. c:function:: char* PyByteArray_AsString(PyObject *bytearray) - - Return the contents of *bytearray* as a char array after checking for a - ``NULL`` pointer. The returned array always has an extra - null byte appended. - - -.. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) - - Resize the internal buffer of *bytearray* to *len*. - -Macros -^^^^^^ - -These macros trade safety for speed and they don't check pointers. - -.. c:function:: char* PyByteArray_AS_STRING(PyObject *bytearray) - - Similar to :c:func:`PyByteArray_AsString`, but without error checking. - - -.. c:function:: Py_ssize_t PyByteArray_GET_SIZE(PyObject *bytearray) - - Similar to :c:func:`PyByteArray_Size`, but without error checking. diff --git a/Doc/c-api/bytes.rst b/Doc/c-api/bytes.rst deleted file mode 100644 index 33b7d23ff85a3d..00000000000000 --- a/Doc/c-api/bytes.rst +++ /dev/null @@ -1,205 +0,0 @@ -.. highlight:: c - -.. _bytesobjects: - -Bytes Objects -------------- - -These functions raise :exc:`TypeError` when expecting a bytes parameter and -called with a non-bytes parameter. - -.. index:: pair: object; bytes - - -.. c:type:: PyBytesObject - - This subtype of :c:type:`PyObject` represents a Python bytes object. - - -.. c:var:: PyTypeObject PyBytes_Type - - This instance of :c:type:`PyTypeObject` represents the Python bytes type; it - is the same object as :class:`bytes` in the Python layer. - - -.. c:function:: int PyBytes_Check(PyObject *o) - - Return true if the object *o* is a bytes object or an instance of a subtype - of the bytes type. This function always succeeds. - - -.. c:function:: int PyBytes_CheckExact(PyObject *o) - - Return true if the object *o* is a bytes object, but not an instance of a - subtype of the bytes type. This function always succeeds. - - -.. c:function:: PyObject* PyBytes_FromString(const char *v) - - Return a new bytes object with a copy of the string *v* as value on success, - and ``NULL`` on failure. The parameter *v* must not be ``NULL``; it will not be - checked. - - -.. c:function:: PyObject* PyBytes_FromStringAndSize(const char *v, Py_ssize_t len) - - Return a new bytes object with a copy of the string *v* as value and length - *len* on success, and ``NULL`` on failure. If *v* is ``NULL``, the contents of - the bytes object are uninitialized. - - -.. c:function:: PyObject* PyBytes_FromFormat(const char *format, ...) - - Take a C :c:func:`printf`\ -style *format* string and a variable number of - arguments, calculate the size of the resulting Python bytes object and return - a bytes object with the values formatted into it. The variable arguments - must be C types and must correspond exactly to the format characters in the - *format* string. The following format characters are allowed: - - .. % XXX: This should be exactly the same as the table in PyErr_Format. - .. % One should just refer to the other. - .. % XXX: The descriptions for %zd and %zu are wrong, but the truth is complicated - .. % because not all compilers support the %z width modifier -- we fake it - .. % when necessary via interpolating PY_FORMAT_SIZE_T. - - .. tabularcolumns:: |l|l|L| - - +-------------------+---------------+--------------------------------+ - | Format Characters | Type | Comment | - +===================+===============+================================+ - | ``%%`` | *n/a* | The literal % character. | - +-------------------+---------------+--------------------------------+ - | ``%c`` | int | A single byte, | - | | | represented as a C int. | - +-------------------+---------------+--------------------------------+ - | ``%d`` | int | Equivalent to | - | | | ``printf("%d")``. [1]_ | - +-------------------+---------------+--------------------------------+ - | ``%u`` | unsigned int | Equivalent to | - | | | ``printf("%u")``. [1]_ | - +-------------------+---------------+--------------------------------+ - | ``%ld`` | long | Equivalent to | - | | | ``printf("%ld")``. [1]_ | - +-------------------+---------------+--------------------------------+ - | ``%lu`` | unsigned long | Equivalent to | - | | | ``printf("%lu")``. [1]_ | - +-------------------+---------------+--------------------------------+ - | ``%zd`` | :c:type:`\ | Equivalent to | - | | Py_ssize_t` | ``printf("%zd")``. [1]_ | - +-------------------+---------------+--------------------------------+ - | ``%zu`` | size_t | Equivalent to | - | | | ``printf("%zu")``. [1]_ | - +-------------------+---------------+--------------------------------+ - | ``%i`` | int | Equivalent to | - | | | ``printf("%i")``. [1]_ | - +-------------------+---------------+--------------------------------+ - | ``%x`` | int | Equivalent to | - | | | ``printf("%x")``. [1]_ | - +-------------------+---------------+--------------------------------+ - | ``%s`` | const char\* | A null-terminated C character | - | | | array. | - +-------------------+---------------+--------------------------------+ - | ``%p`` | const void\* | The hex representation of a C | - | | | pointer. Mostly equivalent to | - | | | ``printf("%p")`` except that | - | | | it is guaranteed to start with | - | | | the literal ``0x`` regardless | - | | | of what the platform's | - | | | ``printf`` yields. | - +-------------------+---------------+--------------------------------+ - - An unrecognized format character causes all the rest of the format string to be - copied as-is to the result object, and any extra arguments discarded. - - .. [1] For integer specifiers (d, u, ld, lu, zd, zu, i, x): the 0-conversion - flag has effect even when a precision is given. - - -.. c:function:: PyObject* PyBytes_FromFormatV(const char *format, va_list vargs) - - Identical to :c:func:`PyBytes_FromFormat` except that it takes exactly two - arguments. - - -.. c:function:: PyObject* PyBytes_FromObject(PyObject *o) - - Return the bytes representation of object *o* that implements the buffer - protocol. - - -.. c:function:: Py_ssize_t PyBytes_Size(PyObject *o) - - Return the length of the bytes in bytes object *o*. - - -.. c:function:: Py_ssize_t PyBytes_GET_SIZE(PyObject *o) - - Similar to :c:func:`PyBytes_Size`, but without error checking. - - -.. c:function:: char* PyBytes_AsString(PyObject *o) - - Return a pointer to the contents of *o*. The pointer - refers to the internal buffer of *o*, which consists of ``len(o) + 1`` - bytes. The last byte in the buffer is always null, regardless of - whether there are any other null bytes. The data must not be - modified in any way, unless the object was just created using - ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If - *o* is not a bytes object at all, :c:func:`PyBytes_AsString` returns ``NULL`` - and raises :exc:`TypeError`. - - -.. c:function:: char* PyBytes_AS_STRING(PyObject *string) - - Similar to :c:func:`PyBytes_AsString`, but without error checking. - - -.. c:function:: int PyBytes_AsStringAndSize(PyObject *obj, char **buffer, Py_ssize_t *length) - - Return the null-terminated contents of the object *obj* - through the output variables *buffer* and *length*. - - If *length* is ``NULL``, the bytes object - may not contain embedded null bytes; - if it does, the function returns ``-1`` and a :exc:`ValueError` is raised. - - The buffer refers to an internal buffer of *obj*, which includes an - additional null byte at the end (not counted in *length*). The data - must not be modified in any way, unless the object was just created using - ``PyBytes_FromStringAndSize(NULL, size)``. It must not be deallocated. If - *obj* is not a bytes object at all, :c:func:`PyBytes_AsStringAndSize` - returns ``-1`` and raises :exc:`TypeError`. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when embedded null bytes were - encountered in the bytes object. - - -.. c:function:: void PyBytes_Concat(PyObject **bytes, PyObject *newpart) - - Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*; the caller will own the new reference. The reference to - the old value of *bytes* will be stolen. If the new object cannot be - created, the old reference to *bytes* will still be discarded and the value - of *\*bytes* will be set to ``NULL``; the appropriate exception will be set. - - -.. c:function:: void PyBytes_ConcatAndDel(PyObject **bytes, PyObject *newpart) - - Create a new bytes object in *\*bytes* containing the contents of *newpart* - appended to *bytes*. This version releases the :term:`strong reference` - to *newpart* (i.e. decrements its reference count). - - -.. c:function:: int _PyBytes_Resize(PyObject **bytes, Py_ssize_t newsize) - - A way to resize a bytes object even though it is "immutable". Only use this - to build up a brand new bytes object; don't use this if the bytes may already - be known in other parts of the code. It is an error to call this function if - the refcount on the input bytes object is not one. Pass the address of an - existing bytes object as an lvalue (it may be written into), and the new size - desired. On success, *\*bytes* holds the resized bytes object and ``0`` is - returned; the address in *\*bytes* may differ from its input value. If the - reallocation fails, the original bytes object at *\*bytes* is deallocated, - *\*bytes* is set to ``NULL``, :exc:`MemoryError` is set, and ``-1`` is - returned. diff --git a/Doc/c-api/call.rst b/Doc/c-api/call.rst deleted file mode 100644 index df17c2207e94fd..00000000000000 --- a/Doc/c-api/call.rst +++ /dev/null @@ -1,408 +0,0 @@ -.. highlight:: c - -.. _call: - -Call Protocol -============= - -CPython supports two different calling protocols: -*tp_call* and vectorcall. - -The *tp_call* Protocol ----------------------- - -Instances of classes that set :c:member:`~PyTypeObject.tp_call` are callable. -The signature of the slot is:: - - PyObject *tp_call(PyObject *callable, PyObject *args, PyObject *kwargs); - -A call is made using a tuple for the positional arguments -and a dict for the keyword arguments, similarly to -``callable(*args, **kwargs)`` in Python code. -*args* must be non-NULL (use an empty tuple if there are no arguments) -but *kwargs* may be *NULL* if there are no keyword arguments. - -This convention is not only used by *tp_call*: -:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_init` -also pass arguments this way. - -To call an object, use :c:func:`PyObject_Call` or another -:ref:`call API `. - - -.. _vectorcall: - -The Vectorcall Protocol ------------------------ - -.. versionadded:: 3.9 - -The vectorcall protocol was introduced in :pep:`590` as an additional protocol -for making calls more efficient. - -As rule of thumb, CPython will prefer the vectorcall for internal calls -if the callable supports it. However, this is not a hard rule. -Additionally, some third-party extensions use *tp_call* directly -(rather than using :c:func:`PyObject_Call`). -Therefore, a class supporting vectorcall must also implement -:c:member:`~PyTypeObject.tp_call`. -Moreover, the callable must behave the same -regardless of which protocol is used. -The recommended way to achieve this is by setting -:c:member:`~PyTypeObject.tp_call` to :c:func:`PyVectorcall_Call`. -This bears repeating: - -.. warning:: - - A class supporting vectorcall **must** also implement - :c:member:`~PyTypeObject.tp_call` with the same semantics. - -A class should not implement vectorcall if that would be slower -than *tp_call*. For example, if the callee needs to convert -the arguments to an args tuple and kwargs dict anyway, then there is no point -in implementing vectorcall. - -Classes can implement the vectorcall protocol by enabling the -:c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag and setting -:c:member:`~PyTypeObject.tp_vectorcall_offset` to the offset inside the -object structure where a *vectorcallfunc* appears. -This is a pointer to a function with the following signature: - -.. c:type:: PyObject *(*vectorcallfunc)(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) - -- *callable* is the object being called. -- *args* is a C array consisting of the positional arguments followed by the - values of the keyword arguments. - This can be *NULL* if there are no arguments. -- *nargsf* is the number of positional arguments plus possibly the - :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET` flag. - To get the actual number of positional arguments from *nargsf*, - use :c:func:`PyVectorcall_NARGS`. -- *kwnames* is a tuple containing the names of the keyword arguments; - in other words, the keys of the kwargs dict. - These names must be strings (instances of ``str`` or a subclass) - and they must be unique. - If there are no keyword arguments, then *kwnames* can instead be *NULL*. - -.. c:macro:: PY_VECTORCALL_ARGUMENTS_OFFSET - - If this flag is set in a vectorcall *nargsf* argument, the callee is allowed - to temporarily change ``args[-1]``. In other words, *args* points to - argument 1 (not 0) in the allocated vector. - The callee must restore the value of ``args[-1]`` before returning. - - For :c:func:`PyObject_VectorcallMethod`, this flag means instead that - ``args[0]`` may be changed. - - Whenever they can do so cheaply (without additional allocation), callers - are encouraged to use :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET`. - Doing so will allow callables such as bound methods to make their onward - calls (which include a prepended *self* argument) very efficiently. - - .. versionadded:: 3.8 - -To call an object that implements vectorcall, use a :ref:`call API ` -function as with any other callable. -:c:func:`PyObject_Vectorcall` will usually be most efficient. - - -.. note:: - - In CPython 3.8, the vectorcall API and related functions were available - provisionally under names with a leading underscore: - ``_PyObject_Vectorcall``, ``_Py_TPFLAGS_HAVE_VECTORCALL``, - ``_PyObject_VectorcallMethod``, ``_PyVectorcall_Function``, - ``_PyObject_CallOneArg``, ``_PyObject_CallMethodNoArgs``, - ``_PyObject_CallMethodOneArg``. - Additionally, ``PyObject_VectorcallDict`` was available as - ``_PyObject_FastCallDict``. - The old names are still defined as aliases of the new, non-underscored names. - - -Recursion Control -................. - -When using *tp_call*, callees do not need to worry about -:ref:`recursion `: CPython uses -:c:func:`Py_EnterRecursiveCall` and :c:func:`Py_LeaveRecursiveCall` -for calls made using *tp_call*. - -For efficiency, this is not the case for calls done using vectorcall: -the callee should use *Py_EnterRecursiveCall* and *Py_LeaveRecursiveCall* -if needed. - - -Vectorcall Support API -...................... - -.. c:function:: Py_ssize_t PyVectorcall_NARGS(size_t nargsf) - - Given a vectorcall *nargsf* argument, return the actual number of - arguments. - Currently equivalent to:: - - (Py_ssize_t)(nargsf & ~PY_VECTORCALL_ARGUMENTS_OFFSET) - - However, the function ``PyVectorcall_NARGS`` should be used to allow - for future extensions. - - .. versionadded:: 3.8 - -.. c:function:: vectorcallfunc PyVectorcall_Function(PyObject *op) - - If *op* does not support the vectorcall protocol (either because the type - does not or because the specific instance does not), return *NULL*. - Otherwise, return the vectorcall function pointer stored in *op*. - This function never raises an exception. - - This is mostly useful to check whether or not *op* supports vectorcall, - which can be done by checking ``PyVectorcall_Function(op) != NULL``. - - .. versionadded:: 3.9 - -.. c:function:: PyObject* PyVectorcall_Call(PyObject *callable, PyObject *tuple, PyObject *dict) - - Call *callable*'s :c:type:`vectorcallfunc` with positional and keyword - arguments given in a tuple and dict, respectively. - - This is a specialized function, intended to be put in the - :c:member:`~PyTypeObject.tp_call` slot or be used in an implementation of ``tp_call``. - It does not check the :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag - and it does not fall back to ``tp_call``. - - .. versionadded:: 3.8 - - -.. _capi-call: - -Object Calling API ------------------- - -Various functions are available for calling a Python object. -Each converts its arguments to a convention supported by the called object – -either *tp_call* or vectorcall. -In order to do as little conversion as possible, pick one that best fits -the format of data you have available. - -The following table summarizes the available functions; -please see individual documentation for details. - -+------------------------------------------+------------------+--------------------+---------------+ -| Function | callable | args | kwargs | -+==========================================+==================+====================+===============+ -| :c:func:`PyObject_Call` | ``PyObject *`` | tuple | dict/``NULL`` | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_CallNoArgs` | ``PyObject *`` | --- | --- | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_CallOneArg` | ``PyObject *`` | 1 object | --- | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_CallObject` | ``PyObject *`` | tuple/``NULL`` | --- | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_CallFunction` | ``PyObject *`` | format | --- | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_CallMethod` | obj + ``char*`` | format | --- | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_CallFunctionObjArgs` | ``PyObject *`` | variadic | --- | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_CallMethodObjArgs` | obj + name | variadic | --- | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_CallMethodNoArgs` | obj + name | --- | --- | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_CallMethodOneArg` | obj + name | 1 object | --- | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_Vectorcall` | ``PyObject *`` | vectorcall | vectorcall | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_VectorcallDict` | ``PyObject *`` | vectorcall | dict/``NULL`` | -+------------------------------------------+------------------+--------------------+---------------+ -| :c:func:`PyObject_VectorcallMethod` | arg + name | vectorcall | vectorcall | -+------------------------------------------+------------------+--------------------+---------------+ - - -.. c:function:: PyObject* PyObject_Call(PyObject *callable, PyObject *args, PyObject *kwargs) - - Call a callable Python object *callable*, with arguments given by the - tuple *args*, and named arguments given by the dictionary *kwargs*. - - *args* must not be *NULL*; use an empty tuple if no arguments are needed. - If no named arguments are needed, *kwargs* can be *NULL*. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - This is the equivalent of the Python expression: - ``callable(*args, **kwargs)``. - - -.. c:function:: PyObject* PyObject_CallNoArgs(PyObject *callable) - - Call a callable Python object *callable* without any arguments. It is the - most efficient way to call a callable Python object without any argument. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyObject_CallOneArg(PyObject *callable, PyObject *arg) - - Call a callable Python object *callable* with exactly 1 positional argument - *arg* and no keyword arguments. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyObject_CallObject(PyObject *callable, PyObject *args) - - Call a callable Python object *callable*, with arguments given by the - tuple *args*. If no arguments are needed, then *args* can be *NULL*. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - This is the equivalent of the Python expression: ``callable(*args)``. - - -.. c:function:: PyObject* PyObject_CallFunction(PyObject *callable, const char *format, ...) - - Call a callable Python object *callable*, with a variable number of C arguments. - The C arguments are described using a :c:func:`Py_BuildValue` style format - string. The format can be *NULL*, indicating that no arguments are provided. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - This is the equivalent of the Python expression: ``callable(*args)``. - - Note that if you only pass :c:expr:`PyObject *` args, - :c:func:`PyObject_CallFunctionObjArgs` is a faster alternative. - - .. versionchanged:: 3.4 - The type of *format* was changed from ``char *``. - - -.. c:function:: PyObject* PyObject_CallMethod(PyObject *obj, const char *name, const char *format, ...) - - Call the method named *name* of object *obj* with a variable number of C - arguments. The C arguments are described by a :c:func:`Py_BuildValue` format - string that should produce a tuple. - - The format can be *NULL*, indicating that no arguments are provided. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - This is the equivalent of the Python expression: - ``obj.name(arg1, arg2, ...)``. - - Note that if you only pass :c:expr:`PyObject *` args, - :c:func:`PyObject_CallMethodObjArgs` is a faster alternative. - - .. versionchanged:: 3.4 - The types of *name* and *format* were changed from ``char *``. - - -.. c:function:: PyObject* PyObject_CallFunctionObjArgs(PyObject *callable, ...) - - Call a callable Python object *callable*, with a variable number of - :c:expr:`PyObject *` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - This is the equivalent of the Python expression: - ``callable(arg1, arg2, ...)``. - - -.. c:function:: PyObject* PyObject_CallMethodObjArgs(PyObject *obj, PyObject *name, ...) - - Call a method of the Python object *obj*, where the name of the method is given as a - Python string object in *name*. It is called with a variable number of - :c:expr:`PyObject *` arguments. The arguments are provided as a variable number - of parameters followed by *NULL*. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - -.. c:function:: PyObject* PyObject_CallMethodNoArgs(PyObject *obj, PyObject *name) - - Call a method of the Python object *obj* without arguments, - where the name of the method is given as a Python string object in *name*. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyObject_CallMethodOneArg(PyObject *obj, PyObject *name, PyObject *arg) - - Call a method of the Python object *obj* with a single positional argument - *arg*, where the name of the method is given as a Python string object in - *name*. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyObject_Vectorcall(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwnames) - - Call a callable Python object *callable*. - The arguments are the same as for :c:type:`vectorcallfunc`. - If *callable* supports vectorcall_, this directly calls - the vectorcall function stored in *callable*. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - .. versionadded:: 3.9 - -.. c:function:: PyObject* PyObject_VectorcallDict(PyObject *callable, PyObject *const *args, size_t nargsf, PyObject *kwdict) - - Call *callable* with positional arguments passed exactly as in the vectorcall_ protocol, - but with keyword arguments passed as a dictionary *kwdict*. - The *args* array contains only the positional arguments. - - Regardless of which protocol is used internally, - a conversion of arguments needs to be done. - Therefore, this function should only be used if the caller - already has a dictionary ready to use for the keyword arguments, - but not a tuple for the positional arguments. - - .. versionadded:: 3.9 - -.. c:function:: PyObject* PyObject_VectorcallMethod(PyObject *name, PyObject *const *args, size_t nargsf, PyObject *kwnames) - - Call a method using the vectorcall calling convention. The name of the method - is given as a Python string *name*. The object whose method is called is - *args[0]*, and the *args* array starting at *args[1]* represents the arguments - of the call. There must be at least one positional argument. - *nargsf* is the number of positional arguments including *args[0]*, - plus :c:macro:`PY_VECTORCALL_ARGUMENTS_OFFSET` if the value of ``args[0]`` may - temporarily be changed. Keyword arguments can be passed just like in - :c:func:`PyObject_Vectorcall`. - - If the object has the :c:macro:`Py_TPFLAGS_METHOD_DESCRIPTOR` feature, - this will call the unbound method object with the full - *args* vector as arguments. - - Return the result of the call on success, or raise an exception and return - *NULL* on failure. - - .. versionadded:: 3.9 - - -Call Support API ----------------- - -.. c:function:: int PyCallable_Check(PyObject *o) - - Determine if the object *o* is callable. Return ``1`` if the object is callable - and ``0`` otherwise. This function always succeeds. diff --git a/Doc/c-api/capsule.rst b/Doc/c-api/capsule.rst deleted file mode 100644 index cdb8aa33e9fd32..00000000000000 --- a/Doc/c-api/capsule.rst +++ /dev/null @@ -1,159 +0,0 @@ -.. highlight:: c - -.. _capsules: - -Capsules --------- - -.. index:: pair: object; Capsule - -Refer to :ref:`using-capsules` for more information on using these objects. - -.. versionadded:: 3.1 - - -.. c:type:: PyCapsule - - This subtype of :c:type:`PyObject` represents an opaque value, useful for C - extension modules who need to pass an opaque value (as a :c:expr:`void*` - pointer) through Python code to other C code. It is often used to make a C - function pointer defined in one module available to other modules, so the - regular import mechanism can be used to access C APIs defined in dynamically - loaded modules. - - -.. c:type:: PyCapsule_Destructor - - The type of a destructor callback for a capsule. Defined as:: - - typedef void (*PyCapsule_Destructor)(PyObject *); - - See :c:func:`PyCapsule_New` for the semantics of PyCapsule_Destructor - callbacks. - - -.. c:function:: int PyCapsule_CheckExact(PyObject *p) - - Return true if its argument is a :c:type:`PyCapsule`. This function always - succeeds. - - -.. c:function:: PyObject* PyCapsule_New(void *pointer, const char *name, PyCapsule_Destructor destructor) - - Create a :c:type:`PyCapsule` encapsulating the *pointer*. The *pointer* - argument may not be ``NULL``. - - On failure, set an exception and return ``NULL``. - - The *name* string may either be ``NULL`` or a pointer to a valid C string. If - non-``NULL``, this string must outlive the capsule. (Though it is permitted to - free it inside the *destructor*.) - - If the *destructor* argument is not ``NULL``, it will be called with the - capsule as its argument when it is destroyed. - - If this capsule will be stored as an attribute of a module, the *name* should - be specified as ``modulename.attributename``. This will enable other modules - to import the capsule using :c:func:`PyCapsule_Import`. - - -.. c:function:: void* PyCapsule_GetPointer(PyObject *capsule, const char *name) - - Retrieve the *pointer* stored in the capsule. On failure, set an exception - and return ``NULL``. - - The *name* parameter must compare exactly to the name stored in the capsule. - If the name stored in the capsule is ``NULL``, the *name* passed in must also - be ``NULL``. Python uses the C function :c:func:`!strcmp` to compare capsule - names. - - -.. c:function:: PyCapsule_Destructor PyCapsule_GetDestructor(PyObject *capsule) - - Return the current destructor stored in the capsule. On failure, set an - exception and return ``NULL``. - - It is legal for a capsule to have a ``NULL`` destructor. This makes a ``NULL`` - return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or - :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: void* PyCapsule_GetContext(PyObject *capsule) - - Return the current context stored in the capsule. On failure, set an - exception and return ``NULL``. - - It is legal for a capsule to have a ``NULL`` context. This makes a ``NULL`` - return code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or - :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: const char* PyCapsule_GetName(PyObject *capsule) - - Return the current name stored in the capsule. On failure, set an exception - and return ``NULL``. - - It is legal for a capsule to have a ``NULL`` name. This makes a ``NULL`` return - code somewhat ambiguous; use :c:func:`PyCapsule_IsValid` or - :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: void* PyCapsule_Import(const char *name, int no_block) - - Import a pointer to a C object from a capsule attribute in a module. The - *name* parameter should specify the full name to the attribute, as in - ``module.attribute``. The *name* stored in the capsule must match this - string exactly. - - Return the capsule's internal *pointer* on success. On failure, set an - exception and return ``NULL``. - - .. versionchanged:: 3.3 - *no_block* has no effect anymore. - - -.. c:function:: int PyCapsule_IsValid(PyObject *capsule, const char *name) - - Determines whether or not *capsule* is a valid capsule. A valid capsule is - non-``NULL``, passes :c:func:`PyCapsule_CheckExact`, has a non-``NULL`` pointer - stored in it, and its internal name matches the *name* parameter. (See - :c:func:`PyCapsule_GetPointer` for information on how capsule names are - compared.) - - In other words, if :c:func:`PyCapsule_IsValid` returns a true value, calls to - any of the accessors (any function starting with ``PyCapsule_Get``) are - guaranteed to succeed. - - Return a nonzero value if the object is valid and matches the name passed in. - Return ``0`` otherwise. This function will not fail. - - -.. c:function:: int PyCapsule_SetContext(PyObject *capsule, void *context) - - Set the context pointer inside *capsule* to *context*. - - Return ``0`` on success. Return nonzero and set an exception on failure. - - -.. c:function:: int PyCapsule_SetDestructor(PyObject *capsule, PyCapsule_Destructor destructor) - - Set the destructor inside *capsule* to *destructor*. - - Return ``0`` on success. Return nonzero and set an exception on failure. - - -.. c:function:: int PyCapsule_SetName(PyObject *capsule, const char *name) - - Set the name inside *capsule* to *name*. If non-``NULL``, the name must - outlive the capsule. If the previous *name* stored in the capsule was not - ``NULL``, no attempt is made to free it. - - Return ``0`` on success. Return nonzero and set an exception on failure. - - -.. c:function:: int PyCapsule_SetPointer(PyObject *capsule, void *pointer) - - Set the void pointer inside *capsule* to *pointer*. The pointer may not be - ``NULL``. - - Return ``0`` on success. Return nonzero and set an exception on failure. diff --git a/Doc/c-api/cell.rst b/Doc/c-api/cell.rst deleted file mode 100644 index f8cd0344fdd1c0..00000000000000 --- a/Doc/c-api/cell.rst +++ /dev/null @@ -1,63 +0,0 @@ -.. highlight:: c - -.. _cell-objects: - -Cell Objects ------------- - -"Cell" objects are used to implement variables referenced by multiple scopes. -For each such variable, a cell object is created to store the value; the local -variables of each stack frame that references the value contains a reference to -the cells from outer scopes which also use that variable. When the value is -accessed, the value contained in the cell is used instead of the cell object -itself. This de-referencing of the cell object requires support from the -generated byte-code; these are not automatically de-referenced when accessed. -Cell objects are not likely to be useful elsewhere. - - -.. c:type:: PyCellObject - - The C structure used for cell objects. - - -.. c:var:: PyTypeObject PyCell_Type - - The type object corresponding to cell objects. - - -.. c:function:: int PyCell_Check(PyObject *ob) - - Return true if *ob* is a cell object; *ob* must not be ``NULL``. This - function always succeeds. - - -.. c:function:: PyObject* PyCell_New(PyObject *ob) - - Create and return a new cell object containing the value *ob*. The parameter may - be ``NULL``. - - -.. c:function:: PyObject* PyCell_Get(PyObject *cell) - - Return the contents of the cell *cell*. - - -.. c:function:: PyObject* PyCell_GET(PyObject *cell) - - Return the contents of the cell *cell*, but without checking that *cell* is - non-``NULL`` and a cell object. - - -.. c:function:: int PyCell_Set(PyObject *cell, PyObject *value) - - Set the contents of the cell object *cell* to *value*. This releases the - reference to any current content of the cell. *value* may be ``NULL``. *cell* - must be non-``NULL``; if it is not a cell object, ``-1`` will be returned. On - success, ``0`` will be returned. - - -.. c:function:: void PyCell_SET(PyObject *cell, PyObject *value) - - Sets the value of the cell object *cell* to *value*. No reference counts are - adjusted, and no checks are made for safety; *cell* must be non-``NULL`` and must - be a cell object. diff --git a/Doc/c-api/code.rst b/Doc/c-api/code.rst deleted file mode 100644 index 33682d30dff337..00000000000000 --- a/Doc/c-api/code.rst +++ /dev/null @@ -1,119 +0,0 @@ -.. highlight:: c - -.. index:: object; code, code object - -.. _codeobjects: - -Code Objects ------------- - -.. sectionauthor:: Jeffrey Yasskin - -Code objects are a low-level detail of the CPython implementation. -Each one represents a chunk of executable code that hasn't yet been -bound into a function. - -.. c:type:: PyCodeObject - - The C structure of the objects used to describe code objects. The - fields of this type are subject to change at any time. - - -.. c:var:: PyTypeObject PyCode_Type - - This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`code` type. - - -.. c:function:: int PyCode_Check(PyObject *co) - - Return true if *co* is a :class:`code` object. This function always succeeds. - -.. c:function:: int PyCode_GetNumFree(PyCodeObject *co) - - Return the number of free variables in *co*. - -.. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) - - Return a new code object. If you need a dummy code object to create a frame, - use :c:func:`PyCode_NewEmpty` instead. Calling :c:func:`PyCode_New` directly - will bind you to a precise Python version since the definition of the bytecode - changes often. The many arguments of this function are inter-dependent in complex - ways, meaning that subtle changes to values are likely to result in incorrect - execution or VM crashes. Use this function only with extreme care. - - .. versionchanged:: 3.11 - Added ``qualname`` and ``exceptiontable`` parameters. - -.. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, PyObject *qualname, int firstlineno, PyObject *linetable, PyObject *exceptiontable) - - Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments. - The same caveats that apply to ``PyCode_New`` also apply to this function. - - .. versionadded:: 3.8 - - .. versionchanged:: 3.11 - Added ``qualname`` and ``exceptiontable`` parameters. - -.. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno) - - Return a new empty code object with the specified filename, - function name, and first line number. The resulting code - object will raise an ``Exception`` if executed. - -.. c:function:: int PyCode_Addr2Line(PyCodeObject *co, int byte_offset) - - Return the line number of the instruction that occurs on or before ``byte_offset`` and ends after it. - If you just need the line number of a frame, use :c:func:`PyFrame_GetLineNumber` instead. - - For efficiently iterating over the line numbers in a code object, use `the API described in PEP 626 - `_. - -.. c:function:: int PyCode_Addr2Location(PyObject *co, int byte_offset, int *start_line, int *start_column, int *end_line, int *end_column) - - Sets the passed ``int`` pointers to the source code line and column numbers - for the instruction at ``byte_offset``. Sets the value to ``0`` when - information is not available for any particular element. - - Returns ``1`` if the function succeeds and 0 otherwise. - - .. versionadded:: 3.11 - -.. c:function:: PyObject* PyCode_GetCode(PyCodeObject *co) - - Equivalent to the Python code ``getattr(co, 'co_code')``. - Returns a strong reference to a :c:type:`PyBytesObject` representing the - bytecode in a code object. On error, ``NULL`` is returned and an exception - is raised. - - This ``PyBytesObject`` may be created on-demand by the interpreter and does - not necessarily represent the bytecode actually executed by CPython. The - primary use case for this function is debuggers and profilers. - - .. versionadded:: 3.11 - -.. c:function:: PyObject* PyCode_GetVarnames(PyCodeObject *co) - - Equivalent to the Python code ``getattr(co, 'co_varnames')``. - Returns a new reference to a :c:type:`PyTupleObject` containing the names of - the local variables. On error, ``NULL`` is returned and an exception - is raised. - - .. versionadded:: 3.11 - -.. c:function:: PyObject* PyCode_GetCellvars(PyCodeObject *co) - - Equivalent to the Python code ``getattr(co, 'co_cellvars')``. - Returns a new reference to a :c:type:`PyTupleObject` containing the names of - the local variables that are referenced by nested functions. On error, ``NULL`` - is returned and an exception is raised. - - .. versionadded:: 3.11 - -.. c:function:: PyObject* PyCode_GetFreevars(PyCodeObject *co) - - Equivalent to the Python code ``getattr(co, 'co_freevars')``. - Returns a new reference to a :c:type:`PyTupleObject` containing the names of - the free variables. On error, ``NULL`` is returned and an exception is raised. - - .. versionadded:: 3.11 diff --git a/Doc/c-api/codec.rst b/Doc/c-api/codec.rst deleted file mode 100644 index 8ae5c4fecd6248..00000000000000 --- a/Doc/c-api/codec.rst +++ /dev/null @@ -1,131 +0,0 @@ -.. _codec-registry: - -Codec registry and support functions -==================================== - -.. c:function:: int PyCodec_Register(PyObject *search_function) - - Register a new codec search function. - - As side effect, this tries to load the :mod:`!encodings` package, if not yet - done, to make sure that it is always first in the list of search functions. - -.. c:function:: int PyCodec_Unregister(PyObject *search_function) - - Unregister a codec search function and clear the registry's cache. - If the search function is not registered, do nothing. - Return 0 on success. Raise an exception and return -1 on error. - - .. versionadded:: 3.10 - -.. c:function:: int PyCodec_KnownEncoding(const char *encoding) - - Return ``1`` or ``0`` depending on whether there is a registered codec for - the given *encoding*. This function always succeeds. - -.. c:function:: PyObject* PyCodec_Encode(PyObject *object, const char *encoding, const char *errors) - - Generic codec based encoding API. - - *object* is passed through the encoder function found for the given - *encoding* using the error handling method defined by *errors*. *errors* may - be ``NULL`` to use the default method defined for the codec. Raises a - :exc:`LookupError` if no encoder can be found. - -.. c:function:: PyObject* PyCodec_Decode(PyObject *object, const char *encoding, const char *errors) - - Generic codec based decoding API. - - *object* is passed through the decoder function found for the given - *encoding* using the error handling method defined by *errors*. *errors* may - be ``NULL`` to use the default method defined for the codec. Raises a - :exc:`LookupError` if no encoder can be found. - - -Codec lookup API ----------------- - -In the following functions, the *encoding* string is looked up converted to all -lower-case characters, which makes encodings looked up through this mechanism -effectively case-insensitive. If no codec is found, a :exc:`KeyError` is set -and ``NULL`` returned. - -.. c:function:: PyObject* PyCodec_Encoder(const char *encoding) - - Get an encoder function for the given *encoding*. - -.. c:function:: PyObject* PyCodec_Decoder(const char *encoding) - - Get a decoder function for the given *encoding*. - -.. c:function:: PyObject* PyCodec_IncrementalEncoder(const char *encoding, const char *errors) - - Get an :class:`~codecs.IncrementalEncoder` object for the given *encoding*. - -.. c:function:: PyObject* PyCodec_IncrementalDecoder(const char *encoding, const char *errors) - - Get an :class:`~codecs.IncrementalDecoder` object for the given *encoding*. - -.. c:function:: PyObject* PyCodec_StreamReader(const char *encoding, PyObject *stream, const char *errors) - - Get a :class:`~codecs.StreamReader` factory function for the given *encoding*. - -.. c:function:: PyObject* PyCodec_StreamWriter(const char *encoding, PyObject *stream, const char *errors) - - Get a :class:`~codecs.StreamWriter` factory function for the given *encoding*. - - -Registry API for Unicode encoding error handlers ------------------------------------------------- - -.. c:function:: int PyCodec_RegisterError(const char *name, PyObject *error) - - Register the error handling callback function *error* under the given *name*. - This callback function will be called by a codec when it encounters - unencodable characters/undecodable bytes and *name* is specified as the error - parameter in the call to the encode/decode function. - - The callback gets a single argument, an instance of - :exc:`UnicodeEncodeError`, :exc:`UnicodeDecodeError` or - :exc:`UnicodeTranslateError` that holds information about the problematic - sequence of characters or bytes and their offset in the original string (see - :ref:`unicodeexceptions` for functions to extract this information). The - callback must either raise the given exception, or return a two-item tuple - containing the replacement for the problematic sequence, and an integer - giving the offset in the original string at which encoding/decoding should be - resumed. - - Return ``0`` on success, ``-1`` on error. - -.. c:function:: PyObject* PyCodec_LookupError(const char *name) - - Lookup the error handling callback function registered under *name*. As a - special case ``NULL`` can be passed, in which case the error handling callback - for "strict" will be returned. - -.. c:function:: PyObject* PyCodec_StrictErrors(PyObject *exc) - - Raise *exc* as an exception. - -.. c:function:: PyObject* PyCodec_IgnoreErrors(PyObject *exc) - - Ignore the unicode error, skipping the faulty input. - -.. c:function:: PyObject* PyCodec_ReplaceErrors(PyObject *exc) - - Replace the unicode encode error with ``?`` or ``U+FFFD``. - -.. c:function:: PyObject* PyCodec_XMLCharRefReplaceErrors(PyObject *exc) - - Replace the unicode encode error with XML character references. - -.. c:function:: PyObject* PyCodec_BackslashReplaceErrors(PyObject *exc) - - Replace the unicode encode error with backslash escapes (``\x``, ``\u`` and - ``\U``). - -.. c:function:: PyObject* PyCodec_NameReplaceErrors(PyObject *exc) - - Replace the unicode encode error with ``\N{...}`` escapes. - - .. versionadded:: 3.5 diff --git a/Doc/c-api/complex.rst b/Doc/c-api/complex.rst deleted file mode 100644 index e3fd001c599c80..00000000000000 --- a/Doc/c-api/complex.rst +++ /dev/null @@ -1,138 +0,0 @@ -.. highlight:: c - -.. _complexobjects: - -Complex Number Objects ----------------------- - -.. index:: pair: object; complex number - -Python's complex number objects are implemented as two distinct types when -viewed from the C API: one is the Python object exposed to Python programs, and -the other is a C structure which represents the actual complex number value. -The API provides functions for working with both. - - -Complex Numbers as C Structures -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Note that the functions which accept these structures as parameters and return -them as results do so *by value* rather than dereferencing them through -pointers. This is consistent throughout the API. - - -.. c:type:: Py_complex - - The C structure which corresponds to the value portion of a Python complex - number object. Most of the functions for dealing with complex number objects - use structures of this type as input or output values, as appropriate. It is - defined as:: - - typedef struct { - double real; - double imag; - } Py_complex; - - -.. c:function:: Py_complex _Py_c_sum(Py_complex left, Py_complex right) - - Return the sum of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_diff(Py_complex left, Py_complex right) - - Return the difference between two complex numbers, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_neg(Py_complex num) - - Return the negation of the complex number *num*, using the C - :c:type:`Py_complex` representation. - - -.. c:function:: Py_complex _Py_c_prod(Py_complex left, Py_complex right) - - Return the product of two complex numbers, using the C :c:type:`Py_complex` - representation. - - -.. c:function:: Py_complex _Py_c_quot(Py_complex dividend, Py_complex divisor) - - Return the quotient of two complex numbers, using the C :c:type:`Py_complex` - representation. - - If *divisor* is null, this method returns zero and sets - :c:data:`errno` to :c:macro:`!EDOM`. - - -.. c:function:: Py_complex _Py_c_pow(Py_complex num, Py_complex exp) - - Return the exponentiation of *num* by *exp*, using the C :c:type:`Py_complex` - representation. - - If *num* is null and *exp* is not a positive real number, - this method returns zero and sets :c:data:`errno` to :c:macro:`!EDOM`. - - -Complex Numbers as Python Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - -.. c:type:: PyComplexObject - - This subtype of :c:type:`PyObject` represents a Python complex number object. - - -.. c:var:: PyTypeObject PyComplex_Type - - This instance of :c:type:`PyTypeObject` represents the Python complex number - type. It is the same object as :class:`complex` in the Python layer. - - -.. c:function:: int PyComplex_Check(PyObject *p) - - Return true if its argument is a :c:type:`PyComplexObject` or a subtype of - :c:type:`PyComplexObject`. This function always succeeds. - - -.. c:function:: int PyComplex_CheckExact(PyObject *p) - - Return true if its argument is a :c:type:`PyComplexObject`, but not a subtype of - :c:type:`PyComplexObject`. This function always succeeds. - - -.. c:function:: PyObject* PyComplex_FromCComplex(Py_complex v) - - Create a new Python complex number object from a C :c:type:`Py_complex` value. - - -.. c:function:: PyObject* PyComplex_FromDoubles(double real, double imag) - - Return a new :c:type:`PyComplexObject` object from *real* and *imag*. - - -.. c:function:: double PyComplex_RealAsDouble(PyObject *op) - - Return the real part of *op* as a C :c:expr:`double`. - - -.. c:function:: double PyComplex_ImagAsDouble(PyObject *op) - - Return the imaginary part of *op* as a C :c:expr:`double`. - - -.. c:function:: Py_complex PyComplex_AsCComplex(PyObject *op) - - Return the :c:type:`Py_complex` value of the complex number *op*. - - If *op* is not a Python complex number object but has a :meth:`~object.__complex__` - method, this method will first be called to convert *op* to a Python complex - number object. If :meth:`!__complex__` is not defined then it falls back to - :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back - to :meth:`~object.__index__`. Upon failure, this method returns ``-1.0`` as a real - value. - - .. versionchanged:: 3.8 - Use :meth:`~object.__index__` if available. diff --git a/Doc/c-api/concrete.rst b/Doc/c-api/concrete.rst deleted file mode 100644 index 880f7b15ce68e8..00000000000000 --- a/Doc/c-api/concrete.rst +++ /dev/null @@ -1,119 +0,0 @@ -.. highlight:: c - - -.. _concrete: - -********************** -Concrete Objects Layer -********************** - -The functions in this chapter are specific to certain Python object types. -Passing them an object of the wrong type is not a good idea; if you receive an -object from a Python program and you are not sure that it has the right type, -you must perform a type check first; for example, to check that an object is a -dictionary, use :c:func:`PyDict_Check`. The chapter is structured like the -"family tree" of Python object types. - -.. warning:: - - While the functions described in this chapter carefully check the type of the - objects which are passed in, many of them do not check for ``NULL`` being passed - instead of a valid object. Allowing ``NULL`` to be passed in can cause memory - access violations and immediate termination of the interpreter. - - -.. _fundamental: - -Fundamental Objects -=================== - -This section describes Python type objects and the singleton object ``None``. - -.. toctree:: - - type.rst - none.rst - - -.. _numericobjects: - -Numeric Objects -=============== - -.. index:: pair: object; numeric - -.. toctree:: - - long.rst - bool.rst - float.rst - complex.rst - - -.. _sequenceobjects: - -Sequence Objects -================ - -.. index:: pair: object; sequence - -Generic operations on sequence objects were discussed in the previous chapter; -this section deals with the specific kinds of sequence objects that are -intrinsic to the Python language. - -.. XXX sort out unicode, str, bytes and bytearray - -.. toctree:: - - bytes.rst - bytearray.rst - unicode.rst - tuple.rst - list.rst - - -.. _mapobjects: - -Container Objects -================= - -.. index:: pair: object; mapping - -.. toctree:: - - dict.rst - set.rst - - -.. _otherobjects: - -Function Objects -================ - -.. toctree:: - - function.rst - method.rst - cell.rst - code.rst - - -Other Objects -============= - -.. toctree:: - - file.rst - module.rst - iterator.rst - descriptor.rst - slice.rst - memoryview.rst - weakref.rst - capsule.rst - frame.rst - gen.rst - coro.rst - contextvars.rst - datetime.rst - typehints.rst diff --git a/Doc/c-api/contextvars.rst b/Doc/c-api/contextvars.rst deleted file mode 100644 index d970f5443b1df5..00000000000000 --- a/Doc/c-api/contextvars.rst +++ /dev/null @@ -1,138 +0,0 @@ -.. highlight:: c - -.. _contextvarsobjects: - -Context Variables Objects -------------------------- - -.. _contextvarsobjects_pointertype_change: -.. versionchanged:: 3.7.1 - - .. note:: - - In Python 3.7.1 the signatures of all context variables - C APIs were **changed** to use :c:type:`PyObject` pointers instead - of :c:type:`PyContext`, :c:type:`PyContextVar`, and - :c:type:`PyContextToken`, e.g.:: - - // in 3.7.0: - PyContext *PyContext_New(void); - - // in 3.7.1+: - PyObject *PyContext_New(void); - - See :issue:`34762` for more details. - - -.. versionadded:: 3.7 - -This section details the public C API for the :mod:`contextvars` module. - -.. c:type:: PyContext - - The C structure used to represent a :class:`contextvars.Context` - object. - -.. c:type:: PyContextVar - - The C structure used to represent a :class:`contextvars.ContextVar` - object. - -.. c:type:: PyContextToken - - The C structure used to represent a :class:`contextvars.Token` object. - -.. c:var:: PyTypeObject PyContext_Type - - The type object representing the *context* type. - -.. c:var:: PyTypeObject PyContextVar_Type - - The type object representing the *context variable* type. - -.. c:var:: PyTypeObject PyContextToken_Type - - The type object representing the *context variable token* type. - - -Type-check macros: - -.. c:function:: int PyContext_CheckExact(PyObject *o) - - Return true if *o* is of type :c:data:`PyContext_Type`. *o* must not be - ``NULL``. This function always succeeds. - -.. c:function:: int PyContextVar_CheckExact(PyObject *o) - - Return true if *o* is of type :c:data:`PyContextVar_Type`. *o* must not be - ``NULL``. This function always succeeds. - -.. c:function:: int PyContextToken_CheckExact(PyObject *o) - - Return true if *o* is of type :c:data:`PyContextToken_Type`. - *o* must not be ``NULL``. This function always succeeds. - - -Context object management functions: - -.. c:function:: PyObject *PyContext_New(void) - - Create a new empty context object. Returns ``NULL`` if an error - has occurred. - -.. c:function:: PyObject *PyContext_Copy(PyObject *ctx) - - Create a shallow copy of the passed *ctx* context object. - Returns ``NULL`` if an error has occurred. - -.. c:function:: PyObject *PyContext_CopyCurrent(void) - - Create a shallow copy of the current thread context. - Returns ``NULL`` if an error has occurred. - -.. c:function:: int PyContext_Enter(PyObject *ctx) - - Set *ctx* as the current context for the current thread. - Returns ``0`` on success, and ``-1`` on error. - -.. c:function:: int PyContext_Exit(PyObject *ctx) - - Deactivate the *ctx* context and restore the previous context as the - current context for the current thread. Returns ``0`` on success, - and ``-1`` on error. - - -Context variable functions: - -.. c:function:: PyObject *PyContextVar_New(const char *name, PyObject *def) - - Create a new ``ContextVar`` object. The *name* parameter is used - for introspection and debug purposes. The *def* parameter specifies - a default value for the context variable, or ``NULL`` for no default. - If an error has occurred, this function returns ``NULL``. - -.. c:function:: int PyContextVar_Get(PyObject *var, PyObject *default_value, PyObject **value) - - Get the value of a context variable. Returns ``-1`` if an error has - occurred during lookup, and ``0`` if no error occurred, whether or not - a value was found. - - If the context variable was found, *value* will be a pointer to it. - If the context variable was *not* found, *value* will point to: - - - *default_value*, if not ``NULL``; - - the default value of *var*, if not ``NULL``; - - ``NULL`` - - Except for ``NULL``, the function returns a new reference. - -.. c:function:: PyObject *PyContextVar_Set(PyObject *var, PyObject *value) - - Set the value of *var* to *value* in the current context. Returns - a new token object for this change, or ``NULL`` if an error has occurred. - -.. c:function:: int PyContextVar_Reset(PyObject *var, PyObject *token) - - Reset the state of the *var* context variable to that it was in before - :c:func:`PyContextVar_Set` that returned the *token* was called. - This function returns ``0`` on success and ``-1`` on error. diff --git a/Doc/c-api/conversion.rst b/Doc/c-api/conversion.rst deleted file mode 100644 index c5350123dfdfdc..00000000000000 --- a/Doc/c-api/conversion.rst +++ /dev/null @@ -1,128 +0,0 @@ -.. highlight:: c - -.. _string-conversion: - -String conversion and formatting -================================ - -Functions for number conversion and formatted string output. - - -.. c:function:: int PyOS_snprintf(char *str, size_t size, const char *format, ...) - - Output not more than *size* bytes to *str* according to the format string - *format* and the extra arguments. See the Unix man page :manpage:`snprintf(3)`. - - -.. c:function:: int PyOS_vsnprintf(char *str, size_t size, const char *format, va_list va) - - Output not more than *size* bytes to *str* according to the format string - *format* and the variable argument list *va*. Unix man page - :manpage:`vsnprintf(3)`. - -:c:func:`PyOS_snprintf` and :c:func:`PyOS_vsnprintf` wrap the Standard C library -functions :c:func:`snprintf` and :c:func:`vsnprintf`. Their purpose is to -guarantee consistent behavior in corner cases, which the Standard C functions do -not. - -The wrappers ensure that ``str[size-1]`` is always ``'\0'`` upon return. They -never write more than *size* bytes (including the trailing ``'\0'``) into str. -Both functions require that ``str != NULL``, ``size > 0``, ``format != NULL`` -and ``size < INT_MAX``. Note that this means there is no equivalent to the C99 -``n = snprintf(NULL, 0, ...)`` which would determine the necessary buffer size. - -The return value (*rv*) for these functions should be interpreted as follows: - -* When ``0 <= rv < size``, the output conversion was successful and *rv* - characters were written to *str* (excluding the trailing ``'\0'`` byte at - ``str[rv]``). - -* When ``rv >= size``, the output conversion was truncated and a buffer with - ``rv + 1`` bytes would have been needed to succeed. ``str[size-1]`` is ``'\0'`` - in this case. - -* When ``rv < 0``, "something bad happened." ``str[size-1]`` is ``'\0'`` in - this case too, but the rest of *str* is undefined. The exact cause of the error - depends on the underlying platform. - - -The following functions provide locale-independent string to number conversions. - -.. c:function:: double PyOS_string_to_double(const char *s, char **endptr, PyObject *overflow_exception) - - Convert a string ``s`` to a :c:expr:`double`, raising a Python - exception on failure. The set of accepted strings corresponds to - the set of strings accepted by Python's :func:`float` constructor, - except that ``s`` must not have leading or trailing whitespace. - The conversion is independent of the current locale. - - If ``endptr`` is ``NULL``, convert the whole string. Raise - :exc:`ValueError` and return ``-1.0`` if the string is not a valid - representation of a floating-point number. - - If endptr is not ``NULL``, convert as much of the string as - possible and set ``*endptr`` to point to the first unconverted - character. If no initial segment of the string is the valid - representation of a floating-point number, set ``*endptr`` to point - to the beginning of the string, raise ValueError, and return - ``-1.0``. - - If ``s`` represents a value that is too large to store in a float - (for example, ``"1e500"`` is such a string on many platforms) then - if ``overflow_exception`` is ``NULL`` return ``Py_HUGE_VAL`` (with - an appropriate sign) and don't set any exception. Otherwise, - ``overflow_exception`` must point to a Python exception object; - raise that exception and return ``-1.0``. In both cases, set - ``*endptr`` to point to the first character after the converted value. - - If any other error occurs during the conversion (for example an - out-of-memory error), set the appropriate Python exception and - return ``-1.0``. - - .. versionadded:: 3.1 - - -.. c:function:: char* PyOS_double_to_string(double val, char format_code, int precision, int flags, int *ptype) - - Convert a :c:expr:`double` *val* to a string using supplied - *format_code*, *precision*, and *flags*. - - *format_code* must be one of ``'e'``, ``'E'``, ``'f'``, ``'F'``, - ``'g'``, ``'G'`` or ``'r'``. For ``'r'``, the supplied *precision* - must be 0 and is ignored. The ``'r'`` format code specifies the - standard :func:`repr` format. - - *flags* can be zero or more of the values ``Py_DTSF_SIGN``, - ``Py_DTSF_ADD_DOT_0``, or ``Py_DTSF_ALT``, or-ed together: - - * ``Py_DTSF_SIGN`` means to always precede the returned string with a sign - character, even if *val* is non-negative. - - * ``Py_DTSF_ADD_DOT_0`` means to ensure that the returned string will not look - like an integer. - - * ``Py_DTSF_ALT`` means to apply "alternate" formatting rules. See the - documentation for the :c:func:`PyOS_snprintf` ``'#'`` specifier for - details. - - If *ptype* is non-``NULL``, then the value it points to will be set to one of - ``Py_DTST_FINITE``, ``Py_DTST_INFINITE``, or ``Py_DTST_NAN``, signifying that - *val* is a finite number, an infinite number, or not a number, respectively. - - The return value is a pointer to *buffer* with the converted string or - ``NULL`` if the conversion failed. The caller is responsible for freeing the - returned string by calling :c:func:`PyMem_Free`. - - .. versionadded:: 3.1 - - -.. c:function:: int PyOS_stricmp(const char *s1, const char *s2) - - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strcmp` except that it ignores the case. - - -.. c:function:: int PyOS_strnicmp(const char *s1, const char *s2, Py_ssize_t size) - - Case insensitive comparison of strings. The function works almost - identically to :c:func:`!strncmp` except that it ignores the case. diff --git a/Doc/c-api/coro.rst b/Doc/c-api/coro.rst deleted file mode 100644 index caa855a10d20ca..00000000000000 --- a/Doc/c-api/coro.rst +++ /dev/null @@ -1,35 +0,0 @@ -.. highlight:: c - -.. _coro-objects: - -Coroutine Objects ------------------ - -.. versionadded:: 3.5 - -Coroutine objects are what functions declared with an ``async`` keyword -return. - - -.. c:type:: PyCoroObject - - The C structure used for coroutine objects. - - -.. c:var:: PyTypeObject PyCoro_Type - - The type object corresponding to coroutine objects. - - -.. c:function:: int PyCoro_CheckExact(PyObject *ob) - - Return true if *ob*'s type is :c:type:`PyCoro_Type`; *ob* must not be ``NULL``. - This function always succeeds. - - -.. c:function:: PyObject* PyCoro_New(PyFrameObject *frame, PyObject *name, PyObject *qualname) - - Create and return a new coroutine object based on the *frame* object, - with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. - A reference to *frame* is stolen by this function. The *frame* argument - must not be ``NULL``. diff --git a/Doc/c-api/datetime.rst b/Doc/c-api/datetime.rst deleted file mode 100644 index 97522da773477e..00000000000000 --- a/Doc/c-api/datetime.rst +++ /dev/null @@ -1,327 +0,0 @@ -.. highlight:: c - -.. _datetimeobjects: - -DateTime Objects ----------------- - -Various date and time objects are supplied by the :mod:`datetime` module. -Before using any of these functions, the header file :file:`datetime.h` must be -included in your source (note that this is not included by :file:`Python.h`), -and the macro :c:macro:`!PyDateTime_IMPORT` must be invoked, usually as part of -the module initialisation function. The macro puts a pointer to a C structure -into a static variable, :c:data:`!PyDateTimeAPI`, that is used by the following -macros. - -.. c:type:: PyDateTime_Date - - This subtype of :c:type:`PyObject` represents a Python date object. - -.. c:type:: PyDateTime_DateTime - - This subtype of :c:type:`PyObject` represents a Python datetime object. - -.. c:type:: PyDateTime_Time - - This subtype of :c:type:`PyObject` represents a Python time object. - -.. c:type:: PyDateTime_Delta - - This subtype of :c:type:`PyObject` represents the difference between two datetime values. - -.. c:var:: PyTypeObject PyDateTime_DateType - - This instance of :c:type:`PyTypeObject` represents the Python date type; - it is the same object as :class:`datetime.date` in the Python layer. - -.. c:var:: PyTypeObject PyDateTime_DateTimeType - - This instance of :c:type:`PyTypeObject` represents the Python datetime type; - it is the same object as :class:`datetime.datetime` in the Python layer. - -.. c:var:: PyTypeObject PyDateTime_TimeType - - This instance of :c:type:`PyTypeObject` represents the Python time type; - it is the same object as :class:`datetime.time` in the Python layer. - -.. c:var:: PyTypeObject PyDateTime_DeltaType - - This instance of :c:type:`PyTypeObject` represents Python type for - the difference between two datetime values; - it is the same object as :class:`datetime.timedelta` in the Python layer. - -.. c:var:: PyTypeObject PyDateTime_TZInfoType - - This instance of :c:type:`PyTypeObject` represents the Python time zone info type; - it is the same object as :class:`datetime.tzinfo` in the Python layer. - - -Macro for access to the UTC singleton: - -.. c:var:: PyObject* PyDateTime_TimeZone_UTC - - Returns the time zone singleton representing UTC, the same object as - :attr:`datetime.timezone.utc`. - - .. versionadded:: 3.7 - - -Type-check macros: - -.. c:function:: int PyDate_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DateType` or a subtype of - :c:data:`!PyDateTime_DateType`. *ob* must not be ``NULL``. This function always - succeeds. - - -.. c:function:: int PyDate_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DateType`. *ob* must not be - ``NULL``. This function always succeeds. - - -.. c:function:: int PyDateTime_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType` or a subtype of - :c:data:`!PyDateTime_DateTimeType`. *ob* must not be ``NULL``. This function always - succeeds. - - -.. c:function:: int PyDateTime_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DateTimeType`. *ob* must not - be ``NULL``. This function always succeeds. - - -.. c:function:: int PyTime_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_TimeType` or a subtype of - :c:data:`!PyDateTime_TimeType`. *ob* must not be ``NULL``. This function always - succeeds. - - -.. c:function:: int PyTime_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_TimeType`. *ob* must not be - ``NULL``. This function always succeeds. - - -.. c:function:: int PyDelta_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DeltaType` or a subtype of - :c:data:`!PyDateTime_DeltaType`. *ob* must not be ``NULL``. This function always - succeeds. - - -.. c:function:: int PyDelta_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_DeltaType`. *ob* must not be - ``NULL``. This function always succeeds. - - -.. c:function:: int PyTZInfo_Check(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType` or a subtype of - :c:data:`!PyDateTime_TZInfoType`. *ob* must not be ``NULL``. This function always - succeeds. - - -.. c:function:: int PyTZInfo_CheckExact(PyObject *ob) - - Return true if *ob* is of type :c:data:`PyDateTime_TZInfoType`. *ob* must not be - ``NULL``. This function always succeeds. - - -Macros to create objects: - -.. c:function:: PyObject* PyDate_FromDate(int year, int month, int day) - - Return a :class:`datetime.date` object with the specified year, month and day. - - -.. c:function:: PyObject* PyDateTime_FromDateAndTime(int year, int month, int day, int hour, int minute, int second, int usecond) - - Return a :class:`datetime.datetime` object with the specified year, month, day, hour, - minute, second and microsecond. - - -.. c:function:: PyObject* PyDateTime_FromDateAndTimeAndFold(int year, int month, int day, int hour, int minute, int second, int usecond, int fold) - - Return a :class:`datetime.datetime` object with the specified year, month, day, hour, - minute, second, microsecond and fold. - - .. versionadded:: 3.6 - - -.. c:function:: PyObject* PyTime_FromTime(int hour, int minute, int second, int usecond) - - Return a :class:`datetime.time` object with the specified hour, minute, second and - microsecond. - - -.. c:function:: PyObject* PyTime_FromTimeAndFold(int hour, int minute, int second, int usecond, int fold) - - Return a :class:`datetime.time` object with the specified hour, minute, second, - microsecond and fold. - - .. versionadded:: 3.6 - - -.. c:function:: PyObject* PyDelta_FromDSU(int days, int seconds, int useconds) - - Return a :class:`datetime.timedelta` object representing the given number - of days, seconds and microseconds. Normalization is performed so that the - resulting number of microseconds and seconds lie in the ranges documented for - :class:`datetime.timedelta` objects. - - -.. c:function:: PyObject* PyTimeZone_FromOffset(PyObject *offset) - - Return a :class:`datetime.timezone` object with an unnamed fixed offset - represented by the *offset* argument. - - .. versionadded:: 3.7 - - -.. c:function:: PyObject* PyTimeZone_FromOffsetAndName(PyObject *offset, PyObject *name) - - Return a :class:`datetime.timezone` object with a fixed offset represented - by the *offset* argument and with tzname *name*. - - .. versionadded:: 3.7 - - -Macros to extract fields from date objects. The argument must be an instance of -:c:type:`PyDateTime_Date`, including subclasses (such as -:c:type:`PyDateTime_DateTime`). The argument must not be ``NULL``, and the type is -not checked: - -.. c:function:: int PyDateTime_GET_YEAR(PyDateTime_Date *o) - - Return the year, as a positive int. - - -.. c:function:: int PyDateTime_GET_MONTH(PyDateTime_Date *o) - - Return the month, as an int from 1 through 12. - - -.. c:function:: int PyDateTime_GET_DAY(PyDateTime_Date *o) - - Return the day, as an int from 1 through 31. - - -Macros to extract fields from datetime objects. The argument must be an -instance of :c:type:`PyDateTime_DateTime`, including subclasses. The argument -must not be ``NULL``, and the type is not checked: - -.. c:function:: int PyDateTime_DATE_GET_HOUR(PyDateTime_DateTime *o) - - Return the hour, as an int from 0 through 23. - - -.. c:function:: int PyDateTime_DATE_GET_MINUTE(PyDateTime_DateTime *o) - - Return the minute, as an int from 0 through 59. - - -.. c:function:: int PyDateTime_DATE_GET_SECOND(PyDateTime_DateTime *o) - - Return the second, as an int from 0 through 59. - - -.. c:function:: int PyDateTime_DATE_GET_MICROSECOND(PyDateTime_DateTime *o) - - Return the microsecond, as an int from 0 through 999999. - - -.. c:function:: int PyDateTime_DATE_GET_FOLD(PyDateTime_DateTime *o) - - Return the fold, as an int from 0 through 1. - - .. versionadded:: 3.6 - - -.. c:function:: PyObject* PyDateTime_DATE_GET_TZINFO(PyDateTime_DateTime *o) - - Return the tzinfo (which may be ``None``). - - .. versionadded:: 3.10 - - -Macros to extract fields from time objects. The argument must be an instance of -:c:type:`PyDateTime_Time`, including subclasses. The argument must not be ``NULL``, -and the type is not checked: - -.. c:function:: int PyDateTime_TIME_GET_HOUR(PyDateTime_Time *o) - - Return the hour, as an int from 0 through 23. - - -.. c:function:: int PyDateTime_TIME_GET_MINUTE(PyDateTime_Time *o) - - Return the minute, as an int from 0 through 59. - - -.. c:function:: int PyDateTime_TIME_GET_SECOND(PyDateTime_Time *o) - - Return the second, as an int from 0 through 59. - - -.. c:function:: int PyDateTime_TIME_GET_MICROSECOND(PyDateTime_Time *o) - - Return the microsecond, as an int from 0 through 999999. - - -.. c:function:: int PyDateTime_TIME_GET_FOLD(PyDateTime_Time *o) - - Return the fold, as an int from 0 through 1. - - .. versionadded:: 3.6 - - -.. c:function:: PyObject* PyDateTime_TIME_GET_TZINFO(PyDateTime_Time *o) - - Return the tzinfo (which may be ``None``). - - .. versionadded:: 3.10 - - -Macros to extract fields from time delta objects. The argument must be an -instance of :c:type:`PyDateTime_Delta`, including subclasses. The argument must -not be ``NULL``, and the type is not checked: - -.. c:function:: int PyDateTime_DELTA_GET_DAYS(PyDateTime_Delta *o) - - Return the number of days, as an int from -999999999 to 999999999. - - .. versionadded:: 3.3 - - -.. c:function:: int PyDateTime_DELTA_GET_SECONDS(PyDateTime_Delta *o) - - Return the number of seconds, as an int from 0 through 86399. - - .. versionadded:: 3.3 - - -.. c:function:: int PyDateTime_DELTA_GET_MICROSECONDS(PyDateTime_Delta *o) - - Return the number of microseconds, as an int from 0 through 999999. - - .. versionadded:: 3.3 - - -Macros for the convenience of modules implementing the DB API: - -.. c:function:: PyObject* PyDateTime_FromTimestamp(PyObject *args) - - Create and return a new :class:`datetime.datetime` object given an argument - tuple suitable for passing to :meth:`datetime.datetime.fromtimestamp()`. - - -.. c:function:: PyObject* PyDate_FromTimestamp(PyObject *args) - - Create and return a new :class:`datetime.date` object given an argument - tuple suitable for passing to :meth:`datetime.date.fromtimestamp()`. diff --git a/Doc/c-api/descriptor.rst b/Doc/c-api/descriptor.rst deleted file mode 100644 index b32c113e5f0457..00000000000000 --- a/Doc/c-api/descriptor.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. highlight:: c - -.. _descriptor-objects: - -Descriptor Objects ------------------- - -"Descriptors" are objects that describe some attribute of an object. They are -found in the dictionary of type objects. - -.. XXX document these! - -.. c:var:: PyTypeObject PyProperty_Type - - The type object for the built-in descriptor types. - - -.. c:function:: PyObject* PyDescr_NewGetSet(PyTypeObject *type, struct PyGetSetDef *getset) - - -.. c:function:: PyObject* PyDescr_NewMember(PyTypeObject *type, struct PyMemberDef *meth) - - -.. c:function:: PyObject* PyDescr_NewMethod(PyTypeObject *type, struct PyMethodDef *meth) - - -.. c:function:: PyObject* PyDescr_NewWrapper(PyTypeObject *type, struct wrapperbase *wrapper, void *wrapped) - - -.. c:function:: PyObject* PyDescr_NewClassMethod(PyTypeObject *type, PyMethodDef *method) - - -.. c:function:: int PyDescr_IsData(PyObject *descr) - - Return non-zero if the descriptor objects *descr* describes a data attribute, or - ``0`` if it describes a method. *descr* must be a descriptor object; there is - no error checking. - - -.. c:function:: PyObject* PyWrapper_New(PyObject *, PyObject *) diff --git a/Doc/c-api/dict.rst b/Doc/c-api/dict.rst deleted file mode 100644 index 1575aa50eff5d6..00000000000000 --- a/Doc/c-api/dict.rst +++ /dev/null @@ -1,243 +0,0 @@ -.. highlight:: c - -.. _dictobjects: - -Dictionary Objects ------------------- - -.. index:: pair: object; dictionary - - -.. c:type:: PyDictObject - - This subtype of :c:type:`PyObject` represents a Python dictionary object. - - -.. c:var:: PyTypeObject PyDict_Type - - This instance of :c:type:`PyTypeObject` represents the Python dictionary - type. This is the same object as :class:`dict` in the Python layer. - - -.. c:function:: int PyDict_Check(PyObject *p) - - Return true if *p* is a dict object or an instance of a subtype of the dict - type. This function always succeeds. - - -.. c:function:: int PyDict_CheckExact(PyObject *p) - - Return true if *p* is a dict object, but not an instance of a subtype of - the dict type. This function always succeeds. - - -.. c:function:: PyObject* PyDict_New() - - Return a new empty dictionary, or ``NULL`` on failure. - - -.. c:function:: PyObject* PyDictProxy_New(PyObject *mapping) - - Return a :class:`types.MappingProxyType` object for a mapping which - enforces read-only behavior. This is normally used to create a view to - prevent modification of the dictionary for non-dynamic class types. - - -.. c:function:: void PyDict_Clear(PyObject *p) - - Empty an existing dictionary of all key-value pairs. - - -.. c:function:: int PyDict_Contains(PyObject *p, PyObject *key) - - Determine if dictionary *p* contains *key*. If an item in *p* is matches - *key*, return ``1``, otherwise return ``0``. On error, return ``-1``. - This is equivalent to the Python expression ``key in p``. - - -.. c:function:: PyObject* PyDict_Copy(PyObject *p) - - Return a new dictionary that contains the same key-value pairs as *p*. - - -.. c:function:: int PyDict_SetItem(PyObject *p, PyObject *key, PyObject *val) - - Insert *val* into the dictionary *p* with a key of *key*. *key* must be - :term:`hashable`; if it isn't, :exc:`TypeError` will be raised. Return - ``0`` on success or ``-1`` on failure. This function *does not* steal a - reference to *val*. - - -.. c:function:: int PyDict_SetItemString(PyObject *p, const char *key, PyObject *val) - - This is the same as :c:func:`PyDict_SetItem`, but *key* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - -.. c:function:: int PyDict_DelItem(PyObject *p, PyObject *key) - - Remove the entry in dictionary *p* with key *key*. *key* must be :term:`hashable`; - if it isn't, :exc:`TypeError` is raised. - If *key* is not in the dictionary, :exc:`KeyError` is raised. - Return ``0`` on success or ``-1`` on failure. - - -.. c:function:: int PyDict_DelItemString(PyObject *p, const char *key) - - This is the same as :c:func:`PyDict_DelItem`, but *key* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - -.. c:function:: PyObject* PyDict_GetItem(PyObject *p, PyObject *key) - - Return the object from dictionary *p* which has a key *key*. Return ``NULL`` - if the key *key* is not present, but *without* setting an exception. - - .. note:: - - Exceptions that occur while this calls :meth:`~object.__hash__` and - :meth:`~object.__eq__` methods are silently ignored. - Prefer the :c:func:`PyDict_GetItemWithError` function instead. - - .. versionchanged:: 3.10 - Calling this API without :term:`GIL` held had been allowed for historical - reason. It is no longer allowed. - - -.. c:function:: PyObject* PyDict_GetItemWithError(PyObject *p, PyObject *key) - - Variant of :c:func:`PyDict_GetItem` that does not suppress - exceptions. Return ``NULL`` **with** an exception set if an exception - occurred. Return ``NULL`` **without** an exception set if the key - wasn't present. - - -.. c:function:: PyObject* PyDict_GetItemString(PyObject *p, const char *key) - - This is the same as :c:func:`PyDict_GetItem`, but *key* is specified as a - :c:expr:`const char*` UTF-8 encoded bytes string, rather than a - :c:expr:`PyObject*`. - - .. note:: - - Exceptions that occur while this calls :meth:`~object.__hash__` and - :meth:`~object.__eq__` methods or while creating the temporary :class:`str` - object are silently ignored. - Prefer using the :c:func:`PyDict_GetItemWithError` function with your own - :c:func:`PyUnicode_FromString` *key* instead. - - -.. c:function:: PyObject* PyDict_SetDefault(PyObject *p, PyObject *key, PyObject *defaultobj) - - This is the same as the Python-level :meth:`dict.setdefault`. If present, it - returns the value corresponding to *key* from the dictionary *p*. If the key - is not in the dict, it is inserted with value *defaultobj* and *defaultobj* - is returned. This function evaluates the hash function of *key* only once, - instead of evaluating it independently for the lookup and the insertion. - - .. versionadded:: 3.4 - -.. c:function:: PyObject* PyDict_Items(PyObject *p) - - Return a :c:type:`PyListObject` containing all the items from the dictionary. - - -.. c:function:: PyObject* PyDict_Keys(PyObject *p) - - Return a :c:type:`PyListObject` containing all the keys from the dictionary. - - -.. c:function:: PyObject* PyDict_Values(PyObject *p) - - Return a :c:type:`PyListObject` containing all the values from the dictionary - *p*. - - -.. c:function:: Py_ssize_t PyDict_Size(PyObject *p) - - .. index:: pair: built-in function; len - - Return the number of items in the dictionary. This is equivalent to - ``len(p)`` on a dictionary. - - -.. c:function:: int PyDict_Next(PyObject *p, Py_ssize_t *ppos, PyObject **pkey, PyObject **pvalue) - - Iterate over all key-value pairs in the dictionary *p*. The - :c:type:`Py_ssize_t` referred to by *ppos* must be initialized to ``0`` - prior to the first call to this function to start the iteration; the - function returns true for each pair in the dictionary, and false once all - pairs have been reported. The parameters *pkey* and *pvalue* should either - point to :c:expr:`PyObject*` variables that will be filled in with each key - and value, respectively, or may be ``NULL``. Any references returned through - them are borrowed. *ppos* should not be altered during iteration. Its - value represents offsets within the internal dictionary structure, and - since the structure is sparse, the offsets are not consecutive. - - For example:: - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(self->dict, &pos, &key, &value)) { - /* do something interesting with the values... */ - ... - } - - The dictionary *p* should not be mutated during iteration. It is safe to - modify the values of the keys as you iterate over the dictionary, but only - so long as the set of keys does not change. For example:: - - PyObject *key, *value; - Py_ssize_t pos = 0; - - while (PyDict_Next(self->dict, &pos, &key, &value)) { - long i = PyLong_AsLong(value); - if (i == -1 && PyErr_Occurred()) { - return -1; - } - PyObject *o = PyLong_FromLong(i + 1); - if (o == NULL) - return -1; - if (PyDict_SetItem(self->dict, key, o) < 0) { - Py_DECREF(o); - return -1; - } - Py_DECREF(o); - } - - -.. c:function:: int PyDict_Merge(PyObject *a, PyObject *b, int override) - - Iterate over mapping object *b* adding key-value pairs to dictionary *a*. - *b* may be a dictionary, or any object supporting :c:func:`PyMapping_Keys` - and :c:func:`PyObject_GetItem`. If *override* is true, existing pairs in *a* - will be replaced if a matching key is found in *b*, otherwise pairs will - only be added if there is not a matching key in *a*. Return ``0`` on - success or ``-1`` if an exception was raised. - - -.. c:function:: int PyDict_Update(PyObject *a, PyObject *b) - - This is the same as ``PyDict_Merge(a, b, 1)`` in C, and is similar to - ``a.update(b)`` in Python except that :c:func:`PyDict_Update` doesn't fall - back to the iterating over a sequence of key value pairs if the second - argument has no "keys" attribute. Return ``0`` on success or ``-1`` if an - exception was raised. - - -.. c:function:: int PyDict_MergeFromSeq2(PyObject *a, PyObject *seq2, int override) - - Update or merge into dictionary *a*, from the key-value pairs in *seq2*. - *seq2* must be an iterable object producing iterable objects of length 2, - viewed as key-value pairs. In case of duplicate keys, the last wins if - *override* is true, else the first wins. Return ``0`` on success or ``-1`` - if an exception was raised. Equivalent Python (except for the return - value):: - - def PyDict_MergeFromSeq2(a, seq2, override): - for key, value in seq2: - if override or key not in a: - a[key] = value diff --git a/Doc/c-api/exceptions.rst b/Doc/c-api/exceptions.rst deleted file mode 100644 index 320f1e6dbcc4ea..00000000000000 --- a/Doc/c-api/exceptions.rst +++ /dev/null @@ -1,1124 +0,0 @@ -.. highlight:: c - - -.. _exceptionhandling: - -****************** -Exception Handling -****************** - -The functions described in this chapter will let you handle and raise Python -exceptions. It is important to understand some of the basics of Python -exception handling. It works somewhat like the POSIX :c:data:`errno` variable: -there is a global indicator (per thread) of the last error that occurred. Most -C API functions don't clear this on success, but will set it to indicate the -cause of the error on failure. Most C API functions also return an error -indicator, usually ``NULL`` if they are supposed to return a pointer, or ``-1`` -if they return an integer (exception: the ``PyArg_*`` functions -return ``1`` for success and ``0`` for failure). - -Concretely, the error indicator consists of three object pointers: the -exception's type, the exception's value, and the traceback object. Any -of those pointers can be ``NULL`` if non-set (although some combinations are -forbidden, for example you can't have a non-``NULL`` traceback if the exception -type is ``NULL``). - -When a function must fail because some function it called failed, it generally -doesn't set the error indicator; the function it called already set it. It is -responsible for either handling the error and clearing the exception or -returning after cleaning up any resources it holds (such as object references or -memory allocations); it should *not* continue normally if it is not prepared to -handle the error. If returning due to an error, it is important to indicate to -the caller that an error has been set. If the error is not handled or carefully -propagated, additional calls into the Python/C API may not behave as intended -and may fail in mysterious ways. - -.. note:: - The error indicator is **not** the result of :func:`sys.exc_info()`. - The former corresponds to an exception that is not yet caught (and is - therefore still propagating), while the latter returns an exception after - it is caught (and has therefore stopped propagating). - - -Printing and clearing -===================== - - -.. c:function:: void PyErr_Clear() - - Clear the error indicator. If the error indicator is not set, there is no - effect. - - -.. c:function:: void PyErr_PrintEx(int set_sys_last_vars) - - Print a standard traceback to ``sys.stderr`` and clear the error indicator. - **Unless** the error is a ``SystemExit``, in that case no traceback is - printed and the Python process will exit with the error code specified by - the ``SystemExit`` instance. - - Call this function **only** when the error indicator is set. Otherwise it - will cause a fatal error! - - If *set_sys_last_vars* is nonzero, the variables :data:`sys.last_type`, - :data:`sys.last_value` and :data:`sys.last_traceback` will be set to the - type, value and traceback of the printed exception, respectively. - - -.. c:function:: void PyErr_Print() - - Alias for ``PyErr_PrintEx(1)``. - - -.. c:function:: void PyErr_WriteUnraisable(PyObject *obj) - - Call :func:`sys.unraisablehook` using the current exception and *obj* - argument. - - This utility function prints a warning message to ``sys.stderr`` when an - exception has been set but it is impossible for the interpreter to actually - raise the exception. It is used, for example, when an exception occurs in an - :meth:`~object.__del__` method. - - The function is called with a single argument *obj* that identifies the context - in which the unraisable exception occurred. If possible, - the repr of *obj* will be printed in the warning message. - - An exception must be set when calling this function. - - -Raising exceptions -================== - -These functions help you set the current thread's error indicator. -For convenience, some of these functions will always return a -``NULL`` pointer for use in a ``return`` statement. - - -.. c:function:: void PyErr_SetString(PyObject *type, const char *message) - - This is the most common way to set the error indicator. The first argument - specifies the exception type; it is normally one of the standard exceptions, - e.g. :c:data:`PyExc_RuntimeError`. You need not create a new - :term:`strong reference` to it (e.g. with :c:func:`Py_INCREF`). - The second argument is an error message; it is decoded from ``'utf-8'``. - - -.. c:function:: void PyErr_SetObject(PyObject *type, PyObject *value) - - This function is similar to :c:func:`PyErr_SetString` but lets you specify an - arbitrary Python object for the "value" of the exception. - - -.. c:function:: PyObject* PyErr_Format(PyObject *exception, const char *format, ...) - - This function sets the error indicator and returns ``NULL``. *exception* - should be a Python exception class. The *format* and subsequent - parameters help format the error message; they have the same meaning and - values as in :c:func:`PyUnicode_FromFormat`. *format* is an ASCII-encoded - string. - - -.. c:function:: PyObject* PyErr_FormatV(PyObject *exception, const char *format, va_list vargs) - - Same as :c:func:`PyErr_Format`, but taking a :c:type:`va_list` argument rather - than a variable number of arguments. - - .. versionadded:: 3.5 - - -.. c:function:: void PyErr_SetNone(PyObject *type) - - This is a shorthand for ``PyErr_SetObject(type, Py_None)``. - - -.. c:function:: int PyErr_BadArgument() - - This is a shorthand for ``PyErr_SetString(PyExc_TypeError, message)``, where - *message* indicates that a built-in operation was invoked with an illegal - argument. It is mostly for internal use. - - -.. c:function:: PyObject* PyErr_NoMemory() - - This is a shorthand for ``PyErr_SetNone(PyExc_MemoryError)``; it returns ``NULL`` - so an object allocation function can write ``return PyErr_NoMemory();`` when it - runs out of memory. - - -.. c:function:: PyObject* PyErr_SetFromErrno(PyObject *type) - - .. index:: single: strerror() - - This is a convenience function to raise an exception when a C library function - has returned an error and set the C variable :c:data:`errno`. It constructs a - tuple object whose first item is the integer :c:data:`errno` value and whose - second item is the corresponding error message (gotten from :c:func:`!strerror`), - and then calls ``PyErr_SetObject(type, object)``. On Unix, when the - :c:data:`errno` value is :c:macro:`!EINTR`, indicating an interrupted system call, - this calls :c:func:`PyErr_CheckSignals`, and if that set the error indicator, - leaves it set to that. The function always returns ``NULL``, so a wrapper - function around a system call can write ``return PyErr_SetFromErrno(type);`` - when the system call returns an error. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObject(PyObject *type, PyObject *filenameObject) - - Similar to :c:func:`PyErr_SetFromErrno`, with the additional behavior that if - *filenameObject* is not ``NULL``, it is passed to the constructor of *type* as - a third parameter. In the case of :exc:`OSError` exception, - this is used to define the :attr:`!filename` attribute of the - exception instance. - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilenameObjects(PyObject *type, PyObject *filenameObject, PyObject *filenameObject2) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but takes a second - filename object, for raising errors when a function that takes two filenames - fails. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetFromErrnoWithFilename(PyObject *type, const char *filename) - - Similar to :c:func:`PyErr_SetFromErrnoWithFilenameObject`, but the filename - is given as a C string. *filename* is decoded from the :term:`filesystem - encoding and error handler`. - - -.. c:function:: PyObject* PyErr_SetFromWindowsErr(int ierr) - - This is a convenience function to raise :exc:`WindowsError`. If called with - *ierr* of ``0``, the error code returned by a call to :c:func:`!GetLastError` - is used instead. It calls the Win32 function :c:func:`!FormatMessage` to retrieve - the Windows description of error code given by *ierr* or :c:func:`!GetLastError`, - then it constructs a tuple object whose first item is the *ierr* value and whose - second item is the corresponding error message (gotten from - :c:func:`!FormatMessage`), and then calls ``PyErr_SetObject(PyExc_WindowsError, - object)``. This function always returns ``NULL``. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErr(PyObject *type, int ierr) - - Similar to :c:func:`PyErr_SetFromWindowsErr`, with an additional parameter - specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetFromWindowsErrWithFilename(int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErr`, with the additional behavior - that if *filename* is not ``NULL``, it is decoded from the filesystem - encoding (:func:`os.fsdecode`) and passed to the constructor of - :exc:`OSError` as a third parameter to be used to define the - :attr:`!filename` attribute of the exception instance. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObject(PyObject *type, int ierr, PyObject *filename) - - Similar to :c:func:`PyErr_SetExcFromWindowsErr`, with the additional behavior - that if *filename* is not ``NULL``, it is passed to the constructor of - :exc:`OSError` as a third parameter to be used to define the - :attr:`!filename` attribute of the exception instance. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilenameObjects(PyObject *type, int ierr, PyObject *filename, PyObject *filename2) - - Similar to :c:func:`PyErr_SetExcFromWindowsErrWithFilenameObject`, - but accepts a second filename object. - - .. availability:: Windows. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyErr_SetExcFromWindowsErrWithFilename(PyObject *type, int ierr, const char *filename) - - Similar to :c:func:`PyErr_SetFromWindowsErrWithFilename`, with an additional - parameter specifying the exception type to be raised. - - .. availability:: Windows. - - -.. c:function:: PyObject* PyErr_SetImportError(PyObject *msg, PyObject *name, PyObject *path) - - This is a convenience function to raise :exc:`ImportError`. *msg* will be - set as the exception's message string. *name* and *path*, both of which can - be ``NULL``, will be set as the :exc:`ImportError`'s respective ``name`` - and ``path`` attributes. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyErr_SetImportErrorSubclass(PyObject *exception, PyObject *msg, PyObject *name, PyObject *path) - - Much like :c:func:`PyErr_SetImportError` but this function allows for - specifying a subclass of :exc:`ImportError` to raise. - - .. versionadded:: 3.6 - - -.. c:function:: void PyErr_SyntaxLocationObject(PyObject *filename, int lineno, int col_offset) - - Set file, line, and offset information for the current exception. If the - current exception is not a :exc:`SyntaxError`, then it sets additional - attributes, which make the exception printing subsystem think the exception - is a :exc:`SyntaxError`. - - .. versionadded:: 3.4 - - -.. c:function:: void PyErr_SyntaxLocationEx(const char *filename, int lineno, int col_offset) - - Like :c:func:`PyErr_SyntaxLocationObject`, but *filename* is a byte string - decoded from the :term:`filesystem encoding and error handler`. - - .. versionadded:: 3.2 - - -.. c:function:: void PyErr_SyntaxLocation(const char *filename, int lineno) - - Like :c:func:`PyErr_SyntaxLocationEx`, but the *col_offset* parameter is - omitted. - - -.. c:function:: void PyErr_BadInternalCall() - - This is a shorthand for ``PyErr_SetString(PyExc_SystemError, message)``, - where *message* indicates that an internal operation (e.g. a Python/C API - function) was invoked with an illegal argument. It is mostly for internal - use. - - -Issuing warnings -================ - -Use these functions to issue warnings from C code. They mirror similar -functions exported by the Python :mod:`warnings` module. They normally -print a warning message to *sys.stderr*; however, it is -also possible that the user has specified that warnings are to be turned into -errors, and in that case they will raise an exception. It is also possible that -the functions raise an exception because of a problem with the warning machinery. -The return value is ``0`` if no exception is raised, or ``-1`` if an exception -is raised. (It is not possible to determine whether a warning message is -actually printed, nor what the reason is for the exception; this is -intentional.) If an exception is raised, the caller should do its normal -exception handling (for example, :c:func:`Py_DECREF` owned references and return -an error value). - -.. c:function:: int PyErr_WarnEx(PyObject *category, const char *message, Py_ssize_t stack_level) - - Issue a warning message. The *category* argument is a warning category (see - below) or ``NULL``; the *message* argument is a UTF-8 encoded string. *stack_level* is a - positive number giving a number of stack frames; the warning will be issued from - the currently executing line of code in that stack frame. A *stack_level* of 1 - is the function calling :c:func:`PyErr_WarnEx`, 2 is the function above that, - and so forth. - - Warning categories must be subclasses of :c:data:`PyExc_Warning`; - :c:data:`PyExc_Warning` is a subclass of :c:data:`PyExc_Exception`; - the default warning category is :c:data:`PyExc_RuntimeWarning`. The standard - Python warning categories are available as global variables whose names are - enumerated at :ref:`standardwarningcategories`. - - For information about warning control, see the documentation for the - :mod:`warnings` module and the :option:`-W` option in the command line - documentation. There is no C API for warning control. - - -.. c:function:: int PyErr_WarnExplicitObject(PyObject *category, PyObject *message, PyObject *filename, int lineno, PyObject *module, PyObject *registry) - - Issue a warning message with explicit control over all warning attributes. This - is a straightforward wrapper around the Python function - :func:`warnings.warn_explicit`; see there for more information. The *module* - and *registry* arguments may be set to ``NULL`` to get the default effect - described there. - - .. versionadded:: 3.4 - - -.. c:function:: int PyErr_WarnExplicit(PyObject *category, const char *message, const char *filename, int lineno, const char *module, PyObject *registry) - - Similar to :c:func:`PyErr_WarnExplicitObject` except that *message* and - *module* are UTF-8 encoded strings, and *filename* is decoded from the - :term:`filesystem encoding and error handler`. - - -.. c:function:: int PyErr_WarnFormat(PyObject *category, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnEx`, but use - :c:func:`PyUnicode_FromFormat` to format the warning message. *format* is - an ASCII-encoded string. - - .. versionadded:: 3.2 - - -.. c:function:: int PyErr_ResourceWarning(PyObject *source, Py_ssize_t stack_level, const char *format, ...) - - Function similar to :c:func:`PyErr_WarnFormat`, but *category* is - :exc:`ResourceWarning` and it passes *source* to :func:`warnings.WarningMessage`. - - .. versionadded:: 3.6 - - -Querying the error indicator -============================ - -.. c:function:: PyObject* PyErr_Occurred() - - Test whether the error indicator is set. If set, return the exception *type* - (the first argument to the last call to one of the ``PyErr_Set*`` - functions or to :c:func:`PyErr_Restore`). If not set, return ``NULL``. You do not - own a reference to the return value, so you do not need to :c:func:`Py_DECREF` - it. - - The caller must hold the GIL. - - .. note:: - - Do not compare the return value to a specific exception; use - :c:func:`PyErr_ExceptionMatches` instead, shown below. (The comparison could - easily fail since the exception may be an instance instead of a class, in the - case of a class exception, or it may be a subclass of the expected exception.) - - -.. c:function:: int PyErr_ExceptionMatches(PyObject *exc) - - Equivalent to ``PyErr_GivenExceptionMatches(PyErr_Occurred(), exc)``. This - should only be called when an exception is actually set; a memory access - violation will occur if no exception has been raised. - - -.. c:function:: int PyErr_GivenExceptionMatches(PyObject *given, PyObject *exc) - - Return true if the *given* exception matches the exception type in *exc*. If - *exc* is a class object, this also returns true when *given* is an instance - of a subclass. If *exc* is a tuple, all exception types in the tuple (and - recursively in subtuples) are searched for a match. - - -.. c:function:: void PyErr_Fetch(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the error indicator into three variables whose addresses are passed. - If the error indicator is not set, set all three variables to ``NULL``. If it is - set, it will be cleared and you own a reference to each object retrieved. The - value and traceback object may be ``NULL`` even when the type object is not. - - .. note:: - - This function is normally only used by code that needs to catch exceptions or - by code that needs to save and restore the error indicator temporarily, e.g.:: - - { - PyObject *type, *value, *traceback; - PyErr_Fetch(&type, &value, &traceback); - - /* ... code that might produce other errors ... */ - - PyErr_Restore(type, value, traceback); - } - - -.. c:function:: void PyErr_Restore(PyObject *type, PyObject *value, PyObject *traceback) - - Set the error indicator from the three objects. If the error indicator is - already set, it is cleared first. If the objects are ``NULL``, the error - indicator is cleared. Do not pass a ``NULL`` type and non-``NULL`` value or - traceback. The exception type should be a class. Do not pass an invalid - exception type or value. (Violating these rules will cause subtle problems - later.) This call takes away a reference to each object: you must own a - reference to each object before the call and after the call you no longer own - these references. (If you don't understand this, don't use this function. I - warned you.) - - .. note:: - - This function is normally only used by code that needs to save and restore the - error indicator temporarily. Use :c:func:`PyErr_Fetch` to save the current - error indicator. - - -.. c:function:: void PyErr_NormalizeException(PyObject **exc, PyObject **val, PyObject **tb) - - Under certain circumstances, the values returned by :c:func:`PyErr_Fetch` below - can be "unnormalized", meaning that ``*exc`` is a class object but ``*val`` is - not an instance of the same class. This function can be used to instantiate - the class in that case. If the values are already normalized, nothing happens. - The delayed normalization is implemented to improve performance. - - .. note:: - - This function *does not* implicitly set the ``__traceback__`` - attribute on the exception value. If setting the traceback - appropriately is desired, the following additional snippet is needed:: - - if (tb != NULL) { - PyException_SetTraceback(val, tb); - } - - -.. c:function:: PyObject* PyErr_GetHandledException(void) - - Retrieve the active exception instance, as would be returned by :func:`sys.exception`. - This refers to an exception that was *already caught*, not to an exception that was - freshly raised. Returns a new reference to the exception or ``NULL``. - Does not modify the interpreter's exception state. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_SetHandledException` to restore or - clear the exception state. - - .. versionadded:: 3.11 - -.. c:function:: void PyErr_SetHandledException(PyObject *exc) - - Set the active exception, as known from ``sys.exception()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. - To clear the exception state, pass ``NULL``. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_GetHandledException` to get the exception - state. - - .. versionadded:: 3.11 - -.. c:function:: void PyErr_GetExcInfo(PyObject **ptype, PyObject **pvalue, PyObject **ptraceback) - - Retrieve the old-style representation of the exception info, as known from - :func:`sys.exc_info`. This refers to an exception that was *already caught*, - not to an exception that was freshly raised. Returns new references for the - three objects, any of which may be ``NULL``. Does not modify the exception - info state. This function is kept for backwards compatibility. Prefer using - :c:func:`PyErr_GetHandledException`. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_SetExcInfo` to restore or clear the - exception state. - - .. versionadded:: 3.3 - - -.. c:function:: void PyErr_SetExcInfo(PyObject *type, PyObject *value, PyObject *traceback) - - Set the exception info, as known from ``sys.exc_info()``. This refers - to an exception that was *already caught*, not to an exception that was - freshly raised. This function steals the references of the arguments. - To clear the exception state, pass ``NULL`` for all three arguments. - This function is kept for backwards compatibility. Prefer using - :c:func:`PyErr_SetHandledException`. - - .. note:: - - This function is not normally used by code that wants to handle exceptions. - Rather, it can be used when code needs to save and restore the exception - state temporarily. Use :c:func:`PyErr_GetExcInfo` to read the exception - state. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.11 - The ``type`` and ``traceback`` arguments are no longer used and - can be NULL. The interpreter now derives them from the exception - instance (the ``value`` argument). The function still steals - references of all three arguments. - - -Signal Handling -=============== - - -.. c:function:: int PyErr_CheckSignals() - - .. index:: - pair: module; signal - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - This function interacts with Python's signal handling. - - If the function is called from the main thread and under the main Python - interpreter, it checks whether a signal has been sent to the processes - and if so, invokes the corresponding signal handler. If the :mod:`signal` - module is supported, this can invoke a signal handler written in Python. - - The function attempts to handle all pending signals, and then returns ``0``. - However, if a Python signal handler raises an exception, the error - indicator is set and the function returns ``-1`` immediately (such that - other pending signals may not have been handled yet: they will be on the - next :c:func:`PyErr_CheckSignals()` invocation). - - If the function is called from a non-main thread, or under a non-main - Python interpreter, it does nothing and returns ``0``. - - This function can be called by long-running C code that wants to - be interruptible by user requests (such as by pressing Ctrl-C). - - .. note:: - The default Python signal handler for :c:macro:`!SIGINT` raises the - :exc:`KeyboardInterrupt` exception. - - -.. c:function:: void PyErr_SetInterrupt() - - .. index:: - pair: module; signal - single: SIGINT - single: KeyboardInterrupt (built-in exception) - - Simulate the effect of a :c:macro:`!SIGINT` signal arriving. - This is equivalent to ``PyErr_SetInterruptEx(SIGINT)``. - - .. note:: - This function is async-signal-safe. It can be called without - the :term:`GIL` and from a C signal handler. - - -.. c:function:: int PyErr_SetInterruptEx(int signum) - - .. index:: - pair: module; signal - single: KeyboardInterrupt (built-in exception) - - Simulate the effect of a signal arriving. The next time - :c:func:`PyErr_CheckSignals` is called, the Python signal handler for - the given signal number will be called. - - This function can be called by C code that sets up its own signal handling - and wants Python signal handlers to be invoked as expected when an - interruption is requested (for example when the user presses Ctrl-C - to interrupt an operation). - - If the given signal isn't handled by Python (it was set to - :py:const:`signal.SIG_DFL` or :py:const:`signal.SIG_IGN`), it will be ignored. - - If *signum* is outside of the allowed range of signal numbers, ``-1`` - is returned. Otherwise, ``0`` is returned. The error indicator is - never changed by this function. - - .. note:: - This function is async-signal-safe. It can be called without - the :term:`GIL` and from a C signal handler. - - .. versionadded:: 3.10 - - -.. c:function:: int PySignal_SetWakeupFd(int fd) - - This utility function specifies a file descriptor to which the signal number - is written as a single byte whenever a signal is received. *fd* must be - non-blocking. It returns the previous such file descriptor. - - The value ``-1`` disables the feature; this is the initial state. - This is equivalent to :func:`signal.set_wakeup_fd` in Python, but without any - error checking. *fd* should be a valid file descriptor. The function should - only be called from the main thread. - - .. versionchanged:: 3.5 - On Windows, the function now also supports socket handles. - - -Exception Classes -================= - -.. c:function:: PyObject* PyErr_NewException(const char *name, PyObject *base, PyObject *dict) - - This utility function creates and returns a new exception class. The *name* - argument must be the name of the new exception, a C string of the form - ``module.classname``. The *base* and *dict* arguments are normally ``NULL``. - This creates a class object derived from :exc:`Exception` (accessible in C as - :c:data:`PyExc_Exception`). - - The :attr:`__module__` attribute of the new class is set to the first part (up - to the last dot) of the *name* argument, and the class name is set to the last - part (after the last dot). The *base* argument can be used to specify alternate - base classes; it can either be only one class or a tuple of classes. The *dict* - argument can be used to specify a dictionary of class variables and methods. - - -.. c:function:: PyObject* PyErr_NewExceptionWithDoc(const char *name, const char *doc, PyObject *base, PyObject *dict) - - Same as :c:func:`PyErr_NewException`, except that the new exception class can - easily be given a docstring: If *doc* is non-``NULL``, it will be used as the - docstring for the exception class. - - .. versionadded:: 3.2 - - -Exception Objects -================= - -.. c:function:: PyObject* PyException_GetTraceback(PyObject *ex) - - Return the traceback associated with the exception as a new reference, as - accessible from Python through :attr:`__traceback__`. If there is no - traceback associated, this returns ``NULL``. - - -.. c:function:: int PyException_SetTraceback(PyObject *ex, PyObject *tb) - - Set the traceback associated with the exception to *tb*. Use ``Py_None`` to - clear it. - - -.. c:function:: PyObject* PyException_GetContext(PyObject *ex) - - Return the context (another exception instance during whose handling *ex* was - raised) associated with the exception as a new reference, as accessible from - Python through :attr:`__context__`. If there is no context associated, this - returns ``NULL``. - - -.. c:function:: void PyException_SetContext(PyObject *ex, PyObject *ctx) - - Set the context associated with the exception to *ctx*. Use ``NULL`` to clear - it. There is no type check to make sure that *ctx* is an exception instance. - This steals a reference to *ctx*. - - -.. c:function:: PyObject* PyException_GetCause(PyObject *ex) - - Return the cause (either an exception instance, or ``None``, - set by ``raise ... from ...``) associated with the exception as a new - reference, as accessible from Python through :attr:`__cause__`. - - -.. c:function:: void PyException_SetCause(PyObject *ex, PyObject *cause) - - Set the cause associated with the exception to *cause*. Use ``NULL`` to clear - it. There is no type check to make sure that *cause* is either an exception - instance or ``None``. This steals a reference to *cause*. - - :attr:`__suppress_context__` is implicitly set to ``True`` by this function. - - -.. _unicodeexceptions: - -Unicode Exception Objects -========================= - -The following functions are used to create and modify Unicode exceptions from C. - -.. c:function:: PyObject* PyUnicodeDecodeError_Create(const char *encoding, const char *object, Py_ssize_t length, Py_ssize_t start, Py_ssize_t end, const char *reason) - - Create a :class:`UnicodeDecodeError` object with the attributes *encoding*, - *object*, *length*, *start*, *end* and *reason*. *encoding* and *reason* are - UTF-8 encoded strings. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetEncoding(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetEncoding(PyObject *exc) - - Return the *encoding* attribute of the given exception object. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetObject(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetObject(PyObject *exc) - - Return the *object* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeEncodeError_GetStart(PyObject *exc, Py_ssize_t *start) - int PyUnicodeTranslateError_GetStart(PyObject *exc, Py_ssize_t *start) - - Get the *start* attribute of the given exception object and place it into - *\*start*. *start* must not be ``NULL``. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeEncodeError_SetStart(PyObject *exc, Py_ssize_t start) - int PyUnicodeTranslateError_SetStart(PyObject *exc, Py_ssize_t start) - - Set the *start* attribute of the given exception object to *start*. Return - ``0`` on success, ``-1`` on failure. - -.. c:function:: int PyUnicodeDecodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeEncodeError_GetEnd(PyObject *exc, Py_ssize_t *end) - int PyUnicodeTranslateError_GetEnd(PyObject *exc, Py_ssize_t *end) - - Get the *end* attribute of the given exception object and place it into - *\*end*. *end* must not be ``NULL``. Return ``0`` on success, ``-1`` on - failure. - -.. c:function:: int PyUnicodeDecodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeEncodeError_SetEnd(PyObject *exc, Py_ssize_t end) - int PyUnicodeTranslateError_SetEnd(PyObject *exc, Py_ssize_t end) - - Set the *end* attribute of the given exception object to *end*. Return ``0`` - on success, ``-1`` on failure. - -.. c:function:: PyObject* PyUnicodeDecodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeEncodeError_GetReason(PyObject *exc) - PyObject* PyUnicodeTranslateError_GetReason(PyObject *exc) - - Return the *reason* attribute of the given exception object. - -.. c:function:: int PyUnicodeDecodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeEncodeError_SetReason(PyObject *exc, const char *reason) - int PyUnicodeTranslateError_SetReason(PyObject *exc, const char *reason) - - Set the *reason* attribute of the given exception object to *reason*. Return - ``0`` on success, ``-1`` on failure. - - -.. _recursion: - -Recursion Control -================= - -These two functions provide a way to perform safe recursive calls at the C -level, both in the core and in extension modules. They are needed if the -recursive code does not necessarily invoke Python code (which tracks its -recursion depth automatically). -They are also not needed for *tp_call* implementations -because the :ref:`call protocol ` takes care of recursion handling. - -.. c:function:: int Py_EnterRecursiveCall(const char *where) - - Marks a point where a recursive C-level call is about to be performed. - - If :c:macro:`USE_STACKCHECK` is defined, this function checks if the OS - stack overflowed using :c:func:`PyOS_CheckStack`. In this is the case, it - sets a :exc:`MemoryError` and returns a nonzero value. - - The function then checks if the recursion limit is reached. If this is the - case, a :exc:`RecursionError` is set and a nonzero value is returned. - Otherwise, zero is returned. - - *where* should be a UTF-8 encoded string such as ``" in instance check"`` to - be concatenated to the :exc:`RecursionError` message caused by the recursion - depth limit. - - .. versionchanged:: 3.9 - This function is now also available in the :ref:`limited API `. - -.. c:function:: void Py_LeaveRecursiveCall(void) - - Ends a :c:func:`Py_EnterRecursiveCall`. Must be called once for each - *successful* invocation of :c:func:`Py_EnterRecursiveCall`. - - .. versionchanged:: 3.9 - This function is now also available in the :ref:`limited API `. - -Properly implementing :c:member:`~PyTypeObject.tp_repr` for container types requires -special recursion handling. In addition to protecting the stack, -:c:member:`~PyTypeObject.tp_repr` also needs to track objects to prevent cycles. The -following two functions facilitate this functionality. Effectively, -these are the C equivalent to :func:`reprlib.recursive_repr`. - -.. c:function:: int Py_ReprEnter(PyObject *object) - - Called at the beginning of the :c:member:`~PyTypeObject.tp_repr` implementation to - detect cycles. - - If the object has already been processed, the function returns a - positive integer. In that case the :c:member:`~PyTypeObject.tp_repr` implementation - should return a string object indicating a cycle. As examples, - :class:`dict` objects return ``{...}`` and :class:`list` objects - return ``[...]``. - - The function will return a negative integer if the recursion limit - is reached. In that case the :c:member:`~PyTypeObject.tp_repr` implementation should - typically return ``NULL``. - - Otherwise, the function returns zero and the :c:member:`~PyTypeObject.tp_repr` - implementation can continue normally. - -.. c:function:: void Py_ReprLeave(PyObject *object) - - Ends a :c:func:`Py_ReprEnter`. Must be called once for each - invocation of :c:func:`Py_ReprEnter` that returns zero. - - -.. _standardexceptions: - -Standard Exceptions -=================== - -All standard Python exceptions are available as global variables whose names are -``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_BaseException - single: PyExc_Exception - single: PyExc_ArithmeticError - single: PyExc_AssertionError - single: PyExc_AttributeError - single: PyExc_BlockingIOError - single: PyExc_BrokenPipeError - single: PyExc_BufferError - single: PyExc_ChildProcessError - single: PyExc_ConnectionAbortedError - single: PyExc_ConnectionError - single: PyExc_ConnectionRefusedError - single: PyExc_ConnectionResetError - single: PyExc_EOFError - single: PyExc_FileExistsError - single: PyExc_FileNotFoundError - single: PyExc_FloatingPointError - single: PyExc_GeneratorExit - single: PyExc_ImportError - single: PyExc_IndentationError - single: PyExc_IndexError - single: PyExc_InterruptedError - single: PyExc_IsADirectoryError - single: PyExc_KeyError - single: PyExc_KeyboardInterrupt - single: PyExc_LookupError - single: PyExc_MemoryError - single: PyExc_ModuleNotFoundError - single: PyExc_NameError - single: PyExc_NotADirectoryError - single: PyExc_NotImplementedError - single: PyExc_OSError - single: PyExc_OverflowError - single: PyExc_PermissionError - single: PyExc_ProcessLookupError - single: PyExc_RecursionError - single: PyExc_ReferenceError - single: PyExc_RuntimeError - single: PyExc_StopAsyncIteration - single: PyExc_StopIteration - single: PyExc_SyntaxError - single: PyExc_SystemError - single: PyExc_SystemExit - single: PyExc_TabError - single: PyExc_TimeoutError - single: PyExc_TypeError - single: PyExc_UnboundLocalError - single: PyExc_UnicodeDecodeError - single: PyExc_UnicodeEncodeError - single: PyExc_UnicodeError - single: PyExc_UnicodeTranslateError - single: PyExc_ValueError - single: PyExc_ZeroDivisionError - -+-----------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+=========================================+=================================+==========+ -| :c:data:`PyExc_BaseException` | :exc:`BaseException` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_Exception` | :exc:`Exception` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ArithmeticError` | :exc:`ArithmeticError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AssertionError` | :exc:`AssertionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_AttributeError` | :exc:`AttributeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BlockingIOError` | :exc:`BlockingIOError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BrokenPipeError` | :exc:`BrokenPipeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BufferError` | :exc:`BufferError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ChildProcessError` | :exc:`ChildProcessError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionAbortedError` | :exc:`ConnectionAbortedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionError` | :exc:`ConnectionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionRefusedError` | :exc:`ConnectionRefusedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ConnectionResetError` | :exc:`ConnectionResetError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_EOFError` | :exc:`EOFError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileExistsError` | :exc:`FileExistsError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FileNotFoundError` | :exc:`FileNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FloatingPointError` | :exc:`FloatingPointError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_GeneratorExit` | :exc:`GeneratorExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportError` | :exc:`ImportError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndentationError` | :exc:`IndentationError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IndexError` | :exc:`IndexError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_InterruptedError` | :exc:`InterruptedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_IsADirectoryError` | :exc:`IsADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyError` | :exc:`KeyError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_KeyboardInterrupt` | :exc:`KeyboardInterrupt` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_LookupError` | :exc:`LookupError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_MemoryError` | :exc:`MemoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ModuleNotFoundError` | :exc:`ModuleNotFoundError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NameError` | :exc:`NameError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotADirectoryError` | :exc:`NotADirectoryError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_NotImplementedError` | :exc:`NotImplementedError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OSError` | :exc:`OSError` | [1]_ | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_OverflowError` | :exc:`OverflowError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PermissionError` | :exc:`PermissionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ProcessLookupError` | :exc:`ProcessLookupError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RecursionError` | :exc:`RecursionError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ReferenceError` | :exc:`ReferenceError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeError` | :exc:`RuntimeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopAsyncIteration` | :exc:`StopAsyncIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_StopIteration` | :exc:`StopIteration` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxError` | :exc:`SyntaxError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemError` | :exc:`SystemError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SystemExit` | :exc:`SystemExit` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TabError` | :exc:`TabError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TimeoutError` | :exc:`TimeoutError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_TypeError` | :exc:`TypeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnboundLocalError` | :exc:`UnboundLocalError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeDecodeError` | :exc:`UnicodeDecodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeEncodeError` | :exc:`UnicodeEncodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeError` | :exc:`UnicodeError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeTranslateError` | :exc:`UnicodeTranslateError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ValueError` | :exc:`ValueError` | | -+-----------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ZeroDivisionError` | :exc:`ZeroDivisionError` | | -+-----------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.3 - :c:data:`PyExc_BlockingIOError`, :c:data:`PyExc_BrokenPipeError`, - :c:data:`PyExc_ChildProcessError`, :c:data:`PyExc_ConnectionError`, - :c:data:`PyExc_ConnectionAbortedError`, :c:data:`PyExc_ConnectionRefusedError`, - :c:data:`PyExc_ConnectionResetError`, :c:data:`PyExc_FileExistsError`, - :c:data:`PyExc_FileNotFoundError`, :c:data:`PyExc_InterruptedError`, - :c:data:`PyExc_IsADirectoryError`, :c:data:`PyExc_NotADirectoryError`, - :c:data:`PyExc_PermissionError`, :c:data:`PyExc_ProcessLookupError` - and :c:data:`PyExc_TimeoutError` were introduced following :pep:`3151`. - -.. versionadded:: 3.5 - :c:data:`PyExc_StopAsyncIteration` and :c:data:`PyExc_RecursionError`. - -.. versionadded:: 3.6 - :c:data:`PyExc_ModuleNotFoundError`. - -These are compatibility aliases to :c:data:`PyExc_OSError`: - -.. index:: - single: PyExc_EnvironmentError - single: PyExc_IOError - single: PyExc_WindowsError - -+-------------------------------------+----------+ -| C Name | Notes | -+=====================================+==========+ -| :c:data:`PyExc_EnvironmentError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_IOError` | | -+-------------------------------------+----------+ -| :c:data:`PyExc_WindowsError` | [2]_ | -+-------------------------------------+----------+ - -.. versionchanged:: 3.3 - These aliases used to be separate exception types. - -Notes: - -.. [1] - This is a base class for other standard exceptions. - -.. [2] - Only defined on Windows; protect code that uses this by testing that the - preprocessor macro ``MS_WINDOWS`` is defined. - -.. _standardwarningcategories: - -Standard Warning Categories -=========================== - -All standard Python warning categories are available as global variables whose -names are ``PyExc_`` followed by the Python exception name. These have the type -:c:expr:`PyObject*`; they are all class objects. For completeness, here are all -the variables: - -.. index:: - single: PyExc_Warning - single: PyExc_BytesWarning - single: PyExc_DeprecationWarning - single: PyExc_FutureWarning - single: PyExc_ImportWarning - single: PyExc_PendingDeprecationWarning - single: PyExc_ResourceWarning - single: PyExc_RuntimeWarning - single: PyExc_SyntaxWarning - single: PyExc_UnicodeWarning - single: PyExc_UserWarning - -+------------------------------------------+---------------------------------+----------+ -| C Name | Python Name | Notes | -+==========================================+=================================+==========+ -| :c:data:`PyExc_Warning` | :exc:`Warning` | [3]_ | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_BytesWarning` | :exc:`BytesWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_DeprecationWarning` | :exc:`DeprecationWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_FutureWarning` | :exc:`FutureWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ImportWarning` | :exc:`ImportWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_PendingDeprecationWarning`| :exc:`PendingDeprecationWarning`| | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_ResourceWarning` | :exc:`ResourceWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_RuntimeWarning` | :exc:`RuntimeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_SyntaxWarning` | :exc:`SyntaxWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UnicodeWarning` | :exc:`UnicodeWarning` | | -+------------------------------------------+---------------------------------+----------+ -| :c:data:`PyExc_UserWarning` | :exc:`UserWarning` | | -+------------------------------------------+---------------------------------+----------+ - -.. versionadded:: 3.2 - :c:data:`PyExc_ResourceWarning`. - -Notes: - -.. [3] - This is a base class for other standard warning categories. diff --git a/Doc/c-api/file.rst b/Doc/c-api/file.rst deleted file mode 100644 index b36c800e00444a..00000000000000 --- a/Doc/c-api/file.rst +++ /dev/null @@ -1,104 +0,0 @@ -.. highlight:: c - -.. _fileobjects: - -File Objects ------------- - -.. index:: pair: object; file - -These APIs are a minimal emulation of the Python 2 C API for built-in file -objects, which used to rely on the buffered I/O (:c:expr:`FILE*`) support -from the C standard library. In Python 3, files and streams use the new -:mod:`io` module, which defines several layers over the low-level unbuffered -I/O of the operating system. The functions described below are -convenience C wrappers over these new APIs, and meant mostly for internal -error reporting in the interpreter; third-party code is advised to access -the :mod:`io` APIs instead. - - -.. c:function:: PyObject* PyFile_FromFd(int fd, const char *name, const char *mode, int buffering, const char *encoding, const char *errors, const char *newline, int closefd) - - Create a Python file object from the file descriptor of an already - opened file *fd*. The arguments *name*, *encoding*, *errors* and *newline* - can be ``NULL`` to use the defaults; *buffering* can be *-1* to use the - default. *name* is ignored and kept for backward compatibility. Return - ``NULL`` on failure. For a more comprehensive description of the arguments, - please refer to the :func:`io.open` function documentation. - - .. warning:: - - Since Python streams have their own buffering layer, mixing them with - OS-level file descriptors can produce various issues (such as unexpected - ordering of data). - - .. versionchanged:: 3.2 - Ignore *name* attribute. - - -.. c:function:: int PyObject_AsFileDescriptor(PyObject *p) - - Return the file descriptor associated with *p* as an :c:expr:`int`. If the - object is an integer, its value is returned. If not, the - object's :meth:`~io.IOBase.fileno` method is called if it exists; the - method must return an integer, which is returned as the file descriptor - value. Sets an exception and returns ``-1`` on failure. - - -.. c:function:: PyObject* PyFile_GetLine(PyObject *p, int n) - - .. index:: single: EOFError (built-in exception) - - Equivalent to ``p.readline([n])``, this function reads one line from the - object *p*. *p* may be a file object or any object with a - :meth:`~io.IOBase.readline` - method. If *n* is ``0``, exactly one line is read, regardless of the length of - the line. If *n* is greater than ``0``, no more than *n* bytes will be read - from the file; a partial line can be returned. In both cases, an empty string - is returned if the end of the file is reached immediately. If *n* is less than - ``0``, however, one line is read regardless of length, but :exc:`EOFError` is - raised if the end of the file is reached immediately. - - -.. c:function:: int PyFile_SetOpenCodeHook(Py_OpenCodeHookFunction handler) - - Overrides the normal behavior of :func:`io.open_code` to pass its parameter - through the provided handler. - - The handler is a function of type :c:expr:`PyObject *(\*)(PyObject *path, - void *userData)`, where *path* is guaranteed to be :c:type:`PyUnicodeObject`. - - The *userData* pointer is passed into the hook function. Since hook - functions may be called from different runtimes, this pointer should not - refer directly to Python state. - - As this hook is intentionally used during import, avoid importing new modules - during its execution unless they are known to be frozen or available in - ``sys.modules``. - - Once a hook has been set, it cannot be removed or replaced, and later calls to - :c:func:`PyFile_SetOpenCodeHook` will fail. On failure, the function returns - -1 and sets an exception if the interpreter has been initialized. - - This function is safe to call before :c:func:`Py_Initialize`. - - .. audit-event:: setopencodehook "" c.PyFile_SetOpenCodeHook - - .. versionadded:: 3.8 - - - -.. c:function:: int PyFile_WriteObject(PyObject *obj, PyObject *p, int flags) - - .. index:: single: Py_PRINT_RAW - - Write object *obj* to file object *p*. The only supported flag for *flags* is - :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written - instead of the :func:`repr`. Return ``0`` on success or ``-1`` on failure; the - appropriate exception will be set. - - -.. c:function:: int PyFile_WriteString(const char *s, PyObject *p) - - Write string *s* to file object *p*. Return ``0`` on success or ``-1`` on - failure; the appropriate exception will be set. diff --git a/Doc/c-api/float.rst b/Doc/c-api/float.rst deleted file mode 100644 index 4f6ac0d8175c6b..00000000000000 --- a/Doc/c-api/float.rst +++ /dev/null @@ -1,164 +0,0 @@ -.. highlight:: c - -.. _floatobjects: - -Floating Point Objects -====================== - -.. index:: pair: object; floating point - - -.. c:type:: PyFloatObject - - This subtype of :c:type:`PyObject` represents a Python floating point object. - - -.. c:var:: PyTypeObject PyFloat_Type - - This instance of :c:type:`PyTypeObject` represents the Python floating point - type. This is the same object as :class:`float` in the Python layer. - - -.. c:function:: int PyFloat_Check(PyObject *p) - - Return true if its argument is a :c:type:`PyFloatObject` or a subtype of - :c:type:`PyFloatObject`. This function always succeeds. - - -.. c:function:: int PyFloat_CheckExact(PyObject *p) - - Return true if its argument is a :c:type:`PyFloatObject`, but not a subtype of - :c:type:`PyFloatObject`. This function always succeeds. - - -.. c:function:: PyObject* PyFloat_FromString(PyObject *str) - - Create a :c:type:`PyFloatObject` object based on the string value in *str*, or - ``NULL`` on failure. - - -.. c:function:: PyObject* PyFloat_FromDouble(double v) - - Create a :c:type:`PyFloatObject` object from *v*, or ``NULL`` on failure. - - -.. c:function:: double PyFloat_AsDouble(PyObject *pyfloat) - - Return a C :c:expr:`double` representation of the contents of *pyfloat*. If - *pyfloat* is not a Python floating point object but has a :meth:`~object.__float__` - method, this method will first be called to convert *pyfloat* into a float. - If :meth:`!__float__` is not defined then it falls back to :meth:`~object.__index__`. - This method returns ``-1.0`` upon failure, so one should call - :c:func:`PyErr_Occurred` to check for errors. - - .. versionchanged:: 3.8 - Use :meth:`~object.__index__` if available. - - -.. c:function:: double PyFloat_AS_DOUBLE(PyObject *pyfloat) - - Return a C :c:expr:`double` representation of the contents of *pyfloat*, but - without error checking. - - -.. c:function:: PyObject* PyFloat_GetInfo(void) - - Return a structseq instance which contains information about the - precision, minimum and maximum values of a float. It's a thin wrapper - around the header file :file:`float.h`. - - -.. c:function:: double PyFloat_GetMax() - - Return the maximum representable finite float *DBL_MAX* as C :c:expr:`double`. - - -.. c:function:: double PyFloat_GetMin() - - Return the minimum normalized positive float *DBL_MIN* as C :c:expr:`double`. - - -Pack and Unpack functions -------------------------- - -The pack and unpack functions provide an efficient platform-independent way to -store floating-point values as byte strings. The Pack routines produce a bytes -string from a C :c:expr:`double`, and the Unpack routines produce a C -:c:expr:`double` from such a bytes string. The suffix (2, 4 or 8) specifies the -number of bytes in the bytes string. - -On platforms that appear to use IEEE 754 formats these functions work by -copying bits. On other platforms, the 2-byte format is identical to the IEEE -754 binary16 half-precision format, the 4-byte format (32-bit) is identical to -the IEEE 754 binary32 single precision format, and the 8-byte format to the -IEEE 754 binary64 double precision format, although the packing of INFs and -NaNs (if such things exist on the platform) isn't handled correctly, and -attempting to unpack a bytes string containing an IEEE INF or NaN will raise an -exception. - -On non-IEEE platforms with more precision, or larger dynamic range, than IEEE -754 supports, not all values can be packed; on non-IEEE platforms with less -precision, or smaller dynamic range, not all values can be unpacked. What -happens in such cases is partly accidental (alas). - -.. versionadded:: 3.11 - -Pack functions -^^^^^^^^^^^^^^ - -The pack routines write 2, 4 or 8 bytes, starting at *p*. *le* is an -:c:expr:`int` argument, non-zero if you want the bytes string in little-endian -format (exponent last, at ``p+1``, ``p+3``, or ``p+6`` ``p+7``), zero if you -want big-endian format (exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` -constant can be used to use the native endian: it is equal to ``1`` on big -endian processor, or ``0`` on little endian processor. - -Return value: ``0`` if all is OK, ``-1`` if error (and an exception is set, -most likely :exc:`OverflowError`). - -There are two problems on non-IEEE platforms: - -* What this does is undefined if *x* is a NaN or infinity. -* ``-0.0`` and ``+0.0`` produce the same bytes string. - -.. c:function:: int PyFloat_Pack2(double x, unsigned char *p, int le) - - Pack a C double as the IEEE 754 binary16 half-precision format. - -.. c:function:: int PyFloat_Pack4(double x, unsigned char *p, int le) - - Pack a C double as the IEEE 754 binary32 single precision format. - -.. c:function:: int PyFloat_Pack8(double x, unsigned char *p, int le) - - Pack a C double as the IEEE 754 binary64 double precision format. - - -Unpack functions -^^^^^^^^^^^^^^^^ - -The unpack routines read 2, 4 or 8 bytes, starting at *p*. *le* is an -:c:expr:`int` argument, non-zero if the bytes string is in little-endian format -(exponent last, at ``p+1``, ``p+3`` or ``p+6`` and ``p+7``), zero if big-endian -(exponent first, at *p*). The :c:macro:`PY_BIG_ENDIAN` constant can be used to -use the native endian: it is equal to ``1`` on big endian processor, or ``0`` -on little endian processor. - -Return value: The unpacked double. On error, this is ``-1.0`` and -:c:func:`PyErr_Occurred` is true (and an exception is set, most likely -:exc:`OverflowError`). - -Note that on a non-IEEE platform this will refuse to unpack a bytes string that -represents a NaN or infinity. - -.. c:function:: double PyFloat_Unpack2(const unsigned char *p, int le) - - Unpack the IEEE 754 binary16 half-precision format as a C double. - -.. c:function:: double PyFloat_Unpack4(const unsigned char *p, int le) - - Unpack the IEEE 754 binary32 single precision format as a C double. - -.. c:function:: double PyFloat_Unpack8(const unsigned char *p, int le) - - Unpack the IEEE 754 binary64 double precision format as a C double. diff --git a/Doc/c-api/function.rst b/Doc/c-api/function.rst deleted file mode 100644 index 1f28a685978b56..00000000000000 --- a/Doc/c-api/function.rst +++ /dev/null @@ -1,111 +0,0 @@ -.. highlight:: c - -.. _function-objects: - -Function Objects ----------------- - -.. index:: pair: object; function - -There are a few functions specific to Python functions. - - -.. c:type:: PyFunctionObject - - The C structure used for functions. - - -.. c:var:: PyTypeObject PyFunction_Type - - .. index:: single: MethodType (in module types) - - This is an instance of :c:type:`PyTypeObject` and represents the Python function - type. It is exposed to Python programmers as ``types.FunctionType``. - - -.. c:function:: int PyFunction_Check(PyObject *o) - - Return true if *o* is a function object (has type :c:data:`PyFunction_Type`). - The parameter must not be ``NULL``. This function always succeeds. - - -.. c:function:: PyObject* PyFunction_New(PyObject *code, PyObject *globals) - - Return a new function object associated with the code object *code*. *globals* - must be a dictionary with the global variables accessible to the function. - - The function's docstring and name are retrieved from the code object. *__module__* - is retrieved from *globals*. The argument defaults, annotations and closure are - set to ``NULL``. *__qualname__* is set to the same value as the code object's - ``co_qualname`` field. - - -.. c:function:: PyObject* PyFunction_NewWithQualName(PyObject *code, PyObject *globals, PyObject *qualname) - - As :c:func:`PyFunction_New`, but also allows setting the function object's - ``__qualname__`` attribute. *qualname* should be a unicode object or ``NULL``; - if ``NULL``, the ``__qualname__`` attribute is set to the same value as the - code object's ``co_qualname`` field. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyFunction_GetCode(PyObject *op) - - Return the code object associated with the function object *op*. - - -.. c:function:: PyObject* PyFunction_GetGlobals(PyObject *op) - - Return the globals dictionary associated with the function object *op*. - - -.. c:function:: PyObject* PyFunction_GetModule(PyObject *op) - - Return a :term:`borrowed reference` to the *__module__* attribute of the - function object *op*. It can be *NULL*. - - This is normally a string containing the module name, but can be set to any - other object by Python code. - - -.. c:function:: PyObject* PyFunction_GetDefaults(PyObject *op) - - Return the argument default values of the function object *op*. This can be a - tuple of arguments or ``NULL``. - - -.. c:function:: int PyFunction_SetDefaults(PyObject *op, PyObject *defaults) - - Set the argument default values for the function object *op*. *defaults* must be - ``Py_None`` or a tuple. - - Raises :exc:`SystemError` and returns ``-1`` on failure. - - -.. c:function:: PyObject* PyFunction_GetClosure(PyObject *op) - - Return the closure associated with the function object *op*. This can be ``NULL`` - or a tuple of cell objects. - - -.. c:function:: int PyFunction_SetClosure(PyObject *op, PyObject *closure) - - Set the closure associated with the function object *op*. *closure* must be - ``Py_None`` or a tuple of cell objects. - - Raises :exc:`SystemError` and returns ``-1`` on failure. - - -.. c:function:: PyObject *PyFunction_GetAnnotations(PyObject *op) - - Return the annotations of the function object *op*. This can be a - mutable dictionary or ``NULL``. - - -.. c:function:: int PyFunction_SetAnnotations(PyObject *op, PyObject *annotations) - - Set the annotations for the function object *op*. *annotations* - must be a dictionary or ``Py_None``. - - Raises :exc:`SystemError` and returns ``-1`` on failure. diff --git a/Doc/c-api/gcsupport.rst b/Doc/c-api/gcsupport.rst deleted file mode 100644 index 6bdbb008f8e74a..00000000000000 --- a/Doc/c-api/gcsupport.rst +++ /dev/null @@ -1,228 +0,0 @@ -.. highlight:: c - -.. _supporting-cycle-detection: - -Supporting Cyclic Garbage Collection -==================================== - -Python's support for detecting and collecting garbage which involves circular -references requires support from object types which are "containers" for other -objects which may also be containers. Types which do not store references to -other objects, or which only store references to atomic types (such as numbers -or strings), do not need to provide any explicit support for garbage -collection. - -To create a container type, the :c:member:`~PyTypeObject.tp_flags` field of the type object must -include the :c:macro:`Py_TPFLAGS_HAVE_GC` and provide an implementation of the -:c:member:`~PyTypeObject.tp_traverse` handler. If instances of the type are mutable, a -:c:member:`~PyTypeObject.tp_clear` implementation must also be provided. - - -:c:macro:`Py_TPFLAGS_HAVE_GC` - Objects with a type with this flag set must conform with the rules - documented here. For convenience these objects will be referred to as - container objects. - -Constructors for container types must conform to two rules: - -#. The memory for the object must be allocated using :c:macro:`PyObject_GC_New` - or :c:macro:`PyObject_GC_NewVar`. - -#. Once all the fields which may contain references to other containers are - initialized, it must call :c:func:`PyObject_GC_Track`. - -Similarly, the deallocator for the object must conform to a similar pair of -rules: - -#. Before fields which refer to other containers are invalidated, - :c:func:`PyObject_GC_UnTrack` must be called. - -#. The object's memory must be deallocated using :c:func:`PyObject_GC_Del`. - - .. warning:: - If a type adds the Py_TPFLAGS_HAVE_GC, then it *must* implement at least - a :c:member:`~PyTypeObject.tp_traverse` handler or explicitly use one - from its subclass or subclasses. - - When calling :c:func:`PyType_Ready` or some of the APIs that indirectly - call it like :c:func:`PyType_FromSpecWithBases` or - :c:func:`PyType_FromSpec` the interpreter will automatically populate the - :c:member:`~PyTypeObject.tp_flags`, :c:member:`~PyTypeObject.tp_traverse` - and :c:member:`~PyTypeObject.tp_clear` fields if the type inherits from a - class that implements the garbage collector protocol and the child class - does *not* include the :c:macro:`Py_TPFLAGS_HAVE_GC` flag. - -.. c:macro:: PyObject_GC_New(TYPE, typeobj) - - Analogous to :c:macro:`PyObject_New` but for container objects with the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. - - -.. c:macro:: PyObject_GC_NewVar(TYPE, typeobj, size) - - Analogous to :c:macro:`PyObject_NewVar` but for container objects with the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag set. - - -.. c:function:: TYPE* PyObject_GC_Resize(TYPE, PyVarObject *op, Py_ssize_t newsize) - - Resize an object allocated by :c:macro:`PyObject_NewVar`. Returns the - resized object or ``NULL`` on failure. *op* must not be tracked by the collector yet. - - -.. c:function:: void PyObject_GC_Track(PyObject *op) - - Adds the object *op* to the set of container objects tracked by the - collector. The collector can run at unexpected times so objects must be - valid while being tracked. This should be called once all the fields - followed by the :c:member:`~PyTypeObject.tp_traverse` handler become valid, usually near the - end of the constructor. - - -.. c:function:: int PyObject_IS_GC(PyObject *obj) - - Returns non-zero if the object implements the garbage collector protocol, - otherwise returns 0. - - The object cannot be tracked by the garbage collector if this function returns 0. - - -.. c:function:: int PyObject_GC_IsTracked(PyObject *op) - - Returns 1 if the object type of *op* implements the GC protocol and *op* is being - currently tracked by the garbage collector and 0 otherwise. - - This is analogous to the Python function :func:`gc.is_tracked`. - - .. versionadded:: 3.9 - - -.. c:function:: int PyObject_GC_IsFinalized(PyObject *op) - - Returns 1 if the object type of *op* implements the GC protocol and *op* has been - already finalized by the garbage collector and 0 otherwise. - - This is analogous to the Python function :func:`gc.is_finalized`. - - .. versionadded:: 3.9 - - -.. c:function:: void PyObject_GC_Del(void *op) - - Releases memory allocated to an object using :c:macro:`PyObject_GC_New` or - :c:macro:`PyObject_GC_NewVar`. - - -.. c:function:: void PyObject_GC_UnTrack(void *op) - - Remove the object *op* from the set of container objects tracked by the - collector. Note that :c:func:`PyObject_GC_Track` can be called again on - this object to add it back to the set of tracked objects. The deallocator - (:c:member:`~PyTypeObject.tp_dealloc` handler) should call this for the object before any of - the fields used by the :c:member:`~PyTypeObject.tp_traverse` handler become invalid. - - -.. versionchanged:: 3.8 - - The :c:func:`!_PyObject_GC_TRACK` and :c:func:`!_PyObject_GC_UNTRACK` macros - have been removed from the public C API. - -The :c:member:`~PyTypeObject.tp_traverse` handler accepts a function parameter of this type: - - -.. c:type:: int (*visitproc)(PyObject *object, void *arg) - - Type of the visitor function passed to the :c:member:`~PyTypeObject.tp_traverse` handler. - The function should be called with an object to traverse as *object* and - the third parameter to the :c:member:`~PyTypeObject.tp_traverse` handler as *arg*. The - Python core uses several visitor functions to implement cyclic garbage - detection; it's not expected that users will need to write their own - visitor functions. - -The :c:member:`~PyTypeObject.tp_traverse` handler must have the following type: - - -.. c:type:: int (*traverseproc)(PyObject *self, visitproc visit, void *arg) - - Traversal function for a container object. Implementations must call the - *visit* function for each object directly contained by *self*, with the - parameters to *visit* being the contained object and the *arg* value passed - to the handler. The *visit* function must not be called with a ``NULL`` - object argument. If *visit* returns a non-zero value that value should be - returned immediately. - -To simplify writing :c:member:`~PyTypeObject.tp_traverse` handlers, a :c:func:`Py_VISIT` macro is -provided. In order to use this macro, the :c:member:`~PyTypeObject.tp_traverse` implementation -must name its arguments exactly *visit* and *arg*: - - -.. c:function:: void Py_VISIT(PyObject *o) - - If *o* is not ``NULL``, call the *visit* callback, with arguments *o* - and *arg*. If *visit* returns a non-zero value, then return it. - Using this macro, :c:member:`~PyTypeObject.tp_traverse` handlers - look like:: - - static int - my_traverse(Noddy *self, visitproc visit, void *arg) - { - Py_VISIT(self->foo); - Py_VISIT(self->bar); - return 0; - } - -The :c:member:`~PyTypeObject.tp_clear` handler must be of the :c:type:`inquiry` type, or ``NULL`` -if the object is immutable. - - -.. c:type:: int (*inquiry)(PyObject *self) - - Drop references that may have created reference cycles. Immutable objects - do not have to define this method since they can never directly create - reference cycles. Note that the object must still be valid after calling - this method (don't just call :c:func:`Py_DECREF` on a reference). The - collector will call this method if it detects that this object is involved - in a reference cycle. - - -Controlling the Garbage Collector State ---------------------------------------- - -The C-API provides the following functions for controlling -garbage collection runs. - -.. c:function:: Py_ssize_t PyGC_Collect(void) - - Perform a full garbage collection, if the garbage collector is enabled. - (Note that :func:`gc.collect` runs it unconditionally.) - - Returns the number of collected + unreachable objects which cannot - be collected. - If the garbage collector is disabled or already collecting, - returns ``0`` immediately. - Errors during garbage collection are passed to :data:`sys.unraisablehook`. - This function does not raise exceptions. - - -.. c:function:: int PyGC_Enable(void) - - Enable the garbage collector: similar to :func:`gc.enable`. - Returns the previous state, 0 for disabled and 1 for enabled. - - .. versionadded:: 3.10 - - -.. c:function:: int PyGC_Disable(void) - - Disable the garbage collector: similar to :func:`gc.disable`. - Returns the previous state, 0 for disabled and 1 for enabled. - - .. versionadded:: 3.10 - - -.. c:function:: int PyGC_IsEnabled(void) - - Query the state of the garbage collector: similar to :func:`gc.isenabled`. - Returns the current state, 0 for disabled and 1 for enabled. - - .. versionadded:: 3.10 diff --git a/Doc/c-api/gen.rst b/Doc/c-api/gen.rst deleted file mode 100644 index 0eb5922f6da75f..00000000000000 --- a/Doc/c-api/gen.rst +++ /dev/null @@ -1,46 +0,0 @@ -.. highlight:: c - -.. _gen-objects: - -Generator Objects ------------------ - -Generator objects are what Python uses to implement generator iterators. They -are normally created by iterating over a function that yields values, rather -than explicitly calling :c:func:`PyGen_New` or :c:func:`PyGen_NewWithQualName`. - - -.. c:type:: PyGenObject - - The C structure used for generator objects. - - -.. c:var:: PyTypeObject PyGen_Type - - The type object corresponding to generator objects. - - -.. c:function:: int PyGen_Check(PyObject *ob) - - Return true if *ob* is a generator object; *ob* must not be ``NULL``. This - function always succeeds. - - -.. c:function:: int PyGen_CheckExact(PyObject *ob) - - Return true if *ob*'s type is :c:type:`PyGen_Type`; *ob* must not be - ``NULL``. This function always succeeds. - - -.. c:function:: PyObject* PyGen_New(PyFrameObject *frame) - - Create and return a new generator object based on the *frame* object. - A reference to *frame* is stolen by this function. The argument must not be - ``NULL``. - -.. c:function:: PyObject* PyGen_NewWithQualName(PyFrameObject *frame, PyObject *name, PyObject *qualname) - - Create and return a new generator object based on the *frame* object, - with ``__name__`` and ``__qualname__`` set to *name* and *qualname*. - A reference to *frame* is stolen by this function. The *frame* argument - must not be ``NULL``. diff --git a/Doc/c-api/import.rst b/Doc/c-api/import.rst deleted file mode 100644 index f2a60bdb023644..00000000000000 --- a/Doc/c-api/import.rst +++ /dev/null @@ -1,312 +0,0 @@ -.. highlight:: c - -.. _importing: - -Importing Modules -================= - - -.. c:function:: PyObject* PyImport_ImportModule(const char *name) - - .. index:: - single: package variable; __all__ - single: __all__ (package variable) - single: modules (in module sys) - - This is a simplified interface to :c:func:`PyImport_ImportModuleEx` below, - leaving the *globals* and *locals* arguments set to ``NULL`` and *level* set - to 0. When the *name* - argument contains a dot (when it specifies a submodule of a package), the - *fromlist* argument is set to the list ``['*']`` so that the return value is the - named module rather than the top-level package containing it as would otherwise - be the case. (Unfortunately, this has an additional side effect when *name* in - fact specifies a subpackage instead of a submodule: the submodules specified in - the package's ``__all__`` variable are loaded.) Return a new reference to the - imported module, or ``NULL`` with an exception set on failure. A failing - import of a module doesn't leave the module in :data:`sys.modules`. - - This function always uses absolute imports. - - -.. c:function:: PyObject* PyImport_ImportModuleNoBlock(const char *name) - - This function is a deprecated alias of :c:func:`PyImport_ImportModule`. - - .. versionchanged:: 3.3 - This function used to fail immediately when the import lock was held - by another thread. In Python 3.3 though, the locking scheme switched - to per-module locks for most purposes, so this function's special - behaviour isn't needed anymore. - - -.. c:function:: PyObject* PyImport_ImportModuleEx(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist) - - .. index:: pair: built-in function; __import__ - - Import a module. This is best described by referring to the built-in Python - function :func:`__import__`. - - The return value is a new reference to the imported module or top-level - package, or ``NULL`` with an exception set on failure. Like for - :func:`__import__`, the return value when a submodule of a package was - requested is normally the top-level package, unless a non-empty *fromlist* - was given. - - Failing imports remove incomplete module objects, like with - :c:func:`PyImport_ImportModule`. - - -.. c:function:: PyObject* PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) - - Import a module. This is best described by referring to the built-in Python - function :func:`__import__`, as the standard :func:`__import__` function calls - this function directly. - - The return value is a new reference to the imported module or top-level package, - or ``NULL`` with an exception set on failure. Like for :func:`__import__`, - the return value when a submodule of a package was requested is normally the - top-level package, unless a non-empty *fromlist* was given. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) - - Similar to :c:func:`PyImport_ImportModuleLevelObject`, but the name is a - UTF-8 encoded string instead of a Unicode object. - - .. versionchanged:: 3.3 - Negative values for *level* are no longer accepted. - -.. c:function:: PyObject* PyImport_Import(PyObject *name) - - This is a higher-level interface that calls the current "import hook - function" (with an explicit *level* of 0, meaning absolute import). It - invokes the :func:`__import__` function from the ``__builtins__`` of the - current globals. This means that the import is done using whatever import - hooks are installed in the current environment. - - This function always uses absolute imports. - - -.. c:function:: PyObject* PyImport_ReloadModule(PyObject *m) - - Reload a module. Return a new reference to the reloaded module, or ``NULL`` with - an exception set on failure (the module still exists in this case). - - -.. c:function:: PyObject* PyImport_AddModuleObject(PyObject *name) - - Return the module object corresponding to a module name. The *name* argument - may be of the form ``package.module``. First check the modules dictionary if - there's one there, and if not, create a new one and insert it in the modules - dictionary. Return ``NULL`` with an exception set on failure. - - .. note:: - - This function does not load or import the module; if the module wasn't already - loaded, you will get an empty module object. Use :c:func:`PyImport_ImportModule` - or one of its variants to import a module. Package structures implied by a - dotted name for *name* are not created if not already present. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyImport_AddModule(const char *name) - - Similar to :c:func:`PyImport_AddModuleObject`, but the name is a UTF-8 - encoded string instead of a Unicode object. - - -.. c:function:: PyObject* PyImport_ExecCodeModule(const char *name, PyObject *co) - - .. index:: pair: built-in function; compile - - Given a module name (possibly of the form ``package.module``) and a code object - read from a Python bytecode file or obtained from the built-in function - :func:`compile`, load the module. Return a new reference to the module object, - or ``NULL`` with an exception set if an error occurred. *name* - is removed from :data:`sys.modules` in error cases, even if *name* was already - in :data:`sys.modules` on entry to :c:func:`PyImport_ExecCodeModule`. Leaving - incompletely initialized modules in :data:`sys.modules` is dangerous, as imports of - such modules have no way to know that the module object is an unknown (and - probably damaged with respect to the module author's intents) state. - - The module's :attr:`__spec__` and :attr:`__loader__` will be set, if - not set already, with the appropriate values. The spec's loader will - be set to the module's ``__loader__`` (if set) and to an instance of - :class:`~importlib.machinery.SourceFileLoader` otherwise. - - The module's :attr:`__file__` attribute will be set to the code object's - :attr:`!co_filename`. If applicable, :attr:`__cached__` will also - be set. - - This function will reload the module if it was already imported. See - :c:func:`PyImport_ReloadModule` for the intended way to reload a module. - - If *name* points to a dotted name of the form ``package.module``, any package - structures not already created will still not be created. - - See also :c:func:`PyImport_ExecCodeModuleEx` and - :c:func:`PyImport_ExecCodeModuleWithPathnames`. - - -.. c:function:: PyObject* PyImport_ExecCodeModuleEx(const char *name, PyObject *co, const char *pathname) - - Like :c:func:`PyImport_ExecCodeModule`, but the :attr:`__file__` attribute of - the module object is set to *pathname* if it is non-``NULL``. - - See also :c:func:`PyImport_ExecCodeModuleWithPathnames`. - - -.. c:function:: PyObject* PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyObject *cpathname) - - Like :c:func:`PyImport_ExecCodeModuleEx`, but the :attr:`__cached__` - attribute of the module object is set to *cpathname* if it is - non-``NULL``. Of the three functions, this is the preferred one to use. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyImport_ExecCodeModuleWithPathnames(const char *name, PyObject *co, const char *pathname, const char *cpathname) - - Like :c:func:`PyImport_ExecCodeModuleObject`, but *name*, *pathname* and - *cpathname* are UTF-8 encoded strings. Attempts are also made to figure out - what the value for *pathname* should be from *cpathname* if the former is - set to ``NULL``. - - .. versionadded:: 3.2 - .. versionchanged:: 3.3 - Uses :func:`imp.source_from_cache()` in calculating the source path if - only the bytecode path is provided. - - -.. c:function:: long PyImport_GetMagicNumber() - - Return the magic number for Python bytecode files (a.k.a. :file:`.pyc` file). - The magic number should be present in the first four bytes of the bytecode - file, in little-endian byte order. Returns ``-1`` on error. - - .. versionchanged:: 3.3 - Return value of ``-1`` upon failure. - - -.. c:function:: const char * PyImport_GetMagicTag() - - Return the magic tag string for :pep:`3147` format Python bytecode file - names. Keep in mind that the value at ``sys.implementation.cache_tag`` is - authoritative and should be used instead of this function. - - .. versionadded:: 3.2 - -.. c:function:: PyObject* PyImport_GetModuleDict() - - Return the dictionary used for the module administration (a.k.a. - ``sys.modules``). Note that this is a per-interpreter variable. - -.. c:function:: PyObject* PyImport_GetModule(PyObject *name) - - Return the already imported module with the given name. If the - module has not been imported yet then returns ``NULL`` but does not set - an error. Returns ``NULL`` and sets an error if the lookup failed. - - .. versionadded:: 3.7 - -.. c:function:: PyObject* PyImport_GetImporter(PyObject *path) - - Return a finder object for a :data:`sys.path`/:attr:`!pkg.__path__` item - *path*, possibly by fetching it from the :data:`sys.path_importer_cache` - dict. If it wasn't yet cached, traverse :data:`sys.path_hooks` until a hook - is found that can handle the path item. Return ``None`` if no hook could; - this tells our caller that the :term:`path based finder` could not find a - finder for this path item. Cache the result in :data:`sys.path_importer_cache`. - Return a new reference to the finder object. - - -.. c:function:: int PyImport_ImportFrozenModuleObject(PyObject *name) - - Load a frozen module named *name*. Return ``1`` for success, ``0`` if the - module is not found, and ``-1`` with an exception set if the initialization - failed. To access the imported module on a successful load, use - :c:func:`PyImport_ImportModule`. (Note the misnomer --- this function would - reload the module if it was already imported.) - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The ``__file__`` attribute is no longer set on the module. - - -.. c:function:: int PyImport_ImportFrozenModule(const char *name) - - Similar to :c:func:`PyImport_ImportFrozenModuleObject`, but the name is a - UTF-8 encoded string instead of a Unicode object. - - -.. c:struct:: _frozen - - .. index:: single: freeze utility - - This is the structure type definition for frozen module descriptors, as - generated by the :program:`freeze` utility (see :file:`Tools/freeze/` in the - Python source distribution). Its definition, found in :file:`Include/import.h`, - is:: - - struct _frozen { - const char *name; - const unsigned char *code; - int size; - bool is_package; - }; - - .. versionchanged:: 3.11 - The new ``is_package`` field indicates whether the module is a package or not. - This replaces setting the ``size`` field to a negative value. - -.. c:var:: const struct _frozen* PyImport_FrozenModules - - This pointer is initialized to point to an array of :c:struct:`_frozen` - records, terminated by one whose members are all ``NULL`` or zero. When a frozen - module is imported, it is searched in this table. Third-party code could play - tricks with this to provide a dynamically created collection of frozen modules. - - -.. c:function:: int PyImport_AppendInittab(const char *name, PyObject* (*initfunc)(void)) - - Add a single module to the existing table of built-in modules. This is a - convenience wrapper around :c:func:`PyImport_ExtendInittab`, returning ``-1`` if - the table could not be extended. The new module can be imported by the name - *name*, and uses the function *initfunc* as the initialization function called - on the first attempted import. This should be called before - :c:func:`Py_Initialize`. - - -.. c:struct:: _inittab - - Structure describing a single entry in the list of built-in modules. - Programs which - embed Python may use an array of these structures in conjunction with - :c:func:`PyImport_ExtendInittab` to provide additional built-in modules. - The structure consists of two members: - - .. c:member:: const char *name - - The module name, as an ASCII encoded string. - - .. c: member:: PyObject* (*initfunc)(void) - - Initialization function for a module built into the interpreter. - - -.. c:function:: int PyImport_ExtendInittab(struct _inittab *newtab) - - Add a collection of modules to the table of built-in modules. The *newtab* - array must end with a sentinel entry which contains ``NULL`` for the :c:member:`~_inittab.name` - field; failure to provide the sentinel value can result in a memory fault. - Returns ``0`` on success or ``-1`` if insufficient memory could be allocated to - extend the internal table. In the event of failure, no modules are added to the - internal table. This must be called before :c:func:`Py_Initialize`. - - If Python is initialized multiple times, :c:func:`PyImport_AppendInittab` or - :c:func:`PyImport_ExtendInittab` must be called before each Python - initialization. diff --git a/Doc/c-api/index.rst b/Doc/c-api/index.rst deleted file mode 100644 index 9a8f1507b3f4cc..00000000000000 --- a/Doc/c-api/index.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _c-api-index: - -################################## - Python/C API Reference Manual -################################## - -This manual documents the API used by C and C++ programmers who want to write -extension modules or embed Python. It is a companion to :ref:`extending-index`, -which describes the general principles of extension writing but does not -document the API functions in detail. - -.. toctree:: - :maxdepth: 2 - - intro.rst - stable.rst - veryhigh.rst - refcounting.rst - exceptions.rst - utilities.rst - abstract.rst - concrete.rst - init.rst - init_config.rst - memory.rst - objimpl.rst - apiabiversion.rst diff --git a/Doc/c-api/init.rst b/Doc/c-api/init.rst deleted file mode 100644 index 88f5944db97c4c..00000000000000 --- a/Doc/c-api/init.rst +++ /dev/null @@ -1,1880 +0,0 @@ -.. highlight:: c - - -.. _initialization: - -***************************************** -Initialization, Finalization, and Threads -***************************************** - -See also :ref:`Python Initialization Configuration `. - -.. _pre-init-safe: - -Before Python Initialization -============================ - -In an application embedding Python, the :c:func:`Py_Initialize` function must -be called before using any other Python/C API functions; with the exception of -a few functions and the :ref:`global configuration variables -`. - -The following functions can be safely called before Python is initialized: - -* Configuration functions: - - * :c:func:`PyImport_AppendInittab` - * :c:func:`PyImport_ExtendInittab` - * :c:func:`!PyInitFrozenExtensions` - * :c:func:`PyMem_SetAllocator` - * :c:func:`PyMem_SetupDebugHooks` - * :c:func:`PyObject_SetArenaAllocator` - * :c:func:`Py_SetPath` - * :c:func:`Py_SetProgramName` - * :c:func:`Py_SetPythonHome` - * :c:func:`Py_SetStandardStreamEncoding` - * :c:func:`PySys_AddWarnOption` - * :c:func:`PySys_AddXOption` - * :c:func:`PySys_ResetWarnOptions` - -* Informative functions: - - * :c:func:`Py_IsInitialized` - * :c:func:`PyMem_GetAllocator` - * :c:func:`PyObject_GetArenaAllocator` - * :c:func:`Py_GetBuildInfo` - * :c:func:`Py_GetCompiler` - * :c:func:`Py_GetCopyright` - * :c:func:`Py_GetPlatform` - * :c:func:`Py_GetVersion` - -* Utilities: - - * :c:func:`Py_DecodeLocale` - -* Memory allocators: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - -.. note:: - - The following functions **should not be called** before - :c:func:`Py_Initialize`: :c:func:`Py_EncodeLocale`, :c:func:`Py_GetPath`, - :c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, - :c:func:`Py_GetProgramFullPath`, :c:func:`Py_GetPythonHome`, - :c:func:`Py_GetProgramName` and :c:func:`PyEval_InitThreads`. - - -.. _global-conf-vars: - -Global configuration variables -============================== - -Python has variables for the global configuration to control different features -and options. By default, these flags are controlled by :ref:`command line -options `. - -When a flag is set by an option, the value of the flag is the number of times -that the option was set. For example, ``-b`` sets :c:data:`Py_BytesWarningFlag` -to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2. - -.. c:var:: int Py_BytesWarningFlag - - Issue a warning when comparing :class:`bytes` or :class:`bytearray` with - :class:`str` or :class:`bytes` with :class:`int`. Issue an error if greater - or equal to ``2``. - - Set by the :option:`-b` option. - -.. c:var:: int Py_DebugFlag - - Turn on parser debugging output (for expert only, depending on compilation - options). - - Set by the :option:`-d` option and the :envvar:`PYTHONDEBUG` environment - variable. - -.. c:var:: int Py_DontWriteBytecodeFlag - - If set to non-zero, Python won't try to write ``.pyc`` files on the - import of source modules. - - Set by the :option:`-B` option and the :envvar:`PYTHONDONTWRITEBYTECODE` - environment variable. - -.. c:var:: int Py_FrozenFlag - - Suppress error messages when calculating the module search path in - :c:func:`Py_GetPath`. - - Private flag used by ``_freeze_module`` and ``frozenmain`` programs. - -.. c:var:: int Py_HashRandomizationFlag - - Set to ``1`` if the :envvar:`PYTHONHASHSEED` environment variable is set to - a non-empty string. - - If the flag is non-zero, read the :envvar:`PYTHONHASHSEED` environment - variable to initialize the secret hash seed. - -.. c:var:: int Py_IgnoreEnvironmentFlag - - Ignore all :envvar:`!PYTHON*` environment variables, e.g. - :envvar:`PYTHONPATH` and :envvar:`PYTHONHOME`, that might be set. - - Set by the :option:`-E` and :option:`-I` options. - -.. c:var:: int Py_InspectFlag - - When a script is passed as first argument or the :option:`-c` option is used, - enter interactive mode after executing the script or the command, even when - :data:`sys.stdin` does not appear to be a terminal. - - Set by the :option:`-i` option and the :envvar:`PYTHONINSPECT` environment - variable. - -.. c:var:: int Py_InteractiveFlag - - Set by the :option:`-i` option. - -.. c:var:: int Py_IsolatedFlag - - Run Python in isolated mode. In isolated mode :data:`sys.path` contains - neither the script's directory nor the user's site-packages directory. - - Set by the :option:`-I` option. - - .. versionadded:: 3.4 - -.. c:var:: int Py_LegacyWindowsFSEncodingFlag - - If the flag is non-zero, use the ``mbcs`` encoding with ``replace`` error - handler, instead of the UTF-8 encoding with ``surrogatepass`` error handler, - for the :term:`filesystem encoding and error handler`. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment - variable is set to a non-empty string. - - See :pep:`529` for more details. - - .. availability:: Windows. - -.. c:var:: int Py_LegacyWindowsStdioFlag - - If the flag is non-zero, use :class:`io.FileIO` instead of - :class:`!io._WindowsConsoleIO` for :mod:`sys` standard streams. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment - variable is set to a non-empty string. - - See :pep:`528` for more details. - - .. availability:: Windows. - -.. c:var:: int Py_NoSiteFlag - - Disable the import of the module :mod:`site` and the site-dependent - manipulations of :data:`sys.path` that it entails. Also disable these - manipulations if :mod:`site` is explicitly imported later (call - :func:`site.main` if you want them to be triggered). - - Set by the :option:`-S` option. - -.. c:var:: int Py_NoUserSiteDirectory - - Don't add the :data:`user site-packages directory ` to - :data:`sys.path`. - - Set by the :option:`-s` and :option:`-I` options, and the - :envvar:`PYTHONNOUSERSITE` environment variable. - -.. c:var:: int Py_OptimizeFlag - - Set by the :option:`-O` option and the :envvar:`PYTHONOPTIMIZE` environment - variable. - -.. c:var:: int Py_QuietFlag - - Don't display the copyright and version messages even in interactive mode. - - Set by the :option:`-q` option. - - .. versionadded:: 3.2 - -.. c:var:: int Py_UnbufferedStdioFlag - - Force the stdout and stderr streams to be unbuffered. - - Set by the :option:`-u` option and the :envvar:`PYTHONUNBUFFERED` - environment variable. - -.. c:var:: int Py_VerboseFlag - - Print a message each time a module is initialized, showing the place - (filename or built-in module) from which it is loaded. If greater or equal - to ``2``, print a message for each file that is checked for when - searching for a module. Also provides information on module cleanup at exit. - - Set by the :option:`-v` option and the :envvar:`PYTHONVERBOSE` environment - variable. - - -Initializing and finalizing the interpreter -=========================================== - - -.. c:function:: void Py_Initialize() - - .. index:: - single: Py_SetProgramName() - single: PyEval_InitThreads() - single: modules (in module sys) - single: path (in module sys) - pair: module; builtins - pair: module; __main__ - pair: module; sys - triple: module; search; path - single: PySys_SetArgv() - single: PySys_SetArgvEx() - single: Py_FinalizeEx() - - Initialize the Python interpreter. In an application embedding Python, - this should be called before using any other Python/C API functions; see - :ref:`Before Python Initialization ` for the few exceptions. - - This initializes - the table of loaded modules (``sys.modules``), and creates the fundamental - modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. It also initializes - the module search path (``sys.path``). It does not set ``sys.argv``; use - :c:func:`PySys_SetArgvEx` for that. This is a no-op when called for a second time - (without calling :c:func:`Py_FinalizeEx` first). There is no return value; it is a - fatal error if the initialization fails. - - .. note:: - On Windows, changes the console mode from ``O_TEXT`` to ``O_BINARY``, which will - also affect non-Python uses of the console using the C Runtime. - - -.. c:function:: void Py_InitializeEx(int initsigs) - - This function works like :c:func:`Py_Initialize` if *initsigs* is ``1``. If - *initsigs* is ``0``, it skips initialization registration of signal handlers, which - might be useful when Python is embedded. - - -.. c:function:: int Py_IsInitialized() - - Return true (nonzero) when the Python interpreter has been initialized, false - (zero) if not. After :c:func:`Py_FinalizeEx` is called, this returns false until - :c:func:`Py_Initialize` is called again. - - -.. c:function:: int Py_FinalizeEx() - - Undo all initializations made by :c:func:`Py_Initialize` and subsequent use of - Python/C API functions, and destroy all sub-interpreters (see - :c:func:`Py_NewInterpreter` below) that were created and not yet destroyed since - the last call to :c:func:`Py_Initialize`. Ideally, this frees all memory - allocated by the Python interpreter. This is a no-op when called for a second - time (without calling :c:func:`Py_Initialize` again first). Normally the - return value is ``0``. If there were errors during finalization - (flushing buffered data), ``-1`` is returned. - - This function is provided for a number of reasons. An embedding application - might want to restart Python without having to restart the application itself. - An application that has loaded the Python interpreter from a dynamically - loadable library (or DLL) might want to free all memory allocated by Python - before unloading the DLL. During a hunt for memory leaks in an application a - developer might want to free all memory allocated by Python before exiting from - the application. - - **Bugs and caveats:** The destruction of modules and objects in modules is done - in random order; this may cause destructors (:meth:`~object.__del__` methods) to fail - when they depend on other objects (even functions) or modules. Dynamically - loaded extension modules loaded by Python are not unloaded. Small amounts of - memory allocated by the Python interpreter may not be freed (if you find a leak, - please report it). Memory tied up in circular references between objects is not - freed. Some memory allocated by extension modules may not be freed. Some - extensions may not work properly if their initialization routine is called more - than once; this can happen if an application calls :c:func:`Py_Initialize` and - :c:func:`Py_FinalizeEx` more than once. - - .. audit-event:: cpython._PySys_ClearAuditHooks "" c.Py_FinalizeEx - - .. versionadded:: 3.6 - -.. c:function:: void Py_Finalize() - - This is a backwards-compatible version of :c:func:`Py_FinalizeEx` that - disregards the return value. - - -Process-wide parameters -======================= - - -.. c:function:: int Py_SetStandardStreamEncoding(const char *encoding, const char *errors) - - .. index:: - single: Py_Initialize() - single: main() - triple: stdin; stdout; sdterr - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors` - should be used instead, see :ref:`Python Initialization Configuration - `. - - This function should be called before :c:func:`Py_Initialize`, if it is - called at all. It specifies which encoding and error handling to use - with standard IO, with the same meanings as in :func:`str.encode`. - - It overrides :envvar:`PYTHONIOENCODING` values, and allows embedding code - to control IO encoding when the environment variable does not work. - - *encoding* and/or *errors* may be ``NULL`` to use - :envvar:`PYTHONIOENCODING` and/or default values (depending on other - settings). - - Note that :data:`sys.stderr` always uses the "backslashreplace" error - handler, regardless of this (or any other) setting. - - If :c:func:`Py_FinalizeEx` is called, this function will need to be called - again in order to affect subsequent calls to :c:func:`Py_Initialize`. - - Returns ``0`` if successful, a nonzero value on error (e.g. calling after the - interpreter has already been initialized). - - .. versionadded:: 3.4 - - .. deprecated:: 3.11 - - -.. c:function:: void Py_SetProgramName(const wchar_t *name) - - .. index:: - single: Py_Initialize() - single: main() - single: Py_GetPath() - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.program_name` should be used instead, see :ref:`Python - Initialization Configuration `. - - This function should be called before :c:func:`Py_Initialize` is called for - the first time, if it is called at all. It tells the interpreter the value - of the ``argv[0]`` argument to the :c:func:`main` function of the program - (converted to wide characters). - This is used by :c:func:`Py_GetPath` and some other functions below to find - the Python run-time libraries relative to the interpreter executable. The - default value is ``'python'``. The argument should point to a - zero-terminated wide character string in static storage whose contents will not - change for the duration of the program's execution. No code in the Python - interpreter will change the contents of this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - .. deprecated:: 3.11 - - -.. c:function:: wchar_t* Py_GetProgramName() - - .. index:: single: Py_SetProgramName() - - Return the program name set with :c:func:`Py_SetProgramName`, or the default. - The returned string points into static storage; the caller should not modify its - value. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - -.. c:function:: wchar_t* Py_GetPrefix() - - Return the *prefix* for installed platform-independent files. This is derived - through a number of complicated rules from the program name set with - :c:func:`Py_SetProgramName` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the prefix is ``'/usr/local'``. The - returned string points into static storage; the caller should not modify its - value. This corresponds to the :makevar:`prefix` variable in the top-level - :file:`Makefile` and the :option:`--prefix` argument to the :program:`configure` - script at build time. The value is available to Python code as ``sys.prefix``. - It is only useful on Unix. See also the next function. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - -.. c:function:: wchar_t* Py_GetExecPrefix() - - Return the *exec-prefix* for installed platform-*dependent* files. This is - derived through a number of complicated rules from the program name set with - :c:func:`Py_SetProgramName` and some environment variables; for example, if the - program name is ``'/usr/local/bin/python'``, the exec-prefix is - ``'/usr/local'``. The returned string points into static storage; the caller - should not modify its value. This corresponds to the :makevar:`exec_prefix` - variable in the top-level :file:`Makefile` and the ``--exec-prefix`` - argument to the :program:`configure` script at build time. The value is - available to Python code as ``sys.exec_prefix``. It is only useful on Unix. - - Background: The exec-prefix differs from the prefix when platform dependent - files (such as executables and shared libraries) are installed in a different - directory tree. In a typical installation, platform dependent files may be - installed in the :file:`/usr/local/plat` subtree while platform independent may - be installed in :file:`/usr/local`. - - Generally speaking, a platform is a combination of hardware and software - families, e.g. Sparc machines running the Solaris 2.x operating system are - considered the same platform, but Intel machines running Solaris 2.x are another - platform, and Intel machines running Linux are yet another platform. Different - major revisions of the same operating system generally also form different - platforms. Non-Unix operating systems are a different story; the installation - strategies on those systems are so different that the prefix and exec-prefix are - meaningless, and set to the empty string. Note that compiled Python bytecode - files are platform independent (but not independent from the Python version by - which they were compiled!). - - System administrators will know how to configure the :program:`mount` or - :program:`automount` programs to share :file:`/usr/local` between platforms - while having :file:`/usr/local/plat` be a different filesystem for each - platform. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - -.. c:function:: wchar_t* Py_GetProgramFullPath() - - .. index:: - single: Py_SetProgramName() - single: executable (in module sys) - - Return the full program name of the Python executable; this is computed as a - side-effect of deriving the default module search path from the program name - (set by :c:func:`Py_SetProgramName` above). The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.executable``. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - -.. c:function:: wchar_t* Py_GetPath() - - .. index:: - triple: module; search; path - single: path (in module sys) - single: Py_SetPath() - - Return the default module search path; this is computed from the program name - (set by :c:func:`Py_SetProgramName` above) and some environment variables. - The returned string consists of a series of directory names separated by a - platform dependent delimiter character. The delimiter character is ``':'`` - on Unix and macOS, ``';'`` on Windows. The returned string points into - static storage; the caller should not modify its value. The list - :data:`sys.path` is initialized with this value on interpreter startup; it - can be (and usually is) modified later to change the search path for loading - modules. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. XXX should give the exact rules - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - -.. c:function:: void Py_SetPath(const wchar_t *) - - .. index:: - triple: module; search; path - single: path (in module sys) - single: Py_GetPath() - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.module_search_paths` and - :c:member:`PyConfig.module_search_paths_set` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set the default module search path. If this function is called before - :c:func:`Py_Initialize`, then :c:func:`Py_GetPath` won't attempt to compute a - default search path but uses the one provided instead. This is useful if - Python is embedded by an application that has full knowledge of the location - of all modules. The path components should be separated by the platform - dependent delimiter character, which is ``':'`` on Unix and macOS, ``';'`` - on Windows. - - This also causes :data:`sys.executable` to be set to the program - full path (see :c:func:`Py_GetProgramFullPath`) and for :data:`sys.prefix` and - :data:`sys.exec_prefix` to be empty. It is up to the caller to modify these - if required after calling :c:func:`Py_Initialize`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - The path argument is copied internally, so the caller may free it after the - call completes. - - .. versionchanged:: 3.8 - The program full path is now used for :data:`sys.executable`, instead - of the program name. - - .. deprecated:: 3.11 - - -.. c:function:: const char* Py_GetVersion() - - Return the version of this Python interpreter. This is a string that looks - something like :: - - "3.0a5+ (py3k:63103M, May 12 2008, 00:53:55) \n[GCC 4.2.3]" - - .. index:: single: version (in module sys) - - The first word (up to the first space character) is the current Python version; - the first characters are the major and minor version separated by a - period. The returned string points into static storage; the caller should not - modify its value. The value is available to Python code as :data:`sys.version`. - - See also the :c:var:`Py_Version` constant. - - -.. c:function:: const char* Py_GetPlatform() - - .. index:: single: platform (in module sys) - - Return the platform identifier for the current platform. On Unix, this is - formed from the "official" name of the operating system, converted to lower - case, followed by the major revision number; e.g., for Solaris 2.x, which is - also known as SunOS 5.x, the value is ``'sunos5'``. On macOS, it is - ``'darwin'``. On Windows, it is ``'win'``. The returned string points into - static storage; the caller should not modify its value. The value is available - to Python code as ``sys.platform``. - - -.. c:function:: const char* Py_GetCopyright() - - Return the official copyright string for the current Python version, for example - - ``'Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam'`` - - .. index:: single: copyright (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as ``sys.copyright``. - - -.. c:function:: const char* Py_GetCompiler() - - Return an indication of the compiler used to build the current Python version, - in square brackets, for example:: - - "[GCC 2.7.2.2]" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: const char* Py_GetBuildInfo() - - Return information about the sequence number and build date and time of the - current Python interpreter instance, for example :: - - "#67, Aug 1 1997, 22:34:28" - - .. index:: single: version (in module sys) - - The returned string points into static storage; the caller should not modify its - value. The value is available to Python code as part of the variable - ``sys.version``. - - -.. c:function:: void PySys_SetArgvEx(int argc, wchar_t **argv, int updatepath) - - .. index:: - single: main() - single: Py_FatalError() - single: argv (in module sys) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv`, :c:member:`PyConfig.parse_argv` and - :c:member:`PyConfig.safe_path` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set :data:`sys.argv` based on *argc* and *argv*. These parameters are - similar to those passed to the program's :c:func:`main` function with the - difference that the first entry should refer to the script file to be - executed rather than the executable hosting the Python interpreter. If there - isn't a script that will be run, the first entry in *argv* can be an empty - string. If this function fails to initialize :data:`sys.argv`, a fatal - condition is signalled using :c:func:`Py_FatalError`. - - If *updatepath* is zero, this is all the function does. If *updatepath* - is non-zero, the function also modifies :data:`sys.path` according to the - following algorithm: - - - If the name of an existing script is passed in ``argv[0]``, the absolute - path of the directory where the script is located is prepended to - :data:`sys.path`. - - Otherwise (that is, if *argc* is ``0`` or ``argv[0]`` doesn't point - to an existing file name), an empty string is prepended to - :data:`sys.path`, which is the same as prepending the current working - directory (``"."``). - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. note:: - It is recommended that applications embedding the Python interpreter - for purposes other than executing a single script pass ``0`` as *updatepath*, - and update :data:`sys.path` themselves if desired. - See `CVE-2008-5983 `_. - - On versions before 3.1.3, you can achieve the same effect by manually - popping the first :data:`sys.path` element after having called - :c:func:`PySys_SetArgv`, for example using:: - - PyRun_SimpleString("import sys; sys.path.pop(0)\n"); - - .. versionadded:: 3.1.3 - - .. XXX impl. doesn't seem consistent in allowing ``0``/``NULL`` for the params; - check w/ Guido. - - .. deprecated:: 3.11 - - -.. c:function:: void PySys_SetArgv(int argc, wchar_t **argv) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` should be used - instead, see :ref:`Python Initialization Configuration `. - - This function works like :c:func:`PySys_SetArgvEx` with *updatepath* set - to ``1`` unless the :program:`python` interpreter was started with the - :option:`-I`. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - See also :c:member:`PyConfig.orig_argv` and :c:member:`PyConfig.argv` - members of the :ref:`Python Initialization Configuration `. - - .. versionchanged:: 3.4 The *updatepath* value depends on :option:`-I`. - - .. deprecated:: 3.11 - - -.. c:function:: void Py_SetPythonHome(const wchar_t *home) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.home` should be used instead, see :ref:`Python - Initialization Configuration `. - - Set the default "home" directory, that is, the location of the standard - Python libraries. See :envvar:`PYTHONHOME` for the meaning of the - argument string. - - The argument should point to a zero-terminated character string in static - storage whose contents will not change for the duration of the program's - execution. No code in the Python interpreter will change the contents of - this storage. - - Use :c:func:`Py_DecodeLocale` to decode a bytes string to get a - :c:expr:`wchar_*` string. - - .. deprecated:: 3.11 - - -.. c:function:: wchar_t* Py_GetPythonHome() - - Return the default "home", that is, the value set by a previous call to - :c:func:`Py_SetPythonHome`, or the value of the :envvar:`PYTHONHOME` - environment variable if it is set. - - This function should not be called before :c:func:`Py_Initialize`, otherwise - it returns ``NULL``. - - .. versionchanged:: 3.10 - It now returns ``NULL`` if called before :c:func:`Py_Initialize`. - - -.. _threads: - -Thread State and the Global Interpreter Lock -============================================ - -.. index:: - single: global interpreter lock - single: interpreter lock - single: lock, interpreter - -The Python interpreter is not fully thread-safe. In order to support -multi-threaded Python programs, there's a global lock, called the :term:`global -interpreter lock` or :term:`GIL`, that must be held by the current thread before -it can safely access Python objects. Without the lock, even the simplest -operations could cause problems in a multi-threaded program: for example, when -two threads simultaneously increment the reference count of the same object, the -reference count could end up being incremented only once instead of twice. - -.. index:: single: setswitchinterval() (in module sys) - -Therefore, the rule exists that only the thread that has acquired the -:term:`GIL` may operate on Python objects or call Python/C API functions. -In order to emulate concurrency of execution, the interpreter regularly -tries to switch threads (see :func:`sys.setswitchinterval`). The lock is also -released around potentially blocking I/O operations like reading or writing -a file, so that other Python threads can run in the meantime. - -.. index:: - single: PyThreadState - single: PyThreadState - -The Python interpreter keeps some thread-specific bookkeeping information -inside a data structure called :c:type:`PyThreadState`. There's also one -global variable pointing to the current :c:type:`PyThreadState`: it can -be retrieved using :c:func:`PyThreadState_Get`. - -Releasing the GIL from extension code -------------------------------------- - -Most extension code manipulating the :term:`GIL` has the following simple -structure:: - - Save the thread state in a local variable. - Release the global interpreter lock. - ... Do some blocking I/O operation ... - Reacquire the global interpreter lock. - Restore the thread state from the local variable. - -This is so common that a pair of macros exists to simplify it:: - - Py_BEGIN_ALLOW_THREADS - ... Do some blocking I/O operation ... - Py_END_ALLOW_THREADS - -.. index:: - single: Py_BEGIN_ALLOW_THREADS - single: Py_END_ALLOW_THREADS - -The :c:macro:`Py_BEGIN_ALLOW_THREADS` macro opens a new block and declares a -hidden local variable; the :c:macro:`Py_END_ALLOW_THREADS` macro closes the -block. - -The block above expands to the following code:: - - PyThreadState *_save; - - _save = PyEval_SaveThread(); - ... Do some blocking I/O operation ... - PyEval_RestoreThread(_save); - -.. index:: - single: PyEval_RestoreThread() - single: PyEval_SaveThread() - -Here is how these functions work: the global interpreter lock is used to protect the pointer to the -current thread state. When releasing the lock and saving the thread state, -the current thread state pointer must be retrieved before the lock is released -(since another thread could immediately acquire the lock and store its own thread -state in the global variable). Conversely, when acquiring the lock and restoring -the thread state, the lock must be acquired before storing the thread state -pointer. - -.. note:: - Calling system I/O functions is the most common use case for releasing - the GIL, but it can also be useful before calling long-running computations - which don't need access to Python objects, such as compression or - cryptographic functions operating over memory buffers. For example, the - standard :mod:`zlib` and :mod:`hashlib` modules release the GIL when - compressing or hashing data. - - -.. _gilstate: - -Non-Python created threads --------------------------- - -When threads are created using the dedicated Python APIs (such as the -:mod:`threading` module), a thread state is automatically associated to them -and the code showed above is therefore correct. However, when threads are -created from C (for example by a third-party library with its own thread -management), they don't hold the GIL, nor is there a thread state structure -for them. - -If you need to call Python code from these threads (often this will be part -of a callback API provided by the aforementioned third-party library), -you must first register these threads with the interpreter by -creating a thread state data structure, then acquiring the GIL, and finally -storing their thread state pointer, before you can start using the Python/C -API. When you are done, you should reset the thread state pointer, release -the GIL, and finally free the thread state data structure. - -The :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` functions do -all of the above automatically. The typical idiom for calling into Python -from a C thread is:: - - PyGILState_STATE gstate; - gstate = PyGILState_Ensure(); - - /* Perform Python actions here. */ - result = CallSomeFunction(); - /* evaluate result or handle exception */ - - /* Release the thread. No Python API allowed beyond this point. */ - PyGILState_Release(gstate); - -Note that the ``PyGILState_*`` functions assume there is only one global -interpreter (created automatically by :c:func:`Py_Initialize`). Python -supports the creation of additional interpreters (using -:c:func:`Py_NewInterpreter`), but mixing multiple interpreters and the -``PyGILState_*`` API is unsupported. - - -.. _fork-and-threads: - -Cautions about fork() ---------------------- - -Another important thing to note about threads is their behaviour in the face -of the C :c:func:`fork` call. On most systems with :c:func:`fork`, after a -process forks only the thread that issued the fork will exist. This has a -concrete impact both on how locks must be handled and on all stored state -in CPython's runtime. - -The fact that only the "current" thread remains -means any locks held by other threads will never be released. Python solves -this for :func:`os.fork` by acquiring the locks it uses internally before -the fork, and releasing them afterwards. In addition, it resets any -:ref:`lock-objects` in the child. When extending or embedding Python, there -is no way to inform Python of additional (non-Python) locks that need to be -acquired before or reset after a fork. OS facilities such as -:c:func:`!pthread_atfork` would need to be used to accomplish the same thing. -Additionally, when extending or embedding Python, calling :c:func:`fork` -directly rather than through :func:`os.fork` (and returning to or calling -into Python) may result in a deadlock by one of Python's internal locks -being held by a thread that is defunct after the fork. -:c:func:`PyOS_AfterFork_Child` tries to reset the necessary locks, but is not -always able to. - -The fact that all other threads go away also means that CPython's -runtime state there must be cleaned up properly, which :func:`os.fork` -does. This means finalizing all other :c:type:`PyThreadState` objects -belonging to the current interpreter and all other -:c:type:`PyInterpreterState` objects. Due to this and the special -nature of the :ref:`"main" interpreter `, -:c:func:`fork` should only be called in that interpreter's "main" -thread, where the CPython global runtime was originally initialized. -The only exception is if :c:func:`exec` will be called immediately -after. - - -High-level API --------------- - -These are the most commonly used types and functions when writing C extension -code, or when embedding the Python interpreter: - -.. c:type:: PyInterpreterState - - This data structure represents the state shared by a number of cooperating - threads. Threads belonging to the same interpreter share their module - administration and a few other internal items. There are no public members in - this structure. - - Threads belonging to different interpreters initially share nothing, except - process state like available memory, open file descriptors and such. The global - interpreter lock is also shared by all threads, regardless of to which - interpreter they belong. - - -.. c:type:: PyThreadState - - This data structure represents the state of a single thread. The only public - data member is: - - .. c:member:: PyInterpreterState *interp - - This thread's interpreter state. - - -.. c:function:: void PyEval_InitThreads() - - .. index:: - single: PyEval_AcquireThread() - single: PyEval_ReleaseThread() - single: PyEval_SaveThread() - single: PyEval_RestoreThread() - - Deprecated function which does nothing. - - In Python 3.6 and older, this function created the GIL if it didn't exist. - - .. versionchanged:: 3.9 - The function now does nothing. - - .. versionchanged:: 3.7 - This function is now called by :c:func:`Py_Initialize()`, so you don't - have to call it yourself anymore. - - .. versionchanged:: 3.2 - This function cannot be called before :c:func:`Py_Initialize()` anymore. - - .. deprecated:: 3.9 - - .. index:: pair: module; _thread - - -.. c:function:: int PyEval_ThreadsInitialized() - - Returns a non-zero value if :c:func:`PyEval_InitThreads` has been called. This - function can be called without holding the GIL, and therefore can be used to - avoid calls to the locking API when running single-threaded. - - .. versionchanged:: 3.7 - The :term:`GIL` is now initialized by :c:func:`Py_Initialize()`. - - .. deprecated:: 3.9 - - -.. c:function:: PyThreadState* PyEval_SaveThread() - - Release the global interpreter lock (if it has been created) and reset the - thread state to ``NULL``, returning the previous thread state (which is not - ``NULL``). If the lock has been created, the current thread must have - acquired it. - - -.. c:function:: void PyEval_RestoreThread(PyThreadState *tstate) - - Acquire the global interpreter lock (if it has been created) and set the - thread state to *tstate*, which must not be ``NULL``. If the lock has been - created, the current thread must not have acquired it, otherwise deadlock - ensues. - - .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. - -.. c:function:: PyThreadState* PyThreadState_Get() - - Return the current thread state. The global interpreter lock must be held. - When the current thread state is ``NULL``, this issues a fatal error (so that - the caller needn't check for ``NULL``). - - -.. c:function:: PyThreadState* PyThreadState_Swap(PyThreadState *tstate) - - Swap the current thread state with the thread state given by the argument - *tstate*, which may be ``NULL``. The global interpreter lock must be held - and is not released. - - -The following functions use thread-local storage, and are not compatible -with sub-interpreters: - -.. c:function:: PyGILState_STATE PyGILState_Ensure() - - Ensure that the current thread is ready to call the Python C API regardless - of the current state of Python, or of the global interpreter lock. This may - be called as many times as desired by a thread as long as each call is - matched with a call to :c:func:`PyGILState_Release`. In general, other - thread-related APIs may be used between :c:func:`PyGILState_Ensure` and - :c:func:`PyGILState_Release` calls as long as the thread state is restored to - its previous state before the Release(). For example, normal usage of the - :c:macro:`Py_BEGIN_ALLOW_THREADS` and :c:macro:`Py_END_ALLOW_THREADS` macros is - acceptable. - - The return value is an opaque "handle" to the thread state when - :c:func:`PyGILState_Ensure` was called, and must be passed to - :c:func:`PyGILState_Release` to ensure Python is left in the same state. Even - though recursive calls are allowed, these handles *cannot* be shared - each - unique call to :c:func:`PyGILState_Ensure` must save the handle for its call - to :c:func:`PyGILState_Release`. - - When the function returns, the current thread will hold the GIL and be able - to call arbitrary Python code. Failure is a fatal error. - - .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. - -.. c:function:: void PyGILState_Release(PyGILState_STATE) - - Release any resources previously acquired. After this call, Python's state will - be the same as it was prior to the corresponding :c:func:`PyGILState_Ensure` call - (but generally this state will be unknown to the caller, hence the use of the - GILState API). - - Every call to :c:func:`PyGILState_Ensure` must be matched by a call to - :c:func:`PyGILState_Release` on the same thread. - - -.. c:function:: PyThreadState* PyGILState_GetThisThreadState() - - Get the current thread state for this thread. May return ``NULL`` if no - GILState API has been used on the current thread. Note that the main thread - always has such a thread-state, even if no auto-thread-state call has been - made on the main thread. This is mainly a helper/diagnostic function. - - -.. c:function:: int PyGILState_Check() - - Return ``1`` if the current thread is holding the GIL and ``0`` otherwise. - This function can be called from any thread at any time. - Only if it has had its Python thread state initialized and currently is - holding the GIL will it return ``1``. - This is mainly a helper/diagnostic function. It can be useful - for example in callback contexts or memory allocation functions when - knowing that the GIL is locked can allow the caller to perform sensitive - actions or otherwise behave differently. - - .. versionadded:: 3.4 - - -The following macros are normally used without a trailing semicolon; look for -example usage in the Python source distribution. - - -.. c:macro:: Py_BEGIN_ALLOW_THREADS - - This macro expands to ``{ PyThreadState *_save; _save = PyEval_SaveThread();``. - Note that it contains an opening brace; it must be matched with a following - :c:macro:`Py_END_ALLOW_THREADS` macro. See above for further discussion of this - macro. - - -.. c:macro:: Py_END_ALLOW_THREADS - - This macro expands to ``PyEval_RestoreThread(_save); }``. Note that it contains - a closing brace; it must be matched with an earlier - :c:macro:`Py_BEGIN_ALLOW_THREADS` macro. See above for further discussion of - this macro. - - -.. c:macro:: Py_BLOCK_THREADS - - This macro expands to ``PyEval_RestoreThread(_save);``: it is equivalent to - :c:macro:`Py_END_ALLOW_THREADS` without the closing brace. - - -.. c:macro:: Py_UNBLOCK_THREADS - - This macro expands to ``_save = PyEval_SaveThread();``: it is equivalent to - :c:macro:`Py_BEGIN_ALLOW_THREADS` without the opening brace and variable - declaration. - - -Low-level API -------------- - -All of the following functions must be called after :c:func:`Py_Initialize`. - -.. versionchanged:: 3.7 - :c:func:`Py_Initialize()` now initializes the :term:`GIL`. - - -.. c:function:: PyInterpreterState* PyInterpreterState_New() - - Create a new interpreter state object. The global interpreter lock need not - be held, but may be held if it is necessary to serialize calls to this - function. - - .. audit-event:: cpython.PyInterpreterState_New "" c.PyInterpreterState_New - - -.. c:function:: void PyInterpreterState_Clear(PyInterpreterState *interp) - - Reset all information in an interpreter state object. The global interpreter - lock must be held. - - .. audit-event:: cpython.PyInterpreterState_Clear "" c.PyInterpreterState_Clear - - -.. c:function:: void PyInterpreterState_Delete(PyInterpreterState *interp) - - Destroy an interpreter state object. The global interpreter lock need not be - held. The interpreter state must have been reset with a previous call to - :c:func:`PyInterpreterState_Clear`. - - -.. c:function:: PyThreadState* PyThreadState_New(PyInterpreterState *interp) - - Create a new thread state object belonging to the given interpreter object. - The global interpreter lock need not be held, but may be held if it is - necessary to serialize calls to this function. - - -.. c:function:: void PyThreadState_Clear(PyThreadState *tstate) - - Reset all information in a thread state object. The global interpreter lock - must be held. - - .. versionchanged:: 3.9 - This function now calls the :c:member:`PyThreadState.on_delete` callback. - Previously, that happened in :c:func:`PyThreadState_Delete`. - - -.. c:function:: void PyThreadState_Delete(PyThreadState *tstate) - - Destroy a thread state object. The global interpreter lock need not be held. - The thread state must have been reset with a previous call to - :c:func:`PyThreadState_Clear`. - - -.. c:function:: void PyThreadState_DeleteCurrent(void) - - Destroy the current thread state and release the global interpreter lock. - Like :c:func:`PyThreadState_Delete`, the global interpreter lock need not - be held. The thread state must have been reset with a previous call - to :c:func:`PyThreadState_Clear`. - - -.. c:function:: PyFrameObject* PyThreadState_GetFrame(PyThreadState *tstate) - - Get the current frame of the Python thread state *tstate*. - - Return a :term:`strong reference`. Return ``NULL`` if no frame is currently - executing. - - See also :c:func:`PyEval_GetFrame`. - - *tstate* must not be ``NULL``. - - .. versionadded:: 3.9 - - -.. c:function:: uint64_t PyThreadState_GetID(PyThreadState *tstate) - - Get the unique thread state identifier of the Python thread state *tstate*. - - *tstate* must not be ``NULL``. - - .. versionadded:: 3.9 - - -.. c:function:: PyInterpreterState* PyThreadState_GetInterpreter(PyThreadState *tstate) - - Get the interpreter of the Python thread state *tstate*. - - *tstate* must not be ``NULL``. - - .. versionadded:: 3.9 - - -.. c:function:: void PyThreadState_EnterTracing(PyThreadState *tstate) - - Suspend tracing and profiling in the Python thread state *tstate*. - - Resume them using the :c:func:`PyThreadState_LeaveTracing` function. - - .. versionadded:: 3.11 - - -.. c:function:: void PyThreadState_LeaveTracing(PyThreadState *tstate) - - Resume tracing and profiling in the Python thread state *tstate* suspended - by the :c:func:`PyThreadState_EnterTracing` function. - - See also :c:func:`PyEval_SetTrace` and :c:func:`PyEval_SetProfile` - functions. - - .. versionadded:: 3.11 - - -.. c:function:: PyInterpreterState* PyInterpreterState_Get(void) - - Get the current interpreter. - - Issue a fatal error if there no current Python thread state or no current - interpreter. It cannot return NULL. - - The caller must hold the GIL. - - .. versionadded:: 3.9 - - -.. c:function:: int64_t PyInterpreterState_GetID(PyInterpreterState *interp) - - Return the interpreter's unique ID. If there was any error in doing - so then ``-1`` is returned and an error is set. - - The caller must hold the GIL. - - .. versionadded:: 3.7 - - -.. c:function:: PyObject* PyInterpreterState_GetDict(PyInterpreterState *interp) - - Return a dictionary in which interpreter-specific data may be stored. - If this function returns ``NULL`` then no exception has been raised and - the caller should assume no interpreter-specific dict is available. - - This is not a replacement for :c:func:`PyModule_GetState()`, which - extensions should use to store interpreter-specific state information. - - .. versionadded:: 3.8 - -.. c:type:: PyObject* (*_PyFrameEvalFunction)(PyThreadState *tstate, _PyInterpreterFrame *frame, int throwflag) - - Type of a frame evaluation function. - - The *throwflag* parameter is used by the ``throw()`` method of generators: - if non-zero, handle the current exception. - - .. versionchanged:: 3.9 - The function now takes a *tstate* parameter. - - .. versionchanged:: 3.11 - The *frame* parameter changed from ``PyFrameObject*`` to ``_PyInterpreterFrame*``. - -.. c:function:: _PyFrameEvalFunction _PyInterpreterState_GetEvalFrameFunc(PyInterpreterState *interp) - - Get the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - -.. c:function:: void _PyInterpreterState_SetEvalFrameFunc(PyInterpreterState *interp, _PyFrameEvalFunction eval_frame) - - Set the frame evaluation function. - - See the :pep:`523` "Adding a frame evaluation API to CPython". - - .. versionadded:: 3.9 - - -.. c:function:: PyObject* PyThreadState_GetDict() - - Return a dictionary in which extensions can store thread-specific state - information. Each extension should use a unique key to use to store state in - the dictionary. It is okay to call this function when no current thread state - is available. If this function returns ``NULL``, no exception has been raised and - the caller should assume no current thread state is available. - - -.. c:function:: int PyThreadState_SetAsyncExc(unsigned long id, PyObject *exc) - - Asynchronously raise an exception in a thread. The *id* argument is the thread - id of the target thread; *exc* is the exception object to be raised. This - function does not steal any references to *exc*. To prevent naive misuse, you - must write your own C extension to call this. Must be called with the GIL held. - Returns the number of thread states modified; this is normally one, but will be - zero if the thread id isn't found. If *exc* is ``NULL``, the pending - exception (if any) for the thread is cleared. This raises no exceptions. - - .. versionchanged:: 3.7 - The type of the *id* parameter changed from :c:expr:`long` to - :c:expr:`unsigned long`. - -.. c:function:: void PyEval_AcquireThread(PyThreadState *tstate) - - Acquire the global interpreter lock and set the current thread state to - *tstate*, which must not be ``NULL``. The lock must have been created earlier. - If this thread already has the lock, deadlock ensues. - - .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`!_Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - :c:func:`PyEval_RestoreThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. c:function:: void PyEval_ReleaseThread(PyThreadState *tstate) - - Reset the current thread state to ``NULL`` and release the global interpreter - lock. The lock must have been created earlier and must be held by the current - thread. The *tstate* argument, which must not be ``NULL``, is only used to check - that it represents the current thread state --- if it isn't, a fatal error is - reported. - - :c:func:`PyEval_SaveThread` is a higher-level function which is always - available (even when threads have not been initialized). - - -.. c:function:: void PyEval_AcquireLock() - - Acquire the global interpreter lock. The lock must have been created earlier. - If this thread already has the lock, a deadlock ensues. - - .. deprecated:: 3.2 - This function does not update the current thread state. Please use - :c:func:`PyEval_RestoreThread` or :c:func:`PyEval_AcquireThread` - instead. - - .. note:: - Calling this function from a thread when the runtime is finalizing - will terminate the thread, even if the thread was not created by Python. - You can use :c:func:`_Py_IsFinalizing` or :func:`sys.is_finalizing` to - check if the interpreter is in process of being finalized before calling - this function to avoid unwanted termination. - - .. versionchanged:: 3.8 - Updated to be consistent with :c:func:`PyEval_RestoreThread`, - :c:func:`Py_END_ALLOW_THREADS`, and :c:func:`PyGILState_Ensure`, - and terminate the current thread if called while the interpreter is finalizing. - - -.. c:function:: void PyEval_ReleaseLock() - - Release the global interpreter lock. The lock must have been created earlier. - - .. deprecated:: 3.2 - This function does not update the current thread state. Please use - :c:func:`PyEval_SaveThread` or :c:func:`PyEval_ReleaseThread` - instead. - - -.. _sub-interpreter-support: - -Sub-interpreter support -======================= - -While in most uses, you will only embed a single Python interpreter, there -are cases where you need to create several independent interpreters in the -same process and perhaps even in the same thread. Sub-interpreters allow -you to do that. - -The "main" interpreter is the first one created when the runtime initializes. -It is usually the only Python interpreter in a process. Unlike sub-interpreters, -the main interpreter has unique process-global responsibilities like signal -handling. It is also responsible for execution during runtime initialization and -is usually the active interpreter during runtime finalization. The -:c:func:`PyInterpreterState_Main` function returns a pointer to its state. - -You can switch between sub-interpreters using the :c:func:`PyThreadState_Swap` -function. You can create and destroy them using the following functions: - - -.. c:function:: PyThreadState* Py_NewInterpreter() - - .. index:: - pair: module; builtins - pair: module; __main__ - pair: module; sys - single: stdout (in module sys) - single: stderr (in module sys) - single: stdin (in module sys) - - Create a new sub-interpreter. This is an (almost) totally separate environment - for the execution of Python code. In particular, the new interpreter has - separate, independent versions of all imported modules, including the - fundamental modules :mod:`builtins`, :mod:`__main__` and :mod:`sys`. The - table of loaded modules (``sys.modules``) and the module search path - (``sys.path``) are also separate. The new environment has no ``sys.argv`` - variable. It has new standard I/O stream file objects ``sys.stdin``, - ``sys.stdout`` and ``sys.stderr`` (however these refer to the same underlying - file descriptors). - - The return value points to the first thread state created in the new - sub-interpreter. This thread state is made in the current thread state. - Note that no actual thread is created; see the discussion of thread states - below. If creation of the new interpreter is unsuccessful, ``NULL`` is - returned; no exception is set since the exception state is stored in the - current thread state and there may not be a current thread state. (Like all - other Python/C API functions, the global interpreter lock must be held before - calling this function and is still held when it returns; however, unlike most - other Python/C API functions, there needn't be a current thread state on - entry.) - - .. index:: - single: Py_FinalizeEx() - single: Py_Initialize() - - Extension modules are shared between (sub-)interpreters as follows: - - * For modules using multi-phase initialization, - e.g. :c:func:`PyModule_FromDefAndSpec`, a separate module object is - created and initialized for each interpreter. - Only C-level static and global variables are shared between these - module objects. - - * For modules using single-phase initialization, - e.g. :c:func:`PyModule_Create`, the first time a particular extension - is imported, it is initialized normally, and a (shallow) copy of its - module's dictionary is squirreled away. - When the same extension is imported by another (sub-)interpreter, a new - module is initialized and filled with the contents of this copy; the - extension's ``init`` function is not called. - Objects in the module's dictionary thus end up shared across - (sub-)interpreters, which might cause unwanted behavior (see - `Bugs and caveats`_ below). - - Note that this is different from what happens when an extension is - imported after the interpreter has been completely re-initialized by - calling :c:func:`Py_FinalizeEx` and :c:func:`Py_Initialize`; in that - case, the extension's ``initmodule`` function *is* called again. - As with multi-phase initialization, this means that only C-level static - and global variables are shared between these modules. - - .. index:: single: close() (in module os) - - -.. c:function:: void Py_EndInterpreter(PyThreadState *tstate) - - .. index:: single: Py_FinalizeEx() - - Destroy the (sub-)interpreter represented by the given thread state. The given - thread state must be the current thread state. See the discussion of thread - states below. When the call returns, the current thread state is ``NULL``. All - thread states associated with this interpreter are destroyed. (The global - interpreter lock must be held before calling this function and is still held - when it returns.) :c:func:`Py_FinalizeEx` will destroy all sub-interpreters that - haven't been explicitly destroyed at that point. - - -Bugs and caveats ----------------- - -Because sub-interpreters (and the main interpreter) are part of the same -process, the insulation between them isn't perfect --- for example, using -low-level file operations like :func:`os.close` they can -(accidentally or maliciously) affect each other's open files. Because of the -way extensions are shared between (sub-)interpreters, some extensions may not -work properly; this is especially likely when using single-phase initialization -or (static) global variables. -It is possible to insert objects created in one sub-interpreter into -a namespace of another (sub-)interpreter; this should be avoided if possible. - -Special care should be taken to avoid sharing user-defined functions, -methods, instances or classes between sub-interpreters, since import -operations executed by such objects may affect the wrong (sub-)interpreter's -dictionary of loaded modules. It is equally important to avoid sharing -objects from which the above are reachable. - -Also note that combining this functionality with ``PyGILState_*`` APIs -is delicate, because these APIs assume a bijection between Python thread states -and OS-level threads, an assumption broken by the presence of sub-interpreters. -It is highly recommended that you don't switch sub-interpreters between a pair -of matching :c:func:`PyGILState_Ensure` and :c:func:`PyGILState_Release` calls. -Furthermore, extensions (such as :mod:`ctypes`) using these APIs to allow calling -of Python code from non-Python created threads will probably be broken when using -sub-interpreters. - - -Asynchronous Notifications -========================== - -A mechanism is provided to make asynchronous notifications to the main -interpreter thread. These notifications take the form of a function -pointer and a void pointer argument. - - -.. c:function:: int Py_AddPendingCall(int (*func)(void *), void *arg) - - .. index:: single: Py_AddPendingCall() - - Schedule a function to be called from the main interpreter thread. On - success, ``0`` is returned and *func* is queued for being called in the - main thread. On failure, ``-1`` is returned without setting any exception. - - When successfully queued, *func* will be *eventually* called from the - main interpreter thread with the argument *arg*. It will be called - asynchronously with respect to normally running Python code, but with - both these conditions met: - - * on a :term:`bytecode` boundary; - * with the main thread holding the :term:`global interpreter lock` - (*func* can therefore use the full C API). - - *func* must return ``0`` on success, or ``-1`` on failure with an exception - set. *func* won't be interrupted to perform another asynchronous - notification recursively, but it can still be interrupted to switch - threads if the global interpreter lock is released. - - This function doesn't need a current thread state to run, and it doesn't - need the global interpreter lock. - - To call this function in a subinterpreter, the caller must hold the GIL. - Otherwise, the function *func* can be scheduled to be called from the wrong - interpreter. - - .. warning:: - This is a low-level function, only useful for very special cases. - There is no guarantee that *func* will be called as quick as - possible. If the main thread is busy executing a system call, - *func* won't be called before the system call returns. This - function is generally **not** suitable for calling Python code from - arbitrary C threads. Instead, use the :ref:`PyGILState API`. - - .. versionchanged:: 3.9 - If this function is called in a subinterpreter, the function *func* is - now scheduled to be called from the subinterpreter, rather than being - called from the main interpreter. Each subinterpreter now has its own - list of scheduled calls. - - .. versionadded:: 3.1 - -.. _profiling: - -Profiling and Tracing -===================== - -.. sectionauthor:: Fred L. Drake, Jr. - - -The Python interpreter provides some low-level support for attaching profiling -and execution tracing facilities. These are used for profiling, debugging, and -coverage analysis tools. - -This C interface allows the profiling or tracing code to avoid the overhead of -calling through Python-level callable objects, making a direct C function call -instead. The essential attributes of the facility have not changed; the -interface allows trace functions to be installed per-thread, and the basic -events reported to the trace function are the same as had been reported to the -Python-level trace functions in previous versions. - - -.. c:type:: int (*Py_tracefunc)(PyObject *obj, PyFrameObject *frame, int what, PyObject *arg) - - The type of the trace function registered using :c:func:`PyEval_SetProfile` and - :c:func:`PyEval_SetTrace`. The first parameter is the object passed to the - registration function as *obj*, *frame* is the frame object to which the event - pertains, *what* is one of the constants :c:data:`PyTrace_CALL`, - :c:data:`PyTrace_EXCEPTION`, :c:data:`PyTrace_LINE`, :c:data:`PyTrace_RETURN`, - :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION`, :c:data:`PyTrace_C_RETURN`, - or :c:data:`PyTrace_OPCODE`, and *arg* depends on the value of *what*: - - +-------------------------------+----------------------------------------+ - | Value of *what* | Meaning of *arg* | - +===============================+========================================+ - | :c:data:`PyTrace_CALL` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_EXCEPTION` | Exception information as returned by | - | | :func:`sys.exc_info`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_LINE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_RETURN` | Value being returned to the caller, | - | | or ``NULL`` if caused by an exception. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_CALL` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_EXCEPTION` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_C_RETURN` | Function object being called. | - +-------------------------------+----------------------------------------+ - | :c:data:`PyTrace_OPCODE` | Always :c:data:`Py_None`. | - +-------------------------------+----------------------------------------+ - -.. c:var:: int PyTrace_CALL - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when a new - call to a function or method is being reported, or a new entry into a generator. - Note that the creation of the iterator for a generator function is not reported - as there is no control transfer to the Python bytecode in the corresponding - frame. - - -.. c:var:: int PyTrace_EXCEPTION - - The value of the *what* parameter to a :c:type:`Py_tracefunc` function when an - exception has been raised. The callback function is called with this value for - *what* when after any bytecode is processed after which the exception becomes - set within the frame being executed. The effect of this is that as exception - propagation causes the Python stack to unwind, the callback is called upon - return to each frame as the exception propagates. Only trace functions receives - these events; they are not needed by the profiler. - - -.. c:var:: int PyTrace_LINE - - The value passed as the *what* parameter to a :c:type:`Py_tracefunc` function - (but not a profiling function) when a line-number event is being reported. - It may be disabled for a frame by setting :attr:`f_trace_lines` to *0* on that frame. - - -.. c:var:: int PyTrace_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a - call is about to return. - - -.. c:var:: int PyTrace_C_CALL - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function is about to be called. - - -.. c:var:: int PyTrace_C_EXCEPTION - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has raised an exception. - - -.. c:var:: int PyTrace_C_RETURN - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions when a C - function has returned. - - -.. c:var:: int PyTrace_OPCODE - - The value for the *what* parameter to :c:type:`Py_tracefunc` functions (but not - profiling functions) when a new opcode is about to be executed. This event is - not emitted by default: it must be explicitly requested by setting - :attr:`f_trace_opcodes` to *1* on the frame. - - -.. c:function:: void PyEval_SetProfile(Py_tracefunc func, PyObject *obj) - - Set the profiler function to *func*. The *obj* parameter is passed to the - function as its first parameter, and may be any Python object, or ``NULL``. If - the profile function needs to maintain state, using a different value for *obj* - for each thread provides a convenient and thread-safe place to store it. The - profile function is called for all monitored events except :c:data:`PyTrace_LINE` - :c:data:`PyTrace_OPCODE` and :c:data:`PyTrace_EXCEPTION`. - - See also the :func:`sys.setprofile` function. - - The caller must hold the :term:`GIL`. - - -.. c:function:: void PyEval_SetTrace(Py_tracefunc func, PyObject *obj) - - Set the tracing function to *func*. This is similar to - :c:func:`PyEval_SetProfile`, except the tracing function does receive line-number - events and per-opcode events, but does not receive any event related to C function - objects being called. Any trace function registered using :c:func:`PyEval_SetTrace` - will not receive :c:data:`PyTrace_C_CALL`, :c:data:`PyTrace_C_EXCEPTION` or - :c:data:`PyTrace_C_RETURN` as a value for the *what* parameter. - - See also the :func:`sys.settrace` function. - - The caller must hold the :term:`GIL`. - - -.. _advanced-debugging: - -Advanced Debugger Support -========================= - -.. sectionauthor:: Fred L. Drake, Jr. - - -These functions are only intended to be used by advanced debugging tools. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Head() - - Return the interpreter state object at the head of the list of all such objects. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Main() - - Return the main interpreter state object. - - -.. c:function:: PyInterpreterState* PyInterpreterState_Next(PyInterpreterState *interp) - - Return the next interpreter state object after *interp* from the list of all - such objects. - - -.. c:function:: PyThreadState * PyInterpreterState_ThreadHead(PyInterpreterState *interp) - - Return the pointer to the first :c:type:`PyThreadState` object in the list of - threads associated with the interpreter *interp*. - - -.. c:function:: PyThreadState* PyThreadState_Next(PyThreadState *tstate) - - Return the next thread state object after *tstate* from the list of all such - objects belonging to the same :c:type:`PyInterpreterState` object. - - -.. _thread-local-storage: - -Thread Local Storage Support -============================ - -.. sectionauthor:: Masayuki Yamamoto - -The Python interpreter provides low-level support for thread-local storage -(TLS) which wraps the underlying native TLS implementation to support the -Python-level thread local storage API (:class:`threading.local`). The -CPython C level APIs are similar to those offered by pthreads and Windows: -use a thread key and functions to associate a :c:expr:`void*` value per -thread. - -The GIL does *not* need to be held when calling these functions; they supply -their own locking. - -Note that :file:`Python.h` does not include the declaration of the TLS APIs, -you need to include :file:`pythread.h` to use thread-local storage. - -.. note:: - None of these API functions handle memory management on behalf of the - :c:expr:`void*` values. You need to allocate and deallocate them yourself. - If the :c:expr:`void*` values happen to be :c:expr:`PyObject*`, these - functions don't do refcount operations on them either. - -.. _thread-specific-storage-api: - -Thread Specific Storage (TSS) API ---------------------------------- - -TSS API is introduced to supersede the use of the existing TLS API within the -CPython interpreter. This API uses a new type :c:type:`Py_tss_t` instead of -:c:expr:`int` to represent thread keys. - -.. versionadded:: 3.7 - -.. seealso:: "A New C-API for Thread-Local Storage in CPython" (:pep:`539`) - - -.. c:type:: Py_tss_t - - This data structure represents the state of a thread key, the definition of - which may depend on the underlying TLS implementation, and it has an - internal field representing the key's initialization state. There are no - public members in this structure. - - When :ref:`Py_LIMITED_API ` is not defined, static allocation of - this type by :c:macro:`Py_tss_NEEDS_INIT` is allowed. - - -.. c:macro:: Py_tss_NEEDS_INIT - - This macro expands to the initializer for :c:type:`Py_tss_t` variables. - Note that this macro won't be defined with :ref:`Py_LIMITED_API `. - - -Dynamic Allocation -~~~~~~~~~~~~~~~~~~ - -Dynamic allocation of the :c:type:`Py_tss_t`, required in extension modules -built with :ref:`Py_LIMITED_API `, where static allocation of this type -is not possible due to its implementation being opaque at build time. - - -.. c:function:: Py_tss_t* PyThread_tss_alloc() - - Return a value which is the same state as a value initialized with - :c:macro:`Py_tss_NEEDS_INIT`, or ``NULL`` in the case of dynamic allocation - failure. - - -.. c:function:: void PyThread_tss_free(Py_tss_t *key) - - Free the given *key* allocated by :c:func:`PyThread_tss_alloc`, after - first calling :c:func:`PyThread_tss_delete` to ensure any associated - thread locals have been unassigned. This is a no-op if the *key* - argument is ``NULL``. - - .. note:: - A freed key becomes a dangling pointer. You should reset the key to - ``NULL``. - - -Methods -~~~~~~~ - -The parameter *key* of these functions must not be ``NULL``. Moreover, the -behaviors of :c:func:`PyThread_tss_set` and :c:func:`PyThread_tss_get` are -undefined if the given :c:type:`Py_tss_t` has not been initialized by -:c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_is_created(Py_tss_t *key) - - Return a non-zero value if the given :c:type:`Py_tss_t` has been initialized - by :c:func:`PyThread_tss_create`. - - -.. c:function:: int PyThread_tss_create(Py_tss_t *key) - - Return a zero value on successful initialization of a TSS key. The behavior - is undefined if the value pointed to by the *key* argument is not - initialized by :c:macro:`Py_tss_NEEDS_INIT`. This function can be called - repeatedly on the same key -- calling it on an already initialized key is a - no-op and immediately returns success. - - -.. c:function:: void PyThread_tss_delete(Py_tss_t *key) - - Destroy a TSS key to forget the values associated with the key across all - threads, and change the key's initialization state to uninitialized. A - destroyed key is able to be initialized again by - :c:func:`PyThread_tss_create`. This function can be called repeatedly on - the same key -- calling it on an already destroyed key is a no-op. - - -.. c:function:: int PyThread_tss_set(Py_tss_t *key, void *value) - - Return a zero value to indicate successfully associating a :c:expr:`void*` - value with a TSS key in the current thread. Each thread has a distinct - mapping of the key to a :c:expr:`void*` value. - - -.. c:function:: void* PyThread_tss_get(Py_tss_t *key) - - Return the :c:expr:`void*` value associated with a TSS key in the current - thread. This returns ``NULL`` if no value is associated with the key in the - current thread. - - -.. _thread-local-storage-api: - -Thread Local Storage (TLS) API ------------------------------- - -.. deprecated:: 3.7 - This API is superseded by - :ref:`Thread Specific Storage (TSS) API `. - -.. note:: - This version of the API does not support platforms where the native TLS key - is defined in a way that cannot be safely cast to ``int``. On such platforms, - :c:func:`PyThread_create_key` will return immediately with a failure status, - and the other TLS functions will all be no-ops on such platforms. - -Due to the compatibility problem noted above, this version of the API should not -be used in new code. - -.. c:function:: int PyThread_create_key() -.. c:function:: void PyThread_delete_key(int key) -.. c:function:: int PyThread_set_key_value(int key, void *value) -.. c:function:: void* PyThread_get_key_value(int key) -.. c:function:: void PyThread_delete_key_value(int key) -.. c:function:: void PyThread_ReInitTLS() - diff --git a/Doc/c-api/init_config.rst b/Doc/c-api/init_config.rst deleted file mode 100644 index ad1cd3f4f06681..00000000000000 --- a/Doc/c-api/init_config.rst +++ /dev/null @@ -1,1607 +0,0 @@ -.. highlight:: c - -.. _init-config: - -*********************************** -Python Initialization Configuration -*********************************** - -.. versionadded:: 3.8 - -Python can be initialized with :c:func:`Py_InitializeFromConfig` and the -:c:type:`PyConfig` structure. It can be preinitialized with -:c:func:`Py_PreInitialize` and the :c:type:`PyPreConfig` structure. - -There are two kinds of configuration: - -* The :ref:`Python Configuration ` can be used to build a - customized Python which behaves as the regular Python. For example, - environment variables and command line arguments are used to configure - Python. - -* The :ref:`Isolated Configuration ` can be used to embed - Python into an application. It isolates Python from the system. For example, - environment variables are ignored, the LC_CTYPE locale is left unchanged and - no signal handler is registered. - -The :c:func:`Py_RunMain` function can be used to write a customized Python -program. - -See also :ref:`Initialization, Finalization, and Threads `. - -.. seealso:: - :pep:`587` "Python Initialization Configuration". - - -Example -======= - -Example of customized Python always running in isolated mode:: - - int main(int argc, char **argv) - { - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - config.isolated = 1; - - /* Decode command line arguments. - Implicitly preinitialize Python (in isolated mode). */ - status = PyConfig_SetBytesArgv(&config, argc, argv); - if (PyStatus_Exception(status)) { - goto exception; - } - - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto exception; - } - PyConfig_Clear(&config); - - return Py_RunMain(); - - exception: - PyConfig_Clear(&config); - if (PyStatus_IsExit(status)) { - return status.exitcode; - } - /* Display the error message and exit the process with - non-zero exit code */ - Py_ExitStatusException(status); - } - - -PyWideStringList -================ - -.. c:type:: PyWideStringList - - List of ``wchar_t*`` strings. - - If *length* is non-zero, *items* must be non-``NULL`` and all strings must be - non-``NULL``. - - .. c:namespace:: NULL - - Methods: - - .. c:function:: PyStatus PyWideStringList_Append(PyWideStringList *list, const wchar_t *item) - - Append *item* to *list*. - - Python must be preinitialized to call this function. - - .. c:function:: PyStatus PyWideStringList_Insert(PyWideStringList *list, Py_ssize_t index, const wchar_t *item) - - Insert *item* into *list* at *index*. - - If *index* is greater than or equal to *list* length, append *item* to - *list*. - - *index* must be greater than or equal to ``0``. - - Python must be preinitialized to call this function. - - .. c:namespace:: PyWideStringList - - Structure fields: - - .. c:member:: Py_ssize_t length - - List length. - - .. c:member:: wchar_t** items - - List items. - -PyStatus -======== - -.. c:type:: PyStatus - - Structure to store an initialization function status: success, error - or exit. - - For an error, it can store the C function name which created the error. - - Structure fields: - - .. c:member:: int exitcode - - Exit code. Argument passed to ``exit()``. - - .. c:member:: const char *err_msg - - Error message. - - .. c:member:: const char *func - - Name of the function which created an error, can be ``NULL``. - - .. c:namespace:: NULL - - Functions to create a status: - - .. c:function:: PyStatus PyStatus_Ok(void) - - Success. - - .. c:function:: PyStatus PyStatus_Error(const char *err_msg) - - Initialization error with a message. - - *err_msg* must not be ``NULL``. - - .. c:function:: PyStatus PyStatus_NoMemory(void) - - Memory allocation failure (out of memory). - - .. c:function:: PyStatus PyStatus_Exit(int exitcode) - - Exit Python with the specified exit code. - - Functions to handle a status: - - .. c:function:: int PyStatus_Exception(PyStatus status) - - Is the status an error or an exit? If true, the exception must be - handled; by calling :c:func:`Py_ExitStatusException` for example. - - .. c:function:: int PyStatus_IsError(PyStatus status) - - Is the result an error? - - .. c:function:: int PyStatus_IsExit(PyStatus status) - - Is the result an exit? - - .. c:function:: void Py_ExitStatusException(PyStatus status) - - Call ``exit(exitcode)`` if *status* is an exit. Print the error - message and exit with a non-zero exit code if *status* is an error. Must - only be called if ``PyStatus_Exception(status)`` is non-zero. - -.. note:: - Internally, Python uses macros which set ``PyStatus.func``, - whereas functions to create a status set ``func`` to ``NULL``. - -Example:: - - PyStatus alloc(void **ptr, size_t size) - { - *ptr = PyMem_RawMalloc(size); - if (*ptr == NULL) { - return PyStatus_NoMemory(); - } - return PyStatus_Ok(); - } - - int main(int argc, char **argv) - { - void *ptr; - PyStatus status = alloc(&ptr, 16); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - PyMem_Free(ptr); - return 0; - } - - -PyPreConfig -=========== - -.. c:type:: PyPreConfig - - Structure used to preinitialize Python. - - .. c:namespace:: NULL - - Function to initialize a preconfiguration: - - .. c:function:: void PyPreConfig_InitPythonConfig(PyPreConfig *preconfig) - - Initialize the preconfiguration with :ref:`Python Configuration - `. - - .. c:function:: void PyPreConfig_InitIsolatedConfig(PyPreConfig *preconfig) - - Initialize the preconfiguration with :ref:`Isolated Configuration - `. - - .. c:namespace:: PyPreConfig - - Structure fields: - - .. c:member:: int allocator - - Name of the Python memory allocators: - - * ``PYMEM_ALLOCATOR_NOT_SET`` (``0``): don't change memory allocators - (use defaults). - * ``PYMEM_ALLOCATOR_DEFAULT`` (``1``): :ref:`default memory allocators - `. - * ``PYMEM_ALLOCATOR_DEBUG`` (``2``): :ref:`default memory allocators - ` with :ref:`debug hooks - `. - * ``PYMEM_ALLOCATOR_MALLOC`` (``3``): use ``malloc()`` of the C library. - * ``PYMEM_ALLOCATOR_MALLOC_DEBUG`` (``4``): force usage of - ``malloc()`` with :ref:`debug hooks `. - * ``PYMEM_ALLOCATOR_PYMALLOC`` (``5``): :ref:`Python pymalloc memory - allocator `. - * ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` (``6``): :ref:`Python pymalloc - memory allocator ` with :ref:`debug hooks - `. - - ``PYMEM_ALLOCATOR_PYMALLOC`` and ``PYMEM_ALLOCATOR_PYMALLOC_DEBUG`` are - not supported if Python is :option:`configured using --without-pymalloc - <--without-pymalloc>`. - - See :ref:`Memory Management `. - - Default: ``PYMEM_ALLOCATOR_NOT_SET``. - - .. c:member:: int configure_locale - - Set the LC_CTYPE locale to the user preferred locale. - - If equals to ``0``, set :c:member:`~PyPreConfig.coerce_c_locale` and - :c:member:`~PyPreConfig.coerce_c_locale_warn` members to ``0``. - - See the :term:`locale encoding`. - - Default: ``1`` in Python config, ``0`` in isolated config. - - .. c:member:: int coerce_c_locale - - If equals to ``2``, coerce the C locale. - - If equals to ``1``, read the LC_CTYPE locale to decide if it should be - coerced. - - See the :term:`locale encoding`. - - Default: ``-1`` in Python config, ``0`` in isolated config. - - .. c:member:: int coerce_c_locale_warn - - If non-zero, emit a warning if the C locale is coerced. - - Default: ``-1`` in Python config, ``0`` in isolated config. - - .. c:member:: int dev_mode - - :ref:`Python Development Mode `: see - :c:member:`PyConfig.dev_mode`. - - Default: ``-1`` in Python mode, ``0`` in isolated mode. - - .. c:member:: int isolated - - Isolated mode: see :c:member:`PyConfig.isolated`. - - Default: ``0`` in Python mode, ``1`` in isolated mode. - - .. c:member:: int legacy_windows_fs_encoding - - If non-zero: - - * Set :c:member:`PyPreConfig.utf8_mode` to ``0``, - * Set :c:member:`PyConfig.filesystem_encoding` to ``"mbcs"``, - * Set :c:member:`PyConfig.filesystem_errors` to ``"replace"``. - - Initialized the from :envvar:`PYTHONLEGACYWINDOWSFSENCODING` environment - variable value. - - Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for - Windows specific code. - - Default: ``0``. - - .. c:member:: int parse_argv - - If non-zero, :c:func:`Py_PreInitializeFromArgs` and - :c:func:`Py_PreInitializeFromBytesArgs` parse their ``argv`` argument the - same way the regular Python parses command line arguments: see - :ref:`Command Line Arguments `. - - Default: ``1`` in Python config, ``0`` in isolated config. - - .. c:member:: int use_environment - - Use :ref:`environment variables `? See - :c:member:`PyConfig.use_environment`. - - Default: ``1`` in Python config and ``0`` in isolated config. - - .. c:member:: int utf8_mode - - If non-zero, enable the :ref:`Python UTF-8 Mode `. - - Set to ``0`` or ``1`` by the :option:`-X utf8 <-X>` command line option - and the :envvar:`PYTHONUTF8` environment variable. - - Also set to ``1`` if the ``LC_CTYPE`` locale is ``C`` or ``POSIX``. - - Default: ``-1`` in Python config and ``0`` in isolated config. - - -.. _c-preinit: - -Preinitialize Python with PyPreConfig -===================================== - -The preinitialization of Python: - -* Set the Python memory allocators (:c:member:`PyPreConfig.allocator`) -* Configure the LC_CTYPE locale (:term:`locale encoding`) -* Set the :ref:`Python UTF-8 Mode ` - (:c:member:`PyPreConfig.utf8_mode`) - -The current preconfiguration (``PyPreConfig`` type) is stored in -``_PyRuntime.preconfig``. - -Functions to preinitialize Python: - -.. c:function:: PyStatus Py_PreInitialize(const PyPreConfig *preconfig) - - Preinitialize Python from *preconfig* preconfiguration. - - *preconfig* must not be ``NULL``. - -.. c:function:: PyStatus Py_PreInitializeFromBytesArgs(const PyPreConfig *preconfig, int argc, char * const *argv) - - Preinitialize Python from *preconfig* preconfiguration. - - Parse *argv* command line arguments (bytes strings) if - :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. - - *preconfig* must not be ``NULL``. - -.. c:function:: PyStatus Py_PreInitializeFromArgs(const PyPreConfig *preconfig, int argc, wchar_t * const * argv) - - Preinitialize Python from *preconfig* preconfiguration. - - Parse *argv* command line arguments (wide strings) if - :c:member:`~PyPreConfig.parse_argv` of *preconfig* is non-zero. - - *preconfig* must not be ``NULL``. - -The caller is responsible to handle exceptions (error or exit) using -:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. - -For :ref:`Python Configuration ` -(:c:func:`PyPreConfig_InitPythonConfig`), if Python is initialized with -command line arguments, the command line arguments must also be passed to -preinitialize Python, since they have an effect on the pre-configuration -like encodings. For example, the :option:`-X utf8 <-X>` command line option -enables the :ref:`Python UTF-8 Mode `. - -``PyMem_SetAllocator()`` can be called after :c:func:`Py_PreInitialize` and -before :c:func:`Py_InitializeFromConfig` to install a custom memory allocator. -It can be called before :c:func:`Py_PreInitialize` if -:c:member:`PyPreConfig.allocator` is set to ``PYMEM_ALLOCATOR_NOT_SET``. - -Python memory allocation functions like :c:func:`PyMem_RawMalloc` must not be -used before the Python preinitialization, whereas calling directly ``malloc()`` -and ``free()`` is always safe. :c:func:`Py_DecodeLocale` must not be called -before the Python preinitialization. - -Example using the preinitialization to enable -the :ref:`Python UTF-8 Mode `:: - - PyStatus status; - PyPreConfig preconfig; - PyPreConfig_InitPythonConfig(&preconfig); - - preconfig.utf8_mode = 1; - - status = Py_PreInitialize(&preconfig); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - - /* at this point, Python speaks UTF-8 */ - - Py_Initialize(); - /* ... use Python API here ... */ - Py_Finalize(); - - -PyConfig -======== - -.. c:type:: PyConfig - - Structure containing most parameters to configure Python. - - When done, the :c:func:`PyConfig_Clear` function must be used to release the - configuration memory. - - .. c:namespace:: NULL - - Structure methods: - - .. c:function:: void PyConfig_InitPythonConfig(PyConfig *config) - - Initialize configuration with the :ref:`Python Configuration - `. - - .. c:function:: void PyConfig_InitIsolatedConfig(PyConfig *config) - - Initialize configuration with the :ref:`Isolated Configuration - `. - - .. c:function:: PyStatus PyConfig_SetString(PyConfig *config, wchar_t * const *config_str, const wchar_t *str) - - Copy the wide character string *str* into ``*config_str``. - - :ref:`Preinitialize Python ` if needed. - - .. c:function:: PyStatus PyConfig_SetBytesString(PyConfig *config, wchar_t * const *config_str, const char *str) - - Decode *str* using :c:func:`Py_DecodeLocale` and set the result into - ``*config_str``. - - :ref:`Preinitialize Python ` if needed. - - .. c:function:: PyStatus PyConfig_SetArgv(PyConfig *config, int argc, wchar_t * const *argv) - - Set command line arguments (:c:member:`~PyConfig.argv` member of - *config*) from the *argv* list of wide character strings. - - :ref:`Preinitialize Python ` if needed. - - .. c:function:: PyStatus PyConfig_SetBytesArgv(PyConfig *config, int argc, char * const *argv) - - Set command line arguments (:c:member:`~PyConfig.argv` member of - *config*) from the *argv* list of bytes strings. Decode bytes using - :c:func:`Py_DecodeLocale`. - - :ref:`Preinitialize Python ` if needed. - - .. c:function:: PyStatus PyConfig_SetWideStringList(PyConfig *config, PyWideStringList *list, Py_ssize_t length, wchar_t **items) - - Set the list of wide strings *list* to *length* and *items*. - - :ref:`Preinitialize Python ` if needed. - - .. c:function:: PyStatus PyConfig_Read(PyConfig *config) - - Read all Python configuration. - - Fields which are already initialized are left unchanged. - - Fields for :ref:`path configuration ` are no longer - calculated or modified when calling this function, as of Python 3.11. - - The :c:func:`PyConfig_Read` function only parses - :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` - is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would - parse the application options as Python options. - - :ref:`Preinitialize Python ` if needed. - - .. versionchanged:: 3.10 - The :c:member:`PyConfig.argv` arguments are now only parsed once, - :c:member:`PyConfig.parse_argv` is set to ``2`` after arguments are - parsed, and arguments are only parsed if - :c:member:`PyConfig.parse_argv` equals ``1``. - - .. versionchanged:: 3.11 - :c:func:`PyConfig_Read` no longer calculates all paths, and so fields - listed under :ref:`Python Path Configuration ` may - no longer be updated until :c:func:`Py_InitializeFromConfig` is - called. - - .. c:function:: void PyConfig_Clear(PyConfig *config) - - Release configuration memory. - - Most ``PyConfig`` methods :ref:`preinitialize Python ` if needed. - In that case, the Python preinitialization configuration - (:c:type:`PyPreConfig`) in based on the :c:type:`PyConfig`. If configuration - fields which are in common with :c:type:`PyPreConfig` are tuned, they must - be set before calling a :c:type:`PyConfig` method: - - * :c:member:`PyConfig.dev_mode` - * :c:member:`PyConfig.isolated` - * :c:member:`PyConfig.parse_argv` - * :c:member:`PyConfig.use_environment` - - Moreover, if :c:func:`PyConfig_SetArgv` or :c:func:`PyConfig_SetBytesArgv` - is used, this method must be called before other methods, since the - preinitialization configuration depends on command line arguments (if - :c:member:`~PyConfig.parse_argv` is non-zero). - - The caller of these methods is responsible to handle exceptions (error or - exit) using ``PyStatus_Exception()`` and ``Py_ExitStatusException()``. - - .. c:namespace:: PyConfig - - Structure fields: - - .. c:member:: PyWideStringList argv - - Command line arguments: :data:`sys.argv`. - - Set :c:member:`~PyConfig.parse_argv` to ``1`` to parse - :c:member:`~PyConfig.argv` the same way the regular Python parses Python - command line arguments and then to strip Python arguments from - :c:member:`~PyConfig.argv`. - - If :c:member:`~PyConfig.argv` is empty, an empty string is added to - ensure that :data:`sys.argv` always exists and is never empty. - - Default: ``NULL``. - - See also the :c:member:`~PyConfig.orig_argv` member. - - .. c:member:: int safe_path - - If equals to zero, ``Py_RunMain()`` prepends a potentially unsafe path to - :data:`sys.path` at startup: - - * If :c:member:`argv[0] ` is equal to ``L"-m"`` - (``python -m module``), prepend the current working directory. - * If running a script (``python script.py``), prepend the script's - directory. If it's a symbolic link, resolve symbolic links. - * Otherwise (``python -c code`` and ``python``), prepend an empty string, - which means the current working directory. - - Set to ``1`` by the :option:`-P` command line option and the - :envvar:`PYTHONSAFEPATH` environment variable. - - Default: ``0`` in Python config, ``1`` in isolated config. - - .. versionadded:: 3.11 - - .. c:member:: wchar_t* base_exec_prefix - - :data:`sys.base_exec_prefix`. - - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` output. - - .. c:member:: wchar_t* base_executable - - Python base executable: :data:`sys._base_executable`. - - Set by the :envvar:`__PYVENV_LAUNCHER__` environment variable. - - Set from :c:member:`PyConfig.executable` if ``NULL``. - - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` output. - - .. c:member:: wchar_t* base_prefix - - :data:`sys.base_prefix`. - - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` output. - - .. c:member:: int buffered_stdio - - If equals to ``0`` and :c:member:`~PyConfig.configure_c_stdio` is non-zero, - disable buffering on the C streams stdout and stderr. - - Set to ``0`` by the :option:`-u` command line option and the - :envvar:`PYTHONUNBUFFERED` environment variable. - - stdin is always opened in buffered mode. - - Default: ``1``. - - .. c:member:: int bytes_warning - - If equals to ``1``, issue a warning when comparing :class:`bytes` or - :class:`bytearray` with :class:`str`, or comparing :class:`bytes` with - :class:`int`. - - If equal or greater to ``2``, raise a :exc:`BytesWarning` exception in these - cases. - - Incremented by the :option:`-b` command line option. - - Default: ``0``. - - .. c:member:: int warn_default_encoding - - If non-zero, emit a :exc:`EncodingWarning` warning when :class:`io.TextIOWrapper` - uses its default encoding. See :ref:`io-encoding-warning` for details. - - Default: ``0``. - - .. versionadded:: 3.10 - - .. c:member:: int code_debug_ranges - - If equals to ``0``, disables the inclusion of the end line and column - mappings in code objects. Also disables traceback printing carets to - specific error locations. - - Set to ``0`` by the :envvar:`PYTHONNODEBUGRANGES` environment variable - and by the :option:`-X no_debug_ranges <-X>` command line option. - - Default: ``1``. - - .. versionadded:: 3.11 - - .. c:member:: wchar_t* check_hash_pycs_mode - - Control the validation behavior of hash-based ``.pyc`` files: - value of the :option:`--check-hash-based-pycs` command line option. - - Valid values: - - - ``L"always"``: Hash the source file for invalidation regardless of - value of the 'check_source' flag. - - ``L"never"``: Assume that hash-based pycs always are valid. - - ``L"default"``: The 'check_source' flag in hash-based pycs - determines invalidation. - - Default: ``L"default"``. - - See also :pep:`552` "Deterministic pycs". - - .. c:member:: int configure_c_stdio - - If non-zero, configure C standard streams: - - * On Windows, set the binary mode (``O_BINARY``) on stdin, stdout and - stderr. - * If :c:member:`~PyConfig.buffered_stdio` equals zero, disable buffering - of stdin, stdout and stderr streams. - * If :c:member:`~PyConfig.interactive` is non-zero, enable stream - buffering on stdin and stdout (only stdout on Windows). - - Default: ``1`` in Python config, ``0`` in isolated config. - - .. c:member:: int dev_mode - - If non-zero, enable the :ref:`Python Development Mode `. - - Set to ``1`` by the :option:`-X dev <-X>` option and the - :envvar:`PYTHONDEVMODE` environment variable. - - Default: ``-1`` in Python mode, ``0`` in isolated mode. - - .. c:member:: int dump_refs - - Dump Python references? - - If non-zero, dump all objects which are still alive at exit. - - Set to ``1`` by the :envvar:`PYTHONDUMPREFS` environment variable. - - Need a special build of Python with the ``Py_TRACE_REFS`` macro defined: - see the :option:`configure --with-trace-refs option <--with-trace-refs>`. - - Default: ``0``. - - .. c:member:: wchar_t* exec_prefix - - The site-specific directory prefix where the platform-dependent Python - files are installed: :data:`sys.exec_prefix`. - - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` output. - - .. c:member:: wchar_t* executable - - The absolute path of the executable binary for the Python interpreter: - :data:`sys.executable`. - - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` output. - - .. c:member:: int faulthandler - - Enable faulthandler? - - If non-zero, call :func:`faulthandler.enable` at startup. - - Set to ``1`` by :option:`-X faulthandler <-X>` and the - :envvar:`PYTHONFAULTHANDLER` environment variable. - - Default: ``-1`` in Python mode, ``0`` in isolated mode. - - .. c:member:: wchar_t* filesystem_encoding - - :term:`Filesystem encoding `: - :func:`sys.getfilesystemencoding`. - - On macOS, Android and VxWorks: use ``"utf-8"`` by default. - - On Windows: use ``"utf-8"`` by default, or ``"mbcs"`` if - :c:member:`~PyPreConfig.legacy_windows_fs_encoding` of - :c:type:`PyPreConfig` is non-zero. - - Default encoding on other platforms: - - * ``"utf-8"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero. - * ``"ascii"`` if Python detects that ``nl_langinfo(CODESET)`` announces - the ASCII encoding, whereas the ``mbstowcs()`` function - decodes from a different encoding (usually Latin1). - * ``"utf-8"`` if ``nl_langinfo(CODESET)`` returns an empty string. - * Otherwise, use the :term:`locale encoding`: - ``nl_langinfo(CODESET)`` result. - - At Python startup, the encoding name is normalized to the Python codec - name. For example, ``"ANSI_X3.4-1968"`` is replaced with ``"ascii"``. - - See also the :c:member:`~PyConfig.filesystem_errors` member. - - .. c:member:: wchar_t* filesystem_errors - - :term:`Filesystem error handler `: - :func:`sys.getfilesystemencodeerrors`. - - On Windows: use ``"surrogatepass"`` by default, or ``"replace"`` if - :c:member:`~PyPreConfig.legacy_windows_fs_encoding` of - :c:type:`PyPreConfig` is non-zero. - - On other platforms: use ``"surrogateescape"`` by default. - - Supported error handlers: - - * ``"strict"`` - * ``"surrogateescape"`` - * ``"surrogatepass"`` (only supported with the UTF-8 encoding) - - See also the :c:member:`~PyConfig.filesystem_encoding` member. - - .. c:member:: unsigned long hash_seed - .. c:member:: int use_hash_seed - - Randomized hash function seed. - - If :c:member:`~PyConfig.use_hash_seed` is zero, a seed is chosen randomly - at Python startup, and :c:member:`~PyConfig.hash_seed` is ignored. - - Set by the :envvar:`PYTHONHASHSEED` environment variable. - - Default *use_hash_seed* value: ``-1`` in Python mode, ``0`` in isolated - mode. - - .. c:member:: wchar_t* home - - Python home directory. - - If :c:func:`Py_SetPythonHome` has been called, use its argument if it is - not ``NULL``. - - Set by the :envvar:`PYTHONHOME` environment variable. - - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` input. - - .. c:member:: int import_time - - If non-zero, profile import time. - - Set the ``1`` by the :option:`-X importtime <-X>` option and the - :envvar:`PYTHONPROFILEIMPORTTIME` environment variable. - - Default: ``0``. - - .. c:member:: int inspect - - Enter interactive mode after executing a script or a command. - - If greater than ``0``, enable inspect: when a script is passed as first - argument or the -c option is used, enter interactive mode after executing - the script or the command, even when :data:`sys.stdin` does not appear to - be a terminal. - - Incremented by the :option:`-i` command line option. Set to ``1`` if the - :envvar:`PYTHONINSPECT` environment variable is non-empty. - - Default: ``0``. - - .. c:member:: int install_signal_handlers - - Install Python signal handlers? - - Default: ``1`` in Python mode, ``0`` in isolated mode. - - .. c:member:: int interactive - - If greater than ``0``, enable the interactive mode (REPL). - - Incremented by the :option:`-i` command line option. - - Default: ``0``. - - .. c:member:: int isolated - - If greater than ``0``, enable isolated mode: - - * Set :c:member:`~PyConfig.safe_path` to ``1``: - don't prepend a potentially unsafe path to :data:`sys.path` at Python - startup. - * Set :c:member:`~PyConfig.use_environment` to ``0``. - * Set :c:member:`~PyConfig.user_site_directory` to ``0``: don't add the user - site directory to :data:`sys.path`. - * Python REPL doesn't import :mod:`readline` nor enable default readline - configuration on interactive prompts. - - Set to ``1`` by the :option:`-I` command line option. - - Default: ``0`` in Python mode, ``1`` in isolated mode. - - See also :c:member:`PyPreConfig.isolated`. - - .. c:member:: int legacy_windows_stdio - - If non-zero, use :class:`io.FileIO` instead of - :class:`!io._WindowsConsoleIO` for :data:`sys.stdin`, :data:`sys.stdout` - and :data:`sys.stderr`. - - Set to ``1`` if the :envvar:`PYTHONLEGACYWINDOWSSTDIO` environment - variable is set to a non-empty string. - - Only available on Windows. ``#ifdef MS_WINDOWS`` macro can be used for - Windows specific code. - - Default: ``0``. - - See also the :pep:`528` (Change Windows console encoding to UTF-8). - - .. c:member:: int malloc_stats - - If non-zero, dump statistics on :ref:`Python pymalloc memory allocator - ` at exit. - - Set to ``1`` by the :envvar:`PYTHONMALLOCSTATS` environment variable. - - The option is ignored if Python is :option:`configured using - the --without-pymalloc option <--without-pymalloc>`. - - Default: ``0``. - - .. c:member:: wchar_t* platlibdir - - Platform library directory name: :data:`sys.platlibdir`. - - Set by the :envvar:`PYTHONPLATLIBDIR` environment variable. - - Default: value of the ``PLATLIBDIR`` macro which is set by the - :option:`configure --with-platlibdir option <--with-platlibdir>` - (default: ``"lib"``, or ``"DLLs"`` on Windows). - - Part of the :ref:`Python Path Configuration ` input. - - .. versionadded:: 3.9 - - .. versionchanged:: 3.11 - This macro is now used on Windows to locate the standard - library extension modules, typically under ``DLLs``. However, - for compatibility, note that this value is ignored for any - non-standard layouts, including in-tree builds and virtual - environments. - - .. c:member:: wchar_t* pythonpath_env - - Module search paths (:data:`sys.path`) as a string separated by ``DELIM`` - (:data:`os.pathsep`). - - Set by the :envvar:`PYTHONPATH` environment variable. - - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` input. - - .. c:member:: PyWideStringList module_search_paths - .. c:member:: int module_search_paths_set - - Module search paths: :data:`sys.path`. - - If :c:member:`~PyConfig.module_search_paths_set` is equal to ``0``, - :c:func:`Py_InitializeFromConfig` will replace - :c:member:`~PyConfig.module_search_paths` and sets - :c:member:`~PyConfig.module_search_paths_set` to ``1``. - - Default: empty list (``module_search_paths``) and ``0`` - (``module_search_paths_set``). - - Part of the :ref:`Python Path Configuration ` output. - - .. c:member:: int optimization_level - - Compilation optimization level: - - * ``0``: Peephole optimizer, set ``__debug__`` to ``True``. - * ``1``: Level 0, remove assertions, set ``__debug__`` to ``False``. - * ``2``: Level 1, strip docstrings. - - Incremented by the :option:`-O` command line option. Set to the - :envvar:`PYTHONOPTIMIZE` environment variable value. - - Default: ``0``. - - .. c:member:: PyWideStringList orig_argv - - The list of the original command line arguments passed to the Python - executable: :data:`sys.orig_argv`. - - If :c:member:`~PyConfig.orig_argv` list is empty and - :c:member:`~PyConfig.argv` is not a list only containing an empty - string, :c:func:`PyConfig_Read` copies :c:member:`~PyConfig.argv` into - :c:member:`~PyConfig.orig_argv` before modifying - :c:member:`~PyConfig.argv` (if :c:member:`~PyConfig.parse_argv` is - non-zero). - - See also the :c:member:`~PyConfig.argv` member and the - :c:func:`Py_GetArgcArgv` function. - - Default: empty list. - - .. versionadded:: 3.10 - - .. c:member:: int parse_argv - - Parse command line arguments? - - If equals to ``1``, parse :c:member:`~PyConfig.argv` the same way the regular - Python parses :ref:`command line arguments `, and strip - Python arguments from :c:member:`~PyConfig.argv`. - - The :c:func:`PyConfig_Read` function only parses - :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv` - is set to ``2`` after arguments are parsed. Since Python arguments are - strippped from :c:member:`PyConfig.argv`, parsing arguments twice would - parse the application options as Python options. - - Default: ``1`` in Python mode, ``0`` in isolated mode. - - .. versionchanged:: 3.10 - The :c:member:`PyConfig.argv` arguments are now only parsed if - :c:member:`PyConfig.parse_argv` equals to ``1``. - - .. c:member:: int parser_debug - - Parser debug mode. If greater than ``0``, turn on parser debugging output (for expert only, depending - on compilation options). - - Incremented by the :option:`-d` command line option. Set to the - :envvar:`PYTHONDEBUG` environment variable value. - - Default: ``0``. - - .. c:member:: int pathconfig_warnings - - If non-zero, calculation of path configuration is allowed to log - warnings into ``stderr``. If equals to ``0``, suppress these warnings. - - Default: ``1`` in Python mode, ``0`` in isolated mode. - - Part of the :ref:`Python Path Configuration ` input. - - .. versionchanged:: 3.11 - Now also applies on Windows. - - .. c:member:: wchar_t* prefix - - The site-specific directory prefix where the platform independent Python - files are installed: :data:`sys.prefix`. - - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` output. - - .. c:member:: wchar_t* program_name - - Program name used to initialize :c:member:`~PyConfig.executable` and in - early error messages during Python initialization. - - * If :func:`Py_SetProgramName` has been called, use its argument. - * On macOS, use :envvar:`PYTHONEXECUTABLE` environment variable if set. - * If the ``WITH_NEXT_FRAMEWORK`` macro is defined, use - :envvar:`__PYVENV_LAUNCHER__` environment variable if set. - * Use ``argv[0]`` of :c:member:`~PyConfig.argv` if available and - non-empty. - * Otherwise, use ``L"python"`` on Windows, or ``L"python3"`` on other - platforms. - - Default: ``NULL``. - - Part of the :ref:`Python Path Configuration ` input. - - .. c:member:: wchar_t* pycache_prefix - - Directory where cached ``.pyc`` files are written: - :data:`sys.pycache_prefix`. - - Set by the :option:`-X pycache_prefix=PATH <-X>` command line option and - the :envvar:`PYTHONPYCACHEPREFIX` environment variable. - - If ``NULL``, :data:`sys.pycache_prefix` is set to ``None``. - - Default: ``NULL``. - - .. c:member:: int quiet - - Quiet mode. If greater than ``0``, don't display the copyright and version at - Python startup in interactive mode. - - Incremented by the :option:`-q` command line option. - - Default: ``0``. - - .. c:member:: wchar_t* run_command - - Value of the :option:`-c` command line option. - - Used by :c:func:`Py_RunMain`. - - Default: ``NULL``. - - .. c:member:: wchar_t* run_filename - - Filename passed on the command line: trailing command line argument - without :option:`-c` or :option:`-m`. It is used by the - :c:func:`Py_RunMain` function. - - For example, it is set to ``script.py`` by the ``python3 script.py arg`` - command line. - - See also the :c:member:`PyConfig.skip_source_first_line` option. - - Default: ``NULL``. - - .. c:member:: wchar_t* run_module - - Value of the :option:`-m` command line option. - - Used by :c:func:`Py_RunMain`. - - Default: ``NULL``. - - .. c:member:: int show_ref_count - - Show total reference count at exit? - - Set to ``1`` by :option:`-X showrefcount <-X>` command line option. - - Need a :ref:`debug build of Python ` (the ``Py_REF_DEBUG`` - macro must be defined). - - Default: ``0``. - - .. c:member:: int site_import - - Import the :mod:`site` module at startup? - - If equal to zero, disable the import of the module site and the - site-dependent manipulations of :data:`sys.path` that it entails. - - Also disable these manipulations if the :mod:`site` module is explicitly - imported later (call :func:`site.main` if you want them to be triggered). - - Set to ``0`` by the :option:`-S` command line option. - - :data:`sys.flags.no_site ` is set to the inverted value of - :c:member:`~PyConfig.site_import`. - - Default: ``1``. - - .. c:member:: int skip_source_first_line - - If non-zero, skip the first line of the :c:member:`PyConfig.run_filename` - source. - - It allows the usage of non-Unix forms of ``#!cmd``. This is intended for - a DOS specific hack only. - - Set to ``1`` by the :option:`-x` command line option. - - Default: ``0``. - - .. c:member:: wchar_t* stdio_encoding - .. c:member:: wchar_t* stdio_errors - - Encoding and encoding errors of :data:`sys.stdin`, :data:`sys.stdout` and - :data:`sys.stderr` (but :data:`sys.stderr` always uses - ``"backslashreplace"`` error handler). - - If :c:func:`Py_SetStandardStreamEncoding` has been called, use its - *error* and *errors* arguments if they are not ``NULL``. - - Use the :envvar:`PYTHONIOENCODING` environment variable if it is - non-empty. - - Default encoding: - - * ``"UTF-8"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero. - * Otherwise, use the :term:`locale encoding`. - - Default error handler: - - * On Windows: use ``"surrogateescape"``. - * ``"surrogateescape"`` if :c:member:`PyPreConfig.utf8_mode` is non-zero, - or if the LC_CTYPE locale is "C" or "POSIX". - * ``"strict"`` otherwise. - - .. c:member:: int tracemalloc - - Enable tracemalloc? - - If non-zero, call :func:`tracemalloc.start` at startup. - - Set by :option:`-X tracemalloc=N <-X>` command line option and by the - :envvar:`PYTHONTRACEMALLOC` environment variable. - - Default: ``-1`` in Python mode, ``0`` in isolated mode. - - .. c:member:: int use_environment - - Use :ref:`environment variables `? - - If equals to zero, ignore the :ref:`environment variables - `. - - Set to ``0`` by the :option:`-E` environment variable. - - Default: ``1`` in Python config and ``0`` in isolated config. - - .. c:member:: int user_site_directory - - If non-zero, add the user site directory to :data:`sys.path`. - - Set to ``0`` by the :option:`-s` and :option:`-I` command line options. - - Set to ``0`` by the :envvar:`PYTHONNOUSERSITE` environment variable. - - Default: ``1`` in Python mode, ``0`` in isolated mode. - - .. c:member:: int verbose - - Verbose mode. If greater than ``0``, print a message each time a module is - imported, showing the place (filename or built-in module) from which - it is loaded. - - If greater or equal to ``2``, print a message for each file that is checked - for when searching for a module. Also provides information on module - cleanup at exit. - - Incremented by the :option:`-v` command line option. - - Set to the :envvar:`PYTHONVERBOSE` environment variable value. - - Default: ``0``. - - .. c:member:: PyWideStringList warnoptions - - Options of the :mod:`warnings` module to build warnings filters, lowest - to highest priority: :data:`sys.warnoptions`. - - The :mod:`warnings` module adds :data:`sys.warnoptions` in the reverse - order: the last :c:member:`PyConfig.warnoptions` item becomes the first - item of :data:`warnings.filters` which is checked first (highest - priority). - - The :option:`-W` command line options adds its value to - :c:member:`~PyConfig.warnoptions`, it can be used multiple times. - - The :envvar:`PYTHONWARNINGS` environment variable can also be used to add - warning options. Multiple options can be specified, separated by commas - (``,``). - - Default: empty list. - - .. c:member:: int write_bytecode - - If equal to ``0``, Python won't try to write ``.pyc`` files on the import of - source modules. - - Set to ``0`` by the :option:`-B` command line option and the - :envvar:`PYTHONDONTWRITEBYTECODE` environment variable. - - :data:`sys.dont_write_bytecode` is initialized to the inverted value of - :c:member:`~PyConfig.write_bytecode`. - - Default: ``1``. - - .. c:member:: PyWideStringList xoptions - - Values of the :option:`-X` command line options: :data:`sys._xoptions`. - - Default: empty list. - -If :c:member:`~PyConfig.parse_argv` is non-zero, :c:member:`~PyConfig.argv` -arguments are parsed the same way the regular Python parses :ref:`command line -arguments `, and Python arguments are stripped from -:c:member:`~PyConfig.argv`. - -The :c:member:`~PyConfig.xoptions` options are parsed to set other options: see -the :option:`-X` command line option. - -.. versionchanged:: 3.9 - - The ``show_alloc_count`` field has been removed. - - -Initialization with PyConfig -============================ - -Function to initialize Python: - -.. c:function:: PyStatus Py_InitializeFromConfig(const PyConfig *config) - - Initialize Python from *config* configuration. - -The caller is responsible to handle exceptions (error or exit) using -:c:func:`PyStatus_Exception` and :c:func:`Py_ExitStatusException`. - -If :c:func:`PyImport_FrozenModules`, :c:func:`PyImport_AppendInittab` or -:c:func:`PyImport_ExtendInittab` are used, they must be set or called after -Python preinitialization and before the Python initialization. If Python is -initialized multiple times, :c:func:`PyImport_AppendInittab` or -:c:func:`PyImport_ExtendInittab` must be called before each Python -initialization. - -The current configuration (``PyConfig`` type) is stored in -``PyInterpreterState.config``. - -Example setting the program name:: - - void init_python(void) - { - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - - /* Set the program name. Implicitly preinitialize Python. */ - status = PyConfig_SetString(&config, &config.program_name, - L"/path/to/my_program"); - if (PyStatus_Exception(status)) { - goto exception; - } - - status = Py_InitializeFromConfig(&config); - if (PyStatus_Exception(status)) { - goto exception; - } - PyConfig_Clear(&config); - return; - - exception: - PyConfig_Clear(&config); - Py_ExitStatusException(status); - } - -More complete example modifying the default configuration, read the -configuration, and then override some parameters. Note that since -3.11, many parameters are not calculated until initialization, and -so values cannot be read from the configuration structure. Any values -set before initialize is called will be left unchanged by -initialization:: - - PyStatus init_python(const char *program_name) - { - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - - /* Set the program name before reading the configuration - (decode byte string from the locale encoding). - - Implicitly preinitialize Python. */ - status = PyConfig_SetBytesString(&config, &config.program_name, - program_name); - if (PyStatus_Exception(status)) { - goto done; - } - - /* Read all configuration at once */ - status = PyConfig_Read(&config); - if (PyStatus_Exception(status)) { - goto done; - } - - /* Specify sys.path explicitly */ - /* If you want to modify the default set of paths, finish - initialization first and then use PySys_GetObject("path") */ - config.module_search_paths_set = 1; - status = PyWideStringList_Append(&config.module_search_paths, - L"/path/to/stdlib"); - if (PyStatus_Exception(status)) { - goto done; - } - status = PyWideStringList_Append(&config.module_search_paths, - L"/path/to/more/modules"); - if (PyStatus_Exception(status)) { - goto done; - } - - /* Override executable computed by PyConfig_Read() */ - status = PyConfig_SetString(&config, &config.executable, - L"/path/to/my_executable"); - if (PyStatus_Exception(status)) { - goto done; - } - - status = Py_InitializeFromConfig(&config); - - done: - PyConfig_Clear(&config); - return status; - } - - -.. _init-isolated-conf: - -Isolated Configuration -====================== - -:c:func:`PyPreConfig_InitIsolatedConfig` and -:c:func:`PyConfig_InitIsolatedConfig` functions create a configuration to -isolate Python from the system. For example, to embed Python into an -application. - -This configuration ignores global configuration variables, environment -variables, command line arguments (:c:member:`PyConfig.argv` is not parsed) -and user site directory. The C standard streams (ex: ``stdout``) and the -LC_CTYPE locale are left unchanged. Signal handlers are not installed. - -Configuration files are still used with this configuration to determine -paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified -to avoid computing the default path configuration. - - -.. _init-python-config: - -Python Configuration -==================== - -:c:func:`PyPreConfig_InitPythonConfig` and :c:func:`PyConfig_InitPythonConfig` -functions create a configuration to build a customized Python which behaves as -the regular Python. - -Environments variables and command line arguments are used to configure -Python, whereas global configuration variables are ignored. - -This function enables C locale coercion (:pep:`538`) -and :ref:`Python UTF-8 Mode ` -(:pep:`540`) depending on the LC_CTYPE locale, :envvar:`PYTHONUTF8` and -:envvar:`PYTHONCOERCECLOCALE` environment variables. - - -.. _init-path-config: - -Python Path Configuration -========================= - -:c:type:`PyConfig` contains multiple fields for the path configuration: - -* Path configuration inputs: - - * :c:member:`PyConfig.home` - * :c:member:`PyConfig.platlibdir` - * :c:member:`PyConfig.pathconfig_warnings` - * :c:member:`PyConfig.program_name` - * :c:member:`PyConfig.pythonpath_env` - * current working directory: to get absolute paths - * ``PATH`` environment variable to get the program full path - (from :c:member:`PyConfig.program_name`) - * ``__PYVENV_LAUNCHER__`` environment variable - * (Windows only) Application paths in the registry under - "Software\Python\PythonCore\X.Y\PythonPath" of HKEY_CURRENT_USER and - HKEY_LOCAL_MACHINE (where X.Y is the Python version). - -* Path configuration output fields: - - * :c:member:`PyConfig.base_exec_prefix` - * :c:member:`PyConfig.base_executable` - * :c:member:`PyConfig.base_prefix` - * :c:member:`PyConfig.exec_prefix` - * :c:member:`PyConfig.executable` - * :c:member:`PyConfig.module_search_paths_set`, - :c:member:`PyConfig.module_search_paths` - * :c:member:`PyConfig.prefix` - -If at least one "output field" is not set, Python calculates the path -configuration to fill unset fields. If -:c:member:`~PyConfig.module_search_paths_set` is equal to ``0``, -:c:member:`~PyConfig.module_search_paths` is overridden and -:c:member:`~PyConfig.module_search_paths_set` is set to ``1``. - -It is possible to completely ignore the function calculating the default -path configuration by setting explicitly all path configuration output -fields listed above. A string is considered as set even if it is non-empty. -``module_search_paths`` is considered as set if -``module_search_paths_set`` is set to ``1``. In this case, -``module_search_paths`` will be used without modification. - -Set :c:member:`~PyConfig.pathconfig_warnings` to ``0`` to suppress warnings when -calculating the path configuration (Unix only, Windows does not log any warning). - -If :c:member:`~PyConfig.base_prefix` or :c:member:`~PyConfig.base_exec_prefix` -fields are not set, they inherit their value from :c:member:`~PyConfig.prefix` -and :c:member:`~PyConfig.exec_prefix` respectively. - -:c:func:`Py_RunMain` and :c:func:`Py_Main` modify :data:`sys.path`: - -* If :c:member:`~PyConfig.run_filename` is set and is a directory which contains a - ``__main__.py`` script, prepend :c:member:`~PyConfig.run_filename` to - :data:`sys.path`. -* If :c:member:`~PyConfig.isolated` is zero: - - * If :c:member:`~PyConfig.run_module` is set, prepend the current directory - to :data:`sys.path`. Do nothing if the current directory cannot be read. - * If :c:member:`~PyConfig.run_filename` is set, prepend the directory of the - filename to :data:`sys.path`. - * Otherwise, prepend an empty string to :data:`sys.path`. - -If :c:member:`~PyConfig.site_import` is non-zero, :data:`sys.path` can be -modified by the :mod:`site` module. If -:c:member:`~PyConfig.user_site_directory` is non-zero and the user's -site-package directory exists, the :mod:`site` module appends the user's -site-package directory to :data:`sys.path`. - -The following configuration files are used by the path configuration: - -* ``pyvenv.cfg`` -* ``._pth`` file (ex: ``python._pth``) -* ``pybuilddir.txt`` (Unix only) - -If a ``._pth`` file is present: - -* Set :c:member:`~PyConfig.isolated` to ``1``. -* Set :c:member:`~PyConfig.use_environment` to ``0``. -* Set :c:member:`~PyConfig.site_import` to ``0``. -* Set :c:member:`~PyConfig.safe_path` to ``1``. - -The ``__PYVENV_LAUNCHER__`` environment variable is used to set -:c:member:`PyConfig.base_executable` - - -Py_RunMain() -============ - -.. c:function:: int Py_RunMain(void) - - Execute the command (:c:member:`PyConfig.run_command`), the script - (:c:member:`PyConfig.run_filename`) or the module - (:c:member:`PyConfig.run_module`) specified on the command line or in the - configuration. - - By default and when if :option:`-i` option is used, run the REPL. - - Finally, finalizes Python and returns an exit status that can be passed to - the ``exit()`` function. - -See :ref:`Python Configuration ` for an example of -customized Python always running in isolated mode using -:c:func:`Py_RunMain`. - - -Py_GetArgcArgv() -================ - -.. c:function:: void Py_GetArgcArgv(int *argc, wchar_t ***argv) - - Get the original command line arguments, before Python modified them. - - See also :c:member:`PyConfig.orig_argv` member. - - -Multi-Phase Initialization Private Provisional API -================================================== - -This section is a private provisional API introducing multi-phase -initialization, the core feature of :pep:`432`: - -* "Core" initialization phase, "bare minimum Python": - - * Builtin types; - * Builtin exceptions; - * Builtin and frozen modules; - * The :mod:`sys` module is only partially initialized - (ex: :data:`sys.path` doesn't exist yet). - -* "Main" initialization phase, Python is fully initialized: - - * Install and configure :mod:`importlib`; - * Apply the :ref:`Path Configuration `; - * Install signal handlers; - * Finish :mod:`sys` module initialization (ex: create :data:`sys.stdout` - and :data:`sys.path`); - * Enable optional features like :mod:`faulthandler` and :mod:`tracemalloc`; - * Import the :mod:`site` module; - * etc. - -Private provisional API: - -* :c:member:`PyConfig._init_main`: if set to ``0``, - :c:func:`Py_InitializeFromConfig` stops at the "Core" initialization phase. -* :c:member:`PyConfig._isolated_interpreter`: if non-zero, - disallow threads, subprocesses and fork. - -.. c:function:: PyStatus _Py_InitializeMain(void) - - Move to the "Main" initialization phase, finish the Python initialization. - -No module is imported during the "Core" phase and the ``importlib`` module is -not configured: the :ref:`Path Configuration ` is only -applied during the "Main" phase. It may allow to customize Python in Python to -override or tune the :ref:`Path Configuration `, maybe -install a custom :data:`sys.meta_path` importer or an import hook, etc. - -It may become possible to calculatin the :ref:`Path Configuration -` in Python, after the Core phase and before the Main phase, -which is one of the :pep:`432` motivation. - -The "Core" phase is not properly defined: what should be and what should -not be available at this phase is not specified yet. The API is marked -as private and provisional: the API can be modified or even be removed -anytime until a proper public API is designed. - -Example running Python code between "Core" and "Main" initialization -phases:: - - void init_python(void) - { - PyStatus status; - - PyConfig config; - PyConfig_InitPythonConfig(&config); - config._init_main = 0; - - /* ... customize 'config' configuration ... */ - - status = Py_InitializeFromConfig(&config); - PyConfig_Clear(&config); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - - /* Use sys.stderr because sys.stdout is only created - by _Py_InitializeMain() */ - int res = PyRun_SimpleString( - "import sys; " - "print('Run Python code before _Py_InitializeMain', " - "file=sys.stderr)"); - if (res < 0) { - exit(1); - } - - /* ... put more configuration code here ... */ - - status = _Py_InitializeMain(); - if (PyStatus_Exception(status)) { - Py_ExitStatusException(status); - } - } diff --git a/Doc/c-api/intro.rst b/Doc/c-api/intro.rst deleted file mode 100644 index 39df9ec390d4a9..00000000000000 --- a/Doc/c-api/intro.rst +++ /dev/null @@ -1,835 +0,0 @@ -.. highlight:: c - - -.. _api-intro: - -************ -Introduction -************ - -The Application Programmer's Interface to Python gives C and C++ programmers -access to the Python interpreter at a variety of levels. The API is equally -usable from C++, but for brevity it is generally referred to as the Python/C -API. There are two fundamentally different reasons for using the Python/C API. -The first reason is to write *extension modules* for specific purposes; these -are C modules that extend the Python interpreter. This is probably the most -common use. The second reason is to use Python as a component in a larger -application; this technique is generally referred to as :dfn:`embedding` Python -in an application. - -Writing an extension module is a relatively well-understood process, where a -"cookbook" approach works well. There are several tools that automate the -process to some extent. While people have embedded Python in other -applications since its early existence, the process of embedding Python is -less straightforward than writing an extension. - -Many API functions are useful independent of whether you're embedding or -extending Python; moreover, most applications that embed Python will need to -provide a custom extension as well, so it's probably a good idea to become -familiar with writing an extension before attempting to embed Python in a real -application. - - -Coding standards -================ - -If you're writing C code for inclusion in CPython, you **must** follow the -guidelines and standards defined in :PEP:`7`. These guidelines apply -regardless of the version of Python you are contributing to. Following these -conventions is not necessary for your own third party extension modules, -unless you eventually expect to contribute them to Python. - - -.. _api-includes: - -Include Files -============= - -All function, type and macro definitions needed to use the Python/C API are -included in your code by the following line:: - - #define PY_SSIZE_T_CLEAN - #include - -This implies inclusion of the following standard headers: ````, -````, ````, ````, ```` and ```` -(if available). - -.. note:: - - Since Python may define some pre-processor definitions which affect the standard - headers on some systems, you *must* include :file:`Python.h` before any standard - headers are included. - - It is recommended to always define ``PY_SSIZE_T_CLEAN`` before including - ``Python.h``. See :ref:`arg-parsing` for a description of this macro. - -All user visible names defined by Python.h (except those defined by the included -standard headers) have one of the prefixes ``Py`` or ``_Py``. Names beginning -with ``_Py`` are for internal use by the Python implementation and should not be -used by extension writers. Structure member names do not have a reserved prefix. - -.. note:: - - User code should never define names that begin with ``Py`` or ``_Py``. This - confuses the reader, and jeopardizes the portability of the user code to - future Python versions, which may define additional names beginning with one - of these prefixes. - -The header files are typically installed with Python. On Unix, these are -located in the directories :file:`{prefix}/include/pythonversion/` and -:file:`{exec_prefix}/include/pythonversion/`, where :option:`prefix <--prefix>` and -:option:`exec_prefix <--exec-prefix>` are defined by the corresponding parameters to Python's -:program:`configure` script and *version* is -``'%d.%d' % sys.version_info[:2]``. On Windows, the headers are installed -in :file:`{prefix}/include`, where ``prefix`` is the installation -directory specified to the installer. - -To include the headers, place both directories (if different) on your compiler's -search path for includes. Do *not* place the parent directories on the search -path and then use ``#include ``; this will break on -multi-platform builds since the platform independent headers under -:option:`prefix <--prefix>` include the platform specific headers from -:option:`exec_prefix <--exec-prefix>`. - -C++ users should note that although the API is defined entirely using C, the -header files properly declare the entry points to be ``extern "C"``. As a result, -there is no need to do anything special to use the API from C++. - - -Useful macros -============= - -Several useful macros are defined in the Python header files. Many are -defined closer to where they are useful (e.g. :c:macro:`Py_RETURN_NONE`). -Others of a more general utility are defined here. This is not necessarily a -complete listing. - -.. c:macro:: PyMODINIT_FUNC - - Declare an extension module ``PyInit`` initialization function. The function - return type is :c:expr:`PyObject*`. The macro declares any special linkage - declarations required by the platform, and for C++ declares the function as - ``extern "C"``. - - The initialization function must be named :samp:`PyInit_{name}`, where - *name* is the name of the module, and should be the only non-\ ``static`` - item defined in the module file. Example:: - - static struct PyModuleDef spam_module = { - PyModuleDef_HEAD_INIT, - .m_name = "spam", - ... - }; - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModule_Create(&spam_module); - } - - -.. c:macro:: Py_ABS(x) - - Return the absolute value of ``x``. - - .. versionadded:: 3.3 - -.. c:macro:: Py_ALWAYS_INLINE - - Ask the compiler to always inline a static inline function. The compiler can - ignore it and decides to not inline the function. - - It can be used to inline performance critical static inline functions when - building Python in debug mode with function inlining disabled. For example, - MSC disables function inlining when building in debug mode. - - Marking blindly a static inline function with Py_ALWAYS_INLINE can result in - worse performances (due to increased code size for example). The compiler is - usually smarter than the developer for the cost/benefit analysis. - - If Python is :ref:`built in debug mode ` (if the ``Py_DEBUG`` - macro is defined), the :c:macro:`Py_ALWAYS_INLINE` macro does nothing. - - It must be specified before the function return type. Usage:: - - static inline Py_ALWAYS_INLINE int random(void) { return 4; } - - .. versionadded:: 3.11 - -.. c:macro:: Py_CHARMASK(c) - - Argument must be a character or an integer in the range [-128, 127] or [0, - 255]. This macro returns ``c`` cast to an ``unsigned char``. - -.. c:macro:: Py_DEPRECATED(version) - - Use this for deprecated declarations. The macro must be placed before the - symbol name. - - Example:: - - Py_DEPRECATED(3.8) PyAPI_FUNC(int) Py_OldFunction(void); - - .. versionchanged:: 3.8 - MSVC support was added. - -.. c:macro:: Py_GETENV(s) - - Like ``getenv(s)``, but returns ``NULL`` if :option:`-E` was passed on the - command line (i.e. if ``Py_IgnoreEnvironmentFlag`` is set). - -.. c:macro:: Py_MAX(x, y) - - Return the maximum value between ``x`` and ``y``. - - .. versionadded:: 3.3 - -.. c:macro:: Py_MEMBER_SIZE(type, member) - - Return the size of a structure (``type``) ``member`` in bytes. - - .. versionadded:: 3.6 - -.. c:macro:: Py_MIN(x, y) - - Return the minimum value between ``x`` and ``y``. - - .. versionadded:: 3.3 - -.. c:macro:: Py_NO_INLINE - - Disable inlining on a function. For example, it reduces the C stack - consumption: useful on LTO+PGO builds which heavily inline code (see - :issue:`33720`). - - Usage:: - - Py_NO_INLINE static int random(void) { return 4; } - - .. versionadded:: 3.11 - -.. c:macro:: Py_STRINGIFY(x) - - Convert ``x`` to a C string. E.g. ``Py_STRINGIFY(123)`` returns - ``"123"``. - - .. versionadded:: 3.4 - -.. c:macro:: Py_UNREACHABLE() - - Use this when you have a code path that cannot be reached by design. - For example, in the ``default:`` clause in a ``switch`` statement for which - all possible values are covered in ``case`` statements. Use this in places - where you might be tempted to put an ``assert(0)`` or ``abort()`` call. - - In release mode, the macro helps the compiler to optimize the code, and - avoids a warning about unreachable code. For example, the macro is - implemented with ``__builtin_unreachable()`` on GCC in release mode. - - A use for ``Py_UNREACHABLE()`` is following a call a function that - never returns but that is not declared :c:macro:`_Py_NO_RETURN`. - - If a code path is very unlikely code but can be reached under exceptional - case, this macro must not be used. For example, under low memory condition - or if a system call returns a value out of the expected range. In this - case, it's better to report the error to the caller. If the error cannot - be reported to caller, :c:func:`Py_FatalError` can be used. - - .. versionadded:: 3.7 - -.. c:macro:: Py_UNUSED(arg) - - Use this for unused arguments in a function definition to silence compiler - warnings. Example: ``int func(int a, int Py_UNUSED(b)) { return a; }``. - - .. versionadded:: 3.4 - -.. c:macro:: PyDoc_STRVAR(name, str) - - Creates a variable with name ``name`` that can be used in docstrings. - If Python is built without docstrings, the value will be empty. - - Use :c:macro:`PyDoc_STRVAR` for docstrings to support building - Python without docstrings, as specified in :pep:`7`. - - Example:: - - PyDoc_STRVAR(pop_doc, "Remove and return the rightmost element."); - - static PyMethodDef deque_methods[] = { - // ... - {"pop", (PyCFunction)deque_pop, METH_NOARGS, pop_doc}, - // ... - } - -.. c:macro:: PyDoc_STR(str) - - Creates a docstring for the given input string or an empty string - if docstrings are disabled. - - Use :c:macro:`PyDoc_STR` in specifying docstrings to support - building Python without docstrings, as specified in :pep:`7`. - - Example:: - - static PyMethodDef pysqlite_row_methods[] = { - {"keys", (PyCFunction)pysqlite_row_keys, METH_NOARGS, - PyDoc_STR("Returns the keys of the row.")}, - {NULL, NULL} - }; - - -.. _api-objects: - -Objects, Types and Reference Counts -=================================== - -.. index:: pair: object; type - -Most Python/C API functions have one or more arguments as well as a return value -of type :c:expr:`PyObject*`. This type is a pointer to an opaque data type -representing an arbitrary Python object. Since all Python object types are -treated the same way by the Python language in most situations (e.g., -assignments, scope rules, and argument passing), it is only fitting that they -should be represented by a single C type. Almost all Python objects live on the -heap: you never declare an automatic or static variable of type -:c:type:`PyObject`, only pointer variables of type :c:expr:`PyObject*` can be -declared. The sole exception are the type objects; since these must never be -deallocated, they are typically static :c:type:`PyTypeObject` objects. - -All Python objects (even Python integers) have a :dfn:`type` and a -:dfn:`reference count`. An object's type determines what kind of object it is -(e.g., an integer, a list, or a user-defined function; there are many more as -explained in :ref:`types`). For each of the well-known types there is a macro -to check whether an object is of that type; for instance, ``PyList_Check(a)`` is -true if (and only if) the object pointed to by *a* is a Python list. - - -.. _api-refcounts: - -Reference Counts ----------------- - -The reference count is important because today's computers have a finite -(and often severely limited) memory size; it counts how many different -places there are that have a :term:`strong reference` to an object. -Such a place could be another object, or a global (or static) C variable, -or a local variable in some C function. -When the last :term:`strong reference` to an object is released -(i.e. its reference count becomes zero), the object is deallocated. -If it contains references to other objects, those references are released. -Those other objects may be deallocated in turn, if there are no more -references to them, and so on. (There's an obvious problem with -objects that reference each other here; for now, the solution -is "don't do that.") - -.. index:: - single: Py_INCREF() - single: Py_DECREF() - -Reference counts are always manipulated explicitly. The normal way is -to use the macro :c:func:`Py_INCREF` to take a new reference to an -object (i.e. increment its reference count by one), -and :c:func:`Py_DECREF` to release that reference (i.e. decrement the -reference count by one). The :c:func:`Py_DECREF` macro -is considerably more complex than the incref one, since it must check whether -the reference count becomes zero and then cause the object's deallocator to be -called. The deallocator is a function pointer contained in the object's type -structure. The type-specific deallocator takes care of releasing references -for other objects contained in the object if this is a compound -object type, such as a list, as well as performing any additional finalization -that's needed. There's no chance that the reference count can overflow; at -least as many bits are used to hold the reference count as there are distinct -memory locations in virtual memory (assuming ``sizeof(Py_ssize_t) >= sizeof(void*)``). -Thus, the reference count increment is a simple operation. - -It is not necessary to hold a :term:`strong reference` (i.e. increment -the reference count) for every local variable that contains a pointer -to an object. In theory, the object's -reference count goes up by one when the variable is made to point to it and it -goes down by one when the variable goes out of scope. However, these two -cancel each other out, so at the end the reference count hasn't changed. The -only real reason to use the reference count is to prevent the object from being -deallocated as long as our variable is pointing to it. If we know that there -is at least one other reference to the object that lives at least as long as -our variable, there is no need to take a new :term:`strong reference` -(i.e. increment the reference count) temporarily. -An important situation where this arises is in objects that are passed as -arguments to C functions in an extension module that are called from Python; -the call mechanism guarantees to hold a reference to every argument for the -duration of the call. - -However, a common pitfall is to extract an object from a list and hold on to it -for a while without taking a new reference. Some other operation might -conceivably remove the object from the list, releasing that reference, -and possibly deallocating it. The real danger is that innocent-looking -operations may invoke arbitrary Python code which could do this; there is a code -path which allows control to flow back to the user from a :c:func:`Py_DECREF`, so -almost any operation is potentially dangerous. - -A safe approach is to always use the generic operations (functions whose name -begins with ``PyObject_``, ``PyNumber_``, ``PySequence_`` or ``PyMapping_``). -These operations always create a new :term:`strong reference` -(i.e. increment the reference count) of the object they return. -This leaves the caller with the responsibility to call :c:func:`Py_DECREF` when -they are done with the result; this soon becomes second nature. - - -.. _api-refcountdetails: - -Reference Count Details -^^^^^^^^^^^^^^^^^^^^^^^ - -The reference count behavior of functions in the Python/C API is best explained -in terms of *ownership of references*. Ownership pertains to references, never -to objects (objects are not owned: they are always shared). "Owning a -reference" means being responsible for calling Py_DECREF on it when the -reference is no longer needed. Ownership can also be transferred, meaning that -the code that receives ownership of the reference then becomes responsible for -eventually releasing it by calling :c:func:`Py_DECREF` or :c:func:`Py_XDECREF` -when it's no longer needed---or passing on this responsibility (usually to its -caller). When a function passes ownership of a reference on to its caller, the -caller is said to receive a *new* reference. When no ownership is transferred, -the caller is said to *borrow* the reference. Nothing needs to be done for a -:term:`borrowed reference`. - -Conversely, when a calling function passes in a reference to an object, there -are two possibilities: the function *steals* a reference to the object, or it -does not. *Stealing a reference* means that when you pass a reference to a -function, that function assumes that it now owns that reference, and you are not -responsible for it any longer. - -.. index:: - single: PyList_SetItem() - single: PyTuple_SetItem() - -Few functions steal references; the two notable exceptions are -:c:func:`PyList_SetItem` and :c:func:`PyTuple_SetItem`, which steal a reference -to the item (but not to the tuple or list into which the item is put!). These -functions were designed to steal a reference because of a common idiom for -populating a tuple or list with newly created objects; for example, the code to -create the tuple ``(1, 2, "three")`` could look like this (forgetting about -error handling for the moment; a better way to code this is shown below):: - - PyObject *t; - - t = PyTuple_New(3); - PyTuple_SetItem(t, 0, PyLong_FromLong(1L)); - PyTuple_SetItem(t, 1, PyLong_FromLong(2L)); - PyTuple_SetItem(t, 2, PyUnicode_FromString("three")); - -Here, :c:func:`PyLong_FromLong` returns a new reference which is immediately -stolen by :c:func:`PyTuple_SetItem`. When you want to keep using an object -although the reference to it will be stolen, use :c:func:`Py_INCREF` to grab -another reference before calling the reference-stealing function. - -Incidentally, :c:func:`PyTuple_SetItem` is the *only* way to set tuple items; -:c:func:`PySequence_SetItem` and :c:func:`PyObject_SetItem` refuse to do this -since tuples are an immutable data type. You should only use -:c:func:`PyTuple_SetItem` for tuples that you are creating yourself. - -Equivalent code for populating a list can be written using :c:func:`PyList_New` -and :c:func:`PyList_SetItem`. - -However, in practice, you will rarely use these ways of creating and populating -a tuple or list. There's a generic function, :c:func:`Py_BuildValue`, that can -create most common objects from C values, directed by a :dfn:`format string`. -For example, the above two blocks of code could be replaced by the following -(which also takes care of the error checking):: - - PyObject *tuple, *list; - - tuple = Py_BuildValue("(iis)", 1, 2, "three"); - list = Py_BuildValue("[iis]", 1, 2, "three"); - -It is much more common to use :c:func:`PyObject_SetItem` and friends with items -whose references you are only borrowing, like arguments that were passed in to -the function you are writing. In that case, their behaviour regarding references -is much saner, since you don't have to take a new reference just so you -can give that reference away ("have it be stolen"). For example, this function -sets all items of a list (actually, any mutable sequence) to a given item:: - - int - set_all(PyObject *target, PyObject *item) - { - Py_ssize_t i, n; - - n = PyObject_Length(target); - if (n < 0) - return -1; - for (i = 0; i < n; i++) { - PyObject *index = PyLong_FromSsize_t(i); - if (!index) - return -1; - if (PyObject_SetItem(target, index, item) < 0) { - Py_DECREF(index); - return -1; - } - Py_DECREF(index); - } - return 0; - } - -.. index:: single: set_all() - -The situation is slightly different for function return values. While passing -a reference to most functions does not change your ownership responsibilities -for that reference, many functions that return a reference to an object give -you ownership of the reference. The reason is simple: in many cases, the -returned object is created on the fly, and the reference you get is the only -reference to the object. Therefore, the generic functions that return object -references, like :c:func:`PyObject_GetItem` and :c:func:`PySequence_GetItem`, -always return a new reference (the caller becomes the owner of the reference). - -It is important to realize that whether you own a reference returned by a -function depends on which function you call only --- *the plumage* (the type of -the object passed as an argument to the function) *doesn't enter into it!* -Thus, if you extract an item from a list using :c:func:`PyList_GetItem`, you -don't own the reference --- but if you obtain the same item from the same list -using :c:func:`PySequence_GetItem` (which happens to take exactly the same -arguments), you do own a reference to the returned object. - -.. index:: - single: PyList_GetItem() - single: PySequence_GetItem() - -Here is an example of how you could write a function that computes the sum of -the items in a list of integers; once using :c:func:`PyList_GetItem`, and once -using :c:func:`PySequence_GetItem`. :: - - long - sum_list(PyObject *list) - { - Py_ssize_t i, n; - long total = 0, value; - PyObject *item; - - n = PyList_Size(list); - if (n < 0) - return -1; /* Not a list */ - for (i = 0; i < n; i++) { - item = PyList_GetItem(list, i); /* Can't fail */ - if (!PyLong_Check(item)) continue; /* Skip non-integers */ - value = PyLong_AsLong(item); - if (value == -1 && PyErr_Occurred()) - /* Integer too big to fit in a C long, bail out */ - return -1; - total += value; - } - return total; - } - -.. index:: single: sum_list() - -:: - - long - sum_sequence(PyObject *sequence) - { - Py_ssize_t i, n; - long total = 0, value; - PyObject *item; - n = PySequence_Length(sequence); - if (n < 0) - return -1; /* Has no length */ - for (i = 0; i < n; i++) { - item = PySequence_GetItem(sequence, i); - if (item == NULL) - return -1; /* Not a sequence, or other failure */ - if (PyLong_Check(item)) { - value = PyLong_AsLong(item); - Py_DECREF(item); - if (value == -1 && PyErr_Occurred()) - /* Integer too big to fit in a C long, bail out */ - return -1; - total += value; - } - else { - Py_DECREF(item); /* Discard reference ownership */ - } - } - return total; - } - -.. index:: single: sum_sequence() - - -.. _api-types: - -Types ------ - -There are few other data types that play a significant role in the Python/C -API; most are simple C types such as :c:expr:`int`, :c:expr:`long`, -:c:expr:`double` and :c:expr:`char*`. A few structure types are used to -describe static tables used to list the functions exported by a module or the -data attributes of a new object type, and another is used to describe the value -of a complex number. These will be discussed together with the functions that -use them. - -.. c:type:: Py_ssize_t - - A signed integral type such that ``sizeof(Py_ssize_t) == sizeof(size_t)``. - C99 doesn't define such a thing directly (size_t is an unsigned integral type). - See :pep:`353` for details. ``PY_SSIZE_T_MAX`` is the largest positive value - of type :c:type:`Py_ssize_t`. - - -.. _api-exceptions: - -Exceptions -========== - -The Python programmer only needs to deal with exceptions if specific error -handling is required; unhandled exceptions are automatically propagated to the -caller, then to the caller's caller, and so on, until they reach the top-level -interpreter, where they are reported to the user accompanied by a stack -traceback. - -.. index:: single: PyErr_Occurred() - -For C programmers, however, error checking always has to be explicit. All -functions in the Python/C API can raise exceptions, unless an explicit claim is -made otherwise in a function's documentation. In general, when a function -encounters an error, it sets an exception, discards any object references that -it owns, and returns an error indicator. If not documented otherwise, this -indicator is either ``NULL`` or ``-1``, depending on the function's return type. -A few functions return a Boolean true/false result, with false indicating an -error. Very few functions return no explicit error indicator or have an -ambiguous return value, and require explicit testing for errors with -:c:func:`PyErr_Occurred`. These exceptions are always explicitly documented. - -.. index:: - single: PyErr_SetString() - single: PyErr_Clear() - -Exception state is maintained in per-thread storage (this is equivalent to -using global storage in an unthreaded application). A thread can be in one of -two states: an exception has occurred, or not. The function -:c:func:`PyErr_Occurred` can be used to check for this: it returns a borrowed -reference to the exception type object when an exception has occurred, and -``NULL`` otherwise. There are a number of functions to set the exception state: -:c:func:`PyErr_SetString` is the most common (though not the most general) -function to set the exception state, and :c:func:`PyErr_Clear` clears the -exception state. - -The full exception state consists of three objects (all of which can be -``NULL``): the exception type, the corresponding exception value, and the -traceback. These have the same meanings as the Python result of -``sys.exc_info()``; however, they are not the same: the Python objects represent -the last exception being handled by a Python :keyword:`try` ... -:keyword:`except` statement, while the C level exception state only exists while -an exception is being passed on between C functions until it reaches the Python -bytecode interpreter's main loop, which takes care of transferring it to -``sys.exc_info()`` and friends. - -.. index:: single: exc_info() (in module sys) - -Note that starting with Python 1.5, the preferred, thread-safe way to access the -exception state from Python code is to call the function :func:`sys.exc_info`, -which returns the per-thread exception state for Python code. Also, the -semantics of both ways to access the exception state have changed so that a -function which catches an exception will save and restore its thread's exception -state so as to preserve the exception state of its caller. This prevents common -bugs in exception handling code caused by an innocent-looking function -overwriting the exception being handled; it also reduces the often unwanted -lifetime extension for objects that are referenced by the stack frames in the -traceback. - -As a general principle, a function that calls another function to perform some -task should check whether the called function raised an exception, and if so, -pass the exception state on to its caller. It should discard any object -references that it owns, and return an error indicator, but it should *not* set -another exception --- that would overwrite the exception that was just raised, -and lose important information about the exact cause of the error. - -.. index:: single: sum_sequence() - -A simple example of detecting exceptions and passing them on is shown in the -:c:func:`!sum_sequence` example above. It so happens that this example doesn't -need to clean up any owned references when it detects an error. The following -example function shows some error cleanup. First, to remind you why you like -Python, we show the equivalent Python code:: - - def incr_item(dict, key): - try: - item = dict[key] - except KeyError: - item = 0 - dict[key] = item + 1 - -.. index:: single: incr_item() - -Here is the corresponding C code, in all its glory:: - - int - incr_item(PyObject *dict, PyObject *key) - { - /* Objects all initialized to NULL for Py_XDECREF */ - PyObject *item = NULL, *const_one = NULL, *incremented_item = NULL; - int rv = -1; /* Return value initialized to -1 (failure) */ - - item = PyObject_GetItem(dict, key); - if (item == NULL) { - /* Handle KeyError only: */ - if (!PyErr_ExceptionMatches(PyExc_KeyError)) - goto error; - - /* Clear the error and use zero: */ - PyErr_Clear(); - item = PyLong_FromLong(0L); - if (item == NULL) - goto error; - } - const_one = PyLong_FromLong(1L); - if (const_one == NULL) - goto error; - - incremented_item = PyNumber_Add(item, const_one); - if (incremented_item == NULL) - goto error; - - if (PyObject_SetItem(dict, key, incremented_item) < 0) - goto error; - rv = 0; /* Success */ - /* Continue with cleanup code */ - - error: - /* Cleanup code, shared by success and failure path */ - - /* Use Py_XDECREF() to ignore NULL references */ - Py_XDECREF(item); - Py_XDECREF(const_one); - Py_XDECREF(incremented_item); - - return rv; /* -1 for error, 0 for success */ - } - -.. index:: single: incr_item() - -.. index:: - single: PyErr_ExceptionMatches() - single: PyErr_Clear() - single: Py_XDECREF() - -This example represents an endorsed use of the ``goto`` statement in C! -It illustrates the use of :c:func:`PyErr_ExceptionMatches` and -:c:func:`PyErr_Clear` to handle specific exceptions, and the use of -:c:func:`Py_XDECREF` to dispose of owned references that may be ``NULL`` (note the -``'X'`` in the name; :c:func:`Py_DECREF` would crash when confronted with a -``NULL`` reference). It is important that the variables used to hold owned -references are initialized to ``NULL`` for this to work; likewise, the proposed -return value is initialized to ``-1`` (failure) and only set to success after -the final call made is successful. - - -.. _api-embedding: - -Embedding Python -================ - -The one important task that only embedders (as opposed to extension writers) of -the Python interpreter have to worry about is the initialization, and possibly -the finalization, of the Python interpreter. Most functionality of the -interpreter can only be used after the interpreter has been initialized. - -.. index:: - single: Py_Initialize() - pair: module; builtins - pair: module; __main__ - pair: module; sys - triple: module; search; path - single: path (in module sys) - -The basic initialization function is :c:func:`Py_Initialize`. This initializes -the table of loaded modules, and creates the fundamental modules -:mod:`builtins`, :mod:`__main__`, and :mod:`sys`. It also -initializes the module search path (``sys.path``). - -:c:func:`Py_Initialize` does not set the "script argument list" (``sys.argv``). -If this variable is needed by Python code that will be executed later, setting -:c:member:`PyConfig.argv` and :c:member:`PyConfig.parse_argv` must be set: see -:ref:`Python Initialization Configuration `. - -On most systems (in particular, on Unix and Windows, although the details are -slightly different), :c:func:`Py_Initialize` calculates the module search path -based upon its best guess for the location of the standard Python interpreter -executable, assuming that the Python library is found in a fixed location -relative to the Python interpreter executable. In particular, it looks for a -directory named :file:`lib/python{X.Y}` relative to the parent directory -where the executable named :file:`python` is found on the shell command search -path (the environment variable :envvar:`PATH`). - -For instance, if the Python executable is found in -:file:`/usr/local/bin/python`, it will assume that the libraries are in -:file:`/usr/local/lib/python{X.Y}`. (In fact, this particular path is also -the "fallback" location, used when no executable file named :file:`python` is -found along :envvar:`PATH`.) The user can override this behavior by setting the -environment variable :envvar:`PYTHONHOME`, or insert additional directories in -front of the standard path by setting :envvar:`PYTHONPATH`. - -.. index:: - single: Py_SetProgramName() - single: Py_GetPath() - single: Py_GetPrefix() - single: Py_GetExecPrefix() - single: Py_GetProgramFullPath() - -The embedding application can steer the search by calling -``Py_SetProgramName(file)`` *before* calling :c:func:`Py_Initialize`. Note that -:envvar:`PYTHONHOME` still overrides this and :envvar:`PYTHONPATH` is still -inserted in front of the standard path. An application that requires total -control has to provide its own implementation of :c:func:`Py_GetPath`, -:c:func:`Py_GetPrefix`, :c:func:`Py_GetExecPrefix`, and -:c:func:`Py_GetProgramFullPath` (all defined in :file:`Modules/getpath.c`). - -.. index:: single: Py_IsInitialized() - -Sometimes, it is desirable to "uninitialize" Python. For instance, the -application may want to start over (make another call to -:c:func:`Py_Initialize`) or the application is simply done with its use of -Python and wants to free memory allocated by Python. This can be accomplished -by calling :c:func:`Py_FinalizeEx`. The function :c:func:`Py_IsInitialized` returns -true if Python is currently in the initialized state. More information about -these functions is given in a later chapter. Notice that :c:func:`Py_FinalizeEx` -does *not* free all memory allocated by the Python interpreter, e.g. memory -allocated by extension modules currently cannot be released. - - -.. _api-debugging: - -Debugging Builds -================ - -Python can be built with several macros to enable extra checks of the -interpreter and extension modules. These checks tend to add a large amount of -overhead to the runtime so they are not enabled by default. - -A full list of the various types of debugging builds is in the file -:file:`Misc/SpecialBuilds.txt` in the Python source distribution. Builds are -available that support tracing of reference counts, debugging the memory -allocator, or low-level profiling of the main interpreter loop. Only the most -frequently used builds will be described in the remainder of this section. - -Compiling the interpreter with the :c:macro:`Py_DEBUG` macro defined produces -what is generally meant by :ref:`a debug build of Python `. -:c:macro:`Py_DEBUG` is enabled in the Unix build by adding -:option:`--with-pydebug` to the :file:`./configure` command. -It is also implied by the presence of the -not-Python-specific :c:macro:`_DEBUG` macro. When :c:macro:`Py_DEBUG` is enabled -in the Unix build, compiler optimization is disabled. - -In addition to the reference count debugging described below, extra checks are -performed, see :ref:`Python Debug Build `. - -Defining :c:macro:`Py_TRACE_REFS` enables reference tracing -(see the :option:`configure --with-trace-refs option <--with-trace-refs>`). -When defined, a circular doubly linked list of active objects is maintained by adding two extra -fields to every :c:type:`PyObject`. Total allocations are tracked as well. Upon -exit, all existing references are printed. (In interactive mode this happens -after every statement run by the interpreter.) - -Please refer to :file:`Misc/SpecialBuilds.txt` in the Python source distribution -for more detailed information. - diff --git a/Doc/c-api/iter.rst b/Doc/c-api/iter.rst deleted file mode 100644 index 434d2021cea8e6..00000000000000 --- a/Doc/c-api/iter.rst +++ /dev/null @@ -1,72 +0,0 @@ -.. highlight:: c - -.. _iterator: - -Iterator Protocol -================= - -There are two functions specifically for working with iterators. - -.. c:function:: int PyIter_Check(PyObject *o) - - Return non-zero if the object *o* can be safely passed to - :c:func:`PyIter_Next`, and ``0`` otherwise. This function always succeeds. - -.. c:function:: int PyAIter_Check(PyObject *o) - - Return non-zero if the object *o* provides the :class:`AsyncIterator` - protocol, and ``0`` otherwise. This function always succeeds. - - .. versionadded:: 3.10 - -.. c:function:: PyObject* PyIter_Next(PyObject *o) - - Return the next value from the iterator *o*. The object must be an iterator - according to :c:func:`PyIter_Check` (it is up to the caller to check this). - If there are no remaining values, returns ``NULL`` with no exception set. - If an error occurs while retrieving the item, returns ``NULL`` and passes - along the exception. - -To write a loop which iterates over an iterator, the C code should look -something like this:: - - PyObject *iterator = PyObject_GetIter(obj); - PyObject *item; - - if (iterator == NULL) { - /* propagate error */ - } - - while ((item = PyIter_Next(iterator))) { - /* do something with item */ - ... - /* release reference when done */ - Py_DECREF(item); - } - - Py_DECREF(iterator); - - if (PyErr_Occurred()) { - /* propagate error */ - } - else { - /* continue doing useful work */ - } - - -.. c:type:: PySendResult - - The enum value used to represent different results of :c:func:`PyIter_Send`. - - .. versionadded:: 3.10 - - -.. c:function:: PySendResult PyIter_Send(PyObject *iter, PyObject *arg, PyObject **presult) - - Sends the *arg* value into the iterator *iter*. Returns: - - - ``PYGEN_RETURN`` if iterator returns. Return value is returned via *presult*. - - ``PYGEN_NEXT`` if iterator yields. Yielded value is returned via *presult*. - - ``PYGEN_ERROR`` if iterator has raised and exception. *presult* is set to ``NULL``. - - .. versionadded:: 3.10 diff --git a/Doc/c-api/iterator.rst b/Doc/c-api/iterator.rst deleted file mode 100644 index 6b7ba8c9979163..00000000000000 --- a/Doc/c-api/iterator.rst +++ /dev/null @@ -1,52 +0,0 @@ -.. highlight:: c - -.. _iterator-objects: - -Iterator Objects ----------------- - -Python provides two general-purpose iterator objects. The first, a sequence -iterator, works with an arbitrary sequence supporting the :meth:`~object.__getitem__` -method. The second works with a callable object and a sentinel value, calling -the callable for each item in the sequence, and ending the iteration when the -sentinel value is returned. - - -.. c:var:: PyTypeObject PySeqIter_Type - - Type object for iterator objects returned by :c:func:`PySeqIter_New` and the - one-argument form of the :func:`iter` built-in function for built-in sequence - types. - - -.. c:function:: int PySeqIter_Check(PyObject *op) - - Return true if the type of *op* is :c:data:`PySeqIter_Type`. This function - always succeeds. - - -.. c:function:: PyObject* PySeqIter_New(PyObject *seq) - - Return an iterator that works with a general sequence object, *seq*. The - iteration ends when the sequence raises :exc:`IndexError` for the subscripting - operation. - - -.. c:var:: PyTypeObject PyCallIter_Type - - Type object for iterator objects returned by :c:func:`PyCallIter_New` and the - two-argument form of the :func:`iter` built-in function. - - -.. c:function:: int PyCallIter_Check(PyObject *op) - - Return true if the type of *op* is :c:data:`PyCallIter_Type`. This - function always succeeds. - - -.. c:function:: PyObject* PyCallIter_New(PyObject *callable, PyObject *sentinel) - - Return a new iterator. The first parameter, *callable*, can be any Python - callable object that can be called with no parameters; each call to it should - return the next item in the iteration. When *callable* returns a value equal to - *sentinel*, the iteration will be terminated. diff --git a/Doc/c-api/list.rst b/Doc/c-api/list.rst deleted file mode 100644 index dbf35611eccd3e..00000000000000 --- a/Doc/c-api/list.rst +++ /dev/null @@ -1,144 +0,0 @@ -.. highlight:: c - -.. _listobjects: - -List Objects ------------- - -.. index:: pair: object; list - - -.. c:type:: PyListObject - - This subtype of :c:type:`PyObject` represents a Python list object. - - -.. c:var:: PyTypeObject PyList_Type - - This instance of :c:type:`PyTypeObject` represents the Python list type. - This is the same object as :class:`list` in the Python layer. - - -.. c:function:: int PyList_Check(PyObject *p) - - Return true if *p* is a list object or an instance of a subtype of the list - type. This function always succeeds. - - -.. c:function:: int PyList_CheckExact(PyObject *p) - - Return true if *p* is a list object, but not an instance of a subtype of - the list type. This function always succeeds. - - -.. c:function:: PyObject* PyList_New(Py_ssize_t len) - - Return a new list of length *len* on success, or ``NULL`` on failure. - - .. note:: - - If *len* is greater than zero, the returned list object's items are - set to ``NULL``. Thus you cannot use abstract API functions such as - :c:func:`PySequence_SetItem` or expose the object to Python code before - setting all items to a real object with :c:func:`PyList_SetItem`. - - -.. c:function:: Py_ssize_t PyList_Size(PyObject *list) - - .. index:: pair: built-in function; len - - Return the length of the list object in *list*; this is equivalent to - ``len(list)`` on a list object. - - -.. c:function:: Py_ssize_t PyList_GET_SIZE(PyObject *list) - - Similar to :c:func:`PyList_Size`, but without error checking. - - -.. c:function:: PyObject* PyList_GetItem(PyObject *list, Py_ssize_t index) - - Return the object at position *index* in the list pointed to by *list*. The - position must be non-negative; indexing from the end of the list is not - supported. If *index* is out of bounds (<0 or >=len(list)), - return ``NULL`` and set an :exc:`IndexError` exception. - - -.. c:function:: PyObject* PyList_GET_ITEM(PyObject *list, Py_ssize_t i) - - Similar to :c:func:`PyList_GetItem`, but without error checking. - - -.. c:function:: int PyList_SetItem(PyObject *list, Py_ssize_t index, PyObject *item) - - Set the item at index *index* in list to *item*. Return ``0`` on success. - If *index* is out of bounds, return ``-1`` and set an :exc:`IndexError` - exception. - - .. note:: - - This function "steals" a reference to *item* and discards a reference to - an item already in the list at the affected position. - - -.. c:function:: void PyList_SET_ITEM(PyObject *list, Py_ssize_t i, PyObject *o) - - Macro form of :c:func:`PyList_SetItem` without error checking. This is - normally only used to fill in new lists where there is no previous content. - - .. note:: - - This macro "steals" a reference to *item*, and, unlike - :c:func:`PyList_SetItem`, does *not* discard a reference to any item that - is being replaced; any reference in *list* at position *i* will be - leaked. - - -.. c:function:: int PyList_Insert(PyObject *list, Py_ssize_t index, PyObject *item) - - Insert the item *item* into list *list* in front of index *index*. Return - ``0`` if successful; return ``-1`` and set an exception if unsuccessful. - Analogous to ``list.insert(index, item)``. - - -.. c:function:: int PyList_Append(PyObject *list, PyObject *item) - - Append the object *item* at the end of list *list*. Return ``0`` if - successful; return ``-1`` and set an exception if unsuccessful. Analogous - to ``list.append(item)``. - - -.. c:function:: PyObject* PyList_GetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high) - - Return a list of the objects in *list* containing the objects *between* *low* - and *high*. Return ``NULL`` and set an exception if unsuccessful. Analogous - to ``list[low:high]``. Indexing from the end of the list is not supported. - - -.. c:function:: int PyList_SetSlice(PyObject *list, Py_ssize_t low, Py_ssize_t high, PyObject *itemlist) - - Set the slice of *list* between *low* and *high* to the contents of - *itemlist*. Analogous to ``list[low:high] = itemlist``. The *itemlist* may - be ``NULL``, indicating the assignment of an empty list (slice deletion). - Return ``0`` on success, ``-1`` on failure. Indexing from the end of the - list is not supported. - - -.. c:function:: int PyList_Sort(PyObject *list) - - Sort the items of *list* in place. Return ``0`` on success, ``-1`` on - failure. This is equivalent to ``list.sort()``. - - -.. c:function:: int PyList_Reverse(PyObject *list) - - Reverse the items of *list* in place. Return ``0`` on success, ``-1`` on - failure. This is the equivalent of ``list.reverse()``. - - -.. c:function:: PyObject* PyList_AsTuple(PyObject *list) - - .. index:: pair: built-in function; tuple - - Return a new tuple object containing the contents of *list*; equivalent to - ``tuple(list)``. diff --git a/Doc/c-api/long.rst b/Doc/c-api/long.rst deleted file mode 100644 index 874e96216097f4..00000000000000 --- a/Doc/c-api/long.rst +++ /dev/null @@ -1,323 +0,0 @@ -.. highlight:: c - -.. _longobjects: - -Integer Objects ---------------- - -.. index:: pair: object; long integer - pair: object; integer - -All integers are implemented as "long" integer objects of arbitrary size. - -On error, most ``PyLong_As*`` APIs return ``(return type)-1`` which cannot be -distinguished from a number. Use :c:func:`PyErr_Occurred` to disambiguate. - -.. c:type:: PyLongObject - - This subtype of :c:type:`PyObject` represents a Python integer object. - - -.. c:var:: PyTypeObject PyLong_Type - - This instance of :c:type:`PyTypeObject` represents the Python integer type. - This is the same object as :class:`int` in the Python layer. - - -.. c:function:: int PyLong_Check(PyObject *p) - - Return true if its argument is a :c:type:`PyLongObject` or a subtype of - :c:type:`PyLongObject`. This function always succeeds. - - -.. c:function:: int PyLong_CheckExact(PyObject *p) - - Return true if its argument is a :c:type:`PyLongObject`, but not a subtype of - :c:type:`PyLongObject`. This function always succeeds. - - -.. c:function:: PyObject* PyLong_FromLong(long v) - - Return a new :c:type:`PyLongObject` object from *v*, or ``NULL`` on failure. - - The current implementation keeps an array of integer objects for all integers - between ``-5`` and ``256``. When you create an int in that range you actually - just get back a reference to the existing object. - - -.. c:function:: PyObject* PyLong_FromUnsignedLong(unsigned long v) - - Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long`, or - ``NULL`` on failure. - - -.. c:function:: PyObject* PyLong_FromSsize_t(Py_ssize_t v) - - Return a new :c:type:`PyLongObject` object from a C :c:type:`Py_ssize_t`, or - ``NULL`` on failure. - - -.. c:function:: PyObject* PyLong_FromSize_t(size_t v) - - Return a new :c:type:`PyLongObject` object from a C :c:type:`size_t`, or - ``NULL`` on failure. - - -.. c:function:: PyObject* PyLong_FromLongLong(long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:expr:`long long`, or ``NULL`` - on failure. - - -.. c:function:: PyObject* PyLong_FromUnsignedLongLong(unsigned long long v) - - Return a new :c:type:`PyLongObject` object from a C :c:expr:`unsigned long long`, - or ``NULL`` on failure. - - -.. c:function:: PyObject* PyLong_FromDouble(double v) - - Return a new :c:type:`PyLongObject` object from the integer part of *v*, or - ``NULL`` on failure. - - -.. c:function:: PyObject* PyLong_FromString(const char *str, char **pend, int base) - - Return a new :c:type:`PyLongObject` based on the string value in *str*, which - is interpreted according to the radix in *base*. If *pend* is non-``NULL``, - *\*pend* will point to the first character in *str* which follows the - representation of the number. If *base* is ``0``, *str* is interpreted using - the :ref:`integers` definition; in this case, leading zeros in a - non-zero decimal number raises a :exc:`ValueError`. If *base* is not ``0``, - it must be between ``2`` and ``36``, inclusive. Leading spaces and single - underscores after a base specifier and between digits are ignored. If there - are no digits, :exc:`ValueError` will be raised. - - .. seealso:: Python methods :meth:`int.to_bytes` and :meth:`int.from_bytes` - to convert a :c:type:`PyLongObject` to/from an array of bytes in base - ``256``. You can call those from C using :c:func:`PyObject_CallMethod`. - - -.. c:function:: PyObject* PyLong_FromUnicodeObject(PyObject *u, int base) - - Convert a sequence of Unicode digits in the string *u* to a Python integer - value. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyLong_FromVoidPtr(void *p) - - Create a Python integer from the pointer *p*. The pointer value can be - retrieved from the resulting value using :c:func:`PyLong_AsVoidPtr`. - - -.. XXX alias PyLong_AS_LONG (for now) -.. c:function:: long PyLong_AsLong(PyObject *obj) - - .. index:: - single: LONG_MAX - single: OverflowError (built-in exception) - - Return a C :c:expr:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method - (if present) to convert it to a :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *obj* is out of range for a - :c:expr:`long`. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - .. versionchanged:: 3.8 - Use :meth:`~object.__index__` if available. - - .. versionchanged:: 3.10 - This function will no longer use :meth:`~object.__int__`. - - -.. c:function:: long PyLong_AsLongAndOverflow(PyObject *obj, int *overflow) - - Return a C :c:expr:`long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` - method (if present) to convert it to a :c:type:`PyLongObject`. - - If the value of *obj* is greater than :c:macro:`LONG_MAX` or less than - :c:macro:`LONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, and - return ``-1``; otherwise, set *\*overflow* to ``0``. If any other exception - occurs set *\*overflow* to ``0`` and return ``-1`` as usual. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - .. versionchanged:: 3.8 - Use :meth:`~object.__index__` if available. - - .. versionchanged:: 3.10 - This function will no longer use :meth:`~object.__int__`. - - -.. c:function:: long long PyLong_AsLongLong(PyObject *obj) - - .. index:: - single: OverflowError (built-in exception) - - Return a C :c:expr:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method - (if present) to convert it to a :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *obj* is out of range for a - :c:expr:`long long`. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - .. versionchanged:: 3.8 - Use :meth:`~object.__index__` if available. - - .. versionchanged:: 3.10 - This function will no longer use :meth:`~object.__int__`. - - -.. c:function:: long long PyLong_AsLongLongAndOverflow(PyObject *obj, int *overflow) - - Return a C :c:expr:`long long` representation of *obj*. If *obj* is not an - instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` method - (if present) to convert it to a :c:type:`PyLongObject`. - - If the value of *obj* is greater than :c:macro:`LLONG_MAX` or less than - :c:macro:`LLONG_MIN`, set *\*overflow* to ``1`` or ``-1``, respectively, - and return ``-1``; otherwise, set *\*overflow* to ``0``. If any other - exception occurs set *\*overflow* to ``0`` and return ``-1`` as usual. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.8 - Use :meth:`~object.__index__` if available. - - .. versionchanged:: 3.10 - This function will no longer use :meth:`~object.__int__`. - - -.. c:function:: Py_ssize_t PyLong_AsSsize_t(PyObject *pylong) - - .. index:: - single: PY_SSIZE_T_MAX - single: OverflowError (built-in exception) - - Return a C :c:type:`Py_ssize_t` representation of *pylong*. *pylong* must - be an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`Py_ssize_t`. - - Returns ``-1`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: unsigned long PyLong_AsUnsignedLong(PyObject *pylong) - - .. index:: - single: ULONG_MAX - single: OverflowError (built-in exception) - - Return a C :c:expr:`unsigned long` representation of *pylong*. *pylong* - must be an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:expr:`unsigned long`. - - Returns ``(unsigned long)-1`` on error. - Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: size_t PyLong_AsSize_t(PyObject *pylong) - - .. index:: - single: SIZE_MAX - single: OverflowError (built-in exception) - - Return a C :c:type:`size_t` representation of *pylong*. *pylong* must be - an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:type:`size_t`. - - Returns ``(size_t)-1`` on error. - Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: unsigned long long PyLong_AsUnsignedLongLong(PyObject *pylong) - - .. index:: - single: OverflowError (built-in exception) - - Return a C :c:expr:`unsigned long long` representation of *pylong*. *pylong* - must be an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for an - :c:expr:`unsigned long long`. - - Returns ``(unsigned long long)-1`` on error. - Use :c:func:`PyErr_Occurred` to disambiguate. - - .. versionchanged:: 3.1 - A negative *pylong* now raises :exc:`OverflowError`, not :exc:`TypeError`. - - -.. c:function:: unsigned long PyLong_AsUnsignedLongMask(PyObject *obj) - - Return a C :c:expr:`unsigned long` representation of *obj*. If *obj* is not - an instance of :c:type:`PyLongObject`, first call its :meth:`~object.__index__` - method (if present) to convert it to a :c:type:`PyLongObject`. - - If the value of *obj* is out of range for an :c:expr:`unsigned long`, - return the reduction of that value modulo ``ULONG_MAX + 1``. - - Returns ``(unsigned long)-1`` on error. Use :c:func:`PyErr_Occurred` to - disambiguate. - - .. versionchanged:: 3.8 - Use :meth:`~object.__index__` if available. - - .. versionchanged:: 3.10 - This function will no longer use :meth:`~object.__int__`. - - -.. c:function:: unsigned long long PyLong_AsUnsignedLongLongMask(PyObject *obj) - - Return a C :c:expr:`unsigned long long` representation of *obj*. If *obj* - is not an instance of :c:type:`PyLongObject`, first call its - :meth:`~object.__index__` method (if present) to convert it to a - :c:type:`PyLongObject`. - - If the value of *obj* is out of range for an :c:expr:`unsigned long long`, - return the reduction of that value modulo ``ULLONG_MAX + 1``. - - Returns ``(unsigned long long)-1`` on error. Use :c:func:`PyErr_Occurred` - to disambiguate. - - .. versionchanged:: 3.8 - Use :meth:`~object.__index__` if available. - - .. versionchanged:: 3.10 - This function will no longer use :meth:`~object.__int__`. - - -.. c:function:: double PyLong_AsDouble(PyObject *pylong) - - Return a C :c:expr:`double` representation of *pylong*. *pylong* must be - an instance of :c:type:`PyLongObject`. - - Raise :exc:`OverflowError` if the value of *pylong* is out of range for a - :c:expr:`double`. - - Returns ``-1.0`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. - - -.. c:function:: void* PyLong_AsVoidPtr(PyObject *pylong) - - Convert a Python integer *pylong* to a C :c:expr:`void` pointer. - If *pylong* cannot be converted, an :exc:`OverflowError` will be raised. This - is only assured to produce a usable :c:expr:`void` pointer for values created - with :c:func:`PyLong_FromVoidPtr`. - - Returns ``NULL`` on error. Use :c:func:`PyErr_Occurred` to disambiguate. diff --git a/Doc/c-api/mapping.rst b/Doc/c-api/mapping.rst deleted file mode 100644 index 0c42b9177bb56d..00000000000000 --- a/Doc/c-api/mapping.rst +++ /dev/null @@ -1,106 +0,0 @@ -.. highlight:: c - -.. _mapping: - -Mapping Protocol -================ - -See also :c:func:`PyObject_GetItem`, :c:func:`PyObject_SetItem` and -:c:func:`PyObject_DelItem`. - - -.. c:function:: int PyMapping_Check(PyObject *o) - - Return ``1`` if the object provides the mapping protocol or supports slicing, - and ``0`` otherwise. Note that it returns ``1`` for Python classes with - a :meth:`~object.__getitem__` method, since in general it is impossible to - determine what type of keys the class supports. This function always succeeds. - - -.. c:function:: Py_ssize_t PyMapping_Size(PyObject *o) - Py_ssize_t PyMapping_Length(PyObject *o) - - .. index:: pair: built-in function; len - - Returns the number of keys in object *o* on success, and ``-1`` on failure. - This is equivalent to the Python expression ``len(o)``. - - -.. c:function:: PyObject* PyMapping_GetItemString(PyObject *o, const char *key) - - This is the same as :c:func:`PyObject_GetItem`, but *key* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - -.. c:function:: int PyMapping_SetItemString(PyObject *o, const char *key, PyObject *v) - - This is the same as :c:func:`PyObject_SetItem`, but *key* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - -.. c:function:: int PyMapping_DelItem(PyObject *o, PyObject *key) - - This is an alias of :c:func:`PyObject_DelItem`. - - -.. c:function:: int PyMapping_DelItemString(PyObject *o, const char *key) - - This is the same as :c:func:`PyObject_DelItem`, but *key* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - -.. c:function:: int PyMapping_HasKey(PyObject *o, PyObject *key) - - Return ``1`` if the mapping object has the key *key* and ``0`` otherwise. - This is equivalent to the Python expression ``key in o``. - This function always succeeds. - - .. note:: - - Exceptions which occur when this calls :meth:`~object.__getitem__` - method are silently ignored. - For proper error handling, use :c:func:`PyObject_GetItem()` instead. - - -.. c:function:: int PyMapping_HasKeyString(PyObject *o, const char *key) - - This is the same as :c:func:`PyMapping_HasKey`, but *key* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - .. note:: - - Exceptions that occur when this calls :meth:`~object.__getitem__` - method or while creating the temporary :class:`str` - object are silently ignored. - For proper error handling, use :c:func:`PyMapping_GetItemString` instead. - - -.. c:function:: PyObject* PyMapping_Keys(PyObject *o) - - On success, return a list of the keys in object *o*. On failure, return - ``NULL``. - - .. versionchanged:: 3.7 - Previously, the function returned a list or a tuple. - - -.. c:function:: PyObject* PyMapping_Values(PyObject *o) - - On success, return a list of the values in object *o*. On failure, return - ``NULL``. - - .. versionchanged:: 3.7 - Previously, the function returned a list or a tuple. - - -.. c:function:: PyObject* PyMapping_Items(PyObject *o) - - On success, return a list of the items in object *o*, where each item is a - tuple containing a key-value pair. On failure, return ``NULL``. - - .. versionchanged:: 3.7 - Previously, the function returned a list or a tuple. diff --git a/Doc/c-api/marshal.rst b/Doc/c-api/marshal.rst deleted file mode 100644 index 489f1580a414b2..00000000000000 --- a/Doc/c-api/marshal.rst +++ /dev/null @@ -1,98 +0,0 @@ -.. highlight:: c - -.. _marshalling-utils: - -Data marshalling support -======================== - -These routines allow C code to work with serialized objects using the same -data format as the :mod:`marshal` module. There are functions to write data -into the serialization format, and additional functions that can be used to -read the data back. Files used to store marshalled data must be opened in -binary mode. - -Numeric values are stored with the least significant byte first. - -The module supports two versions of the data format: version 0 is the -historical version, version 1 shares interned strings in the file, and upon -unmarshalling. Version 2 uses a binary format for floating point numbers. -``Py_MARSHAL_VERSION`` indicates the current file format (currently 2). - - -.. c:function:: void PyMarshal_WriteLongToFile(long value, FILE *file, int version) - - Marshal a :c:expr:`long` integer, *value*, to *file*. This will only write - the least-significant 32 bits of *value*; regardless of the size of the - native :c:expr:`long` type. *version* indicates the file format. - - This function can fail, in which case it sets the error indicator. - Use :c:func:`PyErr_Occurred` to check for that. - -.. c:function:: void PyMarshal_WriteObjectToFile(PyObject *value, FILE *file, int version) - - Marshal a Python object, *value*, to *file*. - *version* indicates the file format. - - This function can fail, in which case it sets the error indicator. - Use :c:func:`PyErr_Occurred` to check for that. - -.. c:function:: PyObject* PyMarshal_WriteObjectToString(PyObject *value, int version) - - Return a bytes object containing the marshalled representation of *value*. - *version* indicates the file format. - - -The following functions allow marshalled values to be read back in. - - -.. c:function:: long PyMarshal_ReadLongFromFile(FILE *file) - - Return a C :c:expr:`long` from the data stream in a :c:expr:`FILE*` opened - for reading. Only a 32-bit value can be read in using this function, - regardless of the native size of :c:expr:`long`. - - On error, sets the appropriate exception (:exc:`EOFError`) and returns - ``-1``. - - -.. c:function:: int PyMarshal_ReadShortFromFile(FILE *file) - - Return a C :c:expr:`short` from the data stream in a :c:expr:`FILE*` opened - for reading. Only a 16-bit value can be read in using this function, - regardless of the native size of :c:expr:`short`. - - On error, sets the appropriate exception (:exc:`EOFError`) and returns - ``-1``. - - -.. c:function:: PyObject* PyMarshal_ReadObjectFromFile(FILE *file) - - Return a Python object from the data stream in a :c:expr:`FILE*` opened for - reading. - - On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns ``NULL``. - - -.. c:function:: PyObject* PyMarshal_ReadLastObjectFromFile(FILE *file) - - Return a Python object from the data stream in a :c:expr:`FILE*` opened for - reading. Unlike :c:func:`PyMarshal_ReadObjectFromFile`, this function - assumes that no further objects will be read from the file, allowing it to - aggressively load file data into memory so that the de-serialization can - operate from data in memory rather than reading a byte at a time from the - file. Only use these variant if you are certain that you won't be reading - anything else from the file. - - On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns ``NULL``. - - -.. c:function:: PyObject* PyMarshal_ReadObjectFromString(const char *data, Py_ssize_t len) - - Return a Python object from the data stream in a byte buffer - containing *len* bytes pointed to by *data*. - - On error, sets the appropriate exception (:exc:`EOFError`, :exc:`ValueError` - or :exc:`TypeError`) and returns ``NULL``. - diff --git a/Doc/c-api/memory.rst b/Doc/c-api/memory.rst deleted file mode 100644 index 27005f92dd43e5..00000000000000 --- a/Doc/c-api/memory.rst +++ /dev/null @@ -1,742 +0,0 @@ -.. highlight:: c - - -.. _memory: - -***************** -Memory Management -***************** - -.. sectionauthor:: Vladimir Marangozov - - - -.. _memoryoverview: - -Overview -======== - -Memory management in Python involves a private heap containing all Python -objects and data structures. The management of this private heap is ensured -internally by the *Python memory manager*. The Python memory manager has -different components which deal with various dynamic storage management aspects, -like sharing, segmentation, preallocation or caching. - -At the lowest level, a raw memory allocator ensures that there is enough room in -the private heap for storing all Python-related data by interacting with the -memory manager of the operating system. On top of the raw memory allocator, -several object-specific allocators operate on the same heap and implement -distinct memory management policies adapted to the peculiarities of every object -type. For example, integer objects are managed differently within the heap than -strings, tuples or dictionaries because integers imply different storage -requirements and speed/space tradeoffs. The Python memory manager thus delegates -some of the work to the object-specific allocators, but ensures that the latter -operate within the bounds of the private heap. - -It is important to understand that the management of the Python heap is -performed by the interpreter itself and that the user has no control over it, -even if they regularly manipulate object pointers to memory blocks inside that -heap. The allocation of heap space for Python objects and other internal -buffers is performed on demand by the Python memory manager through the Python/C -API functions listed in this document. - -.. index:: - single: malloc() - single: calloc() - single: realloc() - single: free() - -To avoid memory corruption, extension writers should never try to operate on -Python objects with the functions exported by the C library: :c:func:`malloc`, -:c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. This will result in mixed -calls between the C allocator and the Python memory manager with fatal -consequences, because they implement different algorithms and operate on -different heaps. However, one may safely allocate and release memory blocks -with the C library allocator for individual purposes, as shown in the following -example:: - - PyObject *res; - char *buf = (char *) malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - ...Do some I/O operation involving buf... - res = PyBytes_FromString(buf); - free(buf); /* malloc'ed */ - return res; - -In this example, the memory request for the I/O buffer is handled by the C -library allocator. The Python memory manager is involved only in the allocation -of the bytes object returned as a result. - -In most situations, however, it is recommended to allocate memory from the -Python heap specifically because the latter is under control of the Python -memory manager. For example, this is required when the interpreter is extended -with new object types written in C. Another reason for using the Python heap is -the desire to *inform* the Python memory manager about the memory needs of the -extension module. Even when the requested memory is used exclusively for -internal, highly specific purposes, delegating all memory requests to the Python -memory manager causes the interpreter to have a more accurate image of its -memory footprint as a whole. Consequently, under certain circumstances, the -Python memory manager may or may not trigger appropriate actions, like garbage -collection, memory compaction or other preventive procedures. Note that by using -the C library allocator as shown in the previous example, the allocated memory -for the I/O buffer escapes completely the Python memory manager. - -.. seealso:: - - The :envvar:`PYTHONMALLOC` environment variable can be used to configure - the memory allocators used by Python. - - The :envvar:`PYTHONMALLOCSTATS` environment variable can be used to print - statistics of the :ref:`pymalloc memory allocator ` every time a - new pymalloc object arena is created, and on shutdown. - -Allocator Domains -================= - -.. _allocator-domains: - -All allocating functions belong to one of three different "domains" (see also -:c:type:`PyMemAllocatorDomain`). These domains represent different allocation -strategies and are optimized for different purposes. The specific details on -how every domain allocates memory or what internal functions each domain calls -is considered an implementation detail, but for debugging purposes a simplified -table can be found at :ref:`here `. There is no hard -requirement to use the memory returned by the allocation functions belonging to -a given domain for only the purposes hinted by that domain (although this is the -recommended practice). For example, one could use the memory returned by -:c:func:`PyMem_RawMalloc` for allocating Python objects or the memory returned -by :c:func:`PyObject_Malloc` for allocating memory for buffers. - -The three allocation domains are: - -* Raw domain: intended for allocating memory for general-purpose memory - buffers where the allocation *must* go to the system allocator or where the - allocator can operate without the :term:`GIL`. The memory is requested directly - to the system. - -* "Mem" domain: intended for allocating memory for Python buffers and - general-purpose memory buffers where the allocation must be performed with - the :term:`GIL` held. The memory is taken from the Python private heap. - -* Object domain: intended for allocating memory belonging to Python objects. The - memory is taken from the Python private heap. - -When freeing memory previously allocated by the allocating functions belonging to a -given domain,the matching specific deallocating functions must be used. For example, -:c:func:`PyMem_Free` must be used to free memory allocated using :c:func:`PyMem_Malloc`. - -Raw Memory Interface -==================== - -The following function sets are wrappers to the system allocator. These -functions are thread-safe, the :term:`GIL ` does not -need to be held. - -The :ref:`default raw memory allocator ` uses -the following functions: :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` -and :c:func:`!free`; call ``malloc(1)`` (or ``calloc(1, 1)``) when requesting -zero bytes. - -.. versionadded:: 3.4 - -.. c:function:: void* PyMem_RawMalloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:expr:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyMem_RawMalloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_RawCalloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyMem_RawCalloc(1, 1)`` had been - called instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_RawRealloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will - be unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyMem_RawMalloc(n)``; else if - *n* is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. - - If the request fails, :c:func:`PyMem_RawRealloc` returns ``NULL`` and *p* - remains a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_RawFree(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_RawMalloc`, :c:func:`PyMem_RawRealloc` or - :c:func:`PyMem_RawCalloc`. Otherwise, or if ``PyMem_RawFree(p)`` has been - called before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - - -.. _memoryinterface: - -Memory Interface -================ - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -The :ref:`default memory allocator ` uses the -:ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. versionchanged:: 3.6 - - The default allocator is now pymalloc instead of system :c:func:`malloc`. - -.. c:function:: void* PyMem_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:expr:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyMem_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyMem_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyMem_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyMem_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyMem_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or :c:func:`PyMem_Calloc`. - - If the request fails, :c:func:`PyMem_Realloc` returns ``NULL`` and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyMem_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyMem_Malloc`, :c:func:`PyMem_Realloc` or - :c:func:`PyMem_Calloc`. Otherwise, or if ``PyMem_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - -The following type-oriented macros are provided for convenience. Note that -*TYPE* refers to any C type. - - -.. c:macro:: PyMem_New(TYPE, n) - - Same as :c:func:`PyMem_Malloc`, but allocates ``(n * sizeof(TYPE))`` bytes of - memory. Returns a pointer cast to :c:expr:`TYPE*`. The memory will not have - been initialized in any way. - - -.. c:macro:: PyMem_Resize(p, TYPE, n) - - Same as :c:func:`PyMem_Realloc`, but the memory block is resized to ``(n * - sizeof(TYPE))`` bytes. Returns a pointer cast to :c:expr:`TYPE*`. On return, - *p* will be a pointer to the new memory area, or ``NULL`` in the event of - failure. - - This is a C preprocessor macro; *p* is always reassigned. Save the original - value of *p* to avoid losing memory when handling errors. - - -.. c:function:: void PyMem_Del(void *p) - - Same as :c:func:`PyMem_Free`. - -In addition, the following macro sets are provided for calling the Python memory -allocator directly, without involving the C API functions listed above. However, -note that their use does not preserve binary compatibility across Python -versions and is therefore deprecated in extension modules. - -* ``PyMem_MALLOC(size)`` -* ``PyMem_NEW(type, size)`` -* ``PyMem_REALLOC(ptr, size)`` -* ``PyMem_RESIZE(ptr, type, size)`` -* ``PyMem_FREE(ptr)`` -* ``PyMem_DEL(ptr)`` - - -Object allocators -================= - -The following function sets, modeled after the ANSI C standard, but specifying -behavior when requesting zero bytes, are available for allocating and releasing -memory from the Python heap. - -.. note:: - There is no guarantee that the memory returned by these allocators can be - successfully cast to a Python object when intercepting the allocating - functions in this domain by the methods described in - the :ref:`Customize Memory Allocators ` section. - -The :ref:`default object allocator ` uses the -:ref:`pymalloc memory allocator `. - -.. warning:: - - The :term:`GIL ` must be held when using these - functions. - -.. c:function:: void* PyObject_Malloc(size_t n) - - Allocates *n* bytes and returns a pointer of type :c:expr:`void*` to the - allocated memory, or ``NULL`` if the request fails. - - Requesting zero bytes returns a distinct non-``NULL`` pointer if possible, as - if ``PyObject_Malloc(1)`` had been called instead. The memory will not have - been initialized in any way. - - -.. c:function:: void* PyObject_Calloc(size_t nelem, size_t elsize) - - Allocates *nelem* elements each whose size in bytes is *elsize* and returns - a pointer of type :c:expr:`void*` to the allocated memory, or ``NULL`` if the - request fails. The memory is initialized to zeros. - - Requesting zero elements or elements of size zero bytes returns a distinct - non-``NULL`` pointer if possible, as if ``PyObject_Calloc(1, 1)`` had been called - instead. - - .. versionadded:: 3.5 - - -.. c:function:: void* PyObject_Realloc(void *p, size_t n) - - Resizes the memory block pointed to by *p* to *n* bytes. The contents will be - unchanged to the minimum of the old and the new sizes. - - If *p* is ``NULL``, the call is equivalent to ``PyObject_Malloc(n)``; else if *n* - is equal to zero, the memory block is resized but is not freed, and the - returned pointer is non-``NULL``. - - Unless *p* is ``NULL``, it must have been returned by a previous call to - :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or :c:func:`PyObject_Calloc`. - - If the request fails, :c:func:`PyObject_Realloc` returns ``NULL`` and *p* remains - a valid pointer to the previous memory area. - - -.. c:function:: void PyObject_Free(void *p) - - Frees the memory block pointed to by *p*, which must have been returned by a - previous call to :c:func:`PyObject_Malloc`, :c:func:`PyObject_Realloc` or - :c:func:`PyObject_Calloc`. Otherwise, or if ``PyObject_Free(p)`` has been called - before, undefined behavior occurs. - - If *p* is ``NULL``, no operation is performed. - - -.. _default-memory-allocators: - -Default Memory Allocators -========================= - -Default memory allocators: - -=============================== ==================== ================== ===================== ==================== -Configuration Name PyMem_RawMalloc PyMem_Malloc PyObject_Malloc -=============================== ==================== ================== ===================== ==================== -Release build ``"pymalloc"`` ``malloc`` ``pymalloc`` ``pymalloc`` -Debug build ``"pymalloc_debug"`` ``malloc`` + debug ``pymalloc`` + debug ``pymalloc`` + debug -Release build, without pymalloc ``"malloc"`` ``malloc`` ``malloc`` ``malloc`` -Debug build, without pymalloc ``"malloc_debug"`` ``malloc`` + debug ``malloc`` + debug ``malloc`` + debug -=============================== ==================== ================== ===================== ==================== - -Legend: - -* Name: value for :envvar:`PYTHONMALLOC` environment variable. -* ``malloc``: system allocators from the standard C library, C functions: - :c:func:`malloc`, :c:func:`calloc`, :c:func:`realloc` and :c:func:`free`. -* ``pymalloc``: :ref:`pymalloc memory allocator `. -* "+ debug": with :ref:`debug hooks on the Python memory allocators - `. -* "Debug build": :ref:`Python build in debug mode `. - -.. _customize-memory-allocators: - -Customize Memory Allocators -=========================== - -.. versionadded:: 3.4 - -.. c:type:: PyMemAllocatorEx - - Structure used to describe a memory block allocator. The structure has - the following fields: - - +----------------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==========================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +----------------------------------------------------------+---------------------------------------+ - | ``void* malloc(void *ctx, size_t size)`` | allocate a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void* calloc(void *ctx, size_t nelem, size_t elsize)`` | allocate a memory block initialized | - | | with zeros | - +----------------------------------------------------------+---------------------------------------+ - | ``void* realloc(void *ctx, void *ptr, size_t new_size)`` | allocate or resize a memory block | - +----------------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, void *ptr)`` | free a memory block | - +----------------------------------------------------------+---------------------------------------+ - - .. versionchanged:: 3.5 - The :c:type:`!PyMemAllocator` structure was renamed to - :c:type:`PyMemAllocatorEx` and a new ``calloc`` field was added. - - -.. c:type:: PyMemAllocatorDomain - - Enum used to identify an allocator domain. Domains: - - .. c:namespace:: NULL - - .. c:macro:: PYMEM_DOMAIN_RAW - - Functions: - - * :c:func:`PyMem_RawMalloc` - * :c:func:`PyMem_RawRealloc` - * :c:func:`PyMem_RawCalloc` - * :c:func:`PyMem_RawFree` - - .. c:macro:: PYMEM_DOMAIN_MEM - - Functions: - - * :c:func:`PyMem_Malloc`, - * :c:func:`PyMem_Realloc` - * :c:func:`PyMem_Calloc` - * :c:func:`PyMem_Free` - - .. c:macro:: PYMEM_DOMAIN_OBJ - - Functions: - - * :c:func:`PyObject_Malloc` - * :c:func:`PyObject_Realloc` - * :c:func:`PyObject_Calloc` - * :c:func:`PyObject_Free` - -.. c:function:: void PyMem_GetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Get the memory block allocator of the specified domain. - - -.. c:function:: void PyMem_SetAllocator(PyMemAllocatorDomain domain, PyMemAllocatorEx *allocator) - - Set the memory block allocator of the specified domain. - - The new allocator must return a distinct non-``NULL`` pointer when requesting - zero bytes. - - For the :c:macro:`PYMEM_DOMAIN_RAW` domain, the allocator must be - thread-safe: the :term:`GIL ` is not held when the - allocator is called. - - If the new allocator is not a hook (does not call the previous allocator), - the :c:func:`PyMem_SetupDebugHooks` function must be called to reinstall the - debug hooks on top on the new allocator. - - See also :c:member:`PyPreConfig.allocator` and :ref:`Preinitialize Python - with PyPreConfig `. - - .. warning:: - - :c:func:`PyMem_SetAllocator` does have the following contract: - - * It can be called after :c:func:`Py_PreInitialize` and before - :c:func:`Py_InitializeFromConfig` to install a custom memory - allocator. There are no restrictions over the installed allocator - other than the ones imposed by the domain (for instance, the Raw - Domain allows the allocator to be called without the GIL held). See - :ref:`the section on allocator domains ` for more - information. - - * If called after Python has finish initializing (after - :c:func:`Py_InitializeFromConfig` has been called) the allocator - **must** wrap the existing allocator. Substituting the current - allocator for some other arbitrary one is **not supported**. - - - -.. c:function:: void PyMem_SetupDebugHooks(void) - - Setup :ref:`debug hooks in the Python memory allocators ` - to detect memory errors. - - -.. _pymem-debug-hooks: - -Debug hooks on the Python memory allocators -=========================================== - -When :ref:`Python is built in debug mode `, the -:c:func:`PyMem_SetupDebugHooks` function is called at the :ref:`Python -preinitialization ` to setup debug hooks on Python memory allocators -to detect memory errors. - -The :envvar:`PYTHONMALLOC` environment variable can be used to install debug -hooks on a Python compiled in release mode (ex: ``PYTHONMALLOC=debug``). - -The :c:func:`PyMem_SetupDebugHooks` function can be used to set debug hooks -after calling :c:func:`PyMem_SetAllocator`. - -These debug hooks fill dynamically allocated memory blocks with special, -recognizable bit patterns. Newly allocated memory is filled with the byte -``0xCD`` (``PYMEM_CLEANBYTE``), freed memory is filled with the byte ``0xDD`` -(``PYMEM_DEADBYTE``). Memory blocks are surrounded by "forbidden bytes" -filled with the byte ``0xFD`` (``PYMEM_FORBIDDENBYTE``). Strings of these bytes -are unlikely to be valid addresses, floats, or ASCII strings. - -Runtime checks: - -- Detect API violations. For example, detect if :c:func:`PyObject_Free` is - called on a memory block allocated by :c:func:`PyMem_Malloc`. -- Detect write before the start of the buffer (buffer underflow). -- Detect write after the end of the buffer (buffer overflow). -- Check that the :term:`GIL ` is held when - allocator functions of :c:macro:`PYMEM_DOMAIN_OBJ` (ex: - :c:func:`PyObject_Malloc`) and :c:macro:`PYMEM_DOMAIN_MEM` (ex: - :c:func:`PyMem_Malloc`) domains are called. - -On error, the debug hooks use the :mod:`tracemalloc` module to get the -traceback where a memory block was allocated. The traceback is only displayed -if :mod:`tracemalloc` is tracing Python memory allocations and the memory block -was traced. - -Let *S* = ``sizeof(size_t)``. ``2*S`` bytes are added at each end of each block -of *N* bytes requested. The memory layout is like so, where p represents the -address returned by a malloc-like or realloc-like function (``p[i:j]`` means -the slice of bytes from ``*(p+i)`` inclusive up to ``*(p+j)`` exclusive; note -that the treatment of negative indices differs from a Python slice): - -``p[-2*S:-S]`` - Number of bytes originally asked for. This is a size_t, big-endian (easier - to read in a memory dump). -``p[-S]`` - API identifier (ASCII character): - - * ``'r'`` for :c:macro:`PYMEM_DOMAIN_RAW`. - * ``'m'`` for :c:macro:`PYMEM_DOMAIN_MEM`. - * ``'o'`` for :c:macro:`PYMEM_DOMAIN_OBJ`. - -``p[-S+1:0]`` - Copies of PYMEM_FORBIDDENBYTE. Used to catch under- writes and reads. - -``p[0:N]`` - The requested memory, filled with copies of PYMEM_CLEANBYTE, used to catch - reference to uninitialized memory. When a realloc-like function is called - requesting a larger memory block, the new excess bytes are also filled with - PYMEM_CLEANBYTE. When a free-like function is called, these are - overwritten with PYMEM_DEADBYTE, to catch reference to freed memory. When - a realloc- like function is called requesting a smaller memory block, the - excess old bytes are also filled with PYMEM_DEADBYTE. - -``p[N:N+S]`` - Copies of PYMEM_FORBIDDENBYTE. Used to catch over- writes and reads. - -``p[N+S:N+2*S]`` - Only used if the ``PYMEM_DEBUG_SERIALNO`` macro is defined (not defined by - default). - - A serial number, incremented by 1 on each call to a malloc-like or - realloc-like function. Big-endian :c:type:`size_t`. If "bad memory" is detected - later, the serial number gives an excellent way to set a breakpoint on the - next run, to capture the instant at which this block was passed out. The - static function bumpserialno() in obmalloc.c is the only place the serial - number is incremented, and exists so you can set such a breakpoint easily. - -A realloc-like or free-like function first checks that the PYMEM_FORBIDDENBYTE -bytes at each end are intact. If they've been altered, diagnostic output is -written to stderr, and the program is aborted via Py_FatalError(). The other -main failure mode is provoking a memory error when a program reads up one of -the special bit patterns and tries to use it as an address. If you get in a -debugger then and look at the object, you're likely to see that it's entirely -filled with PYMEM_DEADBYTE (meaning freed memory is getting used) or -PYMEM_CLEANBYTE (meaning uninitialized memory is getting used). - -.. versionchanged:: 3.6 - The :c:func:`PyMem_SetupDebugHooks` function now also works on Python - compiled in release mode. On error, the debug hooks now use - :mod:`tracemalloc` to get the traceback where a memory block was allocated. - The debug hooks now also check if the GIL is held when functions of - :c:macro:`PYMEM_DOMAIN_OBJ` and :c:macro:`PYMEM_DOMAIN_MEM` domains are - called. - -.. versionchanged:: 3.8 - Byte patterns ``0xCB`` (``PYMEM_CLEANBYTE``), ``0xDB`` (``PYMEM_DEADBYTE``) - and ``0xFB`` (``PYMEM_FORBIDDENBYTE``) have been replaced with ``0xCD``, - ``0xDD`` and ``0xFD`` to use the same values than Windows CRT debug - ``malloc()`` and ``free()``. - - -.. _pymalloc: - -The pymalloc allocator -====================== - -Python has a *pymalloc* allocator optimized for small objects (smaller or equal -to 512 bytes) with a short lifetime. It uses memory mappings called "arenas" -with a fixed size of either 256 KiB on 32-bit platforms or 1 MiB on 64-bit -platforms. It falls back to :c:func:`PyMem_RawMalloc` and -:c:func:`PyMem_RawRealloc` for allocations larger than 512 bytes. - -*pymalloc* is the :ref:`default allocator ` of the -:c:macro:`PYMEM_DOMAIN_MEM` (ex: :c:func:`PyMem_Malloc`) and -:c:macro:`PYMEM_DOMAIN_OBJ` (ex: :c:func:`PyObject_Malloc`) domains. - -The arena allocator uses the following functions: - -* :c:func:`!VirtualAlloc` and :c:func:`!VirtualFree` on Windows, -* :c:func:`!mmap` and :c:func:`!munmap` if available, -* :c:func:`malloc` and :c:func:`free` otherwise. - -This allocator is disabled if Python is configured with the -:option:`--without-pymalloc` option. It can also be disabled at runtime using -the :envvar:`PYTHONMALLOC` environment variable (ex: ``PYTHONMALLOC=malloc``). - -Customize pymalloc Arena Allocator ----------------------------------- - -.. versionadded:: 3.4 - -.. c:type:: PyObjectArenaAllocator - - Structure used to describe an arena allocator. The structure has - three fields: - - +--------------------------------------------------+---------------------------------------+ - | Field | Meaning | - +==================================================+=======================================+ - | ``void *ctx`` | user context passed as first argument | - +--------------------------------------------------+---------------------------------------+ - | ``void* alloc(void *ctx, size_t size)`` | allocate an arena of size bytes | - +--------------------------------------------------+---------------------------------------+ - | ``void free(void *ctx, void *ptr, size_t size)`` | free an arena | - +--------------------------------------------------+---------------------------------------+ - -.. c:function:: void PyObject_GetArenaAllocator(PyObjectArenaAllocator *allocator) - - Get the arena allocator. - -.. c:function:: void PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) - - Set the arena allocator. - - -tracemalloc C API -================= - -.. versionadded:: 3.7 - -.. c:function:: int PyTraceMalloc_Track(unsigned int domain, uintptr_t ptr, size_t size) - - Track an allocated memory block in the :mod:`tracemalloc` module. - - Return ``0`` on success, return ``-1`` on error (failed to allocate memory to - store the trace). Return ``-2`` if tracemalloc is disabled. - - If memory block is already tracked, update the existing trace. - -.. c:function:: int PyTraceMalloc_Untrack(unsigned int domain, uintptr_t ptr) - - Untrack an allocated memory block in the :mod:`tracemalloc` module. - Do nothing if the block was not tracked. - - Return ``-2`` if tracemalloc is disabled, otherwise return ``0``. - - -.. _memoryexamples: - -Examples -======== - -Here is the example from section :ref:`memoryoverview`, rewritten so that the -I/O buffer is allocated from the Python heap by using the first function set:: - - PyObject *res; - char *buf = (char *) PyMem_Malloc(BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Free(buf); /* allocated with PyMem_Malloc */ - return res; - -The same code using the type-oriented function set:: - - PyObject *res; - char *buf = PyMem_New(char, BUFSIZ); /* for I/O */ - - if (buf == NULL) - return PyErr_NoMemory(); - /* ...Do some I/O operation involving buf... */ - res = PyBytes_FromString(buf); - PyMem_Del(buf); /* allocated with PyMem_New */ - return res; - -Note that in the two examples above, the buffer is always manipulated via -functions belonging to the same set. Indeed, it is required to use the same -memory API family for a given memory block, so that the risk of mixing different -allocators is reduced to a minimum. The following code sequence contains two -errors, one of which is labeled as *fatal* because it mixes two different -allocators operating on different heaps. :: - - char *buf1 = PyMem_New(char, BUFSIZ); - char *buf2 = (char *) malloc(BUFSIZ); - char *buf3 = (char *) PyMem_Malloc(BUFSIZ); - ... - PyMem_Del(buf3); /* Wrong -- should be PyMem_Free() */ - free(buf2); /* Right -- allocated via malloc() */ - free(buf1); /* Fatal -- should be PyMem_Del() */ - -In addition to the functions aimed at handling raw memory blocks from the Python -heap, objects in Python are allocated and released with :c:macro:`PyObject_New`, -:c:macro:`PyObject_NewVar` and :c:func:`PyObject_Del`. - -These will be explained in the next chapter on defining and implementing new -object types in C. diff --git a/Doc/c-api/memoryview.rst b/Doc/c-api/memoryview.rst deleted file mode 100644 index 2aa43318e7a455..00000000000000 --- a/Doc/c-api/memoryview.rst +++ /dev/null @@ -1,63 +0,0 @@ -.. highlight:: c - -.. _memoryview-objects: - -.. index:: - pair: object; memoryview - -MemoryView objects ------------------- - -A :class:`memoryview` object exposes the C level :ref:`buffer interface -` as a Python object which can then be passed around like -any other object. - - -.. c:function:: PyObject *PyMemoryView_FromObject(PyObject *obj) - - Create a memoryview object from an object that provides the buffer interface. - If *obj* supports writable buffer exports, the memoryview object will be - read/write, otherwise it may be either read-only or read/write at the - discretion of the exporter. - -.. c:function:: PyObject *PyMemoryView_FromMemory(char *mem, Py_ssize_t size, int flags) - - Create a memoryview object using *mem* as the underlying buffer. - *flags* can be one of :c:macro:`PyBUF_READ` or :c:macro:`PyBUF_WRITE`. - - .. versionadded:: 3.3 - -.. c:function:: PyObject *PyMemoryView_FromBuffer(const Py_buffer *view) - - Create a memoryview object wrapping the given buffer structure *view*. - For simple byte buffers, :c:func:`PyMemoryView_FromMemory` is the preferred - function. - -.. c:function:: PyObject *PyMemoryView_GetContiguous(PyObject *obj, int buffertype, char order) - - Create a memoryview object to a :term:`contiguous` chunk of memory (in either - 'C' or 'F'ortran *order*) from an object that defines the buffer - interface. If memory is contiguous, the memoryview object points to the - original memory. Otherwise, a copy is made and the memoryview points to a - new bytes object. - - -.. c:function:: int PyMemoryView_Check(PyObject *obj) - - Return true if the object *obj* is a memoryview object. It is not - currently allowed to create subclasses of :class:`memoryview`. This - function always succeeds. - - -.. c:function:: Py_buffer *PyMemoryView_GET_BUFFER(PyObject *mview) - - Return a pointer to the memoryview's private copy of the exporter's buffer. - *mview* **must** be a memoryview instance; this macro doesn't check its type, - you must do it yourself or you will risk crashes. - -.. c:function:: PyObject *PyMemoryView_GET_BASE(PyObject *mview) - - Return either a pointer to the exporting object that the memoryview is based - on or ``NULL`` if the memoryview has been created by one of the functions - :c:func:`PyMemoryView_FromMemory` or :c:func:`PyMemoryView_FromBuffer`. - *mview* **must** be a memoryview instance. diff --git a/Doc/c-api/method.rst b/Doc/c-api/method.rst deleted file mode 100644 index 0d75ab8e1af111..00000000000000 --- a/Doc/c-api/method.rst +++ /dev/null @@ -1,95 +0,0 @@ -.. highlight:: c - -.. _instancemethod-objects: - -Instance Method Objects ------------------------ - -.. index:: pair: object; instancemethod - -An instance method is a wrapper for a :c:type:`PyCFunction` and the new way -to bind a :c:type:`PyCFunction` to a class object. It replaces the former call -``PyMethod_New(func, NULL, class)``. - - -.. c:var:: PyTypeObject PyInstanceMethod_Type - - This instance of :c:type:`PyTypeObject` represents the Python instance - method type. It is not exposed to Python programs. - - -.. c:function:: int PyInstanceMethod_Check(PyObject *o) - - Return true if *o* is an instance method object (has type - :c:data:`PyInstanceMethod_Type`). The parameter must not be ``NULL``. - This function always succeeds. - - -.. c:function:: PyObject* PyInstanceMethod_New(PyObject *func) - - Return a new instance method object, with *func* being any callable object. - *func* is the function that will be called when the instance method is - called. - - -.. c:function:: PyObject* PyInstanceMethod_Function(PyObject *im) - - Return the function object associated with the instance method *im*. - - -.. c:function:: PyObject* PyInstanceMethod_GET_FUNCTION(PyObject *im) - - Macro version of :c:func:`PyInstanceMethod_Function` which avoids error checking. - - -.. _method-objects: - -Method Objects --------------- - -.. index:: pair: object; method - -Methods are bound function objects. Methods are always bound to an instance of -a user-defined class. Unbound methods (methods bound to a class object) are -no longer available. - - -.. c:var:: PyTypeObject PyMethod_Type - - .. index:: single: MethodType (in module types) - - This instance of :c:type:`PyTypeObject` represents the Python method type. This - is exposed to Python programs as ``types.MethodType``. - - -.. c:function:: int PyMethod_Check(PyObject *o) - - Return true if *o* is a method object (has type :c:data:`PyMethod_Type`). The - parameter must not be ``NULL``. This function always succeeds. - - -.. c:function:: PyObject* PyMethod_New(PyObject *func, PyObject *self) - - Return a new method object, with *func* being any callable object and *self* - the instance the method should be bound. *func* is the function that will - be called when the method is called. *self* must not be ``NULL``. - - -.. c:function:: PyObject* PyMethod_Function(PyObject *meth) - - Return the function object associated with the method *meth*. - - -.. c:function:: PyObject* PyMethod_GET_FUNCTION(PyObject *meth) - - Macro version of :c:func:`PyMethod_Function` which avoids error checking. - - -.. c:function:: PyObject* PyMethod_Self(PyObject *meth) - - Return the instance associated with the method *meth*. - - -.. c:function:: PyObject* PyMethod_GET_SELF(PyObject *meth) - - Macro version of :c:func:`PyMethod_Self` which avoids error checking. diff --git a/Doc/c-api/module.rst b/Doc/c-api/module.rst deleted file mode 100644 index 71a079a8e67f4f..00000000000000 --- a/Doc/c-api/module.rst +++ /dev/null @@ -1,628 +0,0 @@ -.. highlight:: c - -.. _moduleobjects: - -Module Objects --------------- - -.. index:: pair: object; module - - -.. c:var:: PyTypeObject PyModule_Type - - .. index:: single: ModuleType (in module types) - - This instance of :c:type:`PyTypeObject` represents the Python module type. This - is exposed to Python programs as ``types.ModuleType``. - - -.. c:function:: int PyModule_Check(PyObject *p) - - Return true if *p* is a module object, or a subtype of a module object. - This function always succeeds. - - -.. c:function:: int PyModule_CheckExact(PyObject *p) - - Return true if *p* is a module object, but not a subtype of - :c:data:`PyModule_Type`. This function always succeeds. - - -.. c:function:: PyObject* PyModule_NewObject(PyObject *name) - - .. index:: - single: __name__ (module attribute) - single: __doc__ (module attribute) - single: __file__ (module attribute) - single: __package__ (module attribute) - single: __loader__ (module attribute) - - Return a new module object with the :attr:`__name__` attribute set to *name*. - The module's :attr:`__name__`, :attr:`__doc__`, :attr:`__package__`, and - :attr:`__loader__` attributes are filled in (all but :attr:`__name__` are set - to ``None``); the caller is responsible for providing a :attr:`__file__` - attribute. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - :attr:`__package__` and :attr:`__loader__` are set to ``None``. - - -.. c:function:: PyObject* PyModule_New(const char *name) - - Similar to :c:func:`PyModule_NewObject`, but the name is a UTF-8 encoded - string instead of a Unicode object. - - -.. c:function:: PyObject* PyModule_GetDict(PyObject *module) - - .. index:: single: __dict__ (module attribute) - - Return the dictionary object that implements *module*'s namespace; this object - is the same as the :attr:`~object.__dict__` attribute of the module object. - If *module* is not a module object (or a subtype of a module object), - :exc:`SystemError` is raised and ``NULL`` is returned. - - It is recommended extensions use other ``PyModule_*`` and - ``PyObject_*`` functions rather than directly manipulate a module's - :attr:`~object.__dict__`. - - -.. c:function:: PyObject* PyModule_GetNameObject(PyObject *module) - - .. index:: - single: __name__ (module attribute) - single: SystemError (built-in exception) - - Return *module*'s :attr:`__name__` value. If the module does not provide one, - or if it is not a string, :exc:`SystemError` is raised and ``NULL`` is returned. - - .. versionadded:: 3.3 - - -.. c:function:: const char* PyModule_GetName(PyObject *module) - - Similar to :c:func:`PyModule_GetNameObject` but return the name encoded to - ``'utf-8'``. - -.. c:function:: void* PyModule_GetState(PyObject *module) - - Return the "state" of the module, that is, a pointer to the block of memory - allocated at module creation time, or ``NULL``. See - :c:member:`PyModuleDef.m_size`. - - -.. c:function:: PyModuleDef* PyModule_GetDef(PyObject *module) - - Return a pointer to the :c:type:`PyModuleDef` struct from which the module was - created, or ``NULL`` if the module wasn't created from a definition. - - -.. c:function:: PyObject* PyModule_GetFilenameObject(PyObject *module) - - .. index:: - single: __file__ (module attribute) - single: SystemError (built-in exception) - - Return the name of the file from which *module* was loaded using *module*'s - :attr:`__file__` attribute. If this is not defined, or if it is not a - unicode string, raise :exc:`SystemError` and return ``NULL``; otherwise return - a reference to a Unicode object. - - .. versionadded:: 3.2 - - -.. c:function:: const char* PyModule_GetFilename(PyObject *module) - - Similar to :c:func:`PyModule_GetFilenameObject` but return the filename - encoded to 'utf-8'. - - .. deprecated:: 3.2 - :c:func:`PyModule_GetFilename` raises :exc:`UnicodeEncodeError` on - unencodable filenames, use :c:func:`PyModule_GetFilenameObject` instead. - - -.. _initializing-modules: - -Initializing C modules -^^^^^^^^^^^^^^^^^^^^^^ - -Modules objects are usually created from extension modules (shared libraries -which export an initialization function), or compiled-in modules -(where the initialization function is added using :c:func:`PyImport_AppendInittab`). -See :ref:`building` or :ref:`extending-with-embedding` for details. - -The initialization function can either pass a module definition instance -to :c:func:`PyModule_Create`, and return the resulting module object, -or request "multi-phase initialization" by returning the definition struct itself. - -.. c:type:: PyModuleDef - - The module definition struct, which holds all information needed to create - a module object. There is usually only one statically initialized variable - of this type for each module. - - .. c:member:: PyModuleDef_Base m_base - - Always initialize this member to :c:macro:`PyModuleDef_HEAD_INIT`. - - .. c:member:: const char *m_name - - Name for the new module. - - .. c:member:: const char *m_doc - - Docstring for the module; usually a docstring variable created with - :c:macro:`PyDoc_STRVAR` is used. - - .. c:member:: Py_ssize_t m_size - - Module state may be kept in a per-module memory area that can be - retrieved with :c:func:`PyModule_GetState`, rather than in static globals. - This makes modules safe for use in multiple sub-interpreters. - - This memory area is allocated based on *m_size* on module creation, - and freed when the module object is deallocated, after the - :c:member:`~PyModuleDef.m_free` function has been called, if present. - - Setting ``m_size`` to ``-1`` means that the module does not support - sub-interpreters, because it has global state. - - Setting it to a non-negative value means that the module can be - re-initialized and specifies the additional amount of memory it requires - for its state. Non-negative ``m_size`` is required for multi-phase - initialization. - - See :PEP:`3121` for more details. - - .. c:member:: PyMethodDef* m_methods - - A pointer to a table of module-level functions, described by - :c:type:`PyMethodDef` values. Can be ``NULL`` if no functions are present. - - .. c:member:: PyModuleDef_Slot* m_slots - - An array of slot definitions for multi-phase initialization, terminated by - a ``{0, NULL}`` entry. - When using single-phase initialization, *m_slots* must be ``NULL``. - - .. versionchanged:: 3.5 - - Prior to version 3.5, this member was always set to ``NULL``, - and was defined as: - - .. c:member:: inquiry m_reload - - .. c:member:: traverseproc m_traverse - - A traversal function to call during GC traversal of the module object, or - ``NULL`` if not needed. - - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. - - .. versionchanged:: 3.9 - No longer called before the module state is allocated. - - .. c:member:: inquiry m_clear - - A clear function to call during GC clearing of the module object, or - ``NULL`` if not needed. - - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. - - Like :c:member:`PyTypeObject.tp_clear`, this function is not *always* - called before a module is deallocated. For example, when reference - counting is enough to determine that an object is no longer used, - the cyclic garbage collector is not involved and - :c:member:`~PyModuleDef.m_free` is called directly. - - .. versionchanged:: 3.9 - No longer called before the module state is allocated. - - .. c:member:: freefunc m_free - - A function to call during deallocation of the module object, or ``NULL`` - if not needed. - - This function is not called if the module state was requested but is not - allocated yet. This is the case immediately after the module is created - and before the module is executed (:c:data:`Py_mod_exec` function). More - precisely, this function is not called if :c:member:`~PyModuleDef.m_size` is greater - than 0 and the module state (as returned by :c:func:`PyModule_GetState`) - is ``NULL``. - - .. versionchanged:: 3.9 - No longer called before the module state is allocated. - -Single-phase initialization -........................... - -The module initialization function may create and return the module object -directly. This is referred to as "single-phase initialization", and uses one -of the following two module creation functions: - -.. c:function:: PyObject* PyModule_Create(PyModuleDef *def) - - Create a new module object, given the definition in *def*. This behaves - like :c:func:`PyModule_Create2` with *module_api_version* set to - :c:macro:`PYTHON_API_VERSION`. - - -.. c:function:: PyObject* PyModule_Create2(PyModuleDef *def, int module_api_version) - - Create a new module object, given the definition in *def*, assuming the - API version *module_api_version*. If that version does not match the version - of the running interpreter, a :exc:`RuntimeWarning` is emitted. - - .. note:: - - Most uses of this function should be using :c:func:`PyModule_Create` - instead; only use this if you are sure you need it. - -Before it is returned from in the initialization function, the resulting module -object is typically populated using functions like :c:func:`PyModule_AddObjectRef`. - -.. _multi-phase-initialization: - -Multi-phase initialization -.......................... - -An alternate way to specify extensions is to request "multi-phase initialization". -Extension modules created this way behave more like Python modules: the -initialization is split between the *creation phase*, when the module object -is created, and the *execution phase*, when it is populated. -The distinction is similar to the :py:meth:`!__new__` and :py:meth:`!__init__` methods -of classes. - -Unlike modules created using single-phase initialization, these modules are not -singletons: if the *sys.modules* entry is removed and the module is re-imported, -a new module object is created, and the old module is subject to normal garbage -collection -- as with Python modules. -By default, multiple modules created from the same definition should be -independent: changes to one should not affect the others. -This means that all state should be specific to the module object (using e.g. -using :c:func:`PyModule_GetState`), or its contents (such as the module's -:attr:`~object.__dict__` or individual classes created with :c:func:`PyType_FromSpec`). - -All modules created using multi-phase initialization are expected to support -:ref:`sub-interpreters `. Making sure multiple modules -are independent is typically enough to achieve this. - -To request multi-phase initialization, the initialization function -(PyInit_modulename) returns a :c:type:`PyModuleDef` instance with non-empty -:c:member:`~PyModuleDef.m_slots`. Before it is returned, the ``PyModuleDef`` -instance must be initialized with the following function: - -.. c:function:: PyObject* PyModuleDef_Init(PyModuleDef *def) - - Ensures a module definition is a properly initialized Python object that - correctly reports its type and reference count. - - Returns *def* cast to ``PyObject*``, or ``NULL`` if an error occurred. - - .. versionadded:: 3.5 - -The *m_slots* member of the module definition must point to an array of -``PyModuleDef_Slot`` structures: - -.. c:type:: PyModuleDef_Slot - - .. c:member:: int slot - - A slot ID, chosen from the available values explained below. - - .. c:member:: void* value - - Value of the slot, whose meaning depends on the slot ID. - - .. versionadded:: 3.5 - -The *m_slots* array must be terminated by a slot with id 0. - -The available slot types are: - -.. c:macro:: Py_mod_create - - Specifies a function that is called to create the module object itself. - The *value* pointer of this slot must point to a function of the signature: - - .. c:function:: PyObject* create_module(PyObject *spec, PyModuleDef *def) - :noindex: - - The function receives a :py:class:`~importlib.machinery.ModuleSpec` - instance, as defined in :PEP:`451`, and the module definition. - It should return a new module object, or set an error - and return ``NULL``. - - This function should be kept minimal. In particular, it should not - call arbitrary Python code, as trying to import the same module again may - result in an infinite loop. - - Multiple ``Py_mod_create`` slots may not be specified in one module - definition. - - If ``Py_mod_create`` is not specified, the import machinery will create - a normal module object using :c:func:`PyModule_New`. The name is taken from - *spec*, not the definition, to allow extension modules to dynamically adjust - to their place in the module hierarchy and be imported under different - names through symlinks, all while sharing a single module definition. - - There is no requirement for the returned object to be an instance of - :c:type:`PyModule_Type`. Any type can be used, as long as it supports - setting and getting import-related attributes. - However, only ``PyModule_Type`` instances may be returned if the - ``PyModuleDef`` has non-``NULL`` ``m_traverse``, ``m_clear``, - ``m_free``; non-zero ``m_size``; or slots other than ``Py_mod_create``. - -.. c:macro:: Py_mod_exec - - Specifies a function that is called to *execute* the module. - This is equivalent to executing the code of a Python module: typically, - this function adds classes and constants to the module. - The signature of the function is: - - .. c:function:: int exec_module(PyObject* module) - :noindex: - - If multiple ``Py_mod_exec`` slots are specified, they are processed in the - order they appear in the *m_slots* array. - -See :PEP:`489` for more details on multi-phase initialization. - -Low-level module creation functions -................................... - -The following functions are called under the hood when using multi-phase -initialization. They can be used directly, for example when creating module -objects dynamically. Note that both ``PyModule_FromDefAndSpec`` and -``PyModule_ExecDef`` must be called to fully initialize a module. - -.. c:function:: PyObject * PyModule_FromDefAndSpec(PyModuleDef *def, PyObject *spec) - - Create a new module object, given the definition in *def* and the - ModuleSpec *spec*. This behaves like :c:func:`PyModule_FromDefAndSpec2` - with *module_api_version* set to :c:macro:`PYTHON_API_VERSION`. - - .. versionadded:: 3.5 - -.. c:function:: PyObject * PyModule_FromDefAndSpec2(PyModuleDef *def, PyObject *spec, int module_api_version) - - Create a new module object, given the definition in *def* and the - ModuleSpec *spec*, assuming the API version *module_api_version*. - If that version does not match the version of the running interpreter, - a :exc:`RuntimeWarning` is emitted. - - .. note:: - - Most uses of this function should be using :c:func:`PyModule_FromDefAndSpec` - instead; only use this if you are sure you need it. - - .. versionadded:: 3.5 - -.. c:function:: int PyModule_ExecDef(PyObject *module, PyModuleDef *def) - - Process any execution slots (:c:data:`Py_mod_exec`) given in *def*. - - .. versionadded:: 3.5 - -.. c:function:: int PyModule_SetDocString(PyObject *module, const char *docstring) - - Set the docstring for *module* to *docstring*. - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. - - .. versionadded:: 3.5 - -.. c:function:: int PyModule_AddFunctions(PyObject *module, PyMethodDef *functions) - - Add the functions from the ``NULL`` terminated *functions* array to *module*. - Refer to the :c:type:`PyMethodDef` documentation for details on individual - entries (due to the lack of a shared module namespace, module level - "functions" implemented in C typically receive the module as their first - parameter, making them similar to instance methods on Python classes). - This function is called automatically when creating a module from - ``PyModuleDef``, using either ``PyModule_Create`` or - ``PyModule_FromDefAndSpec``. - - .. versionadded:: 3.5 - -Support functions -................. - -The module initialization function (if using single phase initialization) or -a function called from a module execution slot (if using multi-phase -initialization), can use the following functions to help initialize the module -state: - -.. c:function:: int PyModule_AddObjectRef(PyObject *module, const char *name, PyObject *value) - - Add an object to *module* as *name*. This is a convenience function which - can be used from the module's initialization function. - - On success, return ``0``. On error, raise an exception and return ``-1``. - - Return ``NULL`` if *value* is ``NULL``. It must be called with an exception - raised in this case. - - Example usage:: - - static int - add_spam(PyObject *module, int value) - { - PyObject *obj = PyLong_FromLong(value); - if (obj == NULL) { - return -1; - } - int res = PyModule_AddObjectRef(module, "spam", obj); - Py_DECREF(obj); - return res; - } - - The example can also be written without checking explicitly if *obj* is - ``NULL``:: - - static int - add_spam(PyObject *module, int value) - { - PyObject *obj = PyLong_FromLong(value); - int res = PyModule_AddObjectRef(module, "spam", obj); - Py_XDECREF(obj); - return res; - } - - Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in - this case, since *obj* can be ``NULL``. - - .. versionadded:: 3.10 - - -.. c:function:: int PyModule_AddObject(PyObject *module, const char *name, PyObject *value) - - Similar to :c:func:`PyModule_AddObjectRef`, but steals a reference to - *value* on success (if it returns ``0``). - - The new :c:func:`PyModule_AddObjectRef` function is recommended, since it is - easy to introduce reference leaks by misusing the - :c:func:`PyModule_AddObject` function. - - .. note:: - - Unlike other functions that steal references, ``PyModule_AddObject()`` - only releases the reference to *value* **on success**. - - This means that its return value must be checked, and calling code must - :c:func:`Py_DECREF` *value* manually on error. - - Example usage:: - - static int - add_spam(PyObject *module, int value) - { - PyObject *obj = PyLong_FromLong(value); - if (obj == NULL) { - return -1; - } - if (PyModule_AddObject(module, "spam", obj) < 0) { - Py_DECREF(obj); - return -1; - } - // PyModule_AddObject() stole a reference to obj: - // Py_DECREF(obj) is not needed here - return 0; - } - - The example can also be written without checking explicitly if *obj* is - ``NULL``:: - - static int - add_spam(PyObject *module, int value) - { - PyObject *obj = PyLong_FromLong(value); - if (PyModule_AddObject(module, "spam", obj) < 0) { - Py_XDECREF(obj); - return -1; - } - // PyModule_AddObject() stole a reference to obj: - // Py_DECREF(obj) is not needed here - return 0; - } - - Note that ``Py_XDECREF()`` should be used instead of ``Py_DECREF()`` in - this case, since *obj* can be ``NULL``. - - -.. c:function:: int PyModule_AddIntConstant(PyObject *module, const char *name, long value) - - Add an integer constant to *module* as *name*. This convenience function can be - used from the module's initialization function. Return ``-1`` on error, ``0`` on - success. - - -.. c:function:: int PyModule_AddStringConstant(PyObject *module, const char *name, const char *value) - - Add a string constant to *module* as *name*. This convenience function can be - used from the module's initialization function. The string *value* must be - ``NULL``-terminated. Return ``-1`` on error, ``0`` on success. - - -.. c:macro:: PyModule_AddIntMacro(module, macro) - - Add an int constant to *module*. The name and the value are taken from - *macro*. For example ``PyModule_AddIntMacro(module, AF_INET)`` adds the int - constant *AF_INET* with the value of *AF_INET* to *module*. - Return ``-1`` on error, ``0`` on success. - - -.. c:macro:: PyModule_AddStringMacro(module, macro) - - Add a string constant to *module*. - -.. c:function:: int PyModule_AddType(PyObject *module, PyTypeObject *type) - - Add a type object to *module*. - The type object is finalized by calling internally :c:func:`PyType_Ready`. - The name of the type object is taken from the last component of - :c:member:`~PyTypeObject.tp_name` after dot. - Return ``-1`` on error, ``0`` on success. - - .. versionadded:: 3.9 - - -Module lookup -^^^^^^^^^^^^^ - -Single-phase initialization creates singleton modules that can be looked up -in the context of the current interpreter. This allows the module object to be -retrieved later with only a reference to the module definition. - -These functions will not work on modules created using multi-phase initialization, -since multiple such modules can be created from a single definition. - -.. c:function:: PyObject* PyState_FindModule(PyModuleDef *def) - - Returns the module object that was created from *def* for the current interpreter. - This method requires that the module object has been attached to the interpreter state with - :c:func:`PyState_AddModule` beforehand. In case the corresponding module object is not - found or has not been attached to the interpreter state yet, it returns ``NULL``. - -.. c:function:: int PyState_AddModule(PyObject *module, PyModuleDef *def) - - Attaches the module object passed to the function to the interpreter state. This allows - the module object to be accessible via :c:func:`PyState_FindModule`. - - Only effective on modules created using single-phase initialization. - - Python calls ``PyState_AddModule`` automatically after importing a module, - so it is unnecessary (but harmless) to call it from module initialization - code. An explicit call is needed only if the module's own init code - subsequently calls ``PyState_FindModule``. - The function is mainly intended for implementing alternative import - mechanisms (either by calling it directly, or by referring to its - implementation for details of the required state updates). - - The caller must hold the GIL. - - Return 0 on success or -1 on failure. - - .. versionadded:: 3.3 - -.. c:function:: int PyState_RemoveModule(PyModuleDef *def) - - Removes the module object created from *def* from the interpreter state. - Return 0 on success or -1 on failure. - - The caller must hold the GIL. - - .. versionadded:: 3.3 diff --git a/Doc/c-api/none.rst b/Doc/c-api/none.rst deleted file mode 100644 index e238fd1c2d8dc6..00000000000000 --- a/Doc/c-api/none.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. highlight:: c - -.. _noneobject: - -The ``None`` Object -------------------- - -.. index:: pair: object; None - -Note that the :c:type:`PyTypeObject` for ``None`` is not directly exposed in the -Python/C API. Since ``None`` is a singleton, testing for object identity (using -``==`` in C) is sufficient. There is no :c:func:`!PyNone_Check` function for the -same reason. - - -.. c:var:: PyObject* Py_None - - The Python ``None`` object, denoting lack of value. This object has no methods. - It needs to be treated just like any other object with respect to reference - counts. - - -.. c:macro:: Py_RETURN_NONE - - Properly handle returning :c:data:`Py_None` from within a C function (that is, - increment the reference count of ``None`` and return it.) diff --git a/Doc/c-api/number.rst b/Doc/c-api/number.rst deleted file mode 100644 index 13d3c5af956905..00000000000000 --- a/Doc/c-api/number.rst +++ /dev/null @@ -1,291 +0,0 @@ -.. highlight:: c - -.. _number: - -Number Protocol -=============== - - -.. c:function:: int PyNumber_Check(PyObject *o) - - Returns ``1`` if the object *o* provides numeric protocols, and false otherwise. - This function always succeeds. - - .. versionchanged:: 3.8 - Returns ``1`` if *o* is an index integer. - - -.. c:function:: PyObject* PyNumber_Add(PyObject *o1, PyObject *o2) - - Returns the result of adding *o1* and *o2*, or ``NULL`` on failure. This is the - equivalent of the Python expression ``o1 + o2``. - - -.. c:function:: PyObject* PyNumber_Subtract(PyObject *o1, PyObject *o2) - - Returns the result of subtracting *o2* from *o1*, or ``NULL`` on failure. This is - the equivalent of the Python expression ``o1 - o2``. - - -.. c:function:: PyObject* PyNumber_Multiply(PyObject *o1, PyObject *o2) - - Returns the result of multiplying *o1* and *o2*, or ``NULL`` on failure. This is - the equivalent of the Python expression ``o1 * o2``. - - -.. c:function:: PyObject* PyNumber_MatrixMultiply(PyObject *o1, PyObject *o2) - - Returns the result of matrix multiplication on *o1* and *o2*, or ``NULL`` on - failure. This is the equivalent of the Python expression ``o1 @ o2``. - - .. versionadded:: 3.5 - - -.. c:function:: PyObject* PyNumber_FloorDivide(PyObject *o1, PyObject *o2) - - Return the floor of *o1* divided by *o2*, or ``NULL`` on failure. This is - the equivalent of the Python expression ``o1 // o2``. - - -.. c:function:: PyObject* PyNumber_TrueDivide(PyObject *o1, PyObject *o2) - - Return a reasonable approximation for the mathematical value of *o1* divided by - *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when - passed two integers. This is the equivalent of the Python expression ``o1 / o2``. - - -.. c:function:: PyObject* PyNumber_Remainder(PyObject *o1, PyObject *o2) - - Returns the remainder of dividing *o1* by *o2*, or ``NULL`` on failure. This is - the equivalent of the Python expression ``o1 % o2``. - - -.. c:function:: PyObject* PyNumber_Divmod(PyObject *o1, PyObject *o2) - - .. index:: pair: built-in function; divmod - - See the built-in function :func:`divmod`. Returns ``NULL`` on failure. This is - the equivalent of the Python expression ``divmod(o1, o2)``. - - -.. c:function:: PyObject* PyNumber_Power(PyObject *o1, PyObject *o2, PyObject *o3) - - .. index:: pair: built-in function; pow - - See the built-in function :func:`pow`. Returns ``NULL`` on failure. This is the - equivalent of the Python expression ``pow(o1, o2, o3)``, where *o3* is optional. - If *o3* is to be ignored, pass :c:data:`Py_None` in its place (passing ``NULL`` for - *o3* would cause an illegal memory access). - - -.. c:function:: PyObject* PyNumber_Negative(PyObject *o) - - Returns the negation of *o* on success, or ``NULL`` on failure. This is the - equivalent of the Python expression ``-o``. - - -.. c:function:: PyObject* PyNumber_Positive(PyObject *o) - - Returns *o* on success, or ``NULL`` on failure. This is the equivalent of the - Python expression ``+o``. - - -.. c:function:: PyObject* PyNumber_Absolute(PyObject *o) - - .. index:: pair: built-in function; abs - - Returns the absolute value of *o*, or ``NULL`` on failure. This is the equivalent - of the Python expression ``abs(o)``. - - -.. c:function:: PyObject* PyNumber_Invert(PyObject *o) - - Returns the bitwise negation of *o* on success, or ``NULL`` on failure. This is - the equivalent of the Python expression ``~o``. - - -.. c:function:: PyObject* PyNumber_Lshift(PyObject *o1, PyObject *o2) - - Returns the result of left shifting *o1* by *o2* on success, or ``NULL`` on - failure. This is the equivalent of the Python expression ``o1 << o2``. - - -.. c:function:: PyObject* PyNumber_Rshift(PyObject *o1, PyObject *o2) - - Returns the result of right shifting *o1* by *o2* on success, or ``NULL`` on - failure. This is the equivalent of the Python expression ``o1 >> o2``. - - -.. c:function:: PyObject* PyNumber_And(PyObject *o1, PyObject *o2) - - Returns the "bitwise and" of *o1* and *o2* on success and ``NULL`` on failure. - This is the equivalent of the Python expression ``o1 & o2``. - - -.. c:function:: PyObject* PyNumber_Xor(PyObject *o1, PyObject *o2) - - Returns the "bitwise exclusive or" of *o1* by *o2* on success, or ``NULL`` on - failure. This is the equivalent of the Python expression ``o1 ^ o2``. - - -.. c:function:: PyObject* PyNumber_Or(PyObject *o1, PyObject *o2) - - Returns the "bitwise or" of *o1* and *o2* on success, or ``NULL`` on failure. - This is the equivalent of the Python expression ``o1 | o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceAdd(PyObject *o1, PyObject *o2) - - Returns the result of adding *o1* and *o2*, or ``NULL`` on failure. The operation - is done *in-place* when *o1* supports it. This is the equivalent of the Python - statement ``o1 += o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceSubtract(PyObject *o1, PyObject *o2) - - Returns the result of subtracting *o2* from *o1*, or ``NULL`` on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 -= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceMultiply(PyObject *o1, PyObject *o2) - - Returns the result of multiplying *o1* and *o2*, or ``NULL`` on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 *= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceMatrixMultiply(PyObject *o1, PyObject *o2) - - Returns the result of matrix multiplication on *o1* and *o2*, or ``NULL`` on - failure. The operation is done *in-place* when *o1* supports it. This is - the equivalent of the Python statement ``o1 @= o2``. - - .. versionadded:: 3.5 - - -.. c:function:: PyObject* PyNumber_InPlaceFloorDivide(PyObject *o1, PyObject *o2) - - Returns the mathematical floor of dividing *o1* by *o2*, or ``NULL`` on failure. - The operation is done *in-place* when *o1* supports it. This is the equivalent - of the Python statement ``o1 //= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceTrueDivide(PyObject *o1, PyObject *o2) - - Return a reasonable approximation for the mathematical value of *o1* divided by - *o2*, or ``NULL`` on failure. The return value is "approximate" because binary - floating point numbers are approximate; it is not possible to represent all real - numbers in base two. This function can return a floating point value when - passed two integers. The operation is done *in-place* when *o1* supports it. - This is the equivalent of the Python statement ``o1 /= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceRemainder(PyObject *o1, PyObject *o2) - - Returns the remainder of dividing *o1* by *o2*, or ``NULL`` on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 %= o2``. - - -.. c:function:: PyObject* PyNumber_InPlacePower(PyObject *o1, PyObject *o2, PyObject *o3) - - .. index:: pair: built-in function; pow - - See the built-in function :func:`pow`. Returns ``NULL`` on failure. The operation - is done *in-place* when *o1* supports it. This is the equivalent of the Python - statement ``o1 **= o2`` when o3 is :c:data:`Py_None`, or an in-place variant of - ``pow(o1, o2, o3)`` otherwise. If *o3* is to be ignored, pass :c:data:`Py_None` - in its place (passing ``NULL`` for *o3* would cause an illegal memory access). - - -.. c:function:: PyObject* PyNumber_InPlaceLshift(PyObject *o1, PyObject *o2) - - Returns the result of left shifting *o1* by *o2* on success, or ``NULL`` on - failure. The operation is done *in-place* when *o1* supports it. This is the - equivalent of the Python statement ``o1 <<= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceRshift(PyObject *o1, PyObject *o2) - - Returns the result of right shifting *o1* by *o2* on success, or ``NULL`` on - failure. The operation is done *in-place* when *o1* supports it. This is the - equivalent of the Python statement ``o1 >>= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceAnd(PyObject *o1, PyObject *o2) - - Returns the "bitwise and" of *o1* and *o2* on success and ``NULL`` on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 &= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceXor(PyObject *o1, PyObject *o2) - - Returns the "bitwise exclusive or" of *o1* by *o2* on success, or ``NULL`` on - failure. The operation is done *in-place* when *o1* supports it. This is the - equivalent of the Python statement ``o1 ^= o2``. - - -.. c:function:: PyObject* PyNumber_InPlaceOr(PyObject *o1, PyObject *o2) - - Returns the "bitwise or" of *o1* and *o2* on success, or ``NULL`` on failure. The - operation is done *in-place* when *o1* supports it. This is the equivalent of - the Python statement ``o1 |= o2``. - - -.. c:function:: PyObject* PyNumber_Long(PyObject *o) - - .. index:: pair: built-in function; int - - Returns the *o* converted to an integer object on success, or ``NULL`` on - failure. This is the equivalent of the Python expression ``int(o)``. - - -.. c:function:: PyObject* PyNumber_Float(PyObject *o) - - .. index:: pair: built-in function; float - - Returns the *o* converted to a float object on success, or ``NULL`` on failure. - This is the equivalent of the Python expression ``float(o)``. - - -.. c:function:: PyObject* PyNumber_Index(PyObject *o) - - Returns the *o* converted to a Python int on success or ``NULL`` with a - :exc:`TypeError` exception raised on failure. - - .. versionchanged:: 3.10 - The result always has exact type :class:`int`. Previously, the result - could have been an instance of a subclass of ``int``. - - -.. c:function:: PyObject* PyNumber_ToBase(PyObject *n, int base) - - Returns the integer *n* converted to base *base* as a string. The *base* - argument must be one of 2, 8, 10, or 16. For base 2, 8, or 16, the - returned string is prefixed with a base marker of ``'0b'``, ``'0o'``, or - ``'0x'``, respectively. If *n* is not a Python int, it is converted with - :c:func:`PyNumber_Index` first. - - -.. c:function:: Py_ssize_t PyNumber_AsSsize_t(PyObject *o, PyObject *exc) - - Returns *o* converted to a :c:type:`Py_ssize_t` value if *o* can be interpreted as an - integer. If the call fails, an exception is raised and ``-1`` is returned. - - If *o* can be converted to a Python int but the attempt to - convert to a :c:type:`Py_ssize_t` value would raise an :exc:`OverflowError`, then the - *exc* argument is the type of exception that will be raised (usually - :exc:`IndexError` or :exc:`OverflowError`). If *exc* is ``NULL``, then the - exception is cleared and the value is clipped to ``PY_SSIZE_T_MIN`` for a negative - integer or ``PY_SSIZE_T_MAX`` for a positive integer. - - -.. c:function:: int PyIndex_Check(PyObject *o) - - Returns ``1`` if *o* is an index integer (has the ``nb_index`` slot of the - ``tp_as_number`` structure filled in), and ``0`` otherwise. - This function always succeeds. diff --git a/Doc/c-api/objbuffer.rst b/Doc/c-api/objbuffer.rst deleted file mode 100644 index 6b82a642d7ee42..00000000000000 --- a/Doc/c-api/objbuffer.rst +++ /dev/null @@ -1,55 +0,0 @@ -.. highlight:: c - -Old Buffer Protocol -------------------- - -.. deprecated:: 3.0 - -These functions were part of the "old buffer protocol" API in Python 2. -In Python 3, this protocol doesn't exist anymore but the functions are still -exposed to ease porting 2.x code. They act as a compatibility wrapper -around the :ref:`new buffer protocol `, but they don't give -you control over the lifetime of the resources acquired when a buffer is -exported. - -Therefore, it is recommended that you call :c:func:`PyObject_GetBuffer` -(or the ``y*`` or ``w*`` :ref:`format codes ` with the -:c:func:`PyArg_ParseTuple` family of functions) to get a buffer view over -an object, and :c:func:`PyBuffer_Release` when the buffer view can be released. - - -.. c:function:: int PyObject_AsCharBuffer(PyObject *obj, const char **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a read-only memory location usable as character-based - input. The *obj* argument must support the single-segment character buffer - interface. On success, returns ``0``, sets *buffer* to the memory location - and *buffer_len* to the buffer length. Returns ``-1`` and sets a - :exc:`TypeError` on error. - - -.. c:function:: int PyObject_AsReadBuffer(PyObject *obj, const void **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a read-only memory location containing arbitrary data. - The *obj* argument must support the single-segment readable buffer - interface. On success, returns ``0``, sets *buffer* to the memory location - and *buffer_len* to the buffer length. Returns ``-1`` and sets a - :exc:`TypeError` on error. - - -.. c:function:: int PyObject_CheckReadBuffer(PyObject *o) - - Returns ``1`` if *o* supports the single-segment readable buffer interface. - Otherwise returns ``0``. This function always succeeds. - - Note that this function tries to get and release a buffer, and exceptions - which occur while calling corresponding functions will get suppressed. - To get error reporting use :c:func:`PyObject_GetBuffer()` instead. - - -.. c:function:: int PyObject_AsWriteBuffer(PyObject *obj, void **buffer, Py_ssize_t *buffer_len) - - Returns a pointer to a writable memory location. The *obj* argument must - support the single-segment, character buffer interface. On success, - returns ``0``, sets *buffer* to the memory location and *buffer_len* to the - buffer length. Returns ``-1`` and sets a :exc:`TypeError` on error. - diff --git a/Doc/c-api/object.rst b/Doc/c-api/object.rst deleted file mode 100644 index 54e9733b40c7b6..00000000000000 --- a/Doc/c-api/object.rst +++ /dev/null @@ -1,402 +0,0 @@ -.. highlight:: c - -.. _object: - -Object Protocol -=============== - - -.. c:var:: PyObject* Py_NotImplemented - - The ``NotImplemented`` singleton, used to signal that an operation is - not implemented for the given type combination. - - -.. c:macro:: Py_RETURN_NOTIMPLEMENTED - - Properly handle returning :c:data:`Py_NotImplemented` from within a C - function (that is, create a new :term:`strong reference` - to NotImplemented and return it). - - -.. c:function:: int PyObject_Print(PyObject *o, FILE *fp, int flags) - - Print an object *o*, on file *fp*. Returns ``-1`` on error. The flags argument - is used to enable certain printing options. The only option currently supported - is :c:macro:`Py_PRINT_RAW`; if given, the :func:`str` of the object is written - instead of the :func:`repr`. - - -.. c:function:: int PyObject_HasAttr(PyObject *o, PyObject *attr_name) - - Returns ``1`` if *o* has the attribute *attr_name*, and ``0`` otherwise. This - is equivalent to the Python expression ``hasattr(o, attr_name)``. This function - always succeeds. - - .. note:: - - Exceptions that occur when this calls :meth:`~object.__getattr__` and - :meth:`~object.__getattribute__` methods are silently ignored. - For proper error handling, use :c:func:`PyObject_GetAttr` instead. - - -.. c:function:: int PyObject_HasAttrString(PyObject *o, const char *attr_name) - - This is the same as :c:func:`PyObject_HasAttr`, but *attr_name* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - .. note:: - - Exceptions that occur when this calls :meth:`~object.__getattr__` and - :meth:`~object.__getattribute__` methods or while creating the temporary - :class:`str` object are silently ignored. - For proper error handling, use :c:func:`PyObject_GetAttrString` instead. - - -.. c:function:: PyObject* PyObject_GetAttr(PyObject *o, PyObject *attr_name) - - Retrieve an attribute named *attr_name* from object *o*. Returns the attribute - value on success, or ``NULL`` on failure. This is the equivalent of the Python - expression ``o.attr_name``. - - -.. c:function:: PyObject* PyObject_GetAttrString(PyObject *o, const char *attr_name) - - This is the same as :c:func:`PyObject_GetAttr`, but *attr_name* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - -.. c:function:: PyObject* PyObject_GenericGetAttr(PyObject *o, PyObject *name) - - Generic attribute getter function that is meant to be put into a type - object's ``tp_getattro`` slot. It looks for a descriptor in the dictionary - of classes in the object's MRO as well as an attribute in the object's - :attr:`~object.__dict__` (if present). As outlined in :ref:`descriptors`, - data descriptors take preference over instance attributes, while non-data - descriptors don't. Otherwise, an :exc:`AttributeError` is raised. - - -.. c:function:: int PyObject_SetAttr(PyObject *o, PyObject *attr_name, PyObject *v) - - Set the value of the attribute named *attr_name*, for object *o*, to the value - *v*. Raise an exception and return ``-1`` on failure; - return ``0`` on success. This is the equivalent of the Python statement - ``o.attr_name = v``. - - If *v* is ``NULL``, the attribute is deleted. This behaviour is deprecated - in favour of using :c:func:`PyObject_DelAttr`, but there are currently no - plans to remove it. - - -.. c:function:: int PyObject_SetAttrString(PyObject *o, const char *attr_name, PyObject *v) - - This is the same as :c:func:`PyObject_SetAttr`, but *attr_name* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - If *v* is ``NULL``, the attribute is deleted, but this feature is - deprecated in favour of using :c:func:`PyObject_DelAttrString`. - - -.. c:function:: int PyObject_GenericSetAttr(PyObject *o, PyObject *name, PyObject *value) - - Generic attribute setter and deleter function that is meant - to be put into a type object's :c:member:`~PyTypeObject.tp_setattro` - slot. It looks for a data descriptor in the - dictionary of classes in the object's MRO, and if found it takes preference - over setting or deleting the attribute in the instance dictionary. Otherwise, the - attribute is set or deleted in the object's :attr:`~object.__dict__` (if present). - On success, ``0`` is returned, otherwise an :exc:`AttributeError` - is raised and ``-1`` is returned. - - -.. c:function:: int PyObject_DelAttr(PyObject *o, PyObject *attr_name) - - Delete attribute named *attr_name*, for object *o*. Returns ``-1`` on failure. - This is the equivalent of the Python statement ``del o.attr_name``. - - -.. c:function:: int PyObject_DelAttrString(PyObject *o, const char *attr_name) - - This is the same as :c:func:`PyObject_DelAttr`, but *attr_name* is - specified as a :c:expr:`const char*` UTF-8 encoded bytes string, - rather than a :c:expr:`PyObject*`. - - -.. c:function:: PyObject* PyObject_GenericGetDict(PyObject *o, void *context) - - A generic implementation for the getter of a ``__dict__`` descriptor. It - creates the dictionary if necessary. - - This function may also be called to get the :py:attr:`~object.__dict__` - of the object *o*. Pass ``NULL`` for *context* when calling it. - Since this function may need to allocate memory for the - dictionary, it may be more efficient to call :c:func:`PyObject_GetAttr` - when accessing an attribute on the object. - - On failure, returns ``NULL`` with an exception set. - - .. versionadded:: 3.3 - - -.. c:function:: int PyObject_GenericSetDict(PyObject *o, PyObject *value, void *context) - - A generic implementation for the setter of a ``__dict__`` descriptor. This - implementation does not allow the dictionary to be deleted. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject** _PyObject_GetDictPtr(PyObject *obj) - - Return a pointer to :py:attr:`~object.__dict__` of the object *obj*. - If there is no ``__dict__``, return ``NULL`` without setting an exception. - - This function may need to allocate memory for the - dictionary, so it may be more efficient to call :c:func:`PyObject_GetAttr` - when accessing an attribute on the object. - - -.. c:function:: PyObject* PyObject_RichCompare(PyObject *o1, PyObject *o2, int opid) - - Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`, - :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``, - ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. This is the equivalent of - the Python expression ``o1 op o2``, where ``op`` is the operator corresponding - to *opid*. Returns the value of the comparison on success, or ``NULL`` on failure. - - -.. c:function:: int PyObject_RichCompareBool(PyObject *o1, PyObject *o2, int opid) - - Compare the values of *o1* and *o2* using the operation specified by *opid*, - which must be one of :c:macro:`Py_LT`, :c:macro:`Py_LE`, :c:macro:`Py_EQ`, - :c:macro:`Py_NE`, :c:macro:`Py_GT`, or :c:macro:`Py_GE`, corresponding to ``<``, - ``<=``, ``==``, ``!=``, ``>``, or ``>=`` respectively. Returns ``-1`` on error, - ``0`` if the result is false, ``1`` otherwise. This is the equivalent of the - Python expression ``o1 op o2``, where ``op`` is the operator corresponding to - *opid*. - -.. note:: - If *o1* and *o2* are the same object, :c:func:`PyObject_RichCompareBool` - will always return ``1`` for :c:macro:`Py_EQ` and ``0`` for :c:macro:`Py_NE`. - -.. c:function:: PyObject* PyObject_Format(PyObject *obj, PyObject *format_spec) - - Format *obj* using *format_spec*. This is equivalent to the Python - expression ``format(obj, format_spec)``. - - *format_spec* may be ``NULL``. In this case the call is equivalent - to ``format(obj)``. - Returns the formatted string on success, ``NULL`` on failure. - -.. c:function:: PyObject* PyObject_Repr(PyObject *o) - - .. index:: pair: built-in function; repr - - Compute a string representation of object *o*. Returns the string - representation on success, ``NULL`` on failure. This is the equivalent of the - Python expression ``repr(o)``. Called by the :func:`repr` built-in function. - - .. versionchanged:: 3.4 - This function now includes a debug assertion to help ensure that it - does not silently discard an active exception. - -.. c:function:: PyObject* PyObject_ASCII(PyObject *o) - - .. index:: pair: built-in function; ascii - - As :c:func:`PyObject_Repr`, compute a string representation of object *o*, but - escape the non-ASCII characters in the string returned by - :c:func:`PyObject_Repr` with ``\x``, ``\u`` or ``\U`` escapes. This generates - a string similar to that returned by :c:func:`PyObject_Repr` in Python 2. - Called by the :func:`ascii` built-in function. - - .. index:: string; PyObject_Str (C function) - - -.. c:function:: PyObject* PyObject_Str(PyObject *o) - - Compute a string representation of object *o*. Returns the string - representation on success, ``NULL`` on failure. This is the equivalent of the - Python expression ``str(o)``. Called by the :func:`str` built-in function - and, therefore, by the :func:`print` function. - - .. versionchanged:: 3.4 - This function now includes a debug assertion to help ensure that it - does not silently discard an active exception. - - -.. c:function:: PyObject* PyObject_Bytes(PyObject *o) - - .. index:: pair: built-in function; bytes - - Compute a bytes representation of object *o*. ``NULL`` is returned on - failure and a bytes object on success. This is equivalent to the Python - expression ``bytes(o)``, when *o* is not an integer. Unlike ``bytes(o)``, - a TypeError is raised when *o* is an integer instead of a zero-initialized - bytes object. - - -.. c:function:: int PyObject_IsSubclass(PyObject *derived, PyObject *cls) - - Return ``1`` if the class *derived* is identical to or derived from the class - *cls*, otherwise return ``0``. In case of an error, return ``-1``. - - If *cls* is a tuple, the check will be done against every entry in *cls*. - The result will be ``1`` when at least one of the checks returns ``1``, - otherwise it will be ``0``. - - If *cls* has a :meth:`~class.__subclasscheck__` method, it will be called to - determine the subclass status as described in :pep:`3119`. Otherwise, - *derived* is a subclass of *cls* if it is a direct or indirect subclass, - i.e. contained in ``cls.__mro__``. - - Normally only class objects, i.e. instances of :class:`type` or a derived - class, are considered classes. However, objects can override this by having - a :attr:`~class.__bases__` attribute (which must be a tuple of base classes). - - -.. c:function:: int PyObject_IsInstance(PyObject *inst, PyObject *cls) - - Return ``1`` if *inst* is an instance of the class *cls* or a subclass of - *cls*, or ``0`` if not. On error, returns ``-1`` and sets an exception. - - If *cls* is a tuple, the check will be done against every entry in *cls*. - The result will be ``1`` when at least one of the checks returns ``1``, - otherwise it will be ``0``. - - If *cls* has a :meth:`~class.__instancecheck__` method, it will be called to - determine the subclass status as described in :pep:`3119`. Otherwise, *inst* - is an instance of *cls* if its class is a subclass of *cls*. - - An instance *inst* can override what is considered its class by having a - :attr:`~instance.__class__` attribute. - - An object *cls* can override if it is considered a class, and what its base - classes are, by having a :attr:`~class.__bases__` attribute (which must be a tuple - of base classes). - - -.. c:function:: Py_hash_t PyObject_Hash(PyObject *o) - - .. index:: pair: built-in function; hash - - Compute and return the hash value of an object *o*. On failure, return ``-1``. - This is the equivalent of the Python expression ``hash(o)``. - - .. versionchanged:: 3.2 - The return type is now Py_hash_t. This is a signed integer the same size - as :c:type:`Py_ssize_t`. - - -.. c:function:: Py_hash_t PyObject_HashNotImplemented(PyObject *o) - - Set a :exc:`TypeError` indicating that ``type(o)`` is not :term:`hashable` and return ``-1``. - This function receives special treatment when stored in a ``tp_hash`` slot, - allowing a type to explicitly indicate to the interpreter that it is not - hashable. - - -.. c:function:: int PyObject_IsTrue(PyObject *o) - - Returns ``1`` if the object *o* is considered to be true, and ``0`` otherwise. - This is equivalent to the Python expression ``not not o``. On failure, return - ``-1``. - - -.. c:function:: int PyObject_Not(PyObject *o) - - Returns ``0`` if the object *o* is considered to be true, and ``1`` otherwise. - This is equivalent to the Python expression ``not o``. On failure, return - ``-1``. - - -.. c:function:: PyObject* PyObject_Type(PyObject *o) - - .. index:: pair: built-in function; type - - When *o* is non-``NULL``, returns a type object corresponding to the object type - of object *o*. On failure, raises :exc:`SystemError` and returns ``NULL``. This - is equivalent to the Python expression ``type(o)``. - This function creates a new :term:`strong reference` to the return value. - There's really no reason to use this - function instead of the :c:func:`Py_TYPE()` function, which returns a - pointer of type :c:expr:`PyTypeObject*`, except when a new - :term:`strong reference` is needed. - - -.. c:function:: int PyObject_TypeCheck(PyObject *o, PyTypeObject *type) - - Return non-zero if the object *o* is of type *type* or a subtype of *type*, and - ``0`` otherwise. Both parameters must be non-``NULL``. - - -.. c:function:: Py_ssize_t PyObject_Size(PyObject *o) - Py_ssize_t PyObject_Length(PyObject *o) - - .. index:: pair: built-in function; len - - Return the length of object *o*. If the object *o* provides either the sequence - and mapping protocols, the sequence length is returned. On error, ``-1`` is - returned. This is the equivalent to the Python expression ``len(o)``. - - -.. c:function:: Py_ssize_t PyObject_LengthHint(PyObject *o, Py_ssize_t defaultvalue) - - Return an estimated length for the object *o*. First try to return its - actual length, then an estimate using :meth:`~object.__length_hint__`, and - finally return the default value. On error return ``-1``. This is the - equivalent to the Python expression ``operator.length_hint(o, defaultvalue)``. - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* PyObject_GetItem(PyObject *o, PyObject *key) - - Return element of *o* corresponding to the object *key* or ``NULL`` on failure. - This is the equivalent of the Python expression ``o[key]``. - - -.. c:function:: int PyObject_SetItem(PyObject *o, PyObject *key, PyObject *v) - - Map the object *key* to the value *v*. Raise an exception and - return ``-1`` on failure; return ``0`` on success. This is the - equivalent of the Python statement ``o[key] = v``. This function *does - not* steal a reference to *v*. - - -.. c:function:: int PyObject_DelItem(PyObject *o, PyObject *key) - - Remove the mapping for the object *key* from the object *o*. Return ``-1`` - on failure. This is equivalent to the Python statement ``del o[key]``. - - -.. c:function:: PyObject* PyObject_Dir(PyObject *o) - - This is equivalent to the Python expression ``dir(o)``, returning a (possibly - empty) list of strings appropriate for the object argument, or ``NULL`` if there - was an error. If the argument is ``NULL``, this is like the Python ``dir()``, - returning the names of the current locals; in this case, if no execution frame - is active then ``NULL`` is returned but :c:func:`PyErr_Occurred` will return false. - - -.. c:function:: PyObject* PyObject_GetIter(PyObject *o) - - This is equivalent to the Python expression ``iter(o)``. It returns a new - iterator for the object argument, or the object itself if the object is already - an iterator. Raises :exc:`TypeError` and returns ``NULL`` if the object cannot be - iterated. - - -.. c:function:: PyObject* PyObject_GetAIter(PyObject *o) - - This is the equivalent to the Python expression ``aiter(o)``. Takes an - :class:`AsyncIterable` object and returns an :class:`AsyncIterator` for it. - This is typically a new iterator but if the argument is an - :class:`AsyncIterator`, this returns itself. Raises :exc:`TypeError` and - returns ``NULL`` if the object cannot be iterated. - - .. versionadded:: 3.10 diff --git a/Doc/c-api/objimpl.rst b/Doc/c-api/objimpl.rst deleted file mode 100644 index 8bd8c107c98bdf..00000000000000 --- a/Doc/c-api/objimpl.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. highlight:: c - -.. _newtypes: - -***************************** -Object Implementation Support -***************************** - -This chapter describes the functions, types, and macros used when defining new -object types. - -.. toctree:: - - allocation.rst - structures.rst - typeobj.rst - gcsupport.rst diff --git a/Doc/c-api/refcounting.rst b/Doc/c-api/refcounting.rst deleted file mode 100644 index e1782ab27f4f66..00000000000000 --- a/Doc/c-api/refcounting.rst +++ /dev/null @@ -1,140 +0,0 @@ -.. highlight:: c - - -.. _countingrefs: - -****************** -Reference Counting -****************** - -The macros in this section are used for managing reference counts of Python -objects. - - -.. c:function:: void Py_INCREF(PyObject *o) - - Indicate taking a new :term:`strong reference` to object *o*, - indicating it is in use and should not be destroyed. - - This function is usually used to convert a :term:`borrowed reference` to a - :term:`strong reference` in-place. The :c:func:`Py_NewRef` function can be - used to create a new :term:`strong reference`. - - When done using the object, release it by calling :c:func:`Py_DECREF`. - - The object must not be ``NULL``; if you aren't sure that it isn't - ``NULL``, use :c:func:`Py_XINCREF`. - - Do not expect this function to actually modify *o* in any way. - - -.. c:function:: void Py_XINCREF(PyObject *o) - - Similar to :c:func:`Py_INCREF`, but the object *o* can be ``NULL``, - in which case this has no effect. - - See also :c:func:`Py_XNewRef`. - - -.. c:function:: PyObject* Py_NewRef(PyObject *o) - - Create a new :term:`strong reference` to an object: - call :c:func:`Py_INCREF` on *o* and return the object *o*. - - When the :term:`strong reference` is no longer needed, :c:func:`Py_DECREF` - should be called on it to release the reference. - - The object *o* must not be ``NULL``; use :c:func:`Py_XNewRef` if *o* can be - ``NULL``. - - For example:: - - Py_INCREF(obj); - self->attr = obj; - - can be written as:: - - self->attr = Py_NewRef(obj); - - See also :c:func:`Py_INCREF`. - - .. versionadded:: 3.10 - - -.. c:function:: PyObject* Py_XNewRef(PyObject *o) - - Similar to :c:func:`Py_NewRef`, but the object *o* can be NULL. - - If the object *o* is ``NULL``, the function just returns ``NULL``. - - .. versionadded:: 3.10 - - -.. c:function:: void Py_DECREF(PyObject *o) - - Release a :term:`strong reference` to object *o*, indicating the - reference is no longer used. - - Once the last :term:`strong reference` is released - (i.e. the object's reference count reaches 0), - the object's type's deallocation - function (which must not be ``NULL``) is invoked. - - This function is usually used to delete a :term:`strong reference` before - exiting its scope. - - The object must not be ``NULL``; if you aren't sure that it isn't ``NULL``, - use :c:func:`Py_XDECREF`. - - Do not expect this function to actually modify *o* in any way. - - .. warning:: - - The deallocation function can cause arbitrary Python code to be invoked (e.g. - when a class instance with a :meth:`~object.__del__` method is deallocated). While - exceptions in such code are not propagated, the executed code has free access to - all Python global variables. This means that any object that is reachable from - a global variable should be in a consistent state before :c:func:`Py_DECREF` is - invoked. For example, code to delete an object from a list should copy a - reference to the deleted object in a temporary variable, update the list data - structure, and then call :c:func:`Py_DECREF` for the temporary variable. - - -.. c:function:: void Py_XDECREF(PyObject *o) - - Similar to :c:func:`Py_DECREF`, but the object *o* can be ``NULL``, - in which case this has no effect. - The same warning from :c:func:`Py_DECREF` applies here as well. - - -.. c:function:: void Py_CLEAR(PyObject *o) - - Release a :term:`strong reference` for object *o*. - The object may be ``NULL``, in - which case the macro has no effect; otherwise the effect is the same as for - :c:func:`Py_DECREF`, except that the argument is also set to ``NULL``. The warning - for :c:func:`Py_DECREF` does not apply with respect to the object passed because - the macro carefully uses a temporary variable and sets the argument to ``NULL`` - before releasing the reference. - - It is a good idea to use this macro whenever releasing a reference - to an object that might be traversed during garbage collection. - -.. c:function:: void Py_IncRef(PyObject *o) - - Indicate taking a new :term:`strong reference` to object *o*. - A function version of :c:func:`Py_XINCREF`. - It can be used for runtime dynamic embedding of Python. - - -.. c:function:: void Py_DecRef(PyObject *o) - - Release a :term:`strong reference` to object *o*. - A function version of :c:func:`Py_XDECREF`. - It can be used for runtime dynamic embedding of Python. - - -The following functions or macros are only for use within the interpreter core: -:c:func:`_Py_Dealloc`, :c:func:`_Py_ForgetReference`, :c:func:`_Py_NewReference`, -as well as the global variable :c:data:`_Py_RefTotal`. - diff --git a/Doc/c-api/reflection.rst b/Doc/c-api/reflection.rst deleted file mode 100644 index 4b1c4770848a30..00000000000000 --- a/Doc/c-api/reflection.rst +++ /dev/null @@ -1,46 +0,0 @@ -.. highlight:: c - -.. _reflection: - -Reflection -========== - -.. c:function:: PyObject* PyEval_GetBuiltins(void) - - Return a dictionary of the builtins in the current execution frame, - or the interpreter of the thread state if no frame is currently executing. - - -.. c:function:: PyObject* PyEval_GetLocals(void) - - Return a dictionary of the local variables in the current execution frame, - or ``NULL`` if no frame is currently executing. - - -.. c:function:: PyObject* PyEval_GetGlobals(void) - - Return a dictionary of the global variables in the current execution frame, - or ``NULL`` if no frame is currently executing. - - -.. c:function:: PyFrameObject* PyEval_GetFrame(void) - - Return the current thread state's frame, which is ``NULL`` if no frame is - currently executing. - - See also :c:func:`PyThreadState_GetFrame`. - - -.. c:function:: const char* PyEval_GetFuncName(PyObject *func) - - Return the name of *func* if it is a function, class or instance object, else the - name of *func*\s type. - - -.. c:function:: const char* PyEval_GetFuncDesc(PyObject *func) - - Return a description string, depending on the type of *func*. - Return values include "()" for functions and methods, " constructor", - " instance", and " object". Concatenated with the result of - :c:func:`PyEval_GetFuncName`, the result will be a description of - *func*. diff --git a/Doc/c-api/sequence.rst b/Doc/c-api/sequence.rst deleted file mode 100644 index ce28839f5ba739..00000000000000 --- a/Doc/c-api/sequence.rst +++ /dev/null @@ -1,176 +0,0 @@ -.. highlight:: c - -.. _sequence: - -Sequence Protocol -================= - - -.. c:function:: int PySequence_Check(PyObject *o) - - Return ``1`` if the object provides the sequence protocol, and ``0`` otherwise. - Note that it returns ``1`` for Python classes with a :meth:`~object.__getitem__` - method, unless they are :class:`dict` subclasses, since in general it - is impossible to determine what type of keys the class supports. This - function always succeeds. - - -.. c:function:: Py_ssize_t PySequence_Size(PyObject *o) - Py_ssize_t PySequence_Length(PyObject *o) - - .. index:: pair: built-in function; len - - Returns the number of objects in sequence *o* on success, and ``-1`` on - failure. This is equivalent to the Python expression ``len(o)``. - - -.. c:function:: PyObject* PySequence_Concat(PyObject *o1, PyObject *o2) - - Return the concatenation of *o1* and *o2* on success, and ``NULL`` on failure. - This is the equivalent of the Python expression ``o1 + o2``. - - -.. c:function:: PyObject* PySequence_Repeat(PyObject *o, Py_ssize_t count) - - Return the result of repeating sequence object *o* *count* times, or ``NULL`` on - failure. This is the equivalent of the Python expression ``o * count``. - - -.. c:function:: PyObject* PySequence_InPlaceConcat(PyObject *o1, PyObject *o2) - - Return the concatenation of *o1* and *o2* on success, and ``NULL`` on failure. - The operation is done *in-place* when *o1* supports it. This is the equivalent - of the Python expression ``o1 += o2``. - - -.. c:function:: PyObject* PySequence_InPlaceRepeat(PyObject *o, Py_ssize_t count) - - Return the result of repeating sequence object *o* *count* times, or ``NULL`` on - failure. The operation is done *in-place* when *o* supports it. This is the - equivalent of the Python expression ``o *= count``. - - -.. c:function:: PyObject* PySequence_GetItem(PyObject *o, Py_ssize_t i) - - Return the *i*\ th element of *o*, or ``NULL`` on failure. This is the equivalent of - the Python expression ``o[i]``. - - -.. c:function:: PyObject* PySequence_GetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2) - - Return the slice of sequence object *o* between *i1* and *i2*, or ``NULL`` on - failure. This is the equivalent of the Python expression ``o[i1:i2]``. - - -.. c:function:: int PySequence_SetItem(PyObject *o, Py_ssize_t i, PyObject *v) - - Assign object *v* to the *i*\ th element of *o*. Raise an exception - and return ``-1`` on failure; return ``0`` on success. This - is the equivalent of the Python statement ``o[i] = v``. This function *does - not* steal a reference to *v*. - - If *v* is ``NULL``, the element is deleted, but this feature is - deprecated in favour of using :c:func:`PySequence_DelItem`. - - -.. c:function:: int PySequence_DelItem(PyObject *o, Py_ssize_t i) - - Delete the *i*\ th element of object *o*. Returns ``-1`` on failure. This is the - equivalent of the Python statement ``del o[i]``. - - -.. c:function:: int PySequence_SetSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2, PyObject *v) - - Assign the sequence object *v* to the slice in sequence object *o* from *i1* to - *i2*. This is the equivalent of the Python statement ``o[i1:i2] = v``. - - -.. c:function:: int PySequence_DelSlice(PyObject *o, Py_ssize_t i1, Py_ssize_t i2) - - Delete the slice in sequence object *o* from *i1* to *i2*. Returns ``-1`` on - failure. This is the equivalent of the Python statement ``del o[i1:i2]``. - - -.. c:function:: Py_ssize_t PySequence_Count(PyObject *o, PyObject *value) - - Return the number of occurrences of *value* in *o*, that is, return the number - of keys for which ``o[key] == value``. On failure, return ``-1``. This is - equivalent to the Python expression ``o.count(value)``. - - -.. c:function:: int PySequence_Contains(PyObject *o, PyObject *value) - - Determine if *o* contains *value*. If an item in *o* is equal to *value*, - return ``1``, otherwise return ``0``. On error, return ``-1``. This is - equivalent to the Python expression ``value in o``. - - -.. c:function:: Py_ssize_t PySequence_Index(PyObject *o, PyObject *value) - - Return the first index *i* for which ``o[i] == value``. On error, return - ``-1``. This is equivalent to the Python expression ``o.index(value)``. - - -.. c:function:: PyObject* PySequence_List(PyObject *o) - - Return a list object with the same contents as the sequence or iterable *o*, - or ``NULL`` on failure. The returned list is guaranteed to be new. This is - equivalent to the Python expression ``list(o)``. - - -.. c:function:: PyObject* PySequence_Tuple(PyObject *o) - - .. index:: pair: built-in function; tuple - - Return a tuple object with the same contents as the sequence or iterable *o*, - or ``NULL`` on failure. If *o* is a tuple, a new reference will be returned, - otherwise a tuple will be constructed with the appropriate contents. This is - equivalent to the Python expression ``tuple(o)``. - - -.. c:function:: PyObject* PySequence_Fast(PyObject *o, const char *m) - - Return the sequence or iterable *o* as an object usable by the other - ``PySequence_Fast*`` family of functions. If the object is not a sequence or - iterable, raises :exc:`TypeError` with *m* as the message text. Returns - ``NULL`` on failure. - - The ``PySequence_Fast*`` functions are thus named because they assume - *o* is a :c:type:`PyTupleObject` or a :c:type:`PyListObject` and access - the data fields of *o* directly. - - As a CPython implementation detail, if *o* is already a sequence or list, it - will be returned. - - -.. c:function:: Py_ssize_t PySequence_Fast_GET_SIZE(PyObject *o) - - Returns the length of *o*, assuming that *o* was returned by - :c:func:`PySequence_Fast` and that *o* is not ``NULL``. The size can also be - retrieved by calling :c:func:`PySequence_Size` on *o*, but - :c:func:`PySequence_Fast_GET_SIZE` is faster because it can assume *o* is a - list or tuple. - - -.. c:function:: PyObject* PySequence_Fast_GET_ITEM(PyObject *o, Py_ssize_t i) - - Return the *i*\ th element of *o*, assuming that *o* was returned by - :c:func:`PySequence_Fast`, *o* is not ``NULL``, and that *i* is within bounds. - - -.. c:function:: PyObject** PySequence_Fast_ITEMS(PyObject *o) - - Return the underlying array of PyObject pointers. Assumes that *o* was returned - by :c:func:`PySequence_Fast` and *o* is not ``NULL``. - - Note, if a list gets resized, the reallocation may relocate the items array. - So, only use the underlying array pointer in contexts where the sequence - cannot change. - - -.. c:function:: PyObject* PySequence_ITEM(PyObject *o, Py_ssize_t i) - - Return the *i*\ th element of *o* or ``NULL`` on failure. Faster form of - :c:func:`PySequence_GetItem` but without checking that - :c:func:`PySequence_Check` on *o* is true and without adjustment for negative - indices. diff --git a/Doc/c-api/set.rst b/Doc/c-api/set.rst deleted file mode 100644 index 09c0fb6b9c5f23..00000000000000 --- a/Doc/c-api/set.rst +++ /dev/null @@ -1,168 +0,0 @@ -.. highlight:: c - -.. _setobjects: - -Set Objects ------------ - -.. sectionauthor:: Raymond D. Hettinger - - -.. index:: - pair: object; set - pair: object; frozenset - -This section details the public API for :class:`set` and :class:`frozenset` -objects. Any functionality not listed below is best accessed using either -the abstract object protocol (including :c:func:`PyObject_CallMethod`, -:c:func:`PyObject_RichCompareBool`, :c:func:`PyObject_Hash`, -:c:func:`PyObject_Repr`, :c:func:`PyObject_IsTrue`, :c:func:`PyObject_Print`, and -:c:func:`PyObject_GetIter`) or the abstract number protocol (including -:c:func:`PyNumber_And`, :c:func:`PyNumber_Subtract`, :c:func:`PyNumber_Or`, -:c:func:`PyNumber_Xor`, :c:func:`PyNumber_InPlaceAnd`, -:c:func:`PyNumber_InPlaceSubtract`, :c:func:`PyNumber_InPlaceOr`, and -:c:func:`PyNumber_InPlaceXor`). - - -.. c:type:: PySetObject - - This subtype of :c:type:`PyObject` is used to hold the internal data for both - :class:`set` and :class:`frozenset` objects. It is like a :c:type:`PyDictObject` - in that it is a fixed size for small sets (much like tuple storage) and will - point to a separate, variable sized block of memory for medium and large sized - sets (much like list storage). None of the fields of this structure should be - considered public and all are subject to change. All access should be done through - the documented API rather than by manipulating the values in the structure. - - -.. c:var:: PyTypeObject PySet_Type - - This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`set` type. - - -.. c:var:: PyTypeObject PyFrozenSet_Type - - This is an instance of :c:type:`PyTypeObject` representing the Python - :class:`frozenset` type. - -The following type check macros work on pointers to any Python object. Likewise, -the constructor functions work with any iterable Python object. - - -.. c:function:: int PySet_Check(PyObject *p) - - Return true if *p* is a :class:`set` object or an instance of a subtype. - This function always succeeds. - -.. c:function:: int PyFrozenSet_Check(PyObject *p) - - Return true if *p* is a :class:`frozenset` object or an instance of a - subtype. This function always succeeds. - -.. c:function:: int PyAnySet_Check(PyObject *p) - - Return true if *p* is a :class:`set` object, a :class:`frozenset` object, or an - instance of a subtype. This function always succeeds. - -.. c:function:: int PySet_CheckExact(PyObject *p) - - Return true if *p* is a :class:`set` object but not an instance of a - subtype. This function always succeeds. - - .. versionadded:: 3.10 - -.. c:function:: int PyAnySet_CheckExact(PyObject *p) - - Return true if *p* is a :class:`set` object or a :class:`frozenset` object but - not an instance of a subtype. This function always succeeds. - - -.. c:function:: int PyFrozenSet_CheckExact(PyObject *p) - - Return true if *p* is a :class:`frozenset` object but not an instance of a - subtype. This function always succeeds. - - -.. c:function:: PyObject* PySet_New(PyObject *iterable) - - Return a new :class:`set` containing objects returned by the *iterable*. The - *iterable* may be ``NULL`` to create a new empty set. Return the new set on - success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is not - actually iterable. The constructor is also useful for copying a set - (``c=set(s)``). - - -.. c:function:: PyObject* PyFrozenSet_New(PyObject *iterable) - - Return a new :class:`frozenset` containing objects returned by the *iterable*. - The *iterable* may be ``NULL`` to create a new empty frozenset. Return the new - set on success or ``NULL`` on failure. Raise :exc:`TypeError` if *iterable* is - not actually iterable. - - -The following functions and macros are available for instances of :class:`set` -or :class:`frozenset` or instances of their subtypes. - - -.. c:function:: Py_ssize_t PySet_Size(PyObject *anyset) - - .. index:: pair: built-in function; len - - Return the length of a :class:`set` or :class:`frozenset` object. Equivalent to - ``len(anyset)``. Raises a :exc:`SystemError` if *anyset* is not a - :class:`set`, :class:`frozenset`, or an instance of a subtype. - - -.. c:function:: Py_ssize_t PySet_GET_SIZE(PyObject *anyset) - - Macro form of :c:func:`PySet_Size` without error checking. - - -.. c:function:: int PySet_Contains(PyObject *anyset, PyObject *key) - - Return ``1`` if found, ``0`` if not found, and ``-1`` if an error is encountered. Unlike - the Python :meth:`~object.__contains__` method, this function does not automatically - convert unhashable sets into temporary frozensets. Raise a :exc:`TypeError` if - the *key* is unhashable. Raise :exc:`SystemError` if *anyset* is not a - :class:`set`, :class:`frozenset`, or an instance of a subtype. - - -.. c:function:: int PySet_Add(PyObject *set, PyObject *key) - - Add *key* to a :class:`set` instance. Also works with :class:`frozenset` - instances (like :c:func:`PyTuple_SetItem` it can be used to fill in the values - of brand new frozensets before they are exposed to other code). Return ``0`` on - success or ``-1`` on failure. Raise a :exc:`TypeError` if the *key* is - unhashable. Raise a :exc:`MemoryError` if there is no room to grow. Raise a - :exc:`SystemError` if *set* is not an instance of :class:`set` or its - subtype. - - -The following functions are available for instances of :class:`set` or its -subtypes but not for instances of :class:`frozenset` or its subtypes. - - -.. c:function:: int PySet_Discard(PyObject *set, PyObject *key) - - Return ``1`` if found and removed, ``0`` if not found (no action taken), and ``-1`` if an - error is encountered. Does not raise :exc:`KeyError` for missing keys. Raise a - :exc:`TypeError` if the *key* is unhashable. Unlike the Python :meth:`~set.discard` - method, this function does not automatically convert unhashable sets into - temporary frozensets. Raise :exc:`SystemError` if *set* is not an - instance of :class:`set` or its subtype. - - -.. c:function:: PyObject* PySet_Pop(PyObject *set) - - Return a new reference to an arbitrary object in the *set*, and removes the - object from the *set*. Return ``NULL`` on failure. Raise :exc:`KeyError` if the - set is empty. Raise a :exc:`SystemError` if *set* is not an instance of - :class:`set` or its subtype. - - -.. c:function:: int PySet_Clear(PyObject *set) - - Empty an existing set of all elements. Return ``0`` on - success. Return ``-1`` and raise :exc:`SystemError` if *set* is not an instance of - :class:`set` or its subtype. diff --git a/Doc/c-api/slice.rst b/Doc/c-api/slice.rst deleted file mode 100644 index af4c540068cbf4..00000000000000 --- a/Doc/c-api/slice.rst +++ /dev/null @@ -1,123 +0,0 @@ -.. highlight:: c - -.. _slice-objects: - -Slice Objects -------------- - - -.. c:var:: PyTypeObject PySlice_Type - - The type object for slice objects. This is the same as :class:`slice` in the - Python layer. - - -.. c:function:: int PySlice_Check(PyObject *ob) - - Return true if *ob* is a slice object; *ob* must not be ``NULL``. This - function always succeeds. - - -.. c:function:: PyObject* PySlice_New(PyObject *start, PyObject *stop, PyObject *step) - - Return a new slice object with the given values. The *start*, *stop*, and - *step* parameters are used as the values of the slice object attributes of - the same names. Any of the values may be ``NULL``, in which case the - ``None`` will be used for the corresponding attribute. Return ``NULL`` if - the new object could not be allocated. - - -.. c:function:: int PySlice_GetIndices(PyObject *slice, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step) - - Retrieve the start, stop and step indices from the slice object *slice*, - assuming a sequence of length *length*. Treats indices greater than - *length* as errors. - - Returns ``0`` on success and ``-1`` on error with no exception set (unless one of - the indices was not ``None`` and failed to be converted to an integer, - in which case ``-1`` is returned with an exception set). - - You probably do not want to use this function. - - .. versionchanged:: 3.2 - The parameter type for the *slice* parameter was ``PySliceObject*`` - before. - - -.. c:function:: int PySlice_GetIndicesEx(PyObject *slice, Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step, Py_ssize_t *slicelength) - - Usable replacement for :c:func:`PySlice_GetIndices`. Retrieve the start, - stop, and step indices from the slice object *slice* assuming a sequence of - length *length*, and store the length of the slice in *slicelength*. Out - of bounds indices are clipped in a manner consistent with the handling of - normal slices. - - Returns ``0`` on success and ``-1`` on error with exception set. - - .. note:: - This function is considered not safe for resizable sequences. - Its invocation should be replaced by a combination of - :c:func:`PySlice_Unpack` and :c:func:`PySlice_AdjustIndices` where :: - - if (PySlice_GetIndicesEx(slice, length, &start, &stop, &step, &slicelength) < 0) { - // return error - } - - is replaced by :: - - if (PySlice_Unpack(slice, &start, &stop, &step) < 0) { - // return error - } - slicelength = PySlice_AdjustIndices(length, &start, &stop, step); - - .. versionchanged:: 3.2 - The parameter type for the *slice* parameter was ``PySliceObject*`` - before. - - .. versionchanged:: 3.6.1 - If ``Py_LIMITED_API`` is not set or set to the value between ``0x03050400`` - and ``0x03060000`` (not including) or ``0x03060100`` or higher - :c:func:`!PySlice_GetIndicesEx` is implemented as a macro using - :c:func:`!PySlice_Unpack` and :c:func:`!PySlice_AdjustIndices`. - Arguments *start*, *stop* and *step* are evaluated more than once. - - .. deprecated:: 3.6.1 - If ``Py_LIMITED_API`` is set to the value less than ``0x03050400`` or - between ``0x03060000`` and ``0x03060100`` (not including) - :c:func:`!PySlice_GetIndicesEx` is a deprecated function. - - -.. c:function:: int PySlice_Unpack(PyObject *slice, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t *step) - - Extract the start, stop and step data members from a slice object as - C integers. Silently reduce values larger than ``PY_SSIZE_T_MAX`` to - ``PY_SSIZE_T_MAX``, silently boost the start and stop values less than - ``PY_SSIZE_T_MIN`` to ``PY_SSIZE_T_MIN``, and silently boost the step - values less than ``-PY_SSIZE_T_MAX`` to ``-PY_SSIZE_T_MAX``. - - Return ``-1`` on error, ``0`` on success. - - .. versionadded:: 3.6.1 - - -.. c:function:: Py_ssize_t PySlice_AdjustIndices(Py_ssize_t length, Py_ssize_t *start, Py_ssize_t *stop, Py_ssize_t step) - - Adjust start/end slice indices assuming a sequence of the specified length. - Out of bounds indices are clipped in a manner consistent with the handling - of normal slices. - - Return the length of the slice. Always successful. Doesn't call Python - code. - - .. versionadded:: 3.6.1 - - -Ellipsis Object -^^^^^^^^^^^^^^^ - - -.. c:var:: PyObject *Py_Ellipsis - - The Python ``Ellipsis`` object. This object has no methods. It needs to be - treated just like any other object with respect to reference counts. Like - :c:data:`Py_None` it is a singleton object. diff --git a/Doc/c-api/stable.rst b/Doc/c-api/stable.rst deleted file mode 100644 index 4817da41e3e217..00000000000000 --- a/Doc/c-api/stable.rst +++ /dev/null @@ -1,171 +0,0 @@ -.. highlight:: c - -.. _stable: - -*************** -C API Stability -*************** - -Python's C API is covered by the Backwards Compatibility Policy, :pep:`387`. -While the C API will change with every minor release (e.g. from 3.9 to 3.10), -most changes will be source-compatible, typically by only adding new API. -Changing existing API or removing API is only done after a deprecation period -or to fix serious issues. - -CPython's Application Binary Interface (ABI) is forward- and -backwards-compatible across a minor release (if these are compiled the same -way; see :ref:`stable-abi-platform` below). -So, code compiled for Python 3.10.0 will work on 3.10.8 and vice versa, -but will need to be compiled separately for 3.9.x and 3.10.x. - -Names prefixed by an underscore, such as ``_Py_InternalState``, -are private API that can change without notice even in patch releases. - - -Stable Application Binary Interface -=================================== - -For simplicity, this document talks about *extensions*, but the Limited API -and Stable ABI work the same way for all uses of the API – for example, -embedding Python. - -.. _limited-c-api: - -Limited C API -------------- - -Python 3.2 introduced the *Limited API*, a subset of Python's C API. -Extensions that only use the Limited API can be -compiled once and work with multiple versions of Python. -Contents of the Limited API are :ref:`listed below `. - -.. c:macro:: Py_LIMITED_API - - Define this macro before including ``Python.h`` to opt in to only use - the Limited API, and to select the Limited API version. - - Define ``Py_LIMITED_API`` to the value of :c:macro:`PY_VERSION_HEX` - corresponding to the lowest Python version your extension supports. - The extension will work without recompilation with all Python 3 releases - from the specified one onward, and can use Limited API introduced up to that - version. - - Rather than using the ``PY_VERSION_HEX`` macro directly, hardcode a minimum - minor version (e.g. ``0x030A0000`` for Python 3.10) for stability when - compiling with future Python versions. - - You can also define ``Py_LIMITED_API`` to ``3``. This works the same as - ``0x03020000`` (Python 3.2, the version that introduced Limited API). - - -.. _stable-abi: - -Stable ABI ----------- - -To enable this, Python provides a *Stable ABI*: a set of symbols that will -remain compatible across Python 3.x versions. - -The Stable ABI contains symbols exposed in the :ref:`Limited API -`, but also other ones – for example, functions necessary to -support older versions of the Limited API. - -On Windows, extensions that use the Stable ABI should be linked against -``python3.dll`` rather than a version-specific library such as -``python39.dll``. - -On some platforms, Python will look for and load shared library files named -with the ``abi3`` tag (e.g. ``mymodule.abi3.so``). -It does not check if such extensions conform to a Stable ABI. -The user (or their packaging tools) need to ensure that, for example, -extensions built with the 3.10+ Limited API are not installed for lower -versions of Python. - -All functions in the Stable ABI are present as functions in Python's shared -library, not solely as macros. This makes them usable from languages that don't -use the C preprocessor. - - -Limited API Scope and Performance ---------------------------------- - -The goal for the Limited API is to allow everything that is possible with the -full C API, but possibly with a performance penalty. - -For example, while :c:func:`PyList_GetItem` is available, its “unsafe” macro -variant :c:func:`PyList_GET_ITEM` is not. -The macro can be faster because it can rely on version-specific implementation -details of the list object. - -Without ``Py_LIMITED_API`` defined, some C API functions are inlined or -replaced by macros. -Defining ``Py_LIMITED_API`` disables this inlining, allowing stability as -Python's data structures are improved, but possibly reducing performance. - -By leaving out the ``Py_LIMITED_API`` definition, it is possible to compile -a Limited API extension with a version-specific ABI. This can improve -performance for that Python version, but will limit compatibility. -Compiling with ``Py_LIMITED_API`` will then yield an extension that can be -distributed where a version-specific one is not available – for example, -for prereleases of an upcoming Python version. - - -Limited API Caveats -------------------- - -Note that compiling with ``Py_LIMITED_API`` is *not* a complete guarantee that -code conforms to the :ref:`Limited API ` or the :ref:`Stable ABI -`. ``Py_LIMITED_API`` only covers definitions, but an API also -includes other issues, such as expected semantics. - -One issue that ``Py_LIMITED_API`` does not guard against is calling a function -with arguments that are invalid in a lower Python version. -For example, consider a function that starts accepting ``NULL`` for an -argument. In Python 3.9, ``NULL`` now selects a default behavior, but in -Python 3.8, the argument will be used directly, causing a ``NULL`` dereference -and crash. A similar argument works for fields of structs. - -Another issue is that some struct fields are currently not hidden when -``Py_LIMITED_API`` is defined, even though they're part of the Limited API. - -For these reasons, we recommend testing an extension with *all* minor Python -versions it supports, and preferably to build with the *lowest* such version. - -We also recommend reviewing documentation of all used API to check -if it is explicitly part of the Limited API. Even with ``Py_LIMITED_API`` -defined, a few private declarations are exposed for technical reasons (or -even unintentionally, as bugs). - -Also note that the Limited API is not necessarily stable: compiling with -``Py_LIMITED_API`` with Python 3.8 means that the extension will -run with Python 3.12, but it will not necessarily *compile* with Python 3.12. -In particular, parts of the Limited API may be deprecated and removed, -provided that the Stable ABI stays stable. - - -.. _stable-abi-platform: - -Platform Considerations -======================= - -ABI stability depends not only on Python, but also on the compiler used, -lower-level libraries and compiler options. For the purposes of -the :ref:`Stable ABI `, these details define a “platform”. They -usually depend on the OS type and processor architecture - -It is the responsibility of each particular distributor of Python -to ensure that all Python versions on a particular platform are built -in a way that does not break the Stable ABI. -This is the case with Windows and macOS releases from ``python.org`` and many -third-party distributors. - - -.. _limited-api-list: - -Contents of Limited API -======================= - - -Currently, the :ref:`Limited API ` includes the following items: - -.. limited-api-list:: diff --git a/Doc/c-api/structures.rst b/Doc/c-api/structures.rst deleted file mode 100644 index 304cc31575b1dc..00000000000000 --- a/Doc/c-api/structures.rst +++ /dev/null @@ -1,558 +0,0 @@ -.. highlight:: c - -.. _common-structs: - -Common Object Structures -======================== - -There are a large number of structures which are used in the definition of -object types for Python. This section describes these structures and how they -are used. - - -Base object types and macros ----------------------------- - -All Python objects ultimately share a small number of fields at the beginning -of the object's representation in memory. These are represented by the -:c:type:`PyObject` and :c:type:`PyVarObject` types, which are defined, in turn, -by the expansions of some macros also used, whether directly or indirectly, in -the definition of all other Python objects. - - -.. c:type:: PyObject - - All object types are extensions of this type. This is a type which - contains the information Python needs to treat a pointer to an object as an - object. In a normal "release" build, it contains only the object's - reference count and a pointer to the corresponding type object. - Nothing is actually declared to be a :c:type:`PyObject`, but every pointer - to a Python object can be cast to a :c:expr:`PyObject*`. Access to the - members must be done by using the macros :c:macro:`Py_REFCNT` and - :c:macro:`Py_TYPE`. - - -.. c:type:: PyVarObject - - This is an extension of :c:type:`PyObject` that adds the :c:member:`~PyVarObject.ob_size` - field. This is only used for objects that have some notion of *length*. - This type does not often appear in the Python/C API. - Access to the members must be done by using the macros - :c:macro:`Py_REFCNT`, :c:macro:`Py_TYPE`, and :c:macro:`Py_SIZE`. - - -.. c:macro:: PyObject_HEAD - - This is a macro used when declaring new types which represent objects - without a varying length. The PyObject_HEAD macro expands to:: - - PyObject ob_base; - - See documentation of :c:type:`PyObject` above. - - -.. c:macro:: PyObject_VAR_HEAD - - This is a macro used when declaring new types which represent objects - with a length that varies from instance to instance. - The PyObject_VAR_HEAD macro expands to:: - - PyVarObject ob_base; - - See documentation of :c:type:`PyVarObject` above. - - -.. c:function:: int Py_Is(PyObject *x, PyObject *y) - - Test if the *x* object is the *y* object, the same as ``x is y`` in Python. - - .. versionadded:: 3.10 - - -.. c:function:: int Py_IsNone(PyObject *x) - - Test if an object is the ``None`` singleton, - the same as ``x is None`` in Python. - - .. versionadded:: 3.10 - - -.. c:function:: int Py_IsTrue(PyObject *x) - - Test if an object is the ``True`` singleton, - the same as ``x is True`` in Python. - - .. versionadded:: 3.10 - - -.. c:function:: int Py_IsFalse(PyObject *x) - - Test if an object is the ``False`` singleton, - the same as ``x is False`` in Python. - - .. versionadded:: 3.10 - - -.. c:function:: PyTypeObject* Py_TYPE(PyObject *o) - - Get the type of the Python object *o*. - - Return a :term:`borrowed reference`. - - Use the :c:func:`Py_SET_TYPE` function to set an object type. - - .. versionchanged:: 3.11 - :c:func:`Py_TYPE()` is changed to an inline static function. - The parameter type is no longer :c:expr:`const PyObject*`. - - -.. c:function:: int Py_IS_TYPE(PyObject *o, PyTypeObject *type) - - Return non-zero if the object *o* type is *type*. Return zero otherwise. - Equivalent to: ``Py_TYPE(o) == type``. - - .. versionadded:: 3.9 - - -.. c:function:: void Py_SET_TYPE(PyObject *o, PyTypeObject *type) - - Set the object *o* type to *type*. - - .. versionadded:: 3.9 - - -.. c:function:: Py_ssize_t Py_REFCNT(PyObject *o) - - Get the reference count of the Python object *o*. - - Use the :c:func:`Py_SET_REFCNT()` function to set an object reference count. - - .. versionchanged:: 3.11 - The parameter type is no longer :c:expr:`const PyObject*`. - - .. versionchanged:: 3.10 - :c:func:`Py_REFCNT()` is changed to the inline static function. - - -.. c:function:: void Py_SET_REFCNT(PyObject *o, Py_ssize_t refcnt) - - Set the object *o* reference counter to *refcnt*. - - .. versionadded:: 3.9 - - -.. c:function:: Py_ssize_t Py_SIZE(PyVarObject *o) - - Get the size of the Python object *o*. - - Use the :c:func:`Py_SET_SIZE` function to set an object size. - - .. versionchanged:: 3.11 - :c:func:`Py_SIZE()` is changed to an inline static function. - The parameter type is no longer :c:expr:`const PyVarObject*`. - - -.. c:function:: void Py_SET_SIZE(PyVarObject *o, Py_ssize_t size) - - Set the object *o* size to *size*. - - .. versionadded:: 3.9 - - -.. c:macro:: PyObject_HEAD_INIT(type) - - This is a macro which expands to initialization values for a new - :c:type:`PyObject` type. This macro expands to:: - - _PyObject_EXTRA_INIT - 1, type, - - -.. c:macro:: PyVarObject_HEAD_INIT(type, size) - - This is a macro which expands to initialization values for a new - :c:type:`PyVarObject` type, including the :c:member:`~PyVarObject.ob_size` field. - This macro expands to:: - - _PyObject_EXTRA_INIT - 1, type, size, - - -Implementing functions and methods ----------------------------------- - -.. c:type:: PyCFunction - - Type of the functions used to implement most Python callables in C. - Functions of this type take two :c:expr:`PyObject*` parameters and return - one such value. If the return value is ``NULL``, an exception shall have - been set. If not ``NULL``, the return value is interpreted as the return - value of the function as exposed in Python. The function must return a new - reference. - - The function signature is:: - - PyObject *PyCFunction(PyObject *self, - PyObject *args); - -.. c:type:: PyCFunctionWithKeywords - - Type of the functions used to implement Python callables in C - with signature :ref:`METH_VARARGS | METH_KEYWORDS `. - The function signature is:: - - PyObject *PyCFunctionWithKeywords(PyObject *self, - PyObject *args, - PyObject *kwargs); - - -.. c:type:: _PyCFunctionFast - - Type of the functions used to implement Python callables in C - with signature :c:macro:`METH_FASTCALL`. - The function signature is:: - - PyObject *_PyCFunctionFast(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs); - -.. c:type:: _PyCFunctionFastWithKeywords - - Type of the functions used to implement Python callables in C - with signature :ref:`METH_FASTCALL | METH_KEYWORDS `. - The function signature is:: - - PyObject *_PyCFunctionFastWithKeywords(PyObject *self, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames); - -.. c:type:: PyCMethod - - Type of the functions used to implement Python callables in C - with signature :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `. - The function signature is:: - - PyObject *PyCMethod(PyObject *self, - PyTypeObject *defining_class, - PyObject *const *args, - Py_ssize_t nargs, - PyObject *kwnames) - - .. versionadded:: 3.9 - - -.. c:type:: PyMethodDef - - Structure used to describe a method of an extension type. This structure has - four fields: - - .. c:member:: const char *ml_name - - Name of the method. - - .. c:member:: PyCFunction ml_meth - - Pointer to the C implementation. - - .. c:member:: int ml_flags - - Flags bits indicating how the call should be constructed. - - .. c:member:: const char *ml_doc - - Points to the contents of the docstring. - -The :c:member:`~PyMethodDef.ml_meth` is a C function pointer. -The functions may be of different -types, but they always return :c:expr:`PyObject*`. If the function is not of -the :c:type:`PyCFunction`, the compiler will require a cast in the method table. -Even though :c:type:`PyCFunction` defines the first parameter as -:c:expr:`PyObject*`, it is common that the method implementation uses the -specific C type of the *self* object. - -The :c:member:`~PyMethodDef.ml_flags` field is a bitfield which can include -the following flags. -The individual flags indicate either a calling convention or a binding -convention. - -There are these calling conventions: - -.. c:macro:: METH_VARARGS - - This is the typical calling convention, where the methods have the type - :c:type:`PyCFunction`. The function expects two :c:expr:`PyObject*` values. - The first one is the *self* object for methods; for module functions, it is - the module object. The second parameter (often called *args*) is a tuple - object representing all arguments. This parameter is typically processed - using :c:func:`PyArg_ParseTuple` or :c:func:`PyArg_UnpackTuple`. - - -.. c:macro:: METH_KEYWORDS - - Can only be used in certain combinations with other flags: - :ref:`METH_VARARGS | METH_KEYWORDS `, - :ref:`METH_FASTCALL | METH_KEYWORDS ` and - :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `. - - -.. _METH_VARARGS-METH_KEYWORDS: - -:c:expr:`METH_VARARGS | METH_KEYWORDS` - Methods with these flags must be of type :c:type:`PyCFunctionWithKeywords`. - The function expects three parameters: *self*, *args*, *kwargs* where - *kwargs* is a dictionary of all the keyword arguments or possibly ``NULL`` - if there are no keyword arguments. The parameters are typically processed - using :c:func:`PyArg_ParseTupleAndKeywords`. - - -.. c:macro:: METH_FASTCALL - - Fast calling convention supporting only positional arguments. - The methods have the type :c:type:`_PyCFunctionFast`. - The first parameter is *self*, the second parameter is a C array - of :c:expr:`PyObject*` values indicating the arguments and the third - parameter is the number of arguments (the length of the array). - - .. versionadded:: 3.7 - - .. versionchanged:: 3.10 - - ``METH_FASTCALL`` is now part of the :ref:`stable ABI `. - - -.. _METH_FASTCALL-METH_KEYWORDS: - -:c:expr:`METH_FASTCALL | METH_KEYWORDS` - Extension of :c:macro:`METH_FASTCALL` supporting also keyword arguments, - with methods of type :c:type:`_PyCFunctionFastWithKeywords`. - Keyword arguments are passed the same way as in the - :ref:`vectorcall protocol `: - there is an additional fourth :c:expr:`PyObject*` parameter - which is a tuple representing the names of the keyword arguments - (which are guaranteed to be strings) - or possibly ``NULL`` if there are no keywords. The values of the keyword - arguments are stored in the *args* array, after the positional arguments. - - .. versionadded:: 3.7 - - -.. c:macro:: METH_METHOD - - Can only be used in the combination with other flags: - :ref:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS `. - - -.. _METH_METHOD-METH_FASTCALL-METH_KEYWORDS: - -:c:expr:`METH_METHOD | METH_FASTCALL | METH_KEYWORDS` - Extension of :ref:`METH_FASTCALL | METH_KEYWORDS ` - supporting the *defining class*, that is, - the class that contains the method in question. - The defining class might be a superclass of ``Py_TYPE(self)``. - - The method needs to be of type :c:type:`PyCMethod`, the same as for - ``METH_FASTCALL | METH_KEYWORDS`` with ``defining_class`` argument added after - ``self``. - - .. versionadded:: 3.9 - - -.. c:macro:: METH_NOARGS - - Methods without parameters don't need to check whether arguments are given if - they are listed with the :c:macro:`METH_NOARGS` flag. They need to be of type - :c:type:`PyCFunction`. The first parameter is typically named *self* and will - hold a reference to the module or object instance. In all cases the second - parameter will be ``NULL``. - - The function must have 2 parameters. Since the second parameter is unused, - :c:macro:`Py_UNUSED` can be used to prevent a compiler warning. - - -.. c:macro:: METH_O - - Methods with a single object argument can be listed with the :c:macro:`METH_O` - flag, instead of invoking :c:func:`PyArg_ParseTuple` with a ``"O"`` argument. - They have the type :c:type:`PyCFunction`, with the *self* parameter, and a - :c:expr:`PyObject*` parameter representing the single argument. - - -These two constants are not used to indicate the calling convention but the -binding when use with methods of classes. These may not be used for functions -defined for modules. At most one of these flags may be set for any given -method. - - -.. c:macro:: METH_CLASS - - .. index:: pair: built-in function; classmethod - - The method will be passed the type object as the first parameter rather - than an instance of the type. This is used to create *class methods*, - similar to what is created when using the :func:`classmethod` built-in - function. - - -.. c:macro:: METH_STATIC - - .. index:: pair: built-in function; staticmethod - - The method will be passed ``NULL`` as the first parameter rather than an - instance of the type. This is used to create *static methods*, similar to - what is created when using the :func:`staticmethod` built-in function. - -One other constant controls whether a method is loaded in place of another -definition with the same method name. - - -.. c:macro:: METH_COEXIST - - The method will be loaded in place of existing definitions. Without - *METH_COEXIST*, the default is to skip repeated definitions. Since slot - wrappers are loaded before the method table, the existence of a - *sq_contains* slot, for example, would generate a wrapped method named - :meth:`~object.__contains__` and preclude the loading of a corresponding - PyCFunction with the same name. With the flag defined, the PyCFunction - will be loaded in place of the wrapper object and will co-exist with the - slot. This is helpful because calls to PyCFunctions are optimized more - than wrapper object calls. - - -Accessing attributes of extension types ---------------------------------------- - -.. c:type:: PyMemberDef - - Structure which describes an attribute of a type which corresponds to a C - struct member. Its fields are: - - +------------------+---------------+-------------------------------+ - | Field | C Type | Meaning | - +==================+===============+===============================+ - | :attr:`name` | const char \* | name of the member | - +------------------+---------------+-------------------------------+ - | :attr:`!type` | int | the type of the member in the | - | | | C struct | - +------------------+---------------+-------------------------------+ - | :attr:`offset` | Py_ssize_t | the offset in bytes that the | - | | | member is located on the | - | | | type's object struct | - +------------------+---------------+-------------------------------+ - | :attr:`flags` | int | flag bits indicating if the | - | | | field should be read-only or | - | | | writable | - +------------------+---------------+-------------------------------+ - | :attr:`doc` | const char \* | points to the contents of the | - | | | docstring | - +------------------+---------------+-------------------------------+ - - :attr:`!type` can be one of many ``T_`` macros corresponding to various C - types. When the member is accessed in Python, it will be converted to the - equivalent Python type. - - =============== ================== - Macro name C type - =============== ================== - T_SHORT short - T_INT int - T_LONG long - T_FLOAT float - T_DOUBLE double - T_STRING const char \* - T_OBJECT PyObject \* - T_OBJECT_EX PyObject \* - T_CHAR char - T_BYTE char - T_UBYTE unsigned char - T_UINT unsigned int - T_USHORT unsigned short - T_ULONG unsigned long - T_BOOL char - T_LONGLONG long long - T_ULONGLONG unsigned long long - T_PYSSIZET Py_ssize_t - =============== ================== - - :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` differ in that - :c:macro:`T_OBJECT` returns ``None`` if the member is ``NULL`` and - :c:macro:`T_OBJECT_EX` raises an :exc:`AttributeError`. Try to use - :c:macro:`T_OBJECT_EX` over :c:macro:`T_OBJECT` because :c:macro:`T_OBJECT_EX` - handles use of the :keyword:`del` statement on that attribute more correctly - than :c:macro:`T_OBJECT`. - - :attr:`flags` can be ``0`` for write and read access or :c:macro:`READONLY` for - read-only access. Using :c:macro:`T_STRING` for :attr:`type` implies - :c:macro:`READONLY`. :c:macro:`T_STRING` data is interpreted as UTF-8. - Only :c:macro:`T_OBJECT` and :c:macro:`T_OBJECT_EX` - members can be deleted. (They are set to ``NULL``). - - .. _pymemberdef-offsets: - - Heap allocated types (created using :c:func:`PyType_FromSpec` or similar), - ``PyMemberDef`` may contain definitions for the special members - ``__dictoffset__``, ``__weaklistoffset__`` and ``__vectorcalloffset__``, - corresponding to - :c:member:`~PyTypeObject.tp_dictoffset`, - :c:member:`~PyTypeObject.tp_weaklistoffset` and - :c:member:`~PyTypeObject.tp_vectorcall_offset` in type objects. - These must be defined with ``T_PYSSIZET`` and ``READONLY``, for example:: - - static PyMemberDef spam_type_members[] = { - {"__dictoffset__", T_PYSSIZET, offsetof(Spam_object, dict), READONLY}, - {NULL} /* Sentinel */ - }; - - -.. c:function:: PyObject* PyMember_GetOne(const char *obj_addr, struct PyMemberDef *m) - - Get an attribute belonging to the object at address *obj_addr*. The - attribute is described by ``PyMemberDef`` *m*. Returns ``NULL`` - on error. - - -.. c:function:: int PyMember_SetOne(char *obj_addr, struct PyMemberDef *m, PyObject *o) - - Set an attribute belonging to the object at address *obj_addr* to object *o*. - The attribute to set is described by ``PyMemberDef`` *m*. Returns ``0`` - if successful and a negative value on failure. - - -.. c:type:: PyGetSetDef - - Structure to define property-like access for a type. See also description of - the :c:member:`PyTypeObject.tp_getset` slot. - - +-------------+------------------+-----------------------------------+ - | Field | C Type | Meaning | - +=============+==================+===================================+ - | name | const char \* | attribute name | - +-------------+------------------+-----------------------------------+ - | get | getter | C function to get the attribute | - +-------------+------------------+-----------------------------------+ - | set | setter | optional C function to set or | - | | | delete the attribute, if omitted | - | | | the attribute is readonly | - +-------------+------------------+-----------------------------------+ - | doc | const char \* | optional docstring | - +-------------+------------------+-----------------------------------+ - | closure | void \* | optional function pointer, | - | | | providing additional data for | - | | | getter and setter | - +-------------+------------------+-----------------------------------+ - - The ``get`` function takes one :c:expr:`PyObject*` parameter (the - instance) and a function pointer (the associated ``closure``):: - - typedef PyObject *(*getter)(PyObject *, void *); - - It should return a new reference on success or ``NULL`` with a set exception - on failure. - - ``set`` functions take two :c:expr:`PyObject*` parameters (the instance and - the value to be set) and a function pointer (the associated ``closure``):: - - typedef int (*setter)(PyObject *, PyObject *, void *); - - In case the attribute should be deleted the second parameter is ``NULL``. - Should return ``0`` on success or ``-1`` with a set exception on failure. diff --git a/Doc/c-api/sys.rst b/Doc/c-api/sys.rst deleted file mode 100644 index a811eb17b328ea..00000000000000 --- a/Doc/c-api/sys.rst +++ /dev/null @@ -1,453 +0,0 @@ -.. highlight:: c - -.. _os: - -Operating System Utilities -========================== - -.. c:function:: PyObject* PyOS_FSPath(PyObject *path) - - Return the file system representation for *path*. If the object is a - :class:`str` or :class:`bytes` object, then a new - :term:`strong reference` is returned. - If the object implements the :class:`os.PathLike` interface, - then :meth:`~os.PathLike.__fspath__` is returned as long as it is a - :class:`str` or :class:`bytes` object. Otherwise :exc:`TypeError` is raised - and ``NULL`` is returned. - - .. versionadded:: 3.6 - - -.. c:function:: int Py_FdIsInteractive(FILE *fp, const char *filename) - - Return true (nonzero) if the standard I/O file *fp* with name *filename* is - deemed interactive. This is the case for files for which ``isatty(fileno(fp))`` - is true. If the global flag :c:data:`Py_InteractiveFlag` is true, this function - also returns true if the *filename* pointer is ``NULL`` or if the name is equal to - one of the strings ``''`` or ``'???'``. - - -.. c:function:: void PyOS_BeforeFork() - - Function to prepare some internal state before a process fork. This - should be called before calling :c:func:`fork` or any similar function - that clones the current process. - Only available on systems where :c:func:`fork` is defined. - - .. warning:: - The C :c:func:`fork` call should only be made from the - :ref:`"main" thread ` (of the - :ref:`"main" interpreter `). The same is - true for ``PyOS_BeforeFork()``. - - .. versionadded:: 3.7 - - -.. c:function:: void PyOS_AfterFork_Parent() - - Function to update some internal state after a process fork. This - should be called from the parent process after calling :c:func:`fork` - or any similar function that clones the current process, regardless - of whether process cloning was successful. - Only available on systems where :c:func:`fork` is defined. - - .. warning:: - The C :c:func:`fork` call should only be made from the - :ref:`"main" thread ` (of the - :ref:`"main" interpreter `). The same is - true for ``PyOS_AfterFork_Parent()``. - - .. versionadded:: 3.7 - - -.. c:function:: void PyOS_AfterFork_Child() - - Function to update internal interpreter state after a process fork. - This must be called from the child process after calling :c:func:`fork`, - or any similar function that clones the current process, if there is - any chance the process will call back into the Python interpreter. - Only available on systems where :c:func:`fork` is defined. - - .. warning:: - The C :c:func:`fork` call should only be made from the - :ref:`"main" thread ` (of the - :ref:`"main" interpreter `). The same is - true for ``PyOS_AfterFork_Child()``. - - .. versionadded:: 3.7 - - .. seealso:: - :func:`os.register_at_fork` allows registering custom Python functions - to be called by :c:func:`PyOS_BeforeFork()`, - :c:func:`PyOS_AfterFork_Parent` and :c:func:`PyOS_AfterFork_Child`. - - -.. c:function:: void PyOS_AfterFork() - - Function to update some internal state after a process fork; this should be - called in the new process if the Python interpreter will continue to be used. - If a new executable is loaded into the new process, this function does not need - to be called. - - .. deprecated:: 3.7 - This function is superseded by :c:func:`PyOS_AfterFork_Child()`. - - -.. c:function:: int PyOS_CheckStack() - - Return true when the interpreter runs out of stack space. This is a reliable - check, but is only available when :c:macro:`USE_STACKCHECK` is defined (currently - on certain versions of Windows using the Microsoft Visual C++ compiler). - :c:macro:`USE_STACKCHECK` will be defined automatically; you should never - change the definition in your own code. - - -.. c:function:: PyOS_sighandler_t PyOS_getsig(int i) - - Return the current signal handler for signal *i*. This is a thin wrapper around - either :c:func:`!sigaction` or :c:func:`!signal`. Do not call those functions - directly! :c:type:`PyOS_sighandler_t` is a typedef alias for :c:expr:`void - (\*)(int)`. - - -.. c:function:: PyOS_sighandler_t PyOS_setsig(int i, PyOS_sighandler_t h) - - Set the signal handler for signal *i* to be *h*; return the old signal handler. - This is a thin wrapper around either :c:func:`!sigaction` or :c:func:`!signal`. Do - not call those functions directly! :c:type:`PyOS_sighandler_t` is a typedef - alias for :c:expr:`void (\*)(int)`. - -.. c:function:: wchar_t* Py_DecodeLocale(const char* arg, size_t *size) - - .. warning:: - This function should not be called directly: use the :c:type:`PyConfig` - API with the :c:func:`PyConfig_SetBytesString` function which ensures - that :ref:`Python is preinitialized `. - - This function must not be called before :ref:`Python is preinitialized - ` and so that the LC_CTYPE locale is properly configured: see - the :c:func:`Py_PreInitialize` function. - - Decode a byte string from the :term:`filesystem encoding and error handler`. - If the error handler is :ref:`surrogateescape error handler - `, undecodable bytes are decoded as characters in range - U+DC80..U+DCFF; and if a byte sequence can be decoded as a surrogate - character, the bytes are escaped using the surrogateescape error handler - instead of decoding them. - - Return a pointer to a newly allocated wide character string, use - :c:func:`PyMem_RawFree` to free the memory. If size is not ``NULL``, write - the number of wide characters excluding the null character into ``*size`` - - Return ``NULL`` on decoding error or memory allocation error. If *size* is - not ``NULL``, ``*size`` is set to ``(size_t)-1`` on memory error or set to - ``(size_t)-2`` on decoding error. - - The :term:`filesystem encoding and error handler` are selected by - :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and - :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. - - Decoding errors should never happen, unless there is a bug in the C - library. - - Use the :c:func:`Py_EncodeLocale` function to encode the character string - back to a byte string. - - .. seealso:: - - The :c:func:`PyUnicode_DecodeFSDefaultAndSize` and - :c:func:`PyUnicode_DecodeLocaleAndSize` functions. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.7 - The function now uses the UTF-8 encoding in the :ref:`Python UTF-8 Mode - `. - - .. versionchanged:: 3.8 - The function now uses the UTF-8 encoding on Windows if - :c:data:`Py_LegacyWindowsFSEncodingFlag` is zero; - - -.. c:function:: char* Py_EncodeLocale(const wchar_t *text, size_t *error_pos) - - Encode a wide character string to the :term:`filesystem encoding and error - handler`. If the error handler is :ref:`surrogateescape error handler - `, surrogate characters in the range U+DC80..U+DCFF are - converted to bytes 0x80..0xFF. - - Return a pointer to a newly allocated byte string, use :c:func:`PyMem_Free` - to free the memory. Return ``NULL`` on encoding error or memory allocation - error. - - If error_pos is not ``NULL``, ``*error_pos`` is set to ``(size_t)-1`` on - success, or set to the index of the invalid character on encoding error. - - The :term:`filesystem encoding and error handler` are selected by - :c:func:`PyConfig_Read`: see :c:member:`~PyConfig.filesystem_encoding` and - :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. - - Use the :c:func:`Py_DecodeLocale` function to decode the bytes string back - to a wide character string. - - .. warning:: - This function must not be called before :ref:`Python is preinitialized - ` and so that the LC_CTYPE locale is properly configured: see - the :c:func:`Py_PreInitialize` function. - - .. seealso:: - - The :c:func:`PyUnicode_EncodeFSDefault` and - :c:func:`PyUnicode_EncodeLocale` functions. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.7 - The function now uses the UTF-8 encoding in the :ref:`Python UTF-8 Mode - `. - - .. versionchanged:: 3.8 - The function now uses the UTF-8 encoding on Windows if - :c:data:`Py_LegacyWindowsFSEncodingFlag` is zero. - - -.. _systemfunctions: - -System Functions -================ - -These are utility functions that make functionality from the :mod:`sys` module -accessible to C code. They all work with the current interpreter thread's -:mod:`sys` module's dict, which is contained in the internal thread state structure. - -.. c:function:: PyObject *PySys_GetObject(const char *name) - - Return the object *name* from the :mod:`sys` module or ``NULL`` if it does - not exist, without setting an exception. - -.. c:function:: int PySys_SetObject(const char *name, PyObject *v) - - Set *name* in the :mod:`sys` module to *v* unless *v* is ``NULL``, in which - case *name* is deleted from the sys module. Returns ``0`` on success, ``-1`` - on error. - -.. c:function:: void PySys_ResetWarnOptions() - - Reset :data:`sys.warnoptions` to an empty list. This function may be - called prior to :c:func:`Py_Initialize`. - -.. c:function:: void PySys_AddWarnOption(const wchar_t *s) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.warnoptions` should be used instead, see :ref:`Python - Initialization Configuration `. - - Append *s* to :data:`sys.warnoptions`. This function must be called prior - to :c:func:`Py_Initialize` in order to affect the warnings filter list. - - .. deprecated:: 3.11 - -.. c:function:: void PySys_AddWarnOptionUnicode(PyObject *unicode) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.warnoptions` should be used instead, see :ref:`Python - Initialization Configuration `. - - Append *unicode* to :data:`sys.warnoptions`. - - Note: this function is not currently usable from outside the CPython - implementation, as it must be called prior to the implicit import of - :mod:`warnings` in :c:func:`Py_Initialize` to be effective, but can't be - called until enough of the runtime has been initialized to permit the - creation of Unicode objects. - - .. deprecated:: 3.11 - -.. c:function:: void PySys_SetPath(const wchar_t *path) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.module_search_paths` and - :c:member:`PyConfig.module_search_paths_set` should be used instead, see - :ref:`Python Initialization Configuration `. - - Set :data:`sys.path` to a list object of paths found in *path* which should - be a list of paths separated with the platform's search path delimiter - (``:`` on Unix, ``;`` on Windows). - - .. deprecated:: 3.11 - -.. c:function:: void PySys_WriteStdout(const char *format, ...) - - Write the output string described by *format* to :data:`sys.stdout`. No - exceptions are raised, even if truncation occurs (see below). - - *format* should limit the total size of the formatted output string to - 1000 bytes or less -- after 1000 bytes, the output string is truncated. - In particular, this means that no unrestricted "%s" formats should occur; - these should be limited using "%.s" where is a decimal number - calculated so that plus the maximum size of other formatted text does not - exceed 1000 bytes. Also watch out for "%f", which can print hundreds of - digits for very large numbers. - - If a problem occurs, or :data:`sys.stdout` is unset, the formatted message - is written to the real (C level) *stdout*. - -.. c:function:: void PySys_WriteStderr(const char *format, ...) - - As :c:func:`PySys_WriteStdout`, but write to :data:`sys.stderr` or *stderr* - instead. - -.. c:function:: void PySys_FormatStdout(const char *format, ...) - - Function similar to PySys_WriteStdout() but format the message using - :c:func:`PyUnicode_FromFormatV` and don't truncate the message to an - arbitrary length. - - .. versionadded:: 3.2 - -.. c:function:: void PySys_FormatStderr(const char *format, ...) - - As :c:func:`PySys_FormatStdout`, but write to :data:`sys.stderr` or *stderr* - instead. - - .. versionadded:: 3.2 - -.. c:function:: void PySys_AddXOption(const wchar_t *s) - - This API is kept for backward compatibility: setting - :c:member:`PyConfig.xoptions` should be used instead, see :ref:`Python - Initialization Configuration `. - - Parse *s* as a set of :option:`-X` options and add them to the current - options mapping as returned by :c:func:`PySys_GetXOptions`. This function - may be called prior to :c:func:`Py_Initialize`. - - .. versionadded:: 3.2 - - .. deprecated:: 3.11 - -.. c:function:: PyObject *PySys_GetXOptions() - - Return the current dictionary of :option:`-X` options, similarly to - :data:`sys._xoptions`. On error, ``NULL`` is returned and an exception is - set. - - .. versionadded:: 3.2 - - -.. c:function:: int PySys_Audit(const char *event, const char *format, ...) - - Raise an auditing event with any active hooks. Return zero for success - and non-zero with an exception set on failure. - - If any hooks have been added, *format* and other arguments will be used - to construct a tuple to pass. Apart from ``N``, the same format characters - as used in :c:func:`Py_BuildValue` are available. If the built value is not - a tuple, it will be added into a single-element tuple. (The ``N`` format - option consumes a reference, but since there is no way to know whether - arguments to this function will be consumed, using it may cause reference - leaks.) - - Note that ``#`` format characters should always be treated as - :c:type:`Py_ssize_t`, regardless of whether ``PY_SSIZE_T_CLEAN`` was defined. - - :func:`sys.audit` performs the same function from Python code. - - .. versionadded:: 3.8 - - .. versionchanged:: 3.8.2 - - Require :c:type:`Py_ssize_t` for ``#`` format characters. Previously, an - unavoidable deprecation warning was raised. - - -.. c:function:: int PySys_AddAuditHook(Py_AuditHookFunction hook, void *userData) - - Append the callable *hook* to the list of active auditing hooks. - Return zero on success - and non-zero on failure. If the runtime has been initialized, also set an - error on failure. Hooks added through this API are called for all - interpreters created by the runtime. - - The *userData* pointer is passed into the hook function. Since hook - functions may be called from different runtimes, this pointer should not - refer directly to Python state. - - This function is safe to call before :c:func:`Py_Initialize`. When called - after runtime initialization, existing audit hooks are notified and may - silently abort the operation by raising an error subclassed from - :class:`Exception` (other errors will not be silenced). - - The hook function is of type :c:expr:`int (*)(const char *event, PyObject - *args, void *userData)`, where *args* is guaranteed to be a - :c:type:`PyTupleObject`. The hook function is always called with the GIL - held by the Python interpreter that raised the event. - - See :pep:`578` for a detailed description of auditing. Functions in the - runtime and standard library that raise events are listed in the - :ref:`audit events table `. - Details are in each function's documentation. - - .. audit-event:: sys.addaudithook "" c.PySys_AddAuditHook - - If the interpreter is initialized, this function raises a auditing event - ``sys.addaudithook`` with no arguments. If any existing hooks raise an - exception derived from :class:`Exception`, the new hook will not be - added and the exception is cleared. As a result, callers cannot assume - that their hook has been added unless they control all existing hooks. - - .. versionadded:: 3.8 - - -.. _processcontrol: - -Process Control -=============== - - -.. c:function:: void Py_FatalError(const char *message) - - .. index:: single: abort() - - Print a fatal error message and kill the process. No cleanup is performed. - This function should only be invoked when a condition is detected that would - make it dangerous to continue using the Python interpreter; e.g., when the - object administration appears to be corrupted. On Unix, the standard C library - function :c:func:`!abort` is called which will attempt to produce a :file:`core` - file. - - The ``Py_FatalError()`` function is replaced with a macro which logs - automatically the name of the current function, unless the - ``Py_LIMITED_API`` macro is defined. - - .. versionchanged:: 3.9 - Log the function name automatically. - - -.. c:function:: void Py_Exit(int status) - - .. index:: - single: Py_FinalizeEx() - single: exit() - - Exit the current process. This calls :c:func:`Py_FinalizeEx` and then calls the - standard C library function ``exit(status)``. If :c:func:`Py_FinalizeEx` - indicates an error, the exit status is set to 120. - - .. versionchanged:: 3.6 - Errors from finalization no longer ignored. - - -.. c:function:: int Py_AtExit(void (*func) ()) - - .. index:: - single: Py_FinalizeEx() - single: cleanup functions - - Register a cleanup function to be called by :c:func:`Py_FinalizeEx`. The cleanup - function will be called with no arguments and should return no value. At most - 32 cleanup functions can be registered. When the registration is successful, - :c:func:`Py_AtExit` returns ``0``; on failure, it returns ``-1``. The cleanup - function registered last is called first. Each cleanup function will be called - at most once. Since Python's internal finalization will have completed before - the cleanup function, no Python APIs should be called by *func*. diff --git a/Doc/c-api/tuple.rst b/Doc/c-api/tuple.rst deleted file mode 100644 index d298aaf3ab5484..00000000000000 --- a/Doc/c-api/tuple.rst +++ /dev/null @@ -1,225 +0,0 @@ -.. highlight:: c - -.. _tupleobjects: - -Tuple Objects -------------- - -.. index:: pair: object; tuple - - -.. c:type:: PyTupleObject - - This subtype of :c:type:`PyObject` represents a Python tuple object. - - -.. c:var:: PyTypeObject PyTuple_Type - - This instance of :c:type:`PyTypeObject` represents the Python tuple type; it - is the same object as :class:`tuple` in the Python layer. - - -.. c:function:: int PyTuple_Check(PyObject *p) - - Return true if *p* is a tuple object or an instance of a subtype of the - tuple type. This function always succeeds. - - -.. c:function:: int PyTuple_CheckExact(PyObject *p) - - Return true if *p* is a tuple object, but not an instance of a subtype of the - tuple type. This function always succeeds. - - -.. c:function:: PyObject* PyTuple_New(Py_ssize_t len) - - Return a new tuple object of size *len*, or ``NULL`` on failure. - - -.. c:function:: PyObject* PyTuple_Pack(Py_ssize_t n, ...) - - Return a new tuple object of size *n*, or ``NULL`` on failure. The tuple values - are initialized to the subsequent *n* C arguments pointing to Python objects. - ``PyTuple_Pack(2, a, b)`` is equivalent to ``Py_BuildValue("(OO)", a, b)``. - - -.. c:function:: Py_ssize_t PyTuple_Size(PyObject *p) - - Take a pointer to a tuple object, and return the size of that tuple. - - -.. c:function:: Py_ssize_t PyTuple_GET_SIZE(PyObject *p) - - Return the size of the tuple *p*, which must be non-``NULL`` and point to a tuple; - no error checking is performed. - - -.. c:function:: PyObject* PyTuple_GetItem(PyObject *p, Py_ssize_t pos) - - Return the object at position *pos* in the tuple pointed to by *p*. If *pos* is - negative or out of bounds, return ``NULL`` and set an :exc:`IndexError` exception. - - -.. c:function:: PyObject* PyTuple_GET_ITEM(PyObject *p, Py_ssize_t pos) - - Like :c:func:`PyTuple_GetItem`, but does no checking of its arguments. - - -.. c:function:: PyObject* PyTuple_GetSlice(PyObject *p, Py_ssize_t low, Py_ssize_t high) - - Return the slice of the tuple pointed to by *p* between *low* and *high*, - or ``NULL`` on failure. This is the equivalent of the Python expression - ``p[low:high]``. Indexing from the end of the list is not supported. - - -.. c:function:: int PyTuple_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) - - Insert a reference to object *o* at position *pos* of the tuple pointed to by - *p*. Return ``0`` on success. If *pos* is out of bounds, return ``-1`` - and set an :exc:`IndexError` exception. - - .. note:: - - This function "steals" a reference to *o* and discards a reference to - an item already in the tuple at the affected position. - - -.. c:function:: void PyTuple_SET_ITEM(PyObject *p, Py_ssize_t pos, PyObject *o) - - Like :c:func:`PyTuple_SetItem`, but does no error checking, and should *only* be - used to fill in brand new tuples. - - .. note:: - - This function "steals" a reference to *o*, and, unlike - :c:func:`PyTuple_SetItem`, does *not* discard a reference to any item that - is being replaced; any reference in the tuple at position *pos* will be - leaked. - - -.. c:function:: int _PyTuple_Resize(PyObject **p, Py_ssize_t newsize) - - Can be used to resize a tuple. *newsize* will be the new length of the tuple. - Because tuples are *supposed* to be immutable, this should only be used if there - is only one reference to the object. Do *not* use this if the tuple may already - be known to some other part of the code. The tuple will always grow or shrink - at the end. Think of this as destroying the old tuple and creating a new one, - only more efficiently. Returns ``0`` on success. Client code should never - assume that the resulting value of ``*p`` will be the same as before calling - this function. If the object referenced by ``*p`` is replaced, the original - ``*p`` is destroyed. On failure, returns ``-1`` and sets ``*p`` to ``NULL``, and - raises :exc:`MemoryError` or :exc:`SystemError`. - - -.. _struct-sequence-objects: - -Struct Sequence Objects ------------------------ - -Struct sequence objects are the C equivalent of :func:`~collections.namedtuple` -objects, i.e. a sequence whose items can also be accessed through attributes. -To create a struct sequence, you first have to create a specific struct sequence -type. - -.. c:function:: PyTypeObject* PyStructSequence_NewType(PyStructSequence_Desc *desc) - - Create a new struct sequence type from the data in *desc*, described below. Instances - of the resulting type can be created with :c:func:`PyStructSequence_New`. - - -.. c:function:: void PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc) - - Initializes a struct sequence type *type* from *desc* in place. - - -.. c:function:: int PyStructSequence_InitType2(PyTypeObject *type, PyStructSequence_Desc *desc) - - The same as ``PyStructSequence_InitType``, but returns ``0`` on success and ``-1`` on - failure. - - .. versionadded:: 3.4 - - -.. c:type:: PyStructSequence_Desc - - Contains the meta information of a struct sequence type to create. - - .. c:member:: const char *name - - Name of the struct sequence type. - - .. c:member:: const char *doc - - Pointer to docstring for the type or ``NULL`` to omit. - - .. c:member:: PyStructSequence_Field *fields - - Pointer to ``NULL``-terminated array with field names of the new type. - - .. c:member:: int n_in_sequence - - Number of fields visible to the Python side (if used as tuple). - - -.. c:type:: PyStructSequence_Field - - Describes a field of a struct sequence. As a struct sequence is modeled as a - tuple, all fields are typed as :c:expr:`PyObject*`. The index in the - :c:member:`~PyStructSequence_Desc.fields` array of - the :c:type:`PyStructSequence_Desc` determines which - field of the struct sequence is described. - - .. c:member:: const char *name - - Name for the field or ``NULL`` to end the list of named fields, - set to :c:data:`PyStructSequence_UnnamedField` to leave unnamed. - - .. c:member:: const char *doc - - Field docstring or ``NULL`` to omit. - - -.. c:var:: const char * const PyStructSequence_UnnamedField - - Special value for a field name to leave it unnamed. - - .. versionchanged:: 3.9 - The type was changed from ``char *``. - - -.. c:function:: PyObject* PyStructSequence_New(PyTypeObject *type) - - Creates an instance of *type*, which must have been created with - :c:func:`PyStructSequence_NewType`. - - -.. c:function:: PyObject* PyStructSequence_GetItem(PyObject *p, Py_ssize_t pos) - - Return the object at position *pos* in the struct sequence pointed to by *p*. - No bounds checking is performed. - - -.. c:function:: PyObject* PyStructSequence_GET_ITEM(PyObject *p, Py_ssize_t pos) - - Macro equivalent of :c:func:`PyStructSequence_GetItem`. - - -.. c:function:: void PyStructSequence_SetItem(PyObject *p, Py_ssize_t pos, PyObject *o) - - Sets the field at index *pos* of the struct sequence *p* to value *o*. Like - :c:func:`PyTuple_SET_ITEM`, this should only be used to fill in brand new - instances. - - .. note:: - - This function "steals" a reference to *o*. - - -.. c:function:: void PyStructSequence_SET_ITEM(PyObject *p, Py_ssize_t *pos, PyObject *o) - - Similar to :c:func:`PyStructSequence_SetItem`, but implemented as a static - inlined function. - - .. note:: - - This function "steals" a reference to *o*. diff --git a/Doc/c-api/type.rst b/Doc/c-api/type.rst deleted file mode 100644 index a7ca19677134d0..00000000000000 --- a/Doc/c-api/type.rst +++ /dev/null @@ -1,311 +0,0 @@ -.. highlight:: c - -.. _typeobjects: - -Type Objects ------------- - -.. index:: pair: object; type - - -.. c:type:: PyTypeObject - - The C structure of the objects used to describe built-in types. - - -.. c:var:: PyTypeObject PyType_Type - - This is the type object for type objects; it is the same object as - :class:`type` in the Python layer. - - -.. c:function:: int PyType_Check(PyObject *o) - - Return non-zero if the object *o* is a type object, including instances of - types derived from the standard type object. Return 0 in all other cases. - This function always succeeds. - - -.. c:function:: int PyType_CheckExact(PyObject *o) - - Return non-zero if the object *o* is a type object, but not a subtype of - the standard type object. Return 0 in all other cases. This function - always succeeds. - - -.. c:function:: unsigned int PyType_ClearCache() - - Clear the internal lookup cache. Return the current version tag. - -.. c:function:: unsigned long PyType_GetFlags(PyTypeObject* type) - - Return the :c:member:`~PyTypeObject.tp_flags` member of *type*. This function is primarily - meant for use with ``Py_LIMITED_API``; the individual flag bits are - guaranteed to be stable across Python releases, but access to - :c:member:`~PyTypeObject.tp_flags` itself is not part of the :ref:`limited API `. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - The return type is now ``unsigned long`` rather than ``long``. - - -.. c:function:: void PyType_Modified(PyTypeObject *type) - - Invalidate the internal lookup cache for the type and all of its - subtypes. This function must be called after any manual - modification of the attributes or base classes of the type. - - -.. c:function:: int PyType_HasFeature(PyTypeObject *o, int feature) - - Return non-zero if the type object *o* sets the feature *feature*. - Type features are denoted by single bit flags. - - -.. c:function:: int PyType_IS_GC(PyTypeObject *o) - - Return true if the type object includes support for the cycle detector; this - tests the type flag :c:macro:`Py_TPFLAGS_HAVE_GC`. - - -.. c:function:: int PyType_IsSubtype(PyTypeObject *a, PyTypeObject *b) - - Return true if *a* is a subtype of *b*. - - This function only checks for actual subtypes, which means that - :meth:`~class.__subclasscheck__` is not called on *b*. Call - :c:func:`PyObject_IsSubclass` to do the same check that :func:`issubclass` - would do. - - -.. c:function:: PyObject* PyType_GenericAlloc(PyTypeObject *type, Py_ssize_t nitems) - - Generic handler for the :c:member:`~PyTypeObject.tp_alloc` slot of a type object. Use - Python's default memory allocation mechanism to allocate a new instance and - initialize all its contents to ``NULL``. - -.. c:function:: PyObject* PyType_GenericNew(PyTypeObject *type, PyObject *args, PyObject *kwds) - - Generic handler for the :c:member:`~PyTypeObject.tp_new` slot of a type object. Create a - new instance using the type's :c:member:`~PyTypeObject.tp_alloc` slot. - -.. c:function:: int PyType_Ready(PyTypeObject *type) - - Finalize a type object. This should be called on all type objects to finish - their initialization. This function is responsible for adding inherited slots - from a type's base class. Return ``0`` on success, or return ``-1`` and sets an - exception on error. - - .. note:: - If some of the base classes implements the GC protocol and the provided - type does not include the :c:macro:`Py_TPFLAGS_HAVE_GC` in its flags, then - the GC protocol will be automatically implemented from its parents. On - the contrary, if the type being created does include - :c:macro:`Py_TPFLAGS_HAVE_GC` in its flags then it **must** implement the - GC protocol itself by at least implementing the - :c:member:`~PyTypeObject.tp_traverse` handle. - -.. c:function:: PyObject* PyType_GetName(PyTypeObject *type) - - Return the type's name. Equivalent to getting the type's ``__name__`` attribute. - - .. versionadded:: 3.11 - -.. c:function:: PyObject* PyType_GetQualName(PyTypeObject *type) - - Return the type's qualified name. Equivalent to getting the - type's ``__qualname__`` attribute. - - .. versionadded:: 3.11 - -.. c:function:: void* PyType_GetSlot(PyTypeObject *type, int slot) - - Return the function pointer stored in the given slot. If the - result is ``NULL``, this indicates that either the slot is ``NULL``, - or that the function was called with invalid parameters. - Callers will typically cast the result pointer into the appropriate - function type. - - See :c:member:`PyType_Slot.slot` for possible values of the *slot* argument. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.10 - :c:func:`PyType_GetSlot` can now accept all types. - Previously, it was limited to :ref:`heap types `. - -.. c:function:: PyObject* PyType_GetModule(PyTypeObject *type) - - Return the module object associated with the given type when the type was - created using :c:func:`PyType_FromModuleAndSpec`. - - If no module is associated with the given type, sets :py:class:`TypeError` - and returns ``NULL``. - - This function is usually used to get the module in which a method is defined. - Note that in such a method, ``PyType_GetModule(Py_TYPE(self))`` - may not return the intended result. - ``Py_TYPE(self)`` may be a *subclass* of the intended class, and subclasses - are not necessarily defined in the same module as their superclass. - See :c:type:`PyCMethod` to get the class that defines the method. - See :c:func:`PyType_GetModuleByDef` for cases when :c:type:`!PyCMethod` cannot - be used. - - .. versionadded:: 3.9 - -.. c:function:: void* PyType_GetModuleState(PyTypeObject *type) - - Return the state of the module object associated with the given type. - This is a shortcut for calling :c:func:`PyModule_GetState()` on the result - of :c:func:`PyType_GetModule`. - - If no module is associated with the given type, sets :py:class:`TypeError` - and returns ``NULL``. - - If the *type* has an associated module but its state is ``NULL``, - returns ``NULL`` without setting an exception. - - .. versionadded:: 3.9 - -.. c:function:: PyObject* PyType_GetModuleByDef(PyTypeObject *type, struct PyModuleDef *def) - - Find the first superclass whose module was created from - the given :c:type:`PyModuleDef` *def*, and return that module. - - If no module is found, raises a :py:class:`TypeError` and returns ``NULL``. - - This function is intended to be used together with - :c:func:`PyModule_GetState()` to get module state from slot methods (such as - :c:member:`~PyTypeObject.tp_init` or :c:member:`~PyNumberMethods.nb_add`) - and other places where a method's defining class cannot be passed using the - :c:type:`PyCMethod` calling convention. - - .. versionadded:: 3.11 - - -Creating Heap-Allocated Types -............................. - -The following functions and structs are used to create -:ref:`heap types `. - -.. c:function:: PyObject* PyType_FromModuleAndSpec(PyObject *module, PyType_Spec *spec, PyObject *bases) - - Creates and returns a :ref:`heap type ` from the *spec* - (:c:macro:`Py_TPFLAGS_HEAPTYPE`). - - The *bases* argument can be used to specify base classes; it can either - be only one class or a tuple of classes. - If *bases* is ``NULL``, the *Py_tp_bases* slot is used instead. - If that also is ``NULL``, the *Py_tp_base* slot is used instead. - If that also is ``NULL``, the new type derives from :class:`object`. - - The *module* argument can be used to record the module in which the new - class is defined. It must be a module object or ``NULL``. - If not ``NULL``, the module is associated with the new type and can later be - retrieved with :c:func:`PyType_GetModule`. - The associated module is not inherited by subclasses; it must be specified - for each class individually. - - This function calls :c:func:`PyType_Ready` on the new type. - - .. versionadded:: 3.9 - - .. versionchanged:: 3.10 - - The function now accepts a single class as the *bases* argument and - ``NULL`` as the ``tp_doc`` slot. - -.. c:function:: PyObject* PyType_FromSpecWithBases(PyType_Spec *spec, PyObject *bases) - - Equivalent to ``PyType_FromModuleAndSpec(NULL, spec, bases)``. - - .. versionadded:: 3.3 - -.. c:function:: PyObject* PyType_FromSpec(PyType_Spec *spec) - - Equivalent to ``PyType_FromSpecWithBases(spec, NULL)``. - -.. c:type:: PyType_Spec - - Structure defining a type's behavior. - - .. c:member:: const char* PyType_Spec.name - - Name of the type, used to set :c:member:`PyTypeObject.tp_name`. - - .. c:member:: int PyType_Spec.basicsize - .. c:member:: int PyType_Spec.itemsize - - Size of the instance in bytes, used to set - :c:member:`PyTypeObject.tp_basicsize` and - :c:member:`PyTypeObject.tp_itemsize`. - - .. c:member:: int PyType_Spec.flags - - Type flags, used to set :c:member:`PyTypeObject.tp_flags`. - - If the ``Py_TPFLAGS_HEAPTYPE`` flag is not set, - :c:func:`PyType_FromSpecWithBases` sets it automatically. - - .. c:member:: PyType_Slot *PyType_Spec.slots - - Array of :c:type:`PyType_Slot` structures. - Terminated by the special slot value ``{0, NULL}``. - -.. c:type:: PyType_Slot - - Structure defining optional functionality of a type, containing a slot ID - and a value pointer. - - .. c:member:: int PyType_Slot.slot - - A slot ID. - - Slot IDs are named like the field names of the structures - :c:type:`PyTypeObject`, :c:type:`PyNumberMethods`, - :c:type:`PySequenceMethods`, :c:type:`PyMappingMethods` and - :c:type:`PyAsyncMethods` with an added ``Py_`` prefix. - For example, use: - - * ``Py_tp_dealloc`` to set :c:member:`PyTypeObject.tp_dealloc` - * ``Py_nb_add`` to set :c:member:`PyNumberMethods.nb_add` - * ``Py_sq_length`` to set :c:member:`PySequenceMethods.sq_length` - - The following fields cannot be set at all using :c:type:`PyType_Spec` and - :c:type:`PyType_Slot`: - - * :c:member:`~PyTypeObject.tp_dict` - * :c:member:`~PyTypeObject.tp_mro` - * :c:member:`~PyTypeObject.tp_cache` - * :c:member:`~PyTypeObject.tp_subclasses` - * :c:member:`~PyTypeObject.tp_weaklist` - * :c:member:`~PyTypeObject.tp_vectorcall` - * :c:member:`~PyTypeObject.tp_weaklistoffset` - (see :ref:`PyMemberDef `) - * :c:member:`~PyTypeObject.tp_dictoffset` - (see :ref:`PyMemberDef `) - * :c:member:`~PyTypeObject.tp_vectorcall_offset` - (see :ref:`PyMemberDef `) - - Setting :c:data:`Py_tp_bases` or :c:data:`Py_tp_base` may be - problematic on some platforms. - To avoid issues, use the *bases* argument of - :c:func:`PyType_FromSpecWithBases` instead. - - .. versionchanged:: 3.9 - - Slots in :c:type:`PyBufferProcs` may be set in the unlimited API. - - .. versionchanged:: 3.11 - :c:member:`~PyBufferProcs.bf_getbuffer` and - :c:member:`~PyBufferProcs.bf_releasebuffer` are now available - under the :ref:`limited API `. - - .. c:member:: void *PyType_Slot.pfunc - - The desired value of the slot. In most cases, this is a pointer - to a function. - - Slots other than ``Py_tp_doc`` may not be ``NULL``. diff --git a/Doc/c-api/typehints.rst b/Doc/c-api/typehints.rst deleted file mode 100644 index 98fe68737deb81..00000000000000 --- a/Doc/c-api/typehints.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. highlight:: c - -.. _typehintobjects: - -Objects for Type Hinting ------------------------- - -Various built-in types for type hinting are provided. Currently, -two types exist -- :ref:`GenericAlias ` and -:ref:`Union `. Only ``GenericAlias`` is exposed to C. - -.. c:function:: PyObject* Py_GenericAlias(PyObject *origin, PyObject *args) - - Create a :ref:`GenericAlias ` object. - Equivalent to calling the Python class - :class:`types.GenericAlias`. The *origin* and *args* arguments set the - ``GenericAlias``\ 's ``__origin__`` and ``__args__`` attributes respectively. - *origin* should be a :c:expr:`PyTypeObject*`, and *args* can be a - :c:expr:`PyTupleObject*` or any ``PyObject*``. If *args* passed is - not a tuple, a 1-tuple is automatically constructed and ``__args__`` is set - to ``(args,)``. - Minimal checking is done for the arguments, so the function will succeed even - if *origin* is not a type. - The ``GenericAlias``\ 's ``__parameters__`` attribute is constructed lazily - from ``__args__``. On failure, an exception is raised and ``NULL`` is - returned. - - Here's an example of how to make an extension type generic:: - - ... - static PyMethodDef my_obj_methods[] = { - // Other methods. - ... - {"__class_getitem__", Py_GenericAlias, METH_O|METH_CLASS, "See PEP 585"} - ... - } - - .. seealso:: The data model method :meth:`~object.__class_getitem__`. - - .. versionadded:: 3.9 - -.. c:var:: PyTypeObject Py_GenericAliasType - - The C type of the object returned by :c:func:`Py_GenericAlias`. Equivalent to - :class:`types.GenericAlias` in Python. - - .. versionadded:: 3.9 diff --git a/Doc/c-api/typeobj.rst b/Doc/c-api/typeobj.rst deleted file mode 100644 index 8b662ceac84e7b..00000000000000 --- a/Doc/c-api/typeobj.rst +++ /dev/null @@ -1,2747 +0,0 @@ -.. highlight:: c - -.. _type-structs: - -Type Objects -============ - -Perhaps one of the most important structures of the Python object system is the -structure that defines a new type: the :c:type:`PyTypeObject` structure. Type -objects can be handled using any of the ``PyObject_*`` or -``PyType_*`` functions, but do not offer much that's interesting to most -Python applications. These objects are fundamental to how objects behave, so -they are very important to the interpreter itself and to any extension module -that implements new types. - -Type objects are fairly large compared to most of the standard types. The reason -for the size is that each type object stores a large number of values, mostly C -function pointers, each of which implements a small part of the type's -functionality. The fields of the type object are examined in detail in this -section. The fields will be described in the order in which they occur in the -structure. - -In addition to the following quick reference, the :ref:`typedef-examples` -section provides at-a-glance insight into the meaning and use of -:c:type:`PyTypeObject`. - - -Quick Reference ---------------- - -.. _tp-slots-table: - -"tp slots" -^^^^^^^^^^ - -.. table:: - :widths: 18,18,18,1,1,1,1 - - +------------------------------------------------+-----------------------------------+-------------------+---------------+ - | PyTypeObject Slot [#slots]_ | :ref:`Type ` | special | Info [#cols]_ | - | | | methods/attrs +---+---+---+---+ - | | | | O | T | D | I | - +================================================+===================================+===================+===+===+===+===+ - | :c:member:`~PyTypeObject.tp_name` | const char * | __name__ | X | X | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_basicsize` | :c:type:`Py_ssize_t` | | X | X | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_itemsize` | :c:type:`Py_ssize_t` | | | X | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dealloc` | :c:type:`destructor` | | X | X | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_vectorcall_offset` | :c:type:`Py_ssize_t` | | | X | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_getattr`) | :c:type:`getattrfunc` | __getattribute__, | | | | G | - | | | __getattr__ | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_setattr`) | :c:type:`setattrfunc` | __setattr__, | | | | G | - | | | __delattr__ | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_async` | :c:type:`PyAsyncMethods` * | :ref:`sub-slots` | | | | % | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_repr` | :c:type:`reprfunc` | __repr__ | X | X | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_number` | :c:type:`PyNumberMethods` * | :ref:`sub-slots` | | | | % | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_sequence` | :c:type:`PySequenceMethods` * | :ref:`sub-slots` | | | | % | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_mapping` | :c:type:`PyMappingMethods` * | :ref:`sub-slots` | | | | % | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_hash` | :c:type:`hashfunc` | __hash__ | X | | | G | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_call` | :c:type:`ternaryfunc` | __call__ | | X | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_str` | :c:type:`reprfunc` | __str__ | X | | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getattro` | :c:type:`getattrofunc` | __getattribute__, | X | X | | G | - | | | __getattr__ | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_setattro` | :c:type:`setattrofunc` | __setattr__, | X | X | | G | - | | | __delattr__ | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_as_buffer` | :c:type:`PyBufferProcs` * | | | | | % | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_flags` | unsigned long | | X | X | | ? | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_doc` | const char * | __doc__ | X | X | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_traverse` | :c:type:`traverseproc` | | | X | | G | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_clear` | :c:type:`inquiry` | | | X | | G | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_richcompare` | :c:type:`richcmpfunc` | __lt__, | X | | | G | - | | | __le__, | | | | | - | | | __eq__, | | | | | - | | | __ne__, | | | | | - | | | __gt__, | | | | | - | | | __ge__ | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_weaklistoffset` | :c:type:`Py_ssize_t` | | | X | | ? | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iter` | :c:type:`getiterfunc` | __iter__ | | | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_iternext` | :c:type:`iternextfunc` | __next__ | | | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_methods` | :c:type:`PyMethodDef` [] | | X | X | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_members` | :c:type:`PyMemberDef` [] | | | X | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_getset` | :c:type:`PyGetSetDef` [] | | X | X | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_base` | :c:type:`PyTypeObject` * | __base__ | | | X | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dict` | :c:type:`PyObject` * | __dict__ | | | ? | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_get` | :c:type:`descrgetfunc` | __get__ | | | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_descr_set` | :c:type:`descrsetfunc` | __set__, | | | | X | - | | | __delete__ | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_dictoffset` | :c:type:`Py_ssize_t` | | | X | | ? | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_init` | :c:type:`initproc` | __init__ | X | X | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_alloc` | :c:type:`allocfunc` | | X | | ? | ? | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_new` | :c:type:`newfunc` | __new__ | X | X | ? | ? | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_free` | :c:type:`freefunc` | | X | X | ? | ? | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_is_gc` | :c:type:`inquiry` | | | X | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_bases`> | :c:type:`PyObject` * | __bases__ | | | ~ | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | <:c:member:`~PyTypeObject.tp_mro`> | :c:type:`PyObject` * | __mro__ | | | ~ | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_cache`] | :c:type:`PyObject` * | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_subclasses`] | :c:type:`PyObject` * | __subclasses__ | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_weaklist`] | :c:type:`PyObject` * | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | (:c:member:`~PyTypeObject.tp_del`) | :c:type:`destructor` | | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | [:c:member:`~PyTypeObject.tp_version_tag`] | unsigned int | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_finalize` | :c:type:`destructor` | __del__ | | | | X | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - | :c:member:`~PyTypeObject.tp_vectorcall` | :c:type:`vectorcallfunc` | | | | | | - +------------------------------------------------+-----------------------------------+-------------------+---+---+---+---+ - -.. [#slots] - - **()**: A slot name in parentheses indicates it is (effectively) deprecated. - - **<>**: Names in angle brackets should be initially set to ``NULL`` and - treated as read-only. - - **[]**: Names in square brackets are for internal use only. - - **** (as a prefix) means the field is required (must be non-``NULL``). - -.. [#cols] Columns: - - **"O"**: set on :c:data:`PyBaseObject_Type` - - **"T"**: set on :c:data:`PyType_Type` - - **"D"**: default (if slot is set to ``NULL``) - - .. code-block:: none - - X - PyType_Ready sets this value if it is NULL - ~ - PyType_Ready always sets this value (it should be NULL) - ? - PyType_Ready may set this value depending on other slots - - Also see the inheritance column ("I"). - - **"I"**: inheritance - - .. code-block:: none - - X - type slot is inherited via *PyType_Ready* if defined with a *NULL* value - % - the slots of the sub-struct are inherited individually - G - inherited, but only in combination with other slots; see the slot's description - ? - it's complicated; see the slot's description - - Note that some slots are effectively inherited through the normal - attribute lookup chain. - -.. _sub-slots: - -sub-slots -^^^^^^^^^ - -.. table:: - :widths: 26,17,12 - - +---------------------------------------------------------+-----------------------------------+---------------+ - | Slot | :ref:`Type ` | special | - | | | methods | - +=========================================================+===================================+===============+ - | :c:member:`~PyAsyncMethods.am_await` | :c:type:`unaryfunc` | __await__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyAsyncMethods.am_aiter` | :c:type:`unaryfunc` | __aiter__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyAsyncMethods.am_anext` | :c:type:`unaryfunc` | __anext__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyAsyncMethods.am_send` | :c:type:`sendfunc` | | - +---------------------------------------------------------+-----------------------------------+---------------+ - | | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_add` | :c:type:`binaryfunc` | __add__ | - | | | __radd__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_add` | :c:type:`binaryfunc` | __iadd__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_subtract` | :c:type:`binaryfunc` | __sub__ | - | | | __rsub__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_subtract` | :c:type:`binaryfunc` | __isub__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_multiply` | :c:type:`binaryfunc` | __mul__ | - | | | __rmul__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_multiply` | :c:type:`binaryfunc` | __imul__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_remainder` | :c:type:`binaryfunc` | __mod__ | - | | | __rmod__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_remainder` | :c:type:`binaryfunc` | __imod__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_divmod` | :c:type:`binaryfunc` | __divmod__ | - | | | __rdivmod__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_power` | :c:type:`ternaryfunc` | __pow__ | - | | | __rpow__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_power` | :c:type:`ternaryfunc` | __ipow__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_negative` | :c:type:`unaryfunc` | __neg__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_positive` | :c:type:`unaryfunc` | __pos__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_absolute` | :c:type:`unaryfunc` | __abs__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_bool` | :c:type:`inquiry` | __bool__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_invert` | :c:type:`unaryfunc` | __invert__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_lshift` | :c:type:`binaryfunc` | __lshift__ | - | | | __rlshift__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_lshift` | :c:type:`binaryfunc` | __ilshift__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_rshift` | :c:type:`binaryfunc` | __rshift__ | - | | | __rrshift__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_rshift` | :c:type:`binaryfunc` | __irshift__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_and` | :c:type:`binaryfunc` | __and__ | - | | | __rand__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_and` | :c:type:`binaryfunc` | __iand__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_xor` | :c:type:`binaryfunc` | __xor__ | - | | | __rxor__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_xor` | :c:type:`binaryfunc` | __ixor__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_or` | :c:type:`binaryfunc` | __or__ | - | | | __ror__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_or` | :c:type:`binaryfunc` | __ior__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_int` | :c:type:`unaryfunc` | __int__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_reserved` | void * | | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_float` | :c:type:`unaryfunc` | __float__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_floor_divide` | :c:type:`binaryfunc` | __floordiv__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_floor_divide` | :c:type:`binaryfunc` | __ifloordiv__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_true_divide` | :c:type:`binaryfunc` | __truediv__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_true_divide` | :c:type:`binaryfunc` | __itruediv__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_index` | :c:type:`unaryfunc` | __index__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_matrix_multiply` | :c:type:`binaryfunc` | __matmul__ | - | | | __rmatmul__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyNumberMethods.nb_inplace_matrix_multiply` | :c:type:`binaryfunc` | __imatmul__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyMappingMethods.mp_length` | :c:type:`lenfunc` | __len__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyMappingMethods.mp_subscript` | :c:type:`binaryfunc` | __getitem__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyMappingMethods.mp_ass_subscript` | :c:type:`objobjargproc` | __setitem__, | - | | | __delitem__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PySequenceMethods.sq_length` | :c:type:`lenfunc` | __len__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PySequenceMethods.sq_concat` | :c:type:`binaryfunc` | __add__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PySequenceMethods.sq_repeat` | :c:type:`ssizeargfunc` | __mul__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PySequenceMethods.sq_item` | :c:type:`ssizeargfunc` | __getitem__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PySequenceMethods.sq_ass_item` | :c:type:`ssizeobjargproc` | __setitem__ | - | | | __delitem__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PySequenceMethods.sq_contains` | :c:type:`objobjproc` | __contains__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PySequenceMethods.sq_inplace_concat` | :c:type:`binaryfunc` | __iadd__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PySequenceMethods.sq_inplace_repeat` | :c:type:`ssizeargfunc` | __imul__ | - +---------------------------------------------------------+-----------------------------------+---------------+ - | | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_getbuffer` | :c:func:`getbufferproc` | | - +---------------------------------------------------------+-----------------------------------+---------------+ - | :c:member:`~PyBufferProcs.bf_releasebuffer` | :c:func:`releasebufferproc` | | - +---------------------------------------------------------+-----------------------------------+---------------+ - -.. _slot-typedefs-table: - -slot typedefs -^^^^^^^^^^^^^ - -+-----------------------------+-----------------------------+----------------------+ -| typedef | Parameter Types | Return Type | -+=============================+=============================+======================+ -| :c:type:`allocfunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyTypeObject` * | | -| | :c:type:`Py_ssize_t` | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`destructor` | void * | void | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`freefunc` | void * | void | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`traverseproc` | .. line-block:: | int | -| | | | -| | void * | | -| | :c:type:`visitproc` | | -| | void * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`newfunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`initproc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`reprfunc` | :c:type:`PyObject` * | :c:type:`PyObject` * | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`getattrfunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyObject` * | | -| | const char * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`setattrfunc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | const char * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`getattrofunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`setattrofunc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`descrgetfunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`descrsetfunc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`hashfunc` | :c:type:`PyObject` * | Py_hash_t | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`richcmpfunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -| | int | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`getiterfunc` | :c:type:`PyObject` * | :c:type:`PyObject` * | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`iternextfunc` | :c:type:`PyObject` * | :c:type:`PyObject` * | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`lenfunc` | :c:type:`PyObject` * | :c:type:`Py_ssize_t` | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`getbufferproc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`Py_buffer` * | | -| | int | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`releasebufferproc` | .. line-block:: | void | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`Py_buffer` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`inquiry` | void * | int | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`unaryfunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`binaryfunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`ternaryfunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`ssizeargfunc` | .. line-block:: | :c:type:`PyObject` * | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`Py_ssize_t` | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`ssizeobjargproc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`Py_ssize_t` | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`objobjproc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ -| :c:type:`objobjargproc` | .. line-block:: | int | -| | | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -| | :c:type:`PyObject` * | | -+-----------------------------+-----------------------------+----------------------+ - -See :ref:`slot-typedefs` below for more detail. - - -PyTypeObject Definition ------------------------ - -The structure definition for :c:type:`PyTypeObject` can be found in -:file:`Include/object.h`. For convenience of reference, this repeats the -definition found there: - -.. XXX Drop this? - -.. literalinclude:: ../includes/typestruct.h - - -PyObject Slots --------------- - -The type object structure extends the :c:type:`PyVarObject` structure. The -:c:member:`~PyVarObject.ob_size` field is used for dynamic types (created by :c:func:`!type_new`, -usually called from a class statement). Note that :c:data:`PyType_Type` (the -metatype) initializes :c:member:`~PyTypeObject.tp_itemsize`, which means that its instances (i.e. -type objects) *must* have the :c:member:`~PyVarObject.ob_size` field. - - -.. c:member:: Py_ssize_t PyObject.ob_refcnt - - This is the type object's reference count, initialized to ``1`` by the - ``PyObject_HEAD_INIT`` macro. Note that for :ref:`statically allocated type - objects `, the type's instances (objects whose :c:member:`~PyObject.ob_type` - points back to the type) do *not* count as references. But for - :ref:`dynamically allocated type objects `, the instances *do* - count as references. - - **Inheritance:** - - This field is not inherited by subtypes. - - -.. c:member:: PyTypeObject* PyObject.ob_type - - This is the type's type, in other words its metatype. It is initialized by the - argument to the ``PyObject_HEAD_INIT`` macro, and its value should normally be - ``&PyType_Type``. However, for dynamically loadable extension modules that must - be usable on Windows (at least), the compiler complains that this is not a valid - initializer. Therefore, the convention is to pass ``NULL`` to the - ``PyObject_HEAD_INIT`` macro and to initialize this field explicitly at the - start of the module's initialization function, before doing anything else. This - is typically done like this:: - - Foo_Type.ob_type = &PyType_Type; - - This should be done before any instances of the type are created. - :c:func:`PyType_Ready` checks if :c:member:`~PyObject.ob_type` is ``NULL``, and if so, - initializes it to the :c:member:`~PyObject.ob_type` field of the base class. - :c:func:`PyType_Ready` will not change this field if it is non-zero. - - **Inheritance:** - - This field is inherited by subtypes. - - -.. c:member:: PyObject* PyObject._ob_next - PyObject* PyObject._ob_prev - - These fields are only present when the macro ``Py_TRACE_REFS`` is defined - (see the :option:`configure --with-trace-refs option <--with-trace-refs>`). - - Their initialization to ``NULL`` is taken care of by the - ``PyObject_HEAD_INIT`` macro. For :ref:`statically allocated objects - `, these fields always remain ``NULL``. For :ref:`dynamically - allocated objects `, these two fields are used to link the - object into a doubly linked list of *all* live objects on the heap. - - This could be used for various debugging purposes; currently the only uses - are the :func:`sys.getobjects` function and to print the objects that are - still alive at the end of a run when the environment variable - :envvar:`PYTHONDUMPREFS` is set. - - **Inheritance:** - - These fields are not inherited by subtypes. - - -PyVarObject Slots ------------------ - -.. c:member:: Py_ssize_t PyVarObject.ob_size - - For :ref:`statically allocated type objects `, this should be - initialized to zero. For :ref:`dynamically allocated type objects - `, this field has a special internal meaning. - - **Inheritance:** - - This field is not inherited by subtypes. - - -PyTypeObject Slots ------------------- - -Each slot has a section describing inheritance. If :c:func:`PyType_Ready` -may set a value when the field is set to ``NULL`` then there will also be -a "Default" section. (Note that many fields set on :c:data:`PyBaseObject_Type` -and :c:data:`PyType_Type` effectively act as defaults.) - -.. c:member:: const char* PyTypeObject.tp_name - - Pointer to a NUL-terminated string containing the name of the type. For types - that are accessible as module globals, the string should be the full module - name, followed by a dot, followed by the type name; for built-in types, it - should be just the type name. If the module is a submodule of a package, the - full package name is part of the full module name. For example, a type named - :class:`!T` defined in module :mod:`!M` in subpackage :mod:`!Q` in package :mod:`!P` - should have the :c:member:`~PyTypeObject.tp_name` initializer ``"P.Q.M.T"``. - - For :ref:`dynamically allocated type objects `, - this should just be the type name, and - the module name explicitly stored in the type dict as the value for key - ``'__module__'``. - - For :ref:`statically allocated type objects `, - the *tp_name* field should contain a dot. - Everything before the last dot is made accessible as the :attr:`__module__` - attribute, and everything after the last dot is made accessible as the - :attr:`~definition.__name__` attribute. - - If no dot is present, the entire :c:member:`~PyTypeObject.tp_name` field is made accessible as the - :attr:`~definition.__name__` attribute, and the :attr:`__module__` attribute is undefined - (unless explicitly set in the dictionary, as explained above). This means your - type will be impossible to pickle. Additionally, it will not be listed in - module documentations created with pydoc. - - This field must not be ``NULL``. It is the only required field - in :c:func:`PyTypeObject` (other than potentially - :c:member:`~PyTypeObject.tp_itemsize`). - - **Inheritance:** - - This field is not inherited by subtypes. - - -.. c:member:: Py_ssize_t PyTypeObject.tp_basicsize - Py_ssize_t PyTypeObject.tp_itemsize - - These fields allow calculating the size in bytes of instances of the type. - - There are two kinds of types: types with fixed-length instances have a zero - :c:member:`~PyTypeObject.tp_itemsize` field, types with variable-length instances have a non-zero - :c:member:`~PyTypeObject.tp_itemsize` field. For a type with fixed-length instances, all - instances have the same size, given in :c:member:`~PyTypeObject.tp_basicsize`. - - For a type with variable-length instances, the instances must have an - :c:member:`~PyVarObject.ob_size` field, and the instance size is :c:member:`~PyTypeObject.tp_basicsize` plus N - times :c:member:`~PyTypeObject.tp_itemsize`, where N is the "length" of the object. The value of - N is typically stored in the instance's :c:member:`~PyVarObject.ob_size` field. There are - exceptions: for example, ints use a negative :c:member:`~PyVarObject.ob_size` to indicate a - negative number, and N is ``abs(ob_size)`` there. Also, the presence of an - :c:member:`~PyVarObject.ob_size` field in the instance layout doesn't mean that the instance - structure is variable-length (for example, the structure for the list type has - fixed-length instances, yet those instances have a meaningful :c:member:`~PyVarObject.ob_size` - field). - - The basic size includes the fields in the instance declared by the macro - :c:macro:`PyObject_HEAD` or :c:macro:`PyObject_VAR_HEAD` (whichever is used to - declare the instance struct) and this in turn includes the :c:member:`~PyObject._ob_prev` and - :c:member:`~PyObject._ob_next` fields if they are present. This means that the only correct - way to get an initializer for the :c:member:`~PyTypeObject.tp_basicsize` is to use the - ``sizeof`` operator on the struct used to declare the instance layout. - The basic size does not include the GC header size. - - A note about alignment: if the variable items require a particular alignment, - this should be taken care of by the value of :c:member:`~PyTypeObject.tp_basicsize`. Example: - suppose a type implements an array of ``double``. :c:member:`~PyTypeObject.tp_itemsize` is - ``sizeof(double)``. It is the programmer's responsibility that - :c:member:`~PyTypeObject.tp_basicsize` is a multiple of ``sizeof(double)`` (assuming this is the - alignment requirement for ``double``). - - For any type with variable-length instances, this field must not be ``NULL``. - - **Inheritance:** - - These fields are inherited separately by subtypes. If the base type has a - non-zero :c:member:`~PyTypeObject.tp_itemsize`, it is generally not safe to set - :c:member:`~PyTypeObject.tp_itemsize` to a different non-zero value in a subtype (though this - depends on the implementation of the base type). - - -.. c:member:: destructor PyTypeObject.tp_dealloc - - A pointer to the instance destructor function. This function must be defined - unless the type guarantees that its instances will never be deallocated (as is - the case for the singletons ``None`` and ``Ellipsis``). The function signature is:: - - void tp_dealloc(PyObject *self); - - The destructor function is called by the :c:func:`Py_DECREF` and - :c:func:`Py_XDECREF` macros when the new reference count is zero. At this point, - the instance is still in existence, but there are no references to it. The - destructor function should free all references which the instance owns, free all - memory buffers owned by the instance (using the freeing function corresponding - to the allocation function used to allocate the buffer), and call the type's - :c:member:`~PyTypeObject.tp_free` function. If the type is not subtypable - (doesn't have the :c:macro:`Py_TPFLAGS_BASETYPE` flag bit set), it is - permissible to call the object deallocator directly instead of via - :c:member:`~PyTypeObject.tp_free`. The object deallocator should be the one used to allocate the - instance; this is normally :c:func:`PyObject_Del` if the instance was allocated - using :c:macro:`PyObject_New` or :c:macro:`PyObject_NewVar`, or - :c:func:`PyObject_GC_Del` if the instance was allocated using - :c:macro:`PyObject_GC_New` or :c:macro:`PyObject_GC_NewVar`. - - If the type supports garbage collection (has the :c:macro:`Py_TPFLAGS_HAVE_GC` - flag bit set), the destructor should call :c:func:`PyObject_GC_UnTrack` - before clearing any member fields. - - .. code-block:: c - - static void foo_dealloc(foo_object *self) { - PyObject_GC_UnTrack(self); - Py_CLEAR(self->ref); - Py_TYPE(self)->tp_free((PyObject *)self); - } - - Finally, if the type is heap allocated (:c:macro:`Py_TPFLAGS_HEAPTYPE`), the - deallocator should release the owned reference to its type object - (via :c:func:`Py_DECREF`) after - calling the type deallocator. In order to avoid dangling pointers, the - recommended way to achieve this is: - - .. code-block:: c - - static void foo_dealloc(foo_object *self) { - PyTypeObject *tp = Py_TYPE(self); - // free references and buffers here - tp->tp_free(self); - Py_DECREF(tp); - } - - - **Inheritance:** - - This field is inherited by subtypes. - - -.. c:member:: Py_ssize_t PyTypeObject.tp_vectorcall_offset - - An optional offset to a per-instance function that implements calling - the object using the :ref:`vectorcall protocol `, - a more efficient alternative - of the simpler :c:member:`~PyTypeObject.tp_call`. - - This field is only used if the flag :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` - is set. If so, this must be a positive integer containing the offset in the - instance of a :c:type:`vectorcallfunc` pointer. - - The *vectorcallfunc* pointer may be ``NULL``, in which case the instance behaves - as if :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` was not set: calling the instance - falls back to :c:member:`~PyTypeObject.tp_call`. - - Any class that sets ``Py_TPFLAGS_HAVE_VECTORCALL`` must also set - :c:member:`~PyTypeObject.tp_call` and make sure its behaviour is consistent - with the *vectorcallfunc* function. - This can be done by setting *tp_call* to :c:func:`PyVectorcall_Call`. - - .. warning:: - - It is not recommended for :ref:`mutable heap types ` to implement - the vectorcall protocol. - When a user sets :attr:`__call__` in Python code, only *tp_call* is updated, - likely making it inconsistent with the vectorcall function. - - .. versionchanged:: 3.8 - - Before version 3.8, this slot was named ``tp_print``. - In Python 2.x, it was used for printing to a file. - In Python 3.0 to 3.7, it was unused. - - **Inheritance:** - - This field is always inherited. - However, the :c:macro:`Py_TPFLAGS_HAVE_VECTORCALL` flag is not - always inherited. If it's not, then the subclass won't use - :ref:`vectorcall `, except when - :c:func:`PyVectorcall_Call` is explicitly called. - This is in particular the case for types without the - :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag set (including subclasses defined in - Python). - - -.. c:member:: getattrfunc PyTypeObject.tp_getattr - - An optional pointer to the get-attribute-string function. - - This field is deprecated. When it is defined, it should point to a function - that acts the same as the :c:member:`~PyTypeObject.tp_getattro` function, but taking a C string - instead of a Python string object to give the attribute name. - - **Inheritance:** - - Group: :c:member:`~PyTypeObject.tp_getattr`, :c:member:`~PyTypeObject.tp_getattro` - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattro`: a subtype - inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both ``NULL``. - - -.. c:member:: setattrfunc PyTypeObject.tp_setattr - - An optional pointer to the function for setting and deleting attributes. - - This field is deprecated. When it is defined, it should point to a function - that acts the same as the :c:member:`~PyTypeObject.tp_setattro` function, but taking a C string - instead of a Python string object to give the attribute name. - - **Inheritance:** - - Group: :c:member:`~PyTypeObject.tp_setattr`, :c:member:`~PyTypeObject.tp_setattro` - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattro`: a subtype - inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both ``NULL``. - - -.. c:member:: PyAsyncMethods* PyTypeObject.tp_as_async - - Pointer to an additional structure that contains fields relevant only to - objects which implement :term:`awaitable` and :term:`asynchronous iterator` - protocols at the C-level. See :ref:`async-structs` for details. - - .. versionadded:: 3.5 - Formerly known as ``tp_compare`` and ``tp_reserved``. - - **Inheritance:** - - The :c:member:`~PyTypeObject.tp_as_async` field is not inherited, - but the contained fields are inherited individually. - - -.. c:member:: reprfunc PyTypeObject.tp_repr - - .. index:: pair: built-in function; repr - - An optional pointer to a function that implements the built-in function - :func:`repr`. - - The signature is the same as for :c:func:`PyObject_Repr`:: - - PyObject *tp_repr(PyObject *self); - - The function must return a string or a Unicode object. Ideally, - this function should return a string that, when passed to - :func:`eval`, given a suitable environment, returns an object with the - same value. If this is not feasible, it should return a string starting with - ``'<'`` and ending with ``'>'`` from which both the type and the value of the - object can be deduced. - - **Inheritance:** - - This field is inherited by subtypes. - - **Default:** - - When this field is not set, a string of the form ``<%s object at %p>`` is - returned, where ``%s`` is replaced by the type name, and ``%p`` by the object's - memory address. - - -.. c:member:: PyNumberMethods* PyTypeObject.tp_as_number - - Pointer to an additional structure that contains fields relevant only to - objects which implement the number protocol. These fields are documented in - :ref:`number-structs`. - - **Inheritance:** - - The :c:member:`~PyTypeObject.tp_as_number` field is not inherited, but the contained fields are - inherited individually. - - -.. c:member:: PySequenceMethods* PyTypeObject.tp_as_sequence - - Pointer to an additional structure that contains fields relevant only to - objects which implement the sequence protocol. These fields are documented - in :ref:`sequence-structs`. - - **Inheritance:** - - The :c:member:`~PyTypeObject.tp_as_sequence` field is not inherited, but the contained fields - are inherited individually. - - -.. c:member:: PyMappingMethods* PyTypeObject.tp_as_mapping - - Pointer to an additional structure that contains fields relevant only to - objects which implement the mapping protocol. These fields are documented in - :ref:`mapping-structs`. - - **Inheritance:** - - The :c:member:`~PyTypeObject.tp_as_mapping` field is not inherited, but the contained fields - are inherited individually. - - -.. c:member:: hashfunc PyTypeObject.tp_hash - - .. index:: pair: built-in function; hash - - An optional pointer to a function that implements the built-in function - :func:`hash`. - - The signature is the same as for :c:func:`PyObject_Hash`:: - - Py_hash_t tp_hash(PyObject *); - - The value ``-1`` should not be returned as a - normal return value; when an error occurs during the computation of the hash - value, the function should set an exception and return ``-1``. - - When this field is not set (*and* :c:member:`~PyTypeObject.tp_richcompare` is not set), - an attempt to take the hash of the object raises :exc:`TypeError`. - This is the same as setting it to :c:func:`PyObject_HashNotImplemented`. - - This field can be set explicitly to :c:func:`PyObject_HashNotImplemented` to - block inheritance of the hash method from a parent type. This is interpreted - as the equivalent of ``__hash__ = None`` at the Python level, causing - ``isinstance(o, collections.Hashable)`` to correctly return ``False``. Note - that the converse is also true - setting ``__hash__ = None`` on a class at - the Python level will result in the ``tp_hash`` slot being set to - :c:func:`PyObject_HashNotImplemented`. - - **Inheritance:** - - Group: :c:member:`~PyTypeObject.tp_hash`, :c:member:`~PyTypeObject.tp_richcompare` - - This field is inherited by subtypes together with - :c:member:`~PyTypeObject.tp_richcompare`: a subtype inherits both of - :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash`, when the subtype's - :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both ``NULL``. - - -.. c:member:: ternaryfunc PyTypeObject.tp_call - - An optional pointer to a function that implements calling the object. This - should be ``NULL`` if the object is not callable. The signature is the same as - for :c:func:`PyObject_Call`:: - - PyObject *tp_call(PyObject *self, PyObject *args, PyObject *kwargs); - - **Inheritance:** - - This field is inherited by subtypes. - - -.. c:member:: reprfunc PyTypeObject.tp_str - - An optional pointer to a function that implements the built-in operation - :func:`str`. (Note that :class:`str` is a type now, and :func:`str` calls the - constructor for that type. This constructor calls :c:func:`PyObject_Str` to do - the actual work, and :c:func:`PyObject_Str` will call this handler.) - - The signature is the same as for :c:func:`PyObject_Str`:: - - PyObject *tp_str(PyObject *self); - - The function must return a string or a Unicode object. It should be a "friendly" string - representation of the object, as this is the representation that will be used, - among other things, by the :func:`print` function. - - **Inheritance:** - - This field is inherited by subtypes. - - **Default:** - - When this field is not set, :c:func:`PyObject_Repr` is called to return a string - representation. - - -.. c:member:: getattrofunc PyTypeObject.tp_getattro - - An optional pointer to the get-attribute function. - - The signature is the same as for :c:func:`PyObject_GetAttr`:: - - PyObject *tp_getattro(PyObject *self, PyObject *attr); - - It is usually convenient to set this field to :c:func:`PyObject_GenericGetAttr`, - which implements the normal way of looking for object attributes. - - **Inheritance:** - - Group: :c:member:`~PyTypeObject.tp_getattr`, :c:member:`~PyTypeObject.tp_getattro` - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_getattr`: a subtype - inherits both :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_getattr` and :c:member:`~PyTypeObject.tp_getattro` are both ``NULL``. - - **Default:** - - :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericGetAttr`. - - -.. c:member:: setattrofunc PyTypeObject.tp_setattro - - An optional pointer to the function for setting and deleting attributes. - - The signature is the same as for :c:func:`PyObject_SetAttr`:: - - int tp_setattro(PyObject *self, PyObject *attr, PyObject *value); - - In addition, setting *value* to ``NULL`` to delete an attribute must be - supported. It is usually convenient to set this field to - :c:func:`PyObject_GenericSetAttr`, which implements the normal - way of setting object attributes. - - **Inheritance:** - - Group: :c:member:`~PyTypeObject.tp_setattr`, :c:member:`~PyTypeObject.tp_setattro` - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_setattr`: a subtype - inherits both :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` from its base type when - the subtype's :c:member:`~PyTypeObject.tp_setattr` and :c:member:`~PyTypeObject.tp_setattro` are both ``NULL``. - - **Default:** - - :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_GenericSetAttr`. - - -.. c:member:: PyBufferProcs* PyTypeObject.tp_as_buffer - - Pointer to an additional structure that contains fields relevant only to objects - which implement the buffer interface. These fields are documented in - :ref:`buffer-structs`. - - **Inheritance:** - - The :c:member:`~PyTypeObject.tp_as_buffer` field is not inherited, - but the contained fields are inherited individually. - - -.. c:member:: unsigned long PyTypeObject.tp_flags - - This field is a bit mask of various flags. Some flags indicate variant - semantics for certain situations; others are used to indicate that certain - fields in the type object (or in the extension structures referenced via - :c:member:`~PyTypeObject.tp_as_number`, :c:member:`~PyTypeObject.tp_as_sequence`, :c:member:`~PyTypeObject.tp_as_mapping`, and - :c:member:`~PyTypeObject.tp_as_buffer`) that were historically not always present are valid; if - such a flag bit is clear, the type fields it guards must not be accessed and - must be considered to have a zero or ``NULL`` value instead. - - **Inheritance:** - - Inheritance of this field is complicated. Most flag bits are inherited - individually, i.e. if the base type has a flag bit set, the subtype inherits - this flag bit. The flag bits that pertain to extension structures are strictly - inherited if the extension structure is inherited, i.e. the base type's value of - the flag bit is copied into the subtype together with a pointer to the extension - structure. The :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is inherited together with - the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields, i.e. if the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is clear in the subtype and the - :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have - ``NULL`` values. - - .. XXX are most flag bits *really* inherited individually? - - **Default:** - - :c:data:`PyBaseObject_Type` uses - ``Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE``. - - **Bit Masks:** - - .. c:namespace:: NULL - - The following bit masks are currently defined; these can be ORed together using - the ``|`` operator to form the value of the :c:member:`~PyTypeObject.tp_flags` field. The macro - :c:func:`PyType_HasFeature` takes a type and a flags value, *tp* and *f*, and - checks whether ``tp->tp_flags & f`` is non-zero. - - .. c:macro:: Py_TPFLAGS_HEAPTYPE - - This bit is set when the type object itself is allocated on the heap, for - example, types created dynamically using :c:func:`PyType_FromSpec`. In this - case, the :c:member:`~PyObject.ob_type` field of its instances is considered a reference to - the type, and the type object is INCREF'ed when a new instance is created, and - DECREF'ed when an instance is destroyed (this does not apply to instances of - subtypes; only the type referenced by the instance's ob_type gets INCREF'ed or - DECREF'ed). - - **Inheritance:** - - ??? - - - .. c:macro:: Py_TPFLAGS_BASETYPE - - This bit is set when the type can be used as the base type of another type. If - this bit is clear, the type cannot be subtyped (similar to a "final" class in - Java). - - **Inheritance:** - - ??? - - - .. c:macro:: Py_TPFLAGS_READY - - This bit is set when the type object has been fully initialized by - :c:func:`PyType_Ready`. - - **Inheritance:** - - ??? - - - .. c:macro:: Py_TPFLAGS_READYING - - This bit is set while :c:func:`PyType_Ready` is in the process of initializing - the type object. - - **Inheritance:** - - ??? - - - .. c:macro:: Py_TPFLAGS_HAVE_GC - - This bit is set when the object supports garbage collection. If this bit - is set, instances must be created using :c:macro:`PyObject_GC_New` and - destroyed using :c:func:`PyObject_GC_Del`. More information in section - :ref:`supporting-cycle-detection`. This bit also implies that the - GC-related fields :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` are present in - the type object. - - **Inheritance:** - - Group: :c:macro:`Py_TPFLAGS_HAVE_GC`, :c:member:`~PyTypeObject.tp_traverse`, :c:member:`~PyTypeObject.tp_clear` - - The :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is inherited - together with the :c:member:`~PyTypeObject.tp_traverse` and :c:member:`~PyTypeObject.tp_clear` - fields, i.e. if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is - clear in the subtype and the :c:member:`~PyTypeObject.tp_traverse` and - :c:member:`~PyTypeObject.tp_clear` fields in the subtype exist and have ``NULL`` - values. - - - .. c:macro:: Py_TPFLAGS_DEFAULT - - This is a bitmask of all the bits that pertain to the existence of certain - fields in the type object and its extension structures. Currently, it includes - the following bits: :c:macro:`Py_TPFLAGS_HAVE_STACKLESS_EXTENSION`. - - **Inheritance:** - - ??? - - - .. c:macro:: Py_TPFLAGS_METHOD_DESCRIPTOR - - This bit indicates that objects behave like unbound methods. - - If this flag is set for ``type(meth)``, then: - - - ``meth.__get__(obj, cls)(*args, **kwds)`` (with ``obj`` not None) - must be equivalent to ``meth(obj, *args, **kwds)``. - - - ``meth.__get__(None, cls)(*args, **kwds)`` - must be equivalent to ``meth(*args, **kwds)``. - - This flag enables an optimization for typical method calls like - ``obj.meth()``: it avoids creating a temporary "bound method" object for - ``obj.meth``. - - .. versionadded:: 3.8 - - **Inheritance:** - - This flag is never inherited by types without the - :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag set. For extension types, it is - inherited whenever :c:member:`~PyTypeObject.tp_descr_get` is inherited. - - - .. XXX Document more flags here? - - - .. c:macro:: Py_TPFLAGS_LONG_SUBCLASS - .. c:macro:: Py_TPFLAGS_LIST_SUBCLASS - .. c:macro:: Py_TPFLAGS_TUPLE_SUBCLASS - .. c:macro:: Py_TPFLAGS_BYTES_SUBCLASS - .. c:macro:: Py_TPFLAGS_UNICODE_SUBCLASS - .. c:macro:: Py_TPFLAGS_DICT_SUBCLASS - .. c:macro:: Py_TPFLAGS_BASE_EXC_SUBCLASS - .. c:macro:: Py_TPFLAGS_TYPE_SUBCLASS - - These flags are used by functions such as - :c:func:`PyLong_Check` to quickly determine if a type is a subclass - of a built-in type; such specific checks are faster than a generic - check, like :c:func:`PyObject_IsInstance`. Custom types that inherit - from built-ins should have their :c:member:`~PyTypeObject.tp_flags` - set appropriately, or the code that interacts with such types - will behave differently depending on what kind of check is used. - - - .. c:macro:: Py_TPFLAGS_HAVE_FINALIZE - - This bit is set when the :c:member:`~PyTypeObject.tp_finalize` slot is present in the - type structure. - - .. versionadded:: 3.4 - - .. deprecated:: 3.8 - This flag isn't necessary anymore, as the interpreter assumes the - :c:member:`~PyTypeObject.tp_finalize` slot is always present in the - type structure. - - - .. c:macro:: Py_TPFLAGS_HAVE_VECTORCALL - - This bit is set when the class implements - the :ref:`vectorcall protocol `. - See :c:member:`~PyTypeObject.tp_vectorcall_offset` for details. - - **Inheritance:** - - This bit is inherited for types with the - :c:macro:`Py_TPFLAGS_IMMUTABLETYPE` flag set, if - :c:member:`~PyTypeObject.tp_call` is also inherited. - - .. versionadded:: 3.9 - - .. c:macro:: Py_TPFLAGS_IMMUTABLETYPE - - This bit is set for type objects that are immutable: type attributes cannot be set nor deleted. - - :c:func:`PyType_Ready` automatically applies this flag to - :ref:`static types `. - - **Inheritance:** - - This flag is not inherited. - - .. versionadded:: 3.10 - - .. c:macro:: Py_TPFLAGS_DISALLOW_INSTANTIATION - - Disallow creating instances of the type: set - :c:member:`~PyTypeObject.tp_new` to NULL and don't create the ``__new__`` - key in the type dictionary. - - The flag must be set before creating the type, not after. For example, it - must be set before :c:func:`PyType_Ready` is called on the type. - - The flag is set automatically on :ref:`static types ` if - :c:member:`~PyTypeObject.tp_base` is NULL or ``&PyBaseObject_Type`` and - :c:member:`~PyTypeObject.tp_new` is NULL. - - **Inheritance:** - - This flag is not inherited. - However, subclasses will not be instantiable unless they provide a - non-NULL :c:member:`~PyTypeObject.tp_new` (which is only possible - via the C API). - - .. note:: - - To disallow instantiating a class directly but allow instantiating - its subclasses (e.g. for an :term:`abstract base class`), - do not use this flag. - Instead, make :c:member:`~PyTypeObject.tp_new` only succeed for - subclasses. - - .. versionadded:: 3.10 - - - .. c:macro:: Py_TPFLAGS_MAPPING - - This bit indicates that instances of the class may match mapping patterns - when used as the subject of a :keyword:`match` block. It is automatically - set when registering or subclassing :class:`collections.abc.Mapping`, and - unset when registering :class:`collections.abc.Sequence`. - - .. note:: - - :c:macro:`Py_TPFLAGS_MAPPING` and :c:macro:`Py_TPFLAGS_SEQUENCE` are - mutually exclusive; it is an error to enable both flags simultaneously. - - **Inheritance:** - - This flag is inherited by types that do not already set - :c:macro:`Py_TPFLAGS_SEQUENCE`. - - .. seealso:: :pep:`634` -- Structural Pattern Matching: Specification - - .. versionadded:: 3.10 - - - .. c:macro:: Py_TPFLAGS_SEQUENCE - - This bit indicates that instances of the class may match sequence patterns - when used as the subject of a :keyword:`match` block. It is automatically - set when registering or subclassing :class:`collections.abc.Sequence`, and - unset when registering :class:`collections.abc.Mapping`. - - .. note:: - - :c:macro:`Py_TPFLAGS_MAPPING` and :c:macro:`Py_TPFLAGS_SEQUENCE` are - mutually exclusive; it is an error to enable both flags simultaneously. - - **Inheritance:** - - This flag is inherited by types that do not already set - :c:macro:`Py_TPFLAGS_MAPPING`. - - .. seealso:: :pep:`634` -- Structural Pattern Matching: Specification - - .. versionadded:: 3.10 - - -.. c:member:: const char* PyTypeObject.tp_doc - - An optional pointer to a NUL-terminated C string giving the docstring for this - type object. This is exposed as the :attr:`__doc__` attribute on the type and - instances of the type. - - **Inheritance:** - - This field is *not* inherited by subtypes. - - -.. c:member:: traverseproc PyTypeObject.tp_traverse - - An optional pointer to a traversal function for the garbage collector. This is - only used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: - - int tp_traverse(PyObject *self, visitproc visit, void *arg); - - More information about Python's garbage collection scheme can be found - in section :ref:`supporting-cycle-detection`. - - The :c:member:`~PyTypeObject.tp_traverse` pointer is used by the garbage collector to detect - reference cycles. A typical implementation of a :c:member:`~PyTypeObject.tp_traverse` function - simply calls :c:func:`Py_VISIT` on each of the instance's members that are Python - objects that the instance owns. For example, this is function :c:func:`!local_traverse` from the - :mod:`!_thread` extension module:: - - static int - local_traverse(localobject *self, visitproc visit, void *arg) - { - Py_VISIT(self->args); - Py_VISIT(self->kw); - Py_VISIT(self->dict); - return 0; - } - - Note that :c:func:`Py_VISIT` is called only on those members that can participate - in reference cycles. Although there is also a ``self->key`` member, it can only - be ``NULL`` or a Python string and therefore cannot be part of a reference cycle. - - On the other hand, even if you know a member can never be part of a cycle, as a - debugging aid you may want to visit it anyway just so the :mod:`gc` module's - :func:`~gc.get_referents` function will include it. - - .. warning:: - When implementing :c:member:`~PyTypeObject.tp_traverse`, only the - members that the instance *owns* (by having :term:`strong references - ` to them) must be - visited. For instance, if an object supports weak references via the - :c:member:`~PyTypeObject.tp_weaklist` slot, the pointer supporting - the linked list (what *tp_weaklist* points to) must **not** be - visited as the instance does not directly own the weak references to itself - (the weakreference list is there to support the weak reference machinery, - but the instance has no strong reference to the elements inside it, as they - are allowed to be removed even if the instance is still alive). - - Note that :c:func:`Py_VISIT` requires the *visit* and *arg* parameters to - :c:func:`local_traverse` to have these specific names; don't name them just - anything. - - Instances of :ref:`heap-allocated types ` hold a reference to - their type. Their traversal function must therefore either visit - :c:func:`Py_TYPE(self) `, or delegate this responsibility by - calling ``tp_traverse`` of another heap-allocated type (such as a - heap-allocated superclass). - If they do not, the type object may not be garbage-collected. - - .. versionchanged:: 3.9 - - Heap-allocated types are expected to visit ``Py_TYPE(self)`` in - ``tp_traverse``. In earlier versions of Python, due to - `bug 40217 `_, doing this - may lead to crashes in subclasses. - - **Inheritance:** - - Group: :c:macro:`Py_TPFLAGS_HAVE_GC`, :c:member:`~PyTypeObject.tp_traverse`, :c:member:`~PyTypeObject.tp_clear` - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_clear` and the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and - :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in - the subtype. - - -.. c:member:: inquiry PyTypeObject.tp_clear - - An optional pointer to a clear function for the garbage collector. This is only - used if the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit is set. The signature is:: - - int tp_clear(PyObject *); - - The :c:member:`~PyTypeObject.tp_clear` member function is used to break reference cycles in cyclic - garbage detected by the garbage collector. Taken together, all :c:member:`~PyTypeObject.tp_clear` - functions in the system must combine to break all reference cycles. This is - subtle, and if in any doubt supply a :c:member:`~PyTypeObject.tp_clear` function. For example, - the tuple type does not implement a :c:member:`~PyTypeObject.tp_clear` function, because it's - possible to prove that no reference cycle can be composed entirely of tuples. - Therefore the :c:member:`~PyTypeObject.tp_clear` functions of other types must be sufficient to - break any cycle containing a tuple. This isn't immediately obvious, and there's - rarely a good reason to avoid implementing :c:member:`~PyTypeObject.tp_clear`. - - Implementations of :c:member:`~PyTypeObject.tp_clear` should drop the instance's references to - those of its members that may be Python objects, and set its pointers to those - members to ``NULL``, as in the following example:: - - static int - local_clear(localobject *self) - { - Py_CLEAR(self->key); - Py_CLEAR(self->args); - Py_CLEAR(self->kw); - Py_CLEAR(self->dict); - return 0; - } - - The :c:func:`Py_CLEAR` macro should be used, because clearing references is - delicate: the reference to the contained object must not be released - (via :c:func:`Py_DECREF`) until - after the pointer to the contained object is set to ``NULL``. This is because - releasing the reference may cause the contained object to become trash, - triggering a chain of reclamation activity that may include invoking arbitrary - Python code (due to finalizers, or weakref callbacks, associated with the - contained object). If it's possible for such code to reference *self* again, - it's important that the pointer to the contained object be ``NULL`` at that time, - so that *self* knows the contained object can no longer be used. The - :c:func:`Py_CLEAR` macro performs the operations in a safe order. - - Note that :c:member:`~PyTypeObject.tp_clear` is not *always* called - before an instance is deallocated. For example, when reference counting - is enough to determine that an object is no longer used, the cyclic garbage - collector is not involved and :c:member:`~PyTypeObject.tp_dealloc` is - called directly. - - Because the goal of :c:member:`~PyTypeObject.tp_clear` functions is to break reference cycles, - it's not necessary to clear contained objects like Python strings or Python - integers, which can't participate in reference cycles. On the other hand, it may - be convenient to clear all contained Python objects, and write the type's - :c:member:`~PyTypeObject.tp_dealloc` function to invoke :c:member:`~PyTypeObject.tp_clear`. - - More information about Python's garbage collection scheme can be found in - section :ref:`supporting-cycle-detection`. - - **Inheritance:** - - Group: :c:macro:`Py_TPFLAGS_HAVE_GC`, :c:member:`~PyTypeObject.tp_traverse`, :c:member:`~PyTypeObject.tp_clear` - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_traverse` and the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit: the flag bit, :c:member:`~PyTypeObject.tp_traverse`, and - :c:member:`~PyTypeObject.tp_clear` are all inherited from the base type if they are all zero in - the subtype. - - -.. c:member:: richcmpfunc PyTypeObject.tp_richcompare - - An optional pointer to the rich comparison function, whose signature is:: - - PyObject *tp_richcompare(PyObject *self, PyObject *other, int op); - - The first parameter is guaranteed to be an instance of the type - that is defined by :c:type:`PyTypeObject`. - - The function should return the result of the comparison (usually ``Py_True`` - or ``Py_False``). If the comparison is undefined, it must return - ``Py_NotImplemented``, if another error occurred it must return ``NULL`` and - set an exception condition. - - The following constants are defined to be used as the third argument for - :c:member:`~PyTypeObject.tp_richcompare` and for :c:func:`PyObject_RichCompare`: - - .. c:namespace:: NULL - - +--------------------+------------+ - | Constant | Comparison | - +====================+============+ - | .. c:macro:: Py_LT | ``<`` | - +--------------------+------------+ - | .. c:macro:: Py_LE | ``<=`` | - +--------------------+------------+ - | .. c:macro:: Py_EQ | ``==`` | - +--------------------+------------+ - | .. c:macro:: Py_NE | ``!=`` | - +--------------------+------------+ - | .. c:macro:: Py_GT | ``>`` | - +--------------------+------------+ - | .. c:macro:: Py_GE | ``>=`` | - +--------------------+------------+ - - The following macro is defined to ease writing rich comparison functions: - - .. c:macro:: Py_RETURN_RICHCOMPARE(VAL_A, VAL_B, op) - - Return ``Py_True`` or ``Py_False`` from the function, depending on the - result of a comparison. - VAL_A and VAL_B must be orderable by C comparison operators (for example, - they may be C ints or floats). The third argument specifies the requested - operation, as for :c:func:`PyObject_RichCompare`. - - The returned value is a new :term:`strong reference`. - - On error, sets an exception and returns ``NULL`` from the function. - - .. versionadded:: 3.7 - - **Inheritance:** - - Group: :c:member:`~PyTypeObject.tp_hash`, :c:member:`~PyTypeObject.tp_richcompare` - - This field is inherited by subtypes together with :c:member:`~PyTypeObject.tp_hash`: - a subtype inherits :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` when - the subtype's :c:member:`~PyTypeObject.tp_richcompare` and :c:member:`~PyTypeObject.tp_hash` are both - ``NULL``. - - **Default:** - - :c:data:`PyBaseObject_Type` provides a :c:member:`~PyTypeObject.tp_richcompare` - implementation, which may be inherited. However, if only - :c:member:`~PyTypeObject.tp_hash` is defined, not even the inherited function is used - and instances of the type will not be able to participate in any - comparisons. - - -.. c:member:: Py_ssize_t PyTypeObject.tp_weaklistoffset - - If the instances of this type are weakly referenceable, this field is greater - than zero and contains the offset in the instance structure of the weak - reference list head (ignoring the GC header, if present); this offset is used by - :c:func:`PyObject_ClearWeakRefs` and the ``PyWeakref_*`` functions. The - instance structure needs to include a field of type :c:expr:`PyObject*` which is - initialized to ``NULL``. - - Do not confuse this field with :c:member:`~PyTypeObject.tp_weaklist`; that is the list head for - weak references to the type object itself. - - **Inheritance:** - - This field is inherited by subtypes, but see the rules listed below. A subtype - may override this offset; this means that the subtype uses a different weak - reference list head than the base type. Since the list head is always found via - :c:member:`~PyTypeObject.tp_weaklistoffset`, this should not be a problem. - - When a type defined by a class statement has no :attr:`~object.__slots__` declaration, - and none of its base types are weakly referenceable, the type is made weakly - referenceable by adding a weak reference list head slot to the instance layout - and setting the :c:member:`~PyTypeObject.tp_weaklistoffset` of that slot's offset. - - When a type's :attr:`__slots__` declaration contains a slot named - :attr:`__weakref__`, that slot becomes the weak reference list head for - instances of the type, and the slot's offset is stored in the type's - :c:member:`~PyTypeObject.tp_weaklistoffset`. - - When a type's :attr:`__slots__` declaration does not contain a slot named - :attr:`__weakref__`, the type inherits its :c:member:`~PyTypeObject.tp_weaklistoffset` from its - base type. - - -.. c:member:: getiterfunc PyTypeObject.tp_iter - - An optional pointer to a function that returns an :term:`iterator` for the - object. Its presence normally signals that the instances of this type are - :term:`iterable` (although sequences may be iterable without this function). - - This function has the same signature as :c:func:`PyObject_GetIter`:: - - PyObject *tp_iter(PyObject *self); - - **Inheritance:** - - This field is inherited by subtypes. - - -.. c:member:: iternextfunc PyTypeObject.tp_iternext - - An optional pointer to a function that returns the next item in an - :term:`iterator`. The signature is:: - - PyObject *tp_iternext(PyObject *self); - - When the iterator is exhausted, it must return ``NULL``; a :exc:`StopIteration` - exception may or may not be set. When another error occurs, it must return - ``NULL`` too. Its presence signals that the instances of this type are - iterators. - - Iterator types should also define the :c:member:`~PyTypeObject.tp_iter` function, and that - function should return the iterator instance itself (not a new iterator - instance). - - This function has the same signature as :c:func:`PyIter_Next`. - - **Inheritance:** - - This field is inherited by subtypes. - - -.. c:member:: struct PyMethodDef* PyTypeObject.tp_methods - - An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMethodDef` - structures, declaring regular methods of this type. - - For each entry in the array, an entry is added to the type's dictionary (see - :c:member:`~PyTypeObject.tp_dict` below) containing a method descriptor. - - **Inheritance:** - - This field is not inherited by subtypes (methods are inherited through a - different mechanism). - - -.. c:member:: struct PyMemberDef* PyTypeObject.tp_members - - An optional pointer to a static ``NULL``-terminated array of :c:type:`PyMemberDef` - structures, declaring regular data members (fields or slots) of instances of - this type. - - For each entry in the array, an entry is added to the type's dictionary (see - :c:member:`~PyTypeObject.tp_dict` below) containing a member descriptor. - - **Inheritance:** - - This field is not inherited by subtypes (members are inherited through a - different mechanism). - - -.. c:member:: struct PyGetSetDef* PyTypeObject.tp_getset - - An optional pointer to a static ``NULL``-terminated array of :c:type:`PyGetSetDef` - structures, declaring computed attributes of instances of this type. - - For each entry in the array, an entry is added to the type's dictionary (see - :c:member:`~PyTypeObject.tp_dict` below) containing a getset descriptor. - - **Inheritance:** - - This field is not inherited by subtypes (computed attributes are inherited - through a different mechanism). - - -.. c:member:: PyTypeObject* PyTypeObject.tp_base - - An optional pointer to a base type from which type properties are inherited. At - this level, only single inheritance is supported; multiple inheritance require - dynamically creating a type object by calling the metatype. - - .. note:: - - .. from Modules/xxmodule.c - - Slot initialization is subject to the rules of initializing globals. - C99 requires the initializers to be "address constants". Function - designators like :c:func:`PyType_GenericNew`, with implicit conversion - to a pointer, are valid C99 address constants. - - However, the unary '&' operator applied to a non-static variable - like :c:data:`PyBaseObject_Type` is not required to produce an address - constant. Compilers may support this (gcc does), MSVC does not. - Both compilers are strictly standard conforming in this particular - behavior. - - Consequently, :c:member:`~PyTypeObject.tp_base` should be set in - the extension module's init function. - - **Inheritance:** - - This field is not inherited by subtypes (obviously). - - **Default:** - - This field defaults to ``&PyBaseObject_Type`` (which to Python - programmers is known as the type :class:`object`). - - -.. c:member:: PyObject* PyTypeObject.tp_dict - - The type's dictionary is stored here by :c:func:`PyType_Ready`. - - This field should normally be initialized to ``NULL`` before PyType_Ready is - called; it may also be initialized to a dictionary containing initial attributes - for the type. Once :c:func:`PyType_Ready` has initialized the type, extra - attributes for the type may be added to this dictionary only if they don't - correspond to overloaded operations (like :meth:`~object.__add__`). - - **Inheritance:** - - This field is not inherited by subtypes (though the attributes defined in here - are inherited through a different mechanism). - - **Default:** - - If this field is ``NULL``, :c:func:`PyType_Ready` will assign a new - dictionary to it. - - .. warning:: - - It is not safe to use :c:func:`PyDict_SetItem` on or otherwise modify - :c:member:`~PyTypeObject.tp_dict` with the dictionary C-API. - - -.. c:member:: descrgetfunc PyTypeObject.tp_descr_get - - An optional pointer to a "descriptor get" function. - - The function signature is:: - - PyObject * tp_descr_get(PyObject *self, PyObject *obj, PyObject *type); - - .. XXX explain more? - - **Inheritance:** - - This field is inherited by subtypes. - - -.. c:member:: descrsetfunc PyTypeObject.tp_descr_set - - An optional pointer to a function for setting and deleting - a descriptor's value. - - The function signature is:: - - int tp_descr_set(PyObject *self, PyObject *obj, PyObject *value); - - The *value* argument is set to ``NULL`` to delete the value. - - .. XXX explain more? - - **Inheritance:** - - This field is inherited by subtypes. - - -.. c:member:: Py_ssize_t PyTypeObject.tp_dictoffset - - If the instances of this type have a dictionary containing instance variables, - this field is non-zero and contains the offset in the instances of the type of - the instance variable dictionary; this offset is used by - :c:func:`PyObject_GenericGetAttr`. - - Do not confuse this field with :c:member:`~PyTypeObject.tp_dict`; that is the dictionary for - attributes of the type object itself. - - If the value of this field is greater than zero, it specifies the offset from - the start of the instance structure. If the value is less than zero, it - specifies the offset from the *end* of the instance structure. A negative - offset is more expensive to use, and should only be used when the instance - structure contains a variable-length part. This is used for example to add an - instance variable dictionary to subtypes of :class:`str` or :class:`tuple`. Note - that the :c:member:`~PyTypeObject.tp_basicsize` field should account for the dictionary added to - the end in that case, even though the dictionary is not included in the basic - object layout. On a system with a pointer size of 4 bytes, - :c:member:`~PyTypeObject.tp_dictoffset` should be set to ``-4`` to indicate that the dictionary is - at the very end of the structure. - - The :c:member:`~PyTypeObject.tp_dictoffset` should be regarded as write-only. - To get the pointer to the dictionary call :c:func:`PyObject_GenericGetDict`. - Calling :c:func:`PyObject_GenericGetDict` may need to allocate memory for the - dictionary, so it is may be more efficient to call :c:func:`PyObject_GetAttr` - when accessing an attribute on the object. - - **Inheritance:** - - This field is inherited by subtypes, but see the rules listed below. A subtype - may override this offset; this means that the subtype instances store the - dictionary at a difference offset than the base type. Since the dictionary is - always found via :c:member:`~PyTypeObject.tp_dictoffset`, this should not be a problem. - - When a type defined by a class statement has no :attr:`~object.__slots__` declaration, - and none of its base types has an instance variable dictionary, a dictionary - slot is added to the instance layout and the :c:member:`~PyTypeObject.tp_dictoffset` is set to - that slot's offset. - - When a type defined by a class statement has a :attr:`__slots__` declaration, - the type inherits its :c:member:`~PyTypeObject.tp_dictoffset` from its base type. - - (Adding a slot named :attr:`~object.__dict__` to the :attr:`__slots__` declaration does - not have the expected effect, it just causes confusion. Maybe this should be - added as a feature just like :attr:`__weakref__` though.) - - **Default:** - - This slot has no default. For :ref:`static types `, if the - field is ``NULL`` then no :attr:`~object.__dict__` gets created for instances. - - -.. c:member:: initproc PyTypeObject.tp_init - - An optional pointer to an instance initialization function. - - This function corresponds to the :meth:`~object.__init__` method of classes. Like - :meth:`!__init__`, it is possible to create an instance without calling - :meth:`!__init__`, and it is possible to reinitialize an instance by calling its - :meth:`!__init__` method again. - - The function signature is:: - - int tp_init(PyObject *self, PyObject *args, PyObject *kwds); - - The self argument is the instance to be initialized; the *args* and *kwds* - arguments represent positional and keyword arguments of the call to - :meth:`~object.__init__`. - - The :c:member:`~PyTypeObject.tp_init` function, if not ``NULL``, is called when an instance is - created normally by calling its type, after the type's :c:member:`~PyTypeObject.tp_new` function - has returned an instance of the type. If the :c:member:`~PyTypeObject.tp_new` function returns an - instance of some other type that is not a subtype of the original type, no - :c:member:`~PyTypeObject.tp_init` function is called; if :c:member:`~PyTypeObject.tp_new` returns an instance of a - subtype of the original type, the subtype's :c:member:`~PyTypeObject.tp_init` is called. - - Returns ``0`` on success, ``-1`` and sets an exception on error. - - **Inheritance:** - - This field is inherited by subtypes. - - **Default:** - - For :ref:`static types ` this field does not have a default. - - -.. c:member:: allocfunc PyTypeObject.tp_alloc - - An optional pointer to an instance allocation function. - - The function signature is:: - - PyObject *tp_alloc(PyTypeObject *self, Py_ssize_t nitems); - - **Inheritance:** - - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement). - - **Default:** - - For dynamic subtypes, this field is always set to - :c:func:`PyType_GenericAlloc`, to force a standard heap - allocation strategy. - - For static subtypes, :c:data:`PyBaseObject_Type` uses - :c:func:`PyType_GenericAlloc`. That is the recommended value - for all statically defined types. - - -.. c:member:: newfunc PyTypeObject.tp_new - - An optional pointer to an instance creation function. - - The function signature is:: - - PyObject *tp_new(PyTypeObject *subtype, PyObject *args, PyObject *kwds); - - The *subtype* argument is the type of the object being created; the *args* and - *kwds* arguments represent positional and keyword arguments of the call to the - type. Note that *subtype* doesn't have to equal the type whose :c:member:`~PyTypeObject.tp_new` - function is called; it may be a subtype of that type (but not an unrelated - type). - - The :c:member:`~PyTypeObject.tp_new` function should call ``subtype->tp_alloc(subtype, nitems)`` - to allocate space for the object, and then do only as much further - initialization as is absolutely necessary. Initialization that can safely be - ignored or repeated should be placed in the :c:member:`~PyTypeObject.tp_init` handler. A good - rule of thumb is that for immutable types, all initialization should take place - in :c:member:`~PyTypeObject.tp_new`, while for mutable types, most initialization should be - deferred to :c:member:`~PyTypeObject.tp_init`. - - Set the :c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag to disallow creating - instances of the type in Python. - - **Inheritance:** - - This field is inherited by subtypes, except it is not inherited by - :ref:`static types ` whose :c:member:`~PyTypeObject.tp_base` - is ``NULL`` or ``&PyBaseObject_Type``. - - **Default:** - - For :ref:`static types ` this field has no default. - This means if the slot is defined as ``NULL``, the type cannot be called - to create new instances; presumably there is some other way to create - instances, like a factory function. - - -.. c:member:: freefunc PyTypeObject.tp_free - - An optional pointer to an instance deallocation function. Its signature is:: - - void tp_free(void *self); - - An initializer that is compatible with this signature is :c:func:`PyObject_Free`. - - **Inheritance:** - - This field is inherited by static subtypes, but not by dynamic - subtypes (subtypes created by a class statement) - - **Default:** - - In dynamic subtypes, this field is set to a deallocator suitable to - match :c:func:`PyType_GenericAlloc` and the value of the - :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. - - For static subtypes, :c:data:`PyBaseObject_Type` uses :c:func:`PyObject_Del`. - - -.. c:member:: inquiry PyTypeObject.tp_is_gc - - An optional pointer to a function called by the garbage collector. - - The garbage collector needs to know whether a particular object is collectible - or not. Normally, it is sufficient to look at the object's type's - :c:member:`~PyTypeObject.tp_flags` field, and check the :c:macro:`Py_TPFLAGS_HAVE_GC` flag bit. But - some types have a mixture of statically and dynamically allocated instances, and - the statically allocated instances are not collectible. Such types should - define this function; it should return ``1`` for a collectible instance, and - ``0`` for a non-collectible instance. The signature is:: - - int tp_is_gc(PyObject *self); - - (The only example of this are types themselves. The metatype, - :c:data:`PyType_Type`, defines this function to distinguish between statically - and :ref:`dynamically allocated types `.) - - **Inheritance:** - - This field is inherited by subtypes. - - **Default:** - - This slot has no default. If this field is ``NULL``, - :c:macro:`Py_TPFLAGS_HAVE_GC` is used as the functional equivalent. - - -.. c:member:: PyObject* PyTypeObject.tp_bases - - Tuple of base types. - - This field should be set to ``NULL`` and treated as read-only. - Python will fill it in when the type is :c:func:`initialized `. - - For dynamically created classes, the ``Py_tp_bases`` - :c:type:`slot ` can be used instead of the *bases* argument - of :c:func:`PyType_FromSpecWithBases`. - The argument form is preferred. - - .. warning:: - - Multiple inheritance does not work well for statically defined types. - If you set ``tp_bases`` to a tuple, Python will not raise an error, - but some slots will only be inherited from the first base. - - **Inheritance:** - - This field is not inherited. - - -.. c:member:: PyObject* PyTypeObject.tp_mro - - Tuple containing the expanded set of base types, starting with the type itself - and ending with :class:`object`, in Method Resolution Order. - - This field should be set to ``NULL`` and treated as read-only. - Python will fill it in when the type is :c:func:`initialized `. - - **Inheritance:** - - This field is not inherited; it is calculated fresh by - :c:func:`PyType_Ready`. - - -.. c:member:: PyObject* PyTypeObject.tp_cache - - Unused. Internal use only. - - **Inheritance:** - - This field is not inherited. - - -.. c:member:: PyObject* PyTypeObject.tp_subclasses - - List of weak references to subclasses. Internal use only. - - **Inheritance:** - - This field is not inherited. - - -.. c:member:: PyObject* PyTypeObject.tp_weaklist - - Weak reference list head, for weak references to this type object. Not - inherited. Internal use only. - - **Inheritance:** - - This field is not inherited. - - -.. c:member:: destructor PyTypeObject.tp_del - - This field is deprecated. Use :c:member:`~PyTypeObject.tp_finalize` instead. - - -.. c:member:: unsigned int PyTypeObject.tp_version_tag - - Used to index into the method cache. Internal use only. - - **Inheritance:** - - This field is not inherited. - - -.. c:member:: destructor PyTypeObject.tp_finalize - - An optional pointer to an instance finalization function. Its signature is:: - - void tp_finalize(PyObject *self); - - If :c:member:`~PyTypeObject.tp_finalize` is set, the interpreter calls it once when - finalizing an instance. It is called either from the garbage - collector (if the instance is part of an isolated reference cycle) or - just before the object is deallocated. Either way, it is guaranteed - to be called before attempting to break reference cycles, ensuring - that it finds the object in a sane state. - - :c:member:`~PyTypeObject.tp_finalize` should not mutate the current exception status; - therefore, a recommended way to write a non-trivial finalizer is:: - - static void - local_finalize(PyObject *self) - { - PyObject *error_type, *error_value, *error_traceback; - - /* Save the current exception, if any. */ - PyErr_Fetch(&error_type, &error_value, &error_traceback); - - /* ... */ - - /* Restore the saved exception. */ - PyErr_Restore(error_type, error_value, error_traceback); - } - - Also, note that, in a garbage collected Python, - :c:member:`~PyTypeObject.tp_dealloc` may be called from - any Python thread, not just the thread which created the object (if the object - becomes part of a refcount cycle, that cycle might be collected by a garbage - collection on any thread). This is not a problem for Python API calls, since - the thread on which tp_dealloc is called will own the Global Interpreter Lock - (GIL). However, if the object being destroyed in turn destroys objects from some - other C or C++ library, care should be taken to ensure that destroying those - objects on the thread which called tp_dealloc will not violate any assumptions - of the library. - - **Inheritance:** - - This field is inherited by subtypes. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.8 - - Before version 3.8 it was necessary to set the - :c:macro:`Py_TPFLAGS_HAVE_FINALIZE` flags bit in order for this field to be - used. This is no longer required. - - .. seealso:: "Safe object finalization" (:pep:`442`) - - -.. c:member:: vectorcallfunc PyTypeObject.tp_vectorcall - - Vectorcall function to use for calls of this type object. - In other words, it is used to implement - :ref:`vectorcall ` for ``type.__call__``. - If ``tp_vectorcall`` is ``NULL``, the default call implementation - using :meth:`~object.__new__` and :meth:`~object.__init__` is used. - - **Inheritance:** - - This field is never inherited. - - .. versionadded:: 3.9 (the field exists since 3.8 but it's only used since 3.9) - - -.. _static-types: - -Static Types ------------- - -Traditionally, types defined in C code are *static*, that is, -a static :c:type:`PyTypeObject` structure is defined directly in code -and initialized using :c:func:`PyType_Ready`. - -This results in types that are limited relative to types defined in Python: - -* Static types are limited to one base, i.e. they cannot use multiple - inheritance. -* Static type objects (but not necessarily their instances) are immutable. - It is not possible to add or modify the type object's attributes from Python. -* Static type objects are shared across - :ref:`sub-interpreters `, so they should not - include any subinterpreter-specific state. - -Also, since :c:type:`PyTypeObject` is only part of the :ref:`Limited API -` as an opaque struct, any extension modules using static types must be -compiled for a specific Python minor version. - - -.. _heap-types: - -Heap Types ----------- - -An alternative to :ref:`static types ` is *heap-allocated types*, -or *heap types* for short, which correspond closely to classes created by -Python's ``class`` statement. Heap types have the :c:macro:`Py_TPFLAGS_HEAPTYPE` -flag set. - -This is done by filling a :c:type:`PyType_Spec` structure and calling -:c:func:`PyType_FromSpec`, :c:func:`PyType_FromSpecWithBases`, -or :c:func:`PyType_FromModuleAndSpec`. - - -.. _number-structs: - -Number Object Structures -======================== - -.. sectionauthor:: Amaury Forgeot d'Arc - - -.. c:type:: PyNumberMethods - - This structure holds pointers to the functions which an object uses to - implement the number protocol. Each function is used by the function of - similar name documented in the :ref:`number` section. - - .. XXX Drop the definition? - - Here is the structure definition:: - - typedef struct { - binaryfunc nb_add; - binaryfunc nb_subtract; - binaryfunc nb_multiply; - binaryfunc nb_remainder; - binaryfunc nb_divmod; - ternaryfunc nb_power; - unaryfunc nb_negative; - unaryfunc nb_positive; - unaryfunc nb_absolute; - inquiry nb_bool; - unaryfunc nb_invert; - binaryfunc nb_lshift; - binaryfunc nb_rshift; - binaryfunc nb_and; - binaryfunc nb_xor; - binaryfunc nb_or; - unaryfunc nb_int; - void *nb_reserved; - unaryfunc nb_float; - - binaryfunc nb_inplace_add; - binaryfunc nb_inplace_subtract; - binaryfunc nb_inplace_multiply; - binaryfunc nb_inplace_remainder; - ternaryfunc nb_inplace_power; - binaryfunc nb_inplace_lshift; - binaryfunc nb_inplace_rshift; - binaryfunc nb_inplace_and; - binaryfunc nb_inplace_xor; - binaryfunc nb_inplace_or; - - binaryfunc nb_floor_divide; - binaryfunc nb_true_divide; - binaryfunc nb_inplace_floor_divide; - binaryfunc nb_inplace_true_divide; - - unaryfunc nb_index; - - binaryfunc nb_matrix_multiply; - binaryfunc nb_inplace_matrix_multiply; - } PyNumberMethods; - - .. note:: - - Binary and ternary functions must check the type of all their operands, - and implement the necessary conversions (at least one of the operands is - an instance of the defined type). If the operation is not defined for the - given operands, binary and ternary functions must return - ``Py_NotImplemented``, if another error occurred they must return ``NULL`` - and set an exception. - - .. note:: - - The :c:member:`~PyNumberMethods.nb_reserved` field should always be ``NULL``. It - was previously called :c:member:`!nb_long`, and was renamed in - Python 3.0.1. - -.. c:member:: binaryfunc PyNumberMethods.nb_add -.. c:member:: binaryfunc PyNumberMethods.nb_subtract -.. c:member:: binaryfunc PyNumberMethods.nb_multiply -.. c:member:: binaryfunc PyNumberMethods.nb_remainder -.. c:member:: binaryfunc PyNumberMethods.nb_divmod -.. c:member:: ternaryfunc PyNumberMethods.nb_power -.. c:member:: unaryfunc PyNumberMethods.nb_negative -.. c:member:: unaryfunc PyNumberMethods.nb_positive -.. c:member:: unaryfunc PyNumberMethods.nb_absolute -.. c:member:: inquiry PyNumberMethods.nb_bool -.. c:member:: unaryfunc PyNumberMethods.nb_invert -.. c:member:: binaryfunc PyNumberMethods.nb_lshift -.. c:member:: binaryfunc PyNumberMethods.nb_rshift -.. c:member:: binaryfunc PyNumberMethods.nb_and -.. c:member:: binaryfunc PyNumberMethods.nb_xor -.. c:member:: binaryfunc PyNumberMethods.nb_or -.. c:member:: unaryfunc PyNumberMethods.nb_int -.. c:member:: void *PyNumberMethods.nb_reserved -.. c:member:: unaryfunc PyNumberMethods.nb_float -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_add -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_subtract -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_multiply -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_remainder -.. c:member:: ternaryfunc PyNumberMethods.nb_inplace_power -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_lshift -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_rshift -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_and -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_xor -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_or -.. c:member:: binaryfunc PyNumberMethods.nb_floor_divide -.. c:member:: binaryfunc PyNumberMethods.nb_true_divide -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_floor_divide -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_true_divide -.. c:member:: unaryfunc PyNumberMethods.nb_index -.. c:member:: binaryfunc PyNumberMethods.nb_matrix_multiply -.. c:member:: binaryfunc PyNumberMethods.nb_inplace_matrix_multiply - - -.. _mapping-structs: - -Mapping Object Structures -========================= - -.. sectionauthor:: Amaury Forgeot d'Arc - - -.. c:type:: PyMappingMethods - - This structure holds pointers to the functions which an object uses to - implement the mapping protocol. It has three members: - -.. c:member:: lenfunc PyMappingMethods.mp_length - - This function is used by :c:func:`PyMapping_Size` and - :c:func:`PyObject_Size`, and has the same signature. This slot may be set to - ``NULL`` if the object has no defined length. - -.. c:member:: binaryfunc PyMappingMethods.mp_subscript - - This function is used by :c:func:`PyObject_GetItem` and - :c:func:`PySequence_GetSlice`, and has the same signature as - :c:func:`!PyObject_GetItem`. This slot must be filled for the - :c:func:`PyMapping_Check` function to return ``1``, it can be ``NULL`` - otherwise. - -.. c:member:: objobjargproc PyMappingMethods.mp_ass_subscript - - This function is used by :c:func:`PyObject_SetItem`, - :c:func:`PyObject_DelItem`, :c:func:`PySequence_SetSlice` and - :c:func:`PySequence_DelSlice`. It has the same signature as - :c:func:`!PyObject_SetItem`, but *v* can also be set to ``NULL`` to delete - an item. If this slot is ``NULL``, the object does not support item - assignment and deletion. - - -.. _sequence-structs: - -Sequence Object Structures -========================== - -.. sectionauthor:: Amaury Forgeot d'Arc - - -.. c:type:: PySequenceMethods - - This structure holds pointers to the functions which an object uses to - implement the sequence protocol. - -.. c:member:: lenfunc PySequenceMethods.sq_length - - This function is used by :c:func:`PySequence_Size` and - :c:func:`PyObject_Size`, and has the same signature. It is also used for - handling negative indices via the :c:member:`~PySequenceMethods.sq_item` - and the :c:member:`~PySequenceMethods.sq_ass_item` slots. - -.. c:member:: binaryfunc PySequenceMethods.sq_concat - - This function is used by :c:func:`PySequence_Concat` and has the same - signature. It is also used by the ``+`` operator, after trying the numeric - addition via the :c:member:`~PyNumberMethods.nb_add` slot. - -.. c:member:: ssizeargfunc PySequenceMethods.sq_repeat - - This function is used by :c:func:`PySequence_Repeat` and has the same - signature. It is also used by the ``*`` operator, after trying numeric - multiplication via the :c:member:`~PyNumberMethods.nb_multiply` slot. - -.. c:member:: ssizeargfunc PySequenceMethods.sq_item - - This function is used by :c:func:`PySequence_GetItem` and has the same - signature. It is also used by :c:func:`PyObject_GetItem`, after trying - the subscription via the :c:member:`~PyMappingMethods.mp_subscript` slot. - This slot must be filled for the :c:func:`PySequence_Check` - function to return ``1``, it can be ``NULL`` otherwise. - - Negative indexes are handled as follows: if the :c:member:`~PySequenceMethods.sq_length` slot is - filled, it is called and the sequence length is used to compute a positive - index which is passed to :c:member:`~PySequenceMethods.sq_item`. If :c:member:`!sq_length` is ``NULL``, - the index is passed as is to the function. - -.. c:member:: ssizeobjargproc PySequenceMethods.sq_ass_item - - This function is used by :c:func:`PySequence_SetItem` and has the same - signature. It is also used by :c:func:`PyObject_SetItem` and - :c:func:`PyObject_DelItem`, after trying the item assignment and deletion - via the :c:member:`~PyMappingMethods.mp_ass_subscript` slot. - This slot may be left to ``NULL`` if the object does not support - item assignment and deletion. - -.. c:member:: objobjproc PySequenceMethods.sq_contains - - This function may be used by :c:func:`PySequence_Contains` and has the same - signature. This slot may be left to ``NULL``, in this case - :c:func:`!PySequence_Contains` simply traverses the sequence until it - finds a match. - -.. c:member:: binaryfunc PySequenceMethods.sq_inplace_concat - - This function is used by :c:func:`PySequence_InPlaceConcat` and has the same - signature. It should modify its first operand, and return it. This slot - may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceConcat` - will fall back to :c:func:`PySequence_Concat`. It is also used by the - augmented assignment ``+=``, after trying numeric in-place addition - via the :c:member:`~PyNumberMethods.nb_inplace_add` slot. - -.. c:member:: ssizeargfunc PySequenceMethods.sq_inplace_repeat - - This function is used by :c:func:`PySequence_InPlaceRepeat` and has the same - signature. It should modify its first operand, and return it. This slot - may be left to ``NULL``, in this case :c:func:`!PySequence_InPlaceRepeat` - will fall back to :c:func:`PySequence_Repeat`. It is also used by the - augmented assignment ``*=``, after trying numeric in-place multiplication - via the :c:member:`~PyNumberMethods.nb_inplace_multiply` slot. - - -.. _buffer-structs: - -Buffer Object Structures -======================== - -.. sectionauthor:: Greg J. Stein -.. sectionauthor:: Benjamin Peterson -.. sectionauthor:: Stefan Krah - -.. c:type:: PyBufferProcs - - This structure holds pointers to the functions required by the - :ref:`Buffer protocol `. The protocol defines how - an exporter object can expose its internal data to consumer objects. - -.. c:member:: getbufferproc PyBufferProcs.bf_getbuffer - - The signature of this function is:: - - int (PyObject *exporter, Py_buffer *view, int flags); - - Handle a request to *exporter* to fill in *view* as specified by *flags*. - Except for point (3), an implementation of this function MUST take these - steps: - - (1) Check if the request can be met. If not, raise :exc:`BufferError`, - set :c:expr:`view->obj` to ``NULL`` and return ``-1``. - - (2) Fill in the requested fields. - - (3) Increment an internal counter for the number of exports. - - (4) Set :c:expr:`view->obj` to *exporter* and increment :c:expr:`view->obj`. - - (5) Return ``0``. - - If *exporter* is part of a chain or tree of buffer providers, two main - schemes can be used: - - * Re-export: Each member of the tree acts as the exporting object and - sets :c:expr:`view->obj` to a new reference to itself. - - * Redirect: The buffer request is redirected to the root object of the - tree. Here, :c:expr:`view->obj` will be a new reference to the root - object. - - The individual fields of *view* are described in section - :ref:`Buffer structure `, the rules how an exporter - must react to specific requests are in section - :ref:`Buffer request types `. - - All memory pointed to in the :c:type:`Py_buffer` structure belongs to - the exporter and must remain valid until there are no consumers left. - :c:member:`~Py_buffer.format`, :c:member:`~Py_buffer.shape`, - :c:member:`~Py_buffer.strides`, :c:member:`~Py_buffer.suboffsets` - and :c:member:`~Py_buffer.internal` - are read-only for the consumer. - - :c:func:`PyBuffer_FillInfo` provides an easy way of exposing a simple - bytes buffer while dealing correctly with all request types. - - :c:func:`PyObject_GetBuffer` is the interface for the consumer that - wraps this function. - -.. c:member:: releasebufferproc PyBufferProcs.bf_releasebuffer - - The signature of this function is:: - - void (PyObject *exporter, Py_buffer *view); - - Handle a request to release the resources of the buffer. If no resources - need to be released, :c:member:`PyBufferProcs.bf_releasebuffer` may be - ``NULL``. Otherwise, a standard implementation of this function will take - these optional steps: - - (1) Decrement an internal counter for the number of exports. - - (2) If the counter is ``0``, free all memory associated with *view*. - - The exporter MUST use the :c:member:`~Py_buffer.internal` field to keep - track of buffer-specific resources. This field is guaranteed to remain - constant, while a consumer MAY pass a copy of the original buffer as the - *view* argument. - - - This function MUST NOT decrement :c:expr:`view->obj`, since that is - done automatically in :c:func:`PyBuffer_Release` (this scheme is - useful for breaking reference cycles). - - - :c:func:`PyBuffer_Release` is the interface for the consumer that - wraps this function. - - -.. _async-structs: - - -Async Object Structures -======================= - -.. sectionauthor:: Yury Selivanov - -.. versionadded:: 3.5 - -.. c:type:: PyAsyncMethods - - This structure holds pointers to the functions required to implement - :term:`awaitable` and :term:`asynchronous iterator` objects. - - Here is the structure definition:: - - typedef struct { - unaryfunc am_await; - unaryfunc am_aiter; - unaryfunc am_anext; - sendfunc am_send; - } PyAsyncMethods; - -.. c:member:: unaryfunc PyAsyncMethods.am_await - - The signature of this function is:: - - PyObject *am_await(PyObject *self); - - The returned object must be an :term:`iterator`, i.e. :c:func:`PyIter_Check` - must return ``1`` for it. - - This slot may be set to ``NULL`` if an object is not an :term:`awaitable`. - -.. c:member:: unaryfunc PyAsyncMethods.am_aiter - - The signature of this function is:: - - PyObject *am_aiter(PyObject *self); - - Must return an :term:`asynchronous iterator` object. - See :meth:`~object.__anext__` for details. - - This slot may be set to ``NULL`` if an object does not implement - asynchronous iteration protocol. - -.. c:member:: unaryfunc PyAsyncMethods.am_anext - - The signature of this function is:: - - PyObject *am_anext(PyObject *self); - - Must return an :term:`awaitable` object. - See :meth:`~object.__anext__` for details. - This slot may be set to ``NULL``. - -.. c:member:: sendfunc PyAsyncMethods.am_send - - The signature of this function is:: - - PySendResult am_send(PyObject *self, PyObject *arg, PyObject **result); - - See :c:func:`PyIter_Send` for details. - This slot may be set to ``NULL``. - - .. versionadded:: 3.10 - - -.. _slot-typedefs: - -Slot Type typedefs -================== - -.. c:type:: PyObject *(*allocfunc)(PyTypeObject *cls, Py_ssize_t nitems) - - The purpose of this function is to separate memory allocation from memory - initialization. It should return a pointer to a block of memory of adequate - length for the instance, suitably aligned, and initialized to zeros, but with - :c:member:`~PyObject.ob_refcnt` set to ``1`` and :c:member:`~PyObject.ob_type` set to the type argument. If - the type's :c:member:`~PyTypeObject.tp_itemsize` is non-zero, the object's :c:member:`~PyVarObject.ob_size` field - should be initialized to *nitems* and the length of the allocated memory block - should be ``tp_basicsize + nitems*tp_itemsize``, rounded up to a multiple of - ``sizeof(void*)``; otherwise, *nitems* is not used and the length of the block - should be :c:member:`~PyTypeObject.tp_basicsize`. - - This function should not do any other instance initialization, not even to - allocate additional memory; that should be done by :c:member:`~PyTypeObject.tp_new`. - -.. c:type:: void (*destructor)(PyObject *) - -.. c:type:: void (*freefunc)(void *) - - See :c:member:`~PyTypeObject.tp_free`. - -.. c:type:: PyObject *(*newfunc)(PyObject *, PyObject *, PyObject *) - - See :c:member:`~PyTypeObject.tp_new`. - -.. c:type:: int (*initproc)(PyObject *, PyObject *, PyObject *) - - See :c:member:`~PyTypeObject.tp_init`. - -.. c:type:: PyObject *(*reprfunc)(PyObject *) - - See :c:member:`~PyTypeObject.tp_repr`. - -.. c:type:: PyObject *(*getattrfunc)(PyObject *self, char *attr) - - Return the value of the named attribute for the object. - -.. c:type:: int (*setattrfunc)(PyObject *self, char *attr, PyObject *value) - - Set the value of the named attribute for the object. - The value argument is set to ``NULL`` to delete the attribute. - -.. c:type:: PyObject *(*getattrofunc)(PyObject *self, PyObject *attr) - - Return the value of the named attribute for the object. - - See :c:member:`~PyTypeObject.tp_getattro`. - -.. c:type:: int (*setattrofunc)(PyObject *self, PyObject *attr, PyObject *value) - - Set the value of the named attribute for the object. - The value argument is set to ``NULL`` to delete the attribute. - - See :c:member:`~PyTypeObject.tp_setattro`. - -.. c:type:: PyObject *(*descrgetfunc)(PyObject *, PyObject *, PyObject *) - - See :c:member:`~PyTypeObject.tp_descr_get`. - -.. c:type:: int (*descrsetfunc)(PyObject *, PyObject *, PyObject *) - - See :c:member:`~PyTypeObject.tp_descr_set`. - -.. c:type:: Py_hash_t (*hashfunc)(PyObject *) - - See :c:member:`~PyTypeObject.tp_hash`. - -.. c:type:: PyObject *(*richcmpfunc)(PyObject *, PyObject *, int) - - See :c:member:`~PyTypeObject.tp_richcompare`. - -.. c:type:: PyObject *(*getiterfunc)(PyObject *) - - See :c:member:`~PyTypeObject.tp_iter`. - -.. c:type:: PyObject *(*iternextfunc)(PyObject *) - - See :c:member:`~PyTypeObject.tp_iternext`. - -.. c:type:: Py_ssize_t (*lenfunc)(PyObject *) - -.. c:type:: int (*getbufferproc)(PyObject *, Py_buffer *, int) - -.. c:type:: void (*releasebufferproc)(PyObject *, Py_buffer *) - -.. c:type:: PyObject *(*unaryfunc)(PyObject *) - -.. c:type:: PyObject *(*binaryfunc)(PyObject *, PyObject *) - -.. c:type:: PySendResult (*sendfunc)(PyObject *, PyObject *, PyObject **) - - See :c:member:`~PyAsyncMethods.am_send`. - -.. c:type:: PyObject *(*ternaryfunc)(PyObject *, PyObject *, PyObject *) - -.. c:type:: PyObject *(*ssizeargfunc)(PyObject *, Py_ssize_t) - -.. c:type:: int (*ssizeobjargproc)(PyObject *, Py_ssize_t, PyObject *) - -.. c:type:: int (*objobjproc)(PyObject *, PyObject *) - -.. c:type:: int (*objobjargproc)(PyObject *, PyObject *, PyObject *) - - -.. _typedef-examples: - -Examples -======== - -The following are simple examples of Python type definitions. They -include common usage you may encounter. Some demonstrate tricky corner -cases. For more examples, practical info, and a tutorial, see -:ref:`defining-new-types` and :ref:`new-types-topics`. - -A basic :ref:`static type `:: - - typedef struct { - PyObject_HEAD - const char *data; - } MyObject; - - static PyTypeObject MyObject_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "mymod.MyObject", - .tp_basicsize = sizeof(MyObject), - .tp_doc = PyDoc_STR("My objects"), - .tp_new = myobj_new, - .tp_dealloc = (destructor)myobj_dealloc, - .tp_repr = (reprfunc)myobj_repr, - }; - -You may also find older code (especially in the CPython code base) -with a more verbose initializer:: - - static PyTypeObject MyObject_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "mymod.MyObject", /* tp_name */ - sizeof(MyObject), /* tp_basicsize */ - 0, /* tp_itemsize */ - (destructor)myobj_dealloc, /* tp_dealloc */ - 0, /* tp_vectorcall_offset */ - 0, /* tp_getattr */ - 0, /* tp_setattr */ - 0, /* tp_as_async */ - (reprfunc)myobj_repr, /* tp_repr */ - 0, /* tp_as_number */ - 0, /* tp_as_sequence */ - 0, /* tp_as_mapping */ - 0, /* tp_hash */ - 0, /* tp_call */ - 0, /* tp_str */ - 0, /* tp_getattro */ - 0, /* tp_setattro */ - 0, /* tp_as_buffer */ - 0, /* tp_flags */ - PyDoc_STR("My objects"), /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - 0, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - 0, /* tp_init */ - 0, /* tp_alloc */ - myobj_new, /* tp_new */ - }; - -A type that supports weakrefs, instance dicts, and hashing:: - - typedef struct { - PyObject_HEAD - const char *data; - PyObject *inst_dict; - PyObject *weakreflist; - } MyObject; - - static PyTypeObject MyObject_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "mymod.MyObject", - .tp_basicsize = sizeof(MyObject), - .tp_doc = PyDoc_STR("My objects"), - .tp_weaklistoffset = offsetof(MyObject, weakreflist), - .tp_dictoffset = offsetof(MyObject, inst_dict), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .tp_new = myobj_new, - .tp_traverse = (traverseproc)myobj_traverse, - .tp_clear = (inquiry)myobj_clear, - .tp_alloc = PyType_GenericNew, - .tp_dealloc = (destructor)myobj_dealloc, - .tp_repr = (reprfunc)myobj_repr, - .tp_hash = (hashfunc)myobj_hash, - .tp_richcompare = PyBaseObject_Type.tp_richcompare, - }; - -A str subclass that cannot be subclassed and cannot be called -to create instances (e.g. uses a separate factory func) using -:c:macro:`Py_TPFLAGS_DISALLOW_INSTANTIATION` flag:: - - typedef struct { - PyUnicodeObject raw; - char *extra; - } MyStr; - - static PyTypeObject MyStr_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "mymod.MyStr", - .tp_basicsize = sizeof(MyStr), - .tp_base = NULL, // set to &PyUnicode_Type in module init - .tp_doc = PyDoc_STR("my custom str"), - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_repr = (reprfunc)myobj_repr, - }; - -The simplest :ref:`static type ` with fixed-length instances:: - - typedef struct { - PyObject_HEAD - } MyObject; - - static PyTypeObject MyObject_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "mymod.MyObject", - }; - -The simplest :ref:`static type ` with variable-length instances:: - - typedef struct { - PyObject_VAR_HEAD - const char *data[1]; - } MyObject; - - static PyTypeObject MyObject_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "mymod.MyObject", - .tp_basicsize = sizeof(MyObject) - sizeof(char *), - .tp_itemsize = sizeof(char *), - }; diff --git a/Doc/c-api/unicode.rst b/Doc/c-api/unicode.rst deleted file mode 100644 index 686252d808653d..00000000000000 --- a/Doc/c-api/unicode.rst +++ /dev/null @@ -1,1553 +0,0 @@ -.. highlight:: c - -.. _unicodeobjects: - -Unicode Objects and Codecs --------------------------- - -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Georg Brandl - -Unicode Objects -^^^^^^^^^^^^^^^ - -Since the implementation of :pep:`393` in Python 3.3, Unicode objects internally -use a variety of representations, in order to allow handling the complete range -of Unicode characters while staying memory efficient. There are special cases -for strings where all code points are below 128, 256, or 65536; otherwise, code -points must be below 1114112 (which is the full Unicode range). - -:c:expr:`Py_UNICODE*` and UTF-8 representations are created on demand and cached -in the Unicode object. The :c:expr:`Py_UNICODE*` representation is deprecated -and inefficient. - -Due to the transition between the old APIs and the new APIs, Unicode objects -can internally be in two states depending on how they were created: - -* "canonical" Unicode objects are all objects created by a non-deprecated - Unicode API. They use the most efficient representation allowed by the - implementation. - -* "legacy" Unicode objects have been created through one of the deprecated - APIs (typically :c:func:`PyUnicode_FromUnicode`) and only bear the - :c:expr:`Py_UNICODE*` representation; you will have to call - :c:func:`PyUnicode_READY` on them before calling any other API. - -.. note:: - The "legacy" Unicode object will be removed in Python 3.12 with deprecated - APIs. All Unicode objects will be "canonical" since then. See :pep:`623` - for more information. - - -Unicode Type -"""""""""""" - -These are the basic Unicode object types used for the Unicode implementation in -Python: - -.. c:type:: Py_UCS4 - Py_UCS2 - Py_UCS1 - - These types are typedefs for unsigned integer types wide enough to contain - characters of 32 bits, 16 bits and 8 bits, respectively. When dealing with - single Unicode characters, use :c:type:`Py_UCS4`. - - .. versionadded:: 3.3 - - -.. c:type:: Py_UNICODE - - This is a typedef of :c:type:`wchar_t`, which is a 16-bit type or 32-bit type - depending on the platform. - - .. versionchanged:: 3.3 - In previous versions, this was a 16-bit type or a 32-bit type depending on - whether you selected a "narrow" or "wide" Unicode version of Python at - build time. - - -.. c:type:: PyASCIIObject - PyCompactUnicodeObject - PyUnicodeObject - - These subtypes of :c:type:`PyObject` represent a Python Unicode object. In - almost all cases, they shouldn't be used directly, since all API functions - that deal with Unicode objects take and return :c:type:`PyObject` pointers. - - .. versionadded:: 3.3 - - -.. c:var:: PyTypeObject PyUnicode_Type - - This instance of :c:type:`PyTypeObject` represents the Python Unicode type. It - is exposed to Python code as ``str``. - - -The following APIs are C macros and static inlined functions for fast checks and -access to internal read-only data of Unicode objects: - -.. c:function:: int PyUnicode_Check(PyObject *o) - - Return true if the object *o* is a Unicode object or an instance of a Unicode - subtype. This function always succeeds. - - -.. c:function:: int PyUnicode_CheckExact(PyObject *o) - - Return true if the object *o* is a Unicode object, but not an instance of a - subtype. This function always succeeds. - - -.. c:function:: int PyUnicode_READY(PyObject *o) - - Ensure the string object *o* is in the "canonical" representation. This is - required before using any of the access macros described below. - - .. XXX expand on when it is not required - - Returns ``0`` on success and ``-1`` with an exception set on failure, which in - particular happens if memory allocation fails. - - .. versionadded:: 3.3 - - .. deprecated-removed:: 3.10 3.12 - This API will be removed with :c:func:`PyUnicode_FromUnicode`. - - -.. c:function:: Py_ssize_t PyUnicode_GET_LENGTH(PyObject *o) - - Return the length of the Unicode string, in code points. *o* has to be a - Unicode object in the "canonical" representation (not checked). - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS1* PyUnicode_1BYTE_DATA(PyObject *o) - Py_UCS2* PyUnicode_2BYTE_DATA(PyObject *o) - Py_UCS4* PyUnicode_4BYTE_DATA(PyObject *o) - - Return a pointer to the canonical representation cast to UCS1, UCS2 or UCS4 - integer types for direct character access. No checks are performed if the - canonical representation has the correct character size; use - :c:func:`PyUnicode_KIND` to select the right macro. Make sure - :c:func:`PyUnicode_READY` has been called before accessing this. - - .. versionadded:: 3.3 - - -.. c:macro:: PyUnicode_WCHAR_KIND - PyUnicode_1BYTE_KIND - PyUnicode_2BYTE_KIND - PyUnicode_4BYTE_KIND - - Return values of the :c:func:`PyUnicode_KIND` macro. - - .. versionadded:: 3.3 - - .. deprecated-removed:: 3.10 3.12 - ``PyUnicode_WCHAR_KIND`` is deprecated. - - -.. c:function:: int PyUnicode_KIND(PyObject *o) - - Return one of the PyUnicode kind constants (see above) that indicate how many - bytes per character this Unicode object uses to store its data. *o* has to - be a Unicode object in the "canonical" representation (not checked). - - .. XXX document "0" return value? - - .. versionadded:: 3.3 - - -.. c:function:: void* PyUnicode_DATA(PyObject *o) - - Return a void pointer to the raw Unicode buffer. *o* has to be a Unicode - object in the "canonical" representation (not checked). - - .. versionadded:: 3.3 - - -.. c:function:: void PyUnicode_WRITE(int kind, void *data, \ - Py_ssize_t index, Py_UCS4 value) - - Write into a canonical representation *data* (as obtained with - :c:func:`PyUnicode_DATA`). This function performs no sanity checks, and is - intended for usage in loops. The caller should cache the *kind* value and - *data* pointer as obtained from other calls. *index* is the index in - the string (starts at 0) and *value* is the new code point value which should - be written to that location. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4 PyUnicode_READ(int kind, void *data, \ - Py_ssize_t index) - - Read a code point from a canonical representation *data* (as obtained with - :c:func:`PyUnicode_DATA`). No checks or ready calls are performed. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4 PyUnicode_READ_CHAR(PyObject *o, Py_ssize_t index) - - Read a character from a Unicode object *o*, which must be in the "canonical" - representation. This is less efficient than :c:func:`PyUnicode_READ` if you - do multiple consecutive reads. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4 PyUnicode_MAX_CHAR_VALUE(PyObject *o) - - Return the maximum code point that is suitable for creating another string - based on *o*, which must be in the "canonical" representation. This is - always an approximation but more efficient than iterating over the string. - - .. versionadded:: 3.3 - - -.. c:function:: Py_ssize_t PyUnicode_GET_SIZE(PyObject *o) - - Return the size of the deprecated :c:type:`Py_UNICODE` representation, in - code units (this includes surrogate pairs as 2 units). *o* has to be a - Unicode object (not checked). - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style Unicode API, please migrate to using - :c:func:`PyUnicode_GET_LENGTH`. - - -.. c:function:: Py_ssize_t PyUnicode_GET_DATA_SIZE(PyObject *o) - - Return the size of the deprecated :c:type:`Py_UNICODE` representation in - bytes. *o* has to be a Unicode object (not checked). - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style Unicode API, please migrate to using - :c:func:`PyUnicode_GET_LENGTH`. - - -.. c:function:: Py_UNICODE* PyUnicode_AS_UNICODE(PyObject *o) - const char* PyUnicode_AS_DATA(PyObject *o) - - Return a pointer to a :c:type:`Py_UNICODE` representation of the object. The - returned buffer is always terminated with an extra null code point. It - may also contain embedded null code points, which would cause the string - to be truncated when used in most C functions. The ``AS_DATA`` form - casts the pointer to :c:expr:`const char *`. The *o* argument has to be - a Unicode object (not checked). - - .. versionchanged:: 3.3 - This function is now inefficient -- because in many cases the - :c:type:`Py_UNICODE` representation does not exist and needs to be created - -- and can fail (return ``NULL`` with an exception set). Try to port the - code to use the new :c:func:`PyUnicode_nBYTE_DATA` macros or use - :c:func:`PyUnicode_WRITE` or :c:func:`PyUnicode_READ`. - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style Unicode API, please migrate to using the - :c:func:`PyUnicode_nBYTE_DATA` family of macros. - - -.. c:function:: int PyUnicode_IsIdentifier(PyObject *o) - - Return ``1`` if the string is a valid identifier according to the language - definition, section :ref:`identifiers`. Return ``0`` otherwise. - - .. versionchanged:: 3.9 - The function does not call :c:func:`Py_FatalError` anymore if the string - is not ready. - - -Unicode Character Properties -"""""""""""""""""""""""""""" - -Unicode provides many different character properties. The most often needed ones -are available through these macros which are mapped to C functions depending on -the Python configuration. - - -.. c:function:: int Py_UNICODE_ISSPACE(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is a whitespace character. - - -.. c:function:: int Py_UNICODE_ISLOWER(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is a lowercase character. - - -.. c:function:: int Py_UNICODE_ISUPPER(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is an uppercase character. - - -.. c:function:: int Py_UNICODE_ISTITLE(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is a titlecase character. - - -.. c:function:: int Py_UNICODE_ISLINEBREAK(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is a linebreak character. - - -.. c:function:: int Py_UNICODE_ISDECIMAL(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is a decimal character. - - -.. c:function:: int Py_UNICODE_ISDIGIT(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is a digit character. - - -.. c:function:: int Py_UNICODE_ISNUMERIC(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is a numeric character. - - -.. c:function:: int Py_UNICODE_ISALPHA(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is an alphabetic character. - - -.. c:function:: int Py_UNICODE_ISALNUM(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is an alphanumeric character. - - -.. c:function:: int Py_UNICODE_ISPRINTABLE(Py_UCS4 ch) - - Return ``1`` or ``0`` depending on whether *ch* is a printable character. - Nonprintable characters are those characters defined in the Unicode character - database as "Other" or "Separator", excepting the ASCII space (0x20) which is - considered printable. (Note that printable characters in this context are - those which should not be escaped when :func:`repr` is invoked on a string. - It has no bearing on the handling of strings written to :data:`sys.stdout` or - :data:`sys.stderr`.) - - -These APIs can be used for fast direct character conversions: - - -.. c:function:: Py_UCS4 Py_UNICODE_TOLOWER(Py_UCS4 ch) - - Return the character *ch* converted to lower case. - - .. deprecated:: 3.3 - This function uses simple case mappings. - - -.. c:function:: Py_UCS4 Py_UNICODE_TOUPPER(Py_UCS4 ch) - - Return the character *ch* converted to upper case. - - .. deprecated:: 3.3 - This function uses simple case mappings. - - -.. c:function:: Py_UCS4 Py_UNICODE_TOTITLE(Py_UCS4 ch) - - Return the character *ch* converted to title case. - - .. deprecated:: 3.3 - This function uses simple case mappings. - - -.. c:function:: int Py_UNICODE_TODECIMAL(Py_UCS4 ch) - - Return the character *ch* converted to a decimal positive integer. Return - ``-1`` if this is not possible. This macro does not raise exceptions. - - -.. c:function:: int Py_UNICODE_TODIGIT(Py_UCS4 ch) - - Return the character *ch* converted to a single digit integer. Return ``-1`` if - this is not possible. This macro does not raise exceptions. - - -.. c:function:: double Py_UNICODE_TONUMERIC(Py_UCS4 ch) - - Return the character *ch* converted to a double. Return ``-1.0`` if this is not - possible. This macro does not raise exceptions. - - -These APIs can be used to work with surrogates: - -.. c:macro:: Py_UNICODE_IS_SURROGATE(ch) - - Check if *ch* is a surrogate (``0xD800 <= ch <= 0xDFFF``). - -.. c:macro:: Py_UNICODE_IS_HIGH_SURROGATE(ch) - - Check if *ch* is a high surrogate (``0xD800 <= ch <= 0xDBFF``). - -.. c:macro:: Py_UNICODE_IS_LOW_SURROGATE(ch) - - Check if *ch* is a low surrogate (``0xDC00 <= ch <= 0xDFFF``). - -.. c:macro:: Py_UNICODE_JOIN_SURROGATES(high, low) - - Join two surrogate characters and return a single Py_UCS4 value. - *high* and *low* are respectively the leading and trailing surrogates in a - surrogate pair. - - -Creating and accessing Unicode strings -"""""""""""""""""""""""""""""""""""""" - -To create Unicode objects and access their basic sequence properties, use these -APIs: - -.. c:function:: PyObject* PyUnicode_New(Py_ssize_t size, Py_UCS4 maxchar) - - Create a new Unicode object. *maxchar* should be the true maximum code point - to be placed in the string. As an approximation, it can be rounded up to the - nearest value in the sequence 127, 255, 65535, 1114111. - - This is the recommended way to allocate a new Unicode object. Objects - created using this function are not resizable. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_FromKindAndData(int kind, const void *buffer, \ - Py_ssize_t size) - - Create a new Unicode object with the given *kind* (possible values are - :c:macro:`PyUnicode_1BYTE_KIND` etc., as returned by - :c:func:`PyUnicode_KIND`). The *buffer* must point to an array of *size* - units of 1, 2 or 4 bytes per character, as given by the kind. - - If necessary, the input *buffer* is copied and transformed into the - canonical representation. For example, if the *buffer* is a UCS4 string - (:c:macro:`PyUnicode_4BYTE_KIND`) and it consists only of codepoints in - the UCS1 range, it will be transformed into UCS1 - (:c:macro:`PyUnicode_1BYTE_KIND`). - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_FromStringAndSize(const char *u, Py_ssize_t size) - - Create a Unicode object from the char buffer *u*. The bytes will be - interpreted as being UTF-8 encoded. The buffer is copied into the new - object. If the buffer is not ``NULL``, the return value might be a shared - object, i.e. modification of the data is not allowed. - - If *u* is ``NULL``, this function behaves like :c:func:`PyUnicode_FromUnicode` - with the buffer set to ``NULL``. This usage is deprecated in favor of - :c:func:`PyUnicode_New`, and will be removed in Python 3.12. - - -.. c:function:: PyObject *PyUnicode_FromString(const char *u) - - Create a Unicode object from a UTF-8 encoded null-terminated char buffer - *u*. - - -.. c:function:: PyObject* PyUnicode_FromFormat(const char *format, ...) - - Take a C :c:func:`printf`\ -style *format* string and a variable number of - arguments, calculate the size of the resulting Python Unicode string and return - a string with the values formatted into it. The variable arguments must be C - types and must correspond exactly to the format characters in the *format* - ASCII-encoded string. The following format characters are allowed: - - .. % This should be exactly the same as the table in PyErr_Format. - .. % The descriptions for %zd and %zu are wrong, but the truth is complicated - .. % because not all compilers support the %z width modifier -- we fake it - .. % when necessary via interpolating PY_FORMAT_SIZE_T. - .. % Similar comments apply to the %ll width modifier and - - .. tabularcolumns:: |l|l|L| - - +-------------------+---------------------+----------------------------------+ - | Format Characters | Type | Comment | - +===================+=====================+==================================+ - | :attr:`%%` | *n/a* | The literal % character. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%c` | int | A single character, | - | | | represented as a C int. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%d` | int | Equivalent to | - | | | ``printf("%d")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%u` | unsigned int | Equivalent to | - | | | ``printf("%u")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%ld` | long | Equivalent to | - | | | ``printf("%ld")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%li` | long | Equivalent to | - | | | ``printf("%li")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lu` | unsigned long | Equivalent to | - | | | ``printf("%lu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lld` | long long | Equivalent to | - | | | ``printf("%lld")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%lli` | long long | Equivalent to | - | | | ``printf("%lli")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%llu` | unsigned long long | Equivalent to | - | | | ``printf("%llu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zd` | :c:type:`\ | Equivalent to | - | | Py_ssize_t` | ``printf("%zd")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zi` | :c:type:`\ | Equivalent to | - | | Py_ssize_t` | ``printf("%zi")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%zu` | size_t | Equivalent to | - | | | ``printf("%zu")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%i` | int | Equivalent to | - | | | ``printf("%i")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%x` | int | Equivalent to | - | | | ``printf("%x")``. [1]_ | - +-------------------+---------------------+----------------------------------+ - | :attr:`%s` | const char\* | A null-terminated C character | - | | | array. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%p` | const void\* | The hex representation of a C | - | | | pointer. Mostly equivalent to | - | | | ``printf("%p")`` except that | - | | | it is guaranteed to start with | - | | | the literal ``0x`` regardless | - | | | of what the platform's | - | | | ``printf`` yields. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%A` | PyObject\* | The result of calling | - | | | :func:`ascii`. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%U` | PyObject\* | A Unicode object. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%V` | PyObject\*, | A Unicode object (which may be | - | | const char\* | ``NULL``) and a null-terminated | - | | | C character array as a second | - | | | parameter (which will be used, | - | | | if the first parameter is | - | | | ``NULL``). | - +-------------------+---------------------+----------------------------------+ - | :attr:`%S` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Str`. | - +-------------------+---------------------+----------------------------------+ - | :attr:`%R` | PyObject\* | The result of calling | - | | | :c:func:`PyObject_Repr`. | - +-------------------+---------------------+----------------------------------+ - - An unrecognized format character causes all the rest of the format string to be - copied as-is to the result string, and any extra arguments discarded. - - .. note:: - The width formatter unit is number of characters rather than bytes. - The precision formatter unit is number of bytes for ``"%s"`` and - ``"%V"`` (if the ``PyObject*`` argument is ``NULL``), and a number of - characters for ``"%A"``, ``"%U"``, ``"%S"``, ``"%R"`` and ``"%V"`` - (if the ``PyObject*`` argument is not ``NULL``). - - .. [1] For integer specifiers (d, u, ld, li, lu, lld, lli, llu, zd, zi, - zu, i, x): the 0-conversion flag has effect even when a precision is given. - - .. versionchanged:: 3.2 - Support for ``"%lld"`` and ``"%llu"`` added. - - .. versionchanged:: 3.3 - Support for ``"%li"``, ``"%lli"`` and ``"%zi"`` added. - - .. versionchanged:: 3.4 - Support width and precision formatter for ``"%s"``, ``"%A"``, ``"%U"``, - ``"%V"``, ``"%S"``, ``"%R"`` added. - - -.. c:function:: PyObject* PyUnicode_FromFormatV(const char *format, va_list vargs) - - Identical to :c:func:`PyUnicode_FromFormat` except that it takes exactly two - arguments. - - -.. c:function:: PyObject* PyUnicode_FromObject(PyObject *obj) - - Copy an instance of a Unicode subtype to a new true Unicode object if - necessary. If *obj* is already a true Unicode object (not a subtype), - return a new :term:`strong reference` to the object. - - Objects other than Unicode or its subtypes will cause a :exc:`TypeError`. - - -.. c:function:: PyObject* PyUnicode_FromEncodedObject(PyObject *obj, \ - const char *encoding, const char *errors) - - Decode an encoded object *obj* to a Unicode object. - - :class:`bytes`, :class:`bytearray` and other - :term:`bytes-like objects ` - are decoded according to the given *encoding* and using the error handling - defined by *errors*. Both can be ``NULL`` to have the interface use the default - values (see :ref:`builtincodecs` for details). - - All other objects, including Unicode objects, cause a :exc:`TypeError` to be - set. - - The API returns ``NULL`` if there was an error. The caller is responsible for - decref'ing the returned objects. - - -.. c:function:: Py_ssize_t PyUnicode_GetLength(PyObject *unicode) - - Return the length of the Unicode object, in code points. - - .. versionadded:: 3.3 - - -.. c:function:: Py_ssize_t PyUnicode_CopyCharacters(PyObject *to, \ - Py_ssize_t to_start, \ - PyObject *from, \ - Py_ssize_t from_start, \ - Py_ssize_t how_many) - - Copy characters from one Unicode object into another. This function performs - character conversion when necessary and falls back to :c:func:`!memcpy` if - possible. Returns ``-1`` and sets an exception on error, otherwise returns - the number of copied characters. - - .. versionadded:: 3.3 - - -.. c:function:: Py_ssize_t PyUnicode_Fill(PyObject *unicode, Py_ssize_t start, \ - Py_ssize_t length, Py_UCS4 fill_char) - - Fill a string with a character: write *fill_char* into - ``unicode[start:start+length]``. - - Fail if *fill_char* is bigger than the string maximum character, or if the - string has more than 1 reference. - - Return the number of written character, or return ``-1`` and raise an - exception on error. - - .. versionadded:: 3.3 - - -.. c:function:: int PyUnicode_WriteChar(PyObject *unicode, Py_ssize_t index, \ - Py_UCS4 character) - - Write a character to a string. The string must have been created through - :c:func:`PyUnicode_New`. Since Unicode strings are supposed to be immutable, - the string must not be shared, or have been hashed yet. - - This function checks that *unicode* is a Unicode object, that the index is - not out of bounds, and that the object can be modified safely (i.e. that it - its reference count is one). - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4 PyUnicode_ReadChar(PyObject *unicode, Py_ssize_t index) - - Read a character from a string. This function checks that *unicode* is a - Unicode object and the index is not out of bounds, in contrast to - :c:func:`PyUnicode_READ_CHAR`, which performs no error checking. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_Substring(PyObject *str, Py_ssize_t start, \ - Py_ssize_t end) - - Return a substring of *str*, from character index *start* (included) to - character index *end* (excluded). Negative indices are not supported. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4* PyUnicode_AsUCS4(PyObject *u, Py_UCS4 *buffer, \ - Py_ssize_t buflen, int copy_null) - - Copy the string *u* into a UCS4 buffer, including a null character, if - *copy_null* is set. Returns ``NULL`` and sets an exception on error (in - particular, a :exc:`SystemError` if *buflen* is smaller than the length of - *u*). *buffer* is returned on success. - - .. versionadded:: 3.3 - - -.. c:function:: Py_UCS4* PyUnicode_AsUCS4Copy(PyObject *u) - - Copy the string *u* into a new UCS4 buffer that is allocated using - :c:func:`PyMem_Malloc`. If this fails, ``NULL`` is returned with a - :exc:`MemoryError` set. The returned buffer always has an extra - null code point appended. - - .. versionadded:: 3.3 - - -Deprecated Py_UNICODE APIs -"""""""""""""""""""""""""" - -.. deprecated-removed:: 3.3 3.12 - -These API functions are deprecated with the implementation of :pep:`393`. -Extension modules can continue using them, as they will not be removed in Python -3.x, but need to be aware that their use can now cause performance and memory hits. - - -.. c:function:: PyObject* PyUnicode_FromUnicode(const Py_UNICODE *u, Py_ssize_t size) - - Create a Unicode object from the Py_UNICODE buffer *u* of the given size. *u* - may be ``NULL`` which causes the contents to be undefined. It is the user's - responsibility to fill in the needed data. The buffer is copied into the new - object. - - If the buffer is not ``NULL``, the return value might be a shared object. - Therefore, modification of the resulting Unicode object is only allowed when - *u* is ``NULL``. - - If the buffer is ``NULL``, :c:func:`PyUnicode_READY` must be called once the - string content has been filled before using any of the access macros such as - :c:func:`PyUnicode_KIND`. - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style Unicode API, please migrate to using - :c:func:`PyUnicode_FromKindAndData`, :c:func:`PyUnicode_FromWideChar`, or - :c:func:`PyUnicode_New`. - - -.. c:function:: Py_UNICODE* PyUnicode_AsUnicode(PyObject *unicode) - - Return a read-only pointer to the Unicode object's internal - :c:type:`Py_UNICODE` buffer, or ``NULL`` on error. This will create the - :c:expr:`Py_UNICODE*` representation of the object if it is not yet - available. The buffer is always terminated with an extra null code point. - Note that the resulting :c:type:`Py_UNICODE` string may also contain - embedded null code points, which would cause the string to be truncated when - used in most C functions. - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style Unicode API, please migrate to using - :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsWideChar`, - :c:func:`PyUnicode_ReadChar` or similar new APIs. - - -.. c:function:: Py_UNICODE* PyUnicode_AsUnicodeAndSize(PyObject *unicode, Py_ssize_t *size) - - Like :c:func:`PyUnicode_AsUnicode`, but also saves the :c:func:`Py_UNICODE` - array length (excluding the extra null terminator) in *size*. - Note that the resulting :c:expr:`Py_UNICODE*` string - may contain embedded null code points, which would cause the string to be - truncated when used in most C functions. - - .. versionadded:: 3.3 - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style Unicode API, please migrate to using - :c:func:`PyUnicode_AsUCS4`, :c:func:`PyUnicode_AsWideChar`, - :c:func:`PyUnicode_ReadChar` or similar new APIs. - - -.. c:function:: Py_ssize_t PyUnicode_GetSize(PyObject *unicode) - - Return the size of the deprecated :c:type:`Py_UNICODE` representation, in - code units (this includes surrogate pairs as 2 units). - - .. deprecated-removed:: 3.3 3.12 - Part of the old-style Unicode API, please migrate to using - :c:func:`PyUnicode_GET_LENGTH`. - - -Locale Encoding -""""""""""""""" - -The current locale encoding can be used to decode text from the operating -system. - -.. c:function:: PyObject* PyUnicode_DecodeLocaleAndSize(const char *str, \ - Py_ssize_t len, \ - const char *errors) - - Decode a string from UTF-8 on Android and VxWorks, or from the current - locale encoding on other platforms. The supported - error handlers are ``"strict"`` and ``"surrogateescape"`` - (:pep:`383`). The decoder uses ``"strict"`` error handler if - *errors* is ``NULL``. *str* must end with a null character but - cannot contain embedded null characters. - - Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` to decode a string from - :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at - Python startup). - - This function ignores the :ref:`Python UTF-8 Mode `. - - .. seealso:: - - The :c:func:`Py_DecodeLocale` function. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.7 - The function now also uses the current locale encoding for the - ``surrogateescape`` error handler, except on Android. Previously, :c:func:`Py_DecodeLocale` - was used for the ``surrogateescape``, and the current locale encoding was - used for ``strict``. - - -.. c:function:: PyObject* PyUnicode_DecodeLocale(const char *str, const char *errors) - - Similar to :c:func:`PyUnicode_DecodeLocaleAndSize`, but compute the string - length using :c:func:`!strlen`. - - .. versionadded:: 3.3 - - -.. c:function:: PyObject* PyUnicode_EncodeLocale(PyObject *unicode, const char *errors) - - Encode a Unicode object to UTF-8 on Android and VxWorks, or to the current - locale encoding on other platforms. The - supported error handlers are ``"strict"`` and ``"surrogateescape"`` - (:pep:`383`). The encoder uses ``"strict"`` error handler if - *errors* is ``NULL``. Return a :class:`bytes` object. *unicode* cannot - contain embedded null characters. - - Use :c:func:`PyUnicode_EncodeFSDefault` to encode a string to - :c:data:`Py_FileSystemDefaultEncoding` (the locale encoding read at - Python startup). - - This function ignores the :ref:`Python UTF-8 Mode `. - - .. seealso:: - - The :c:func:`Py_EncodeLocale` function. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.7 - The function now also uses the current locale encoding for the - ``surrogateescape`` error handler, except on Android. Previously, - :c:func:`Py_EncodeLocale` - was used for the ``surrogateescape``, and the current locale encoding was - used for ``strict``. - - -File System Encoding -"""""""""""""""""""" - -To encode and decode file names and other environment strings, -:c:data:`Py_FileSystemDefaultEncoding` should be used as the encoding, and -:c:data:`Py_FileSystemDefaultEncodeErrors` should be used as the error handler -(:pep:`383` and :pep:`529`). To encode file names to :class:`bytes` during -argument parsing, the ``"O&"`` converter should be used, passing -:c:func:`PyUnicode_FSConverter` as the conversion function: - -.. c:function:: int PyUnicode_FSConverter(PyObject* obj, void* result) - - ParseTuple converter: encode :class:`str` objects -- obtained directly or - through the :class:`os.PathLike` interface -- to :class:`bytes` using - :c:func:`PyUnicode_EncodeFSDefault`; :class:`bytes` objects are output as-is. - *result* must be a :c:expr:`PyBytesObject*` which must be released when it is - no longer used. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -To decode file names to :class:`str` during argument parsing, the ``"O&"`` -converter should be used, passing :c:func:`PyUnicode_FSDecoder` as the -conversion function: - -.. c:function:: int PyUnicode_FSDecoder(PyObject* obj, void* result) - - ParseTuple converter: decode :class:`bytes` objects -- obtained either - directly or indirectly through the :class:`os.PathLike` interface -- to - :class:`str` using :c:func:`PyUnicode_DecodeFSDefaultAndSize`; :class:`str` - objects are output as-is. *result* must be a :c:expr:`PyUnicodeObject*` which - must be released when it is no longer used. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. c:function:: PyObject* PyUnicode_DecodeFSDefaultAndSize(const char *s, Py_ssize_t size) - - Decode a string from the :term:`filesystem encoding and error handler`. - - If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the - locale encoding. - - :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the - locale encoding and cannot be modified later. If you need to decode a string - from the current locale encoding, use - :c:func:`PyUnicode_DecodeLocaleAndSize`. - - .. seealso:: - - The :c:func:`Py_DecodeLocale` function. - - .. versionchanged:: 3.6 - Use :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. - - -.. c:function:: PyObject* PyUnicode_DecodeFSDefault(const char *s) - - Decode a null-terminated string from the :term:`filesystem encoding and - error handler`. - - If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the - locale encoding. - - Use :c:func:`PyUnicode_DecodeFSDefaultAndSize` if you know the string length. - - .. versionchanged:: 3.6 - Use :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. - - -.. c:function:: PyObject* PyUnicode_EncodeFSDefault(PyObject *unicode) - - Encode a Unicode object to :c:data:`Py_FileSystemDefaultEncoding` with the - :c:data:`Py_FileSystemDefaultEncodeErrors` error handler, and return - :class:`bytes`. Note that the resulting :class:`bytes` object may contain - null bytes. - - If :c:data:`Py_FileSystemDefaultEncoding` is not set, fall back to the - locale encoding. - - :c:data:`Py_FileSystemDefaultEncoding` is initialized at startup from the - locale encoding and cannot be modified later. If you need to encode a string - to the current locale encoding, use :c:func:`PyUnicode_EncodeLocale`. - - .. seealso:: - - The :c:func:`Py_EncodeLocale` function. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.6 - Use :c:data:`Py_FileSystemDefaultEncodeErrors` error handler. - -wchar_t Support -""""""""""""""" - -:c:type:`wchar_t` support for platforms which support it: - -.. c:function:: PyObject* PyUnicode_FromWideChar(const wchar_t *w, Py_ssize_t size) - - Create a Unicode object from the :c:type:`wchar_t` buffer *w* of the given *size*. - Passing ``-1`` as the *size* indicates that the function must itself compute the length, - using wcslen. - Return ``NULL`` on failure. - - -.. c:function:: Py_ssize_t PyUnicode_AsWideChar(PyObject *unicode, wchar_t *w, Py_ssize_t size) - - Copy the Unicode object contents into the :c:type:`wchar_t` buffer *w*. At most - *size* :c:type:`wchar_t` characters are copied (excluding a possibly trailing - null termination character). Return the number of :c:type:`wchar_t` characters - copied or ``-1`` in case of an error. Note that the resulting :c:expr:`wchar_t*` - string may or may not be null-terminated. It is the responsibility of the caller - to make sure that the :c:expr:`wchar_t*` string is null-terminated in case this is - required by the application. Also, note that the :c:expr:`wchar_t*` string - might contain null characters, which would cause the string to be truncated - when used with most C functions. - - -.. c:function:: wchar_t* PyUnicode_AsWideCharString(PyObject *unicode, Py_ssize_t *size) - - Convert the Unicode object to a wide character string. The output string - always ends with a null character. If *size* is not ``NULL``, write the number - of wide characters (excluding the trailing null termination character) into - *\*size*. Note that the resulting :c:type:`wchar_t` string might contain - null characters, which would cause the string to be truncated when used with - most C functions. If *size* is ``NULL`` and the :c:expr:`wchar_t*` string - contains null characters a :exc:`ValueError` is raised. - - Returns a buffer allocated by :c:macro:`PyMem_New` (use - :c:func:`PyMem_Free` to free it) on success. On error, returns ``NULL`` - and *\*size* is undefined. Raises a :exc:`MemoryError` if memory allocation - is failed. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.7 - Raises a :exc:`ValueError` if *size* is ``NULL`` and the :c:expr:`wchar_t*` - string contains null characters. - - -.. _builtincodecs: - -Built-in Codecs -^^^^^^^^^^^^^^^ - -Python provides a set of built-in codecs which are written in C for speed. All of -these codecs are directly usable via the following functions. - -Many of the following APIs take two arguments encoding and errors, and they -have the same semantics as the ones of the built-in :func:`str` string object -constructor. - -Setting encoding to ``NULL`` causes the default encoding to be used -which is UTF-8. The file system calls should use -:c:func:`PyUnicode_FSConverter` for encoding file names. This uses the -variable :c:data:`Py_FileSystemDefaultEncoding` internally. This -variable should be treated as read-only: on some systems, it will be a -pointer to a static string, on others, it will change at run-time -(such as when the application invokes setlocale). - -Error handling is set by errors which may also be set to ``NULL`` meaning to use -the default handling defined for the codec. Default error handling for all -built-in codecs is "strict" (:exc:`ValueError` is raised). - -The codecs all use a similar interface. Only deviations from the following -generic ones are documented for simplicity. - - -Generic Codecs -"""""""""""""" - -These are the generic codec APIs: - - -.. c:function:: PyObject* PyUnicode_Decode(const char *s, Py_ssize_t size, \ - const char *encoding, const char *errors) - - Create a Unicode object by decoding *size* bytes of the encoded string *s*. - *encoding* and *errors* have the same meaning as the parameters of the same name - in the :func:`str` built-in function. The codec to be used is looked up - using the Python codec registry. Return ``NULL`` if an exception was raised by - the codec. - - -.. c:function:: PyObject* PyUnicode_AsEncodedString(PyObject *unicode, \ - const char *encoding, const char *errors) - - Encode a Unicode object and return the result as Python bytes object. - *encoding* and *errors* have the same meaning as the parameters of the same - name in the Unicode :meth:`~str.encode` method. The codec to be used is looked up - using the Python codec registry. Return ``NULL`` if an exception was raised by - the codec. - - -UTF-8 Codecs -"""""""""""" - -These are the UTF-8 codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUTF8(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the UTF-8 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeUTF8Stateful(const char *s, Py_ssize_t size, \ - const char *errors, Py_ssize_t *consumed) - - If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF8`. If - *consumed* is not ``NULL``, trailing incomplete UTF-8 byte sequences will not be - treated as an error. Those bytes will not be decoded and the number of bytes - that have been decoded will be stored in *consumed*. - - -.. c:function:: PyObject* PyUnicode_AsUTF8String(PyObject *unicode) - - Encode a Unicode object using UTF-8 and return the result as Python bytes - object. Error handling is "strict". Return ``NULL`` if an exception was - raised by the codec. - - -.. c:function:: const char* PyUnicode_AsUTF8AndSize(PyObject *unicode, Py_ssize_t *size) - - Return a pointer to the UTF-8 encoding of the Unicode object, and - store the size of the encoded representation (in bytes) in *size*. The - *size* argument can be ``NULL``; in this case no size will be stored. The - returned buffer always has an extra null byte appended (not included in - *size*), regardless of whether there are any other null code points. - - In the case of an error, ``NULL`` is returned with an exception set and no - *size* is stored. - - This caches the UTF-8 representation of the string in the Unicode object, and - subsequent calls will return a pointer to the same buffer. The caller is not - responsible for deallocating the buffer. The buffer is deallocated and - pointers to it become invalid when the Unicode object is garbage collected. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. - - .. versionchanged:: 3.10 - This function is a part of the :ref:`limited API `. - - -.. c:function:: const char* PyUnicode_AsUTF8(PyObject *unicode) - - As :c:func:`PyUnicode_AsUTF8AndSize`, but does not store the size. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.7 - The return type is now ``const char *`` rather of ``char *``. - - -UTF-32 Codecs -""""""""""""" - -These are the UTF-32 codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUTF32(const char *s, Py_ssize_t size, \ - const char *errors, int *byteorder) - - Decode *size* bytes from a UTF-32 encoded buffer string and return the - corresponding Unicode object. *errors* (if non-``NULL``) defines the error - handling. It defaults to "strict". - - If *byteorder* is non-``NULL``, the decoder starts decoding using the given byte - order:: - - *byteorder == -1: little endian - *byteorder == 0: native order - *byteorder == 1: big endian - - If ``*byteorder`` is zero, and the first four bytes of the input data are a - byte order mark (BOM), the decoder switches to this byte order and the BOM is - not copied into the resulting Unicode string. If ``*byteorder`` is ``-1`` or - ``1``, any byte order mark is copied to the output. - - After completion, *\*byteorder* is set to the current byte order at the end - of input data. - - If *byteorder* is ``NULL``, the codec starts in native order mode. - - Return ``NULL`` if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeUTF32Stateful(const char *s, Py_ssize_t size, \ - const char *errors, int *byteorder, Py_ssize_t *consumed) - - If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF32`. If - *consumed* is not ``NULL``, :c:func:`PyUnicode_DecodeUTF32Stateful` will not treat - trailing incomplete UTF-32 byte sequences (such as a number of bytes not divisible - by four) as an error. Those bytes will not be decoded and the number of bytes - that have been decoded will be stored in *consumed*. - - -.. c:function:: PyObject* PyUnicode_AsUTF32String(PyObject *unicode) - - Return a Python byte string using the UTF-32 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return ``NULL`` if an exception was raised by the codec. - - -UTF-16 Codecs -""""""""""""" - -These are the UTF-16 codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUTF16(const char *s, Py_ssize_t size, \ - const char *errors, int *byteorder) - - Decode *size* bytes from a UTF-16 encoded buffer string and return the - corresponding Unicode object. *errors* (if non-``NULL``) defines the error - handling. It defaults to "strict". - - If *byteorder* is non-``NULL``, the decoder starts decoding using the given byte - order:: - - *byteorder == -1: little endian - *byteorder == 0: native order - *byteorder == 1: big endian - - If ``*byteorder`` is zero, and the first two bytes of the input data are a - byte order mark (BOM), the decoder switches to this byte order and the BOM is - not copied into the resulting Unicode string. If ``*byteorder`` is ``-1`` or - ``1``, any byte order mark is copied to the output (where it will result in - either a ``\ufeff`` or a ``\ufffe`` character). - - After completion, ``*byteorder`` is set to the current byte order at the end - of input data. - - If *byteorder* is ``NULL``, the codec starts in native order mode. - - Return ``NULL`` if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeUTF16Stateful(const char *s, Py_ssize_t size, \ - const char *errors, int *byteorder, Py_ssize_t *consumed) - - If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF16`. If - *consumed* is not ``NULL``, :c:func:`PyUnicode_DecodeUTF16Stateful` will not treat - trailing incomplete UTF-16 byte sequences (such as an odd number of bytes or a - split surrogate pair) as an error. Those bytes will not be decoded and the - number of bytes that have been decoded will be stored in *consumed*. - - -.. c:function:: PyObject* PyUnicode_AsUTF16String(PyObject *unicode) - - Return a Python byte string using the UTF-16 encoding in native byte - order. The string always starts with a BOM mark. Error handling is "strict". - Return ``NULL`` if an exception was raised by the codec. - - -UTF-7 Codecs -"""""""""""" - -These are the UTF-7 codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUTF7(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the UTF-7 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeUTF7Stateful(const char *s, Py_ssize_t size, \ - const char *errors, Py_ssize_t *consumed) - - If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeUTF7`. If - *consumed* is not ``NULL``, trailing incomplete UTF-7 base-64 sections will not - be treated as an error. Those bytes will not be decoded and the number of - bytes that have been decoded will be stored in *consumed*. - - -Unicode-Escape Codecs -""""""""""""""""""""" - -These are the "Unicode Escape" codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeUnicodeEscape(const char *s, \ - Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the Unicode-Escape encoded - string *s*. Return ``NULL`` if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_AsUnicodeEscapeString(PyObject *unicode) - - Encode a Unicode object using Unicode-Escape and return the result as a - bytes object. Error handling is "strict". Return ``NULL`` if an exception was - raised by the codec. - - -Raw-Unicode-Escape Codecs -""""""""""""""""""""""""" - -These are the "Raw Unicode Escape" codec APIs: - - -.. c:function:: PyObject* PyUnicode_DecodeRawUnicodeEscape(const char *s, \ - Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the Raw-Unicode-Escape - encoded string *s*. Return ``NULL`` if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_AsRawUnicodeEscapeString(PyObject *unicode) - - Encode a Unicode object using Raw-Unicode-Escape and return the result as - a bytes object. Error handling is "strict". Return ``NULL`` if an exception - was raised by the codec. - - -Latin-1 Codecs -"""""""""""""" - -These are the Latin-1 codec APIs: Latin-1 corresponds to the first 256 Unicode -ordinals and only these are accepted by the codecs during encoding. - - -.. c:function:: PyObject* PyUnicode_DecodeLatin1(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the Latin-1 encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_AsLatin1String(PyObject *unicode) - - Encode a Unicode object using Latin-1 and return the result as Python bytes - object. Error handling is "strict". Return ``NULL`` if an exception was - raised by the codec. - - -ASCII Codecs -"""""""""""" - -These are the ASCII codec APIs. Only 7-bit ASCII data is accepted. All other -codes generate errors. - - -.. c:function:: PyObject* PyUnicode_DecodeASCII(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the ASCII encoded string - *s*. Return ``NULL`` if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_AsASCIIString(PyObject *unicode) - - Encode a Unicode object using ASCII and return the result as Python bytes - object. Error handling is "strict". Return ``NULL`` if an exception was - raised by the codec. - - -Character Map Codecs -"""""""""""""""""""" - -This codec is special in that it can be used to implement many different codecs -(and this is in fact what was done to obtain most of the standard codecs -included in the :mod:`!encodings` package). The codec uses mappings to encode and -decode characters. The mapping objects provided must support the -:meth:`~object.__getitem__` mapping interface; dictionaries and sequences work well. - -These are the mapping codec APIs: - -.. c:function:: PyObject* PyUnicode_DecodeCharmap(const char *data, Py_ssize_t size, \ - PyObject *mapping, const char *errors) - - Create a Unicode object by decoding *size* bytes of the encoded string *s* - using the given *mapping* object. Return ``NULL`` if an exception was raised - by the codec. - - If *mapping* is ``NULL``, Latin-1 decoding will be applied. Else - *mapping* must map bytes ordinals (integers in the range from 0 to 255) - to Unicode strings, integers (which are then interpreted as Unicode - ordinals) or ``None``. Unmapped data bytes -- ones which cause a - :exc:`LookupError`, as well as ones which get mapped to ``None``, - ``0xFFFE`` or ``'\ufffe'``, are treated as undefined mappings and cause - an error. - - -.. c:function:: PyObject* PyUnicode_AsCharmapString(PyObject *unicode, PyObject *mapping) - - Encode a Unicode object using the given *mapping* object and return the - result as a bytes object. Error handling is "strict". Return ``NULL`` if an - exception was raised by the codec. - - The *mapping* object must map Unicode ordinal integers to bytes objects, - integers in the range from 0 to 255 or ``None``. Unmapped character - ordinals (ones which cause a :exc:`LookupError`) as well as mapped to - ``None`` are treated as "undefined mapping" and cause an error. - - -The following codec API is special in that maps Unicode to Unicode. - -.. c:function:: PyObject* PyUnicode_Translate(PyObject *str, PyObject *table, const char *errors) - - Translate a string by applying a character mapping table to it and return the - resulting Unicode object. Return ``NULL`` if an exception was raised by the - codec. - - The mapping table must map Unicode ordinal integers to Unicode ordinal integers - or ``None`` (causing deletion of the character). - - Mapping tables need only provide the :meth:`~object.__getitem__` interface; dictionaries - and sequences work well. Unmapped character ordinals (ones which cause a - :exc:`LookupError`) are left untouched and are copied as-is. - - *errors* has the usual meaning for codecs. It may be ``NULL`` which indicates to - use the default error handling. - - -MBCS codecs for Windows -""""""""""""""""""""""" - -These are the MBCS codec APIs. They are currently only available on Windows and -use the Win32 MBCS converters to implement the conversions. Note that MBCS (or -DBCS) is a class of encodings, not just one. The target encoding is defined by -the user settings on the machine running the codec. - -.. c:function:: PyObject* PyUnicode_DecodeMBCS(const char *s, Py_ssize_t size, const char *errors) - - Create a Unicode object by decoding *size* bytes of the MBCS encoded string *s*. - Return ``NULL`` if an exception was raised by the codec. - - -.. c:function:: PyObject* PyUnicode_DecodeMBCSStateful(const char *s, Py_ssize_t size, \ - const char *errors, Py_ssize_t *consumed) - - If *consumed* is ``NULL``, behave like :c:func:`PyUnicode_DecodeMBCS`. If - *consumed* is not ``NULL``, :c:func:`PyUnicode_DecodeMBCSStateful` will not decode - trailing lead byte and the number of bytes that have been decoded will be stored - in *consumed*. - - -.. c:function:: PyObject* PyUnicode_AsMBCSString(PyObject *unicode) - - Encode a Unicode object using MBCS and return the result as Python bytes - object. Error handling is "strict". Return ``NULL`` if an exception was - raised by the codec. - - -.. c:function:: PyObject* PyUnicode_EncodeCodePage(int code_page, PyObject *unicode, const char *errors) - - Encode the Unicode object using the specified code page and return a Python - bytes object. Return ``NULL`` if an exception was raised by the codec. Use - :c:macro:`!CP_ACP` code page to get the MBCS encoder. - - .. versionadded:: 3.3 - - -Methods & Slots -""""""""""""""" - - -.. _unicodemethodsandslots: - -Methods and Slot Functions -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following APIs are capable of handling Unicode objects and strings on input -(we refer to them as strings in the descriptions) and return Unicode objects or -integers as appropriate. - -They all return ``NULL`` or ``-1`` if an exception occurs. - - -.. c:function:: PyObject* PyUnicode_Concat(PyObject *left, PyObject *right) - - Concat two strings giving a new Unicode string. - - -.. c:function:: PyObject* PyUnicode_Split(PyObject *s, PyObject *sep, Py_ssize_t maxsplit) - - Split a string giving a list of Unicode strings. If *sep* is ``NULL``, splitting - will be done at all whitespace substrings. Otherwise, splits occur at the given - separator. At most *maxsplit* splits will be done. If negative, no limit is - set. Separators are not included in the resulting list. - - -.. c:function:: PyObject* PyUnicode_Splitlines(PyObject *s, int keepend) - - Split a Unicode string at line breaks, returning a list of Unicode strings. - CRLF is considered to be one line break. If *keepend* is ``0``, the line break - characters are not included in the resulting strings. - - -.. c:function:: PyObject* PyUnicode_Join(PyObject *separator, PyObject *seq) - - Join a sequence of strings using the given *separator* and return the resulting - Unicode string. - - -.. c:function:: Py_ssize_t PyUnicode_Tailmatch(PyObject *str, PyObject *substr, \ - Py_ssize_t start, Py_ssize_t end, int direction) - - Return ``1`` if *substr* matches ``str[start:end]`` at the given tail end - (*direction* == ``-1`` means to do a prefix match, *direction* == ``1`` a suffix match), - ``0`` otherwise. Return ``-1`` if an error occurred. - - -.. c:function:: Py_ssize_t PyUnicode_Find(PyObject *str, PyObject *substr, \ - Py_ssize_t start, Py_ssize_t end, int direction) - - Return the first position of *substr* in ``str[start:end]`` using the given - *direction* (*direction* == ``1`` means to do a forward search, *direction* == ``-1`` a - backward search). The return value is the index of the first match; a value of - ``-1`` indicates that no match was found, and ``-2`` indicates that an error - occurred and an exception has been set. - - -.. c:function:: Py_ssize_t PyUnicode_FindChar(PyObject *str, Py_UCS4 ch, \ - Py_ssize_t start, Py_ssize_t end, int direction) - - Return the first position of the character *ch* in ``str[start:end]`` using - the given *direction* (*direction* == ``1`` means to do a forward search, - *direction* == ``-1`` a backward search). The return value is the index of the - first match; a value of ``-1`` indicates that no match was found, and ``-2`` - indicates that an error occurred and an exception has been set. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.7 - *start* and *end* are now adjusted to behave like ``str[start:end]``. - - -.. c:function:: Py_ssize_t PyUnicode_Count(PyObject *str, PyObject *substr, \ - Py_ssize_t start, Py_ssize_t end) - - Return the number of non-overlapping occurrences of *substr* in - ``str[start:end]``. Return ``-1`` if an error occurred. - - -.. c:function:: PyObject* PyUnicode_Replace(PyObject *str, PyObject *substr, \ - PyObject *replstr, Py_ssize_t maxcount) - - Replace at most *maxcount* occurrences of *substr* in *str* with *replstr* and - return the resulting Unicode object. *maxcount* == ``-1`` means replace all - occurrences. - - -.. c:function:: int PyUnicode_Compare(PyObject *left, PyObject *right) - - Compare two strings and return ``-1``, ``0``, ``1`` for less than, equal, and greater than, - respectively. - - This function returns ``-1`` upon failure, so one should call - :c:func:`PyErr_Occurred` to check for errors. - - -.. c:function:: int PyUnicode_CompareWithASCIIString(PyObject *uni, const char *string) - - Compare a Unicode object, *uni*, with *string* and return ``-1``, ``0``, ``1`` for less - than, equal, and greater than, respectively. It is best to pass only - ASCII-encoded strings, but the function interprets the input string as - ISO-8859-1 if it contains non-ASCII characters. - - This function does not raise exceptions. - - -.. c:function:: PyObject* PyUnicode_RichCompare(PyObject *left, PyObject *right, int op) - - Rich compare two Unicode strings and return one of the following: - - * ``NULL`` in case an exception was raised - * :c:data:`Py_True` or :c:data:`Py_False` for successful comparisons - * :c:data:`Py_NotImplemented` in case the type combination is unknown - - Possible values for *op* are :c:macro:`Py_GT`, :c:macro:`Py_GE`, :c:macro:`Py_EQ`, - :c:macro:`Py_NE`, :c:macro:`Py_LT`, and :c:macro:`Py_LE`. - - -.. c:function:: PyObject* PyUnicode_Format(PyObject *format, PyObject *args) - - Return a new string object from *format* and *args*; this is analogous to - ``format % args``. - - -.. c:function:: int PyUnicode_Contains(PyObject *container, PyObject *element) - - Check whether *element* is contained in *container* and return true or false - accordingly. - - *element* has to coerce to a one element Unicode string. ``-1`` is returned - if there was an error. - - -.. c:function:: void PyUnicode_InternInPlace(PyObject **string) - - Intern the argument *\*string* in place. The argument must be the address of a - pointer variable pointing to a Python Unicode string object. If there is an - existing interned string that is the same as *\*string*, it sets *\*string* to - it (releasing the reference to the old string object and creating a new - :term:`strong reference` to the interned string object), otherwise it leaves - *\*string* alone and interns it (creating a new :term:`strong reference`). - (Clarification: even though there is a lot of talk about references, think - of this function as reference-neutral; you own the object after the call - if and only if you owned it before the call.) - - -.. c:function:: PyObject* PyUnicode_InternFromString(const char *v) - - A combination of :c:func:`PyUnicode_FromString` and - :c:func:`PyUnicode_InternInPlace`, returning either a new Unicode string - object that has been interned, or a new ("owned") reference to an earlier - interned string object with the same value. diff --git a/Doc/c-api/utilities.rst b/Doc/c-api/utilities.rst deleted file mode 100644 index a805b564763c40..00000000000000 --- a/Doc/c-api/utilities.rst +++ /dev/null @@ -1,21 +0,0 @@ -.. highlight:: c - -.. _utilities: - -********* -Utilities -********* - -The functions in this chapter perform various utility tasks, ranging from -helping C code be more portable across platforms, using Python modules from C, -and parsing function arguments and constructing Python values from C values. - -.. toctree:: - - sys.rst - import.rst - marshal.rst - arg.rst - conversion.rst - reflection.rst - codec.rst diff --git a/Doc/c-api/veryhigh.rst b/Doc/c-api/veryhigh.rst deleted file mode 100644 index 9a1295b558f638..00000000000000 --- a/Doc/c-api/veryhigh.rst +++ /dev/null @@ -1,371 +0,0 @@ -.. highlight:: c - - -.. _veryhigh: - -************************* -The Very High Level Layer -************************* - -The functions in this chapter will let you execute Python source code given in a -file or a buffer, but they will not let you interact in a more detailed way with -the interpreter. - -Several of these functions accept a start symbol from the grammar as a -parameter. The available start symbols are :c:data:`Py_eval_input`, -:c:data:`Py_file_input`, and :c:data:`Py_single_input`. These are described -following the functions which accept them as parameters. - -Note also that several of these functions take :c:expr:`FILE*` parameters. One -particular issue which needs to be handled carefully is that the :c:type:`FILE` -structure for different C libraries can be different and incompatible. Under -Windows (at least), it is possible for dynamically linked extensions to actually -use different libraries, so care should be taken that :c:expr:`FILE*` parameters -are only passed to these functions if it is certain that they were created by -the same library that the Python runtime is using. - - -.. c:function:: int Py_Main(int argc, wchar_t **argv) - - The main program for the standard interpreter. This is made available for - programs which embed Python. The *argc* and *argv* parameters should be - prepared exactly as those which are passed to a C program's :c:func:`main` - function (converted to wchar_t according to the user's locale). It is - important to note that the argument list may be modified (but the contents of - the strings pointed to by the argument list are not). The return value will - be ``0`` if the interpreter exits normally (i.e., without an exception), - ``1`` if the interpreter exits due to an exception, or ``2`` if the parameter - list does not represent a valid Python command line. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``1``, but exit the process, as long as - ``Py_InspectFlag`` is not set. - - -.. c:function:: int Py_BytesMain(int argc, char **argv) - - Similar to :c:func:`Py_Main` but *argv* is an array of bytes strings. - - .. versionadded:: 3.8 - - -.. c:function:: int PyRun_AnyFile(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - *closeit* set to ``0`` and *flags* set to ``NULL``. - - -.. c:function:: int PyRun_AnyFileFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - - This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - the *closeit* argument set to ``0``. - - -.. c:function:: int PyRun_AnyFileEx(FILE *fp, const char *filename, int closeit) - - This is a simplified interface to :c:func:`PyRun_AnyFileExFlags` below, leaving - the *flags* argument set to ``NULL``. - - -.. c:function:: int PyRun_AnyFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) - - If *fp* refers to a file associated with an interactive device (console or - terminal input or Unix pseudo-terminal), return the value of - :c:func:`PyRun_InteractiveLoop`, otherwise return the result of - :c:func:`PyRun_SimpleFile`. *filename* is decoded from the filesystem - encoding (:func:`sys.getfilesystemencoding`). If *filename* is ``NULL``, this - function uses ``"???"`` as the filename. - If *closeit* is true, the file is closed before - ``PyRun_SimpleFileExFlags()`` returns. - - -.. c:function:: int PyRun_SimpleString(const char *command) - - This is a simplified interface to :c:func:`PyRun_SimpleStringFlags` below, - leaving the :c:struct:`PyCompilerFlags`\* argument set to ``NULL``. - - -.. c:function:: int PyRun_SimpleStringFlags(const char *command, PyCompilerFlags *flags) - - Executes the Python source code from *command* in the :mod:`__main__` module - according to the *flags* argument. If :mod:`__main__` does not already exist, it - is created. Returns ``0`` on success or ``-1`` if an exception was raised. If - there was an error, there is no way to get the exception information. For the - meaning of *flags*, see below. - - Note that if an otherwise unhandled :exc:`SystemExit` is raised, this - function will not return ``-1``, but exit the process, as long as - ``Py_InspectFlag`` is not set. - - -.. c:function:: int PyRun_SimpleFile(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, - leaving *closeit* set to ``0`` and *flags* set to ``NULL``. - - -.. c:function:: int PyRun_SimpleFileEx(FILE *fp, const char *filename, int closeit) - - This is a simplified interface to :c:func:`PyRun_SimpleFileExFlags` below, - leaving *flags* set to ``NULL``. - - -.. c:function:: int PyRun_SimpleFileExFlags(FILE *fp, const char *filename, int closeit, PyCompilerFlags *flags) - - Similar to :c:func:`PyRun_SimpleStringFlags`, but the Python source code is read - from *fp* instead of an in-memory string. *filename* should be the name of - the file, it is decoded from :term:`filesystem encoding and error handler`. - If *closeit* is true, the file is closed before - ``PyRun_SimpleFileExFlags()`` returns. - - .. note:: - On Windows, *fp* should be opened as binary mode (e.g. ``fopen(filename, "rb")``). - Otherwise, Python may not handle script file with LF line ending correctly. - - -.. c:function:: int PyRun_InteractiveOne(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveOneFlags` below, - leaving *flags* set to ``NULL``. - - -.. c:function:: int PyRun_InteractiveOneFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - - Read and execute a single statement from a file associated with an - interactive device according to the *flags* argument. The user will be - prompted using ``sys.ps1`` and ``sys.ps2``. *filename* is decoded from the - :term:`filesystem encoding and error handler`. - - Returns ``0`` when the input was - executed successfully, ``-1`` if there was an exception, or an error code - from the :file:`errcode.h` include file distributed as part of Python if - there was a parse error. (Note that :file:`errcode.h` is not included by - :file:`Python.h`, so must be included specifically if needed.) - - -.. c:function:: int PyRun_InteractiveLoop(FILE *fp, const char *filename) - - This is a simplified interface to :c:func:`PyRun_InteractiveLoopFlags` below, - leaving *flags* set to ``NULL``. - - -.. c:function:: int PyRun_InteractiveLoopFlags(FILE *fp, const char *filename, PyCompilerFlags *flags) - - Read and execute statements from a file associated with an interactive device - until EOF is reached. The user will be prompted using ``sys.ps1`` and - ``sys.ps2``. *filename* is decoded from the :term:`filesystem encoding and - error handler`. Returns ``0`` at EOF or a negative number upon failure. - - -.. c:var:: int (*PyOS_InputHook)(void) - - Can be set to point to a function with the prototype - ``int func(void)``. The function will be called when Python's - interpreter prompt is about to become idle and wait for user input - from the terminal. The return value is ignored. Overriding this - hook can be used to integrate the interpreter's prompt with other - event loops, as done in the :file:`Modules/_tkinter.c` in the - Python source code. - - -.. c:var:: char* (*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *) - - Can be set to point to a function with the prototype - ``char *func(FILE *stdin, FILE *stdout, char *prompt)``, - overriding the default function used to read a single line of input - at the interpreter's prompt. The function is expected to output - the string *prompt* if it's not ``NULL``, and then read a line of - input from the provided standard input file, returning the - resulting string. For example, The :mod:`readline` module sets - this hook to provide line-editing and tab-completion features. - - The result must be a string allocated by :c:func:`PyMem_RawMalloc` or - :c:func:`PyMem_RawRealloc`, or ``NULL`` if an error occurred. - - .. versionchanged:: 3.4 - The result must be allocated by :c:func:`PyMem_RawMalloc` or - :c:func:`PyMem_RawRealloc`, instead of being allocated by - :c:func:`PyMem_Malloc` or :c:func:`PyMem_Realloc`. - -.. c:function:: PyObject* PyRun_String(const char *str, int start, PyObject *globals, PyObject *locals) - - This is a simplified interface to :c:func:`PyRun_StringFlags` below, leaving - *flags* set to ``NULL``. - - -.. c:function:: PyObject* PyRun_StringFlags(const char *str, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) - - Execute Python source code from *str* in the context specified by the - objects *globals* and *locals* with the compiler flags specified by - *flags*. *globals* must be a dictionary; *locals* can be any object - that implements the mapping protocol. The parameter *start* specifies - the start token that should be used to parse the source code. - - Returns the result of executing the code as a Python object, or ``NULL`` if an - exception was raised. - - -.. c:function:: PyObject* PyRun_File(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals) - - This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *closeit* set to ``0`` and *flags* set to ``NULL``. - - -.. c:function:: PyObject* PyRun_FileEx(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit) - - This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *flags* set to ``NULL``. - - -.. c:function:: PyObject* PyRun_FileFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, PyCompilerFlags *flags) - - This is a simplified interface to :c:func:`PyRun_FileExFlags` below, leaving - *closeit* set to ``0``. - - -.. c:function:: PyObject* PyRun_FileExFlags(FILE *fp, const char *filename, int start, PyObject *globals, PyObject *locals, int closeit, PyCompilerFlags *flags) - - Similar to :c:func:`PyRun_StringFlags`, but the Python source code is read from - *fp* instead of an in-memory string. *filename* should be the name of the file, - it is decoded from the :term:`filesystem encoding and error handler`. - If *closeit* is true, the file is closed before :c:func:`PyRun_FileExFlags` - returns. - - -.. c:function:: PyObject* Py_CompileString(const char *str, const char *filename, int start) - - This is a simplified interface to :c:func:`Py_CompileStringFlags` below, leaving - *flags* set to ``NULL``. - - -.. c:function:: PyObject* Py_CompileStringFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags) - - This is a simplified interface to :c:func:`Py_CompileStringExFlags` below, with - *optimize* set to ``-1``. - - -.. c:function:: PyObject* Py_CompileStringObject(const char *str, PyObject *filename, int start, PyCompilerFlags *flags, int optimize) - - Parse and compile the Python source code in *str*, returning the resulting code - object. The start token is given by *start*; this can be used to constrain the - code which can be compiled and should be :c:data:`Py_eval_input`, - :c:data:`Py_file_input`, or :c:data:`Py_single_input`. The filename specified by - *filename* is used to construct the code object and may appear in tracebacks or - :exc:`SyntaxError` exception messages. This returns ``NULL`` if the code - cannot be parsed or compiled. - - The integer *optimize* specifies the optimization level of the compiler; a - value of ``-1`` selects the optimization level of the interpreter as given by - :option:`-O` options. Explicit levels are ``0`` (no optimization; - ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) - or ``2`` (docstrings are removed too). - - .. versionadded:: 3.4 - - -.. c:function:: PyObject* Py_CompileStringExFlags(const char *str, const char *filename, int start, PyCompilerFlags *flags, int optimize) - - Like :c:func:`Py_CompileStringObject`, but *filename* is a byte string - decoded from the :term:`filesystem encoding and error handler`. - - .. versionadded:: 3.2 - -.. c:function:: PyObject* PyEval_EvalCode(PyObject *co, PyObject *globals, PyObject *locals) - - This is a simplified interface to :c:func:`PyEval_EvalCodeEx`, with just - the code object, and global and local variables. The other arguments are - set to ``NULL``. - - -.. c:function:: PyObject* PyEval_EvalCodeEx(PyObject *co, PyObject *globals, PyObject *locals, PyObject *const *args, int argcount, PyObject *const *kws, int kwcount, PyObject *const *defs, int defcount, PyObject *kwdefs, PyObject *closure) - - Evaluate a precompiled code object, given a particular environment for its - evaluation. This environment consists of a dictionary of global variables, - a mapping object of local variables, arrays of arguments, keywords and - defaults, a dictionary of default values for :ref:`keyword-only - ` arguments and a closure tuple of cells. - - -.. c:function:: PyObject* PyEval_EvalFrame(PyFrameObject *f) - - Evaluate an execution frame. This is a simplified interface to - :c:func:`PyEval_EvalFrameEx`, for backward compatibility. - - -.. c:function:: PyObject* PyEval_EvalFrameEx(PyFrameObject *f, int throwflag) - - This is the main, unvarnished function of Python interpretation. The code - object associated with the execution frame *f* is executed, interpreting - bytecode and executing calls as needed. The additional *throwflag* - parameter can mostly be ignored - if true, then it causes an exception - to immediately be thrown; this is used for the :meth:`~generator.throw` - methods of generator objects. - - .. versionchanged:: 3.4 - This function now includes a debug assertion to help ensure that it - does not silently discard an active exception. - - -.. c:function:: int PyEval_MergeCompilerFlags(PyCompilerFlags *cf) - - This function changes the flags of the current evaluation frame, and returns - true on success, false on failure. - - -.. c:var:: int Py_eval_input - - .. index:: single: Py_CompileString() - - The start symbol from the Python grammar for isolated expressions; for use with - :c:func:`Py_CompileString`. - - -.. c:var:: int Py_file_input - - .. index:: single: Py_CompileString() - - The start symbol from the Python grammar for sequences of statements as read - from a file or other source; for use with :c:func:`Py_CompileString`. This is - the symbol to use when compiling arbitrarily long Python source code. - - -.. c:var:: int Py_single_input - - .. index:: single: Py_CompileString() - - The start symbol from the Python grammar for a single statement; for use with - :c:func:`Py_CompileString`. This is the symbol used for the interactive - interpreter loop. - - -.. c:struct:: PyCompilerFlags - - This is the structure used to hold compiler flags. In cases where code is only - being compiled, it is passed as ``int flags``, and in cases where code is being - executed, it is passed as ``PyCompilerFlags *flags``. In this case, ``from - __future__ import`` can modify *flags*. - - Whenever ``PyCompilerFlags *flags`` is ``NULL``, :c:member:`~PyCompilerFlags.cf_flags` is treated as - equal to ``0``, and any modification due to ``from __future__ import`` is - discarded. - - .. c:member:: int cf_flags - - Compiler flags. - - .. c:member:: int cf_feature_version - - *cf_feature_version* is the minor Python version. It should be - initialized to ``PY_MINOR_VERSION``. - - The field is ignored by default, it is used if and only if - ``PyCF_ONLY_AST`` flag is set in :c:member:`~PyCompilerFlags.cf_flags`. - - .. versionchanged:: 3.8 - Added *cf_feature_version* field. - - -.. c:var:: int CO_FUTURE_DIVISION - - This bit can be set in *flags* to cause division operator ``/`` to be - interpreted as "true division" according to :pep:`238`. diff --git a/Doc/c-api/weakref.rst b/Doc/c-api/weakref.rst deleted file mode 100644 index f46507608606b9..00000000000000 --- a/Doc/c-api/weakref.rst +++ /dev/null @@ -1,79 +0,0 @@ -.. highlight:: c - -.. _weakrefobjects: - -Weak Reference Objects ----------------------- - -Python supports *weak references* as first-class objects. There are two -specific object types which directly implement weak references. The first is a -simple reference object, and the second acts as a proxy for the original object -as much as it can. - - -.. c:function:: int PyWeakref_Check(PyObject *ob) - - Return true if *ob* is either a reference or proxy object. This function - always succeeds. - - -.. c:function:: int PyWeakref_CheckRef(PyObject *ob) - - Return true if *ob* is a reference object. This function always succeeds. - - -.. c:function:: int PyWeakref_CheckProxy(PyObject *ob) - - Return true if *ob* is a proxy object. This function always succeeds. - - -.. c:function:: PyObject* PyWeakref_NewRef(PyObject *ob, PyObject *callback) - - Return a weak reference object for the object *ob*. This will always return - a new reference, but is not guaranteed to create a new object; an existing - reference object may be returned. The second parameter, *callback*, can be a - callable object that receives notification when *ob* is garbage collected; it - should accept a single parameter, which will be the weak reference object - itself. *callback* may also be ``None`` or ``NULL``. If *ob* is not a - weakly referencable object, or if *callback* is not callable, ``None``, or - ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. - - -.. c:function:: PyObject* PyWeakref_NewProxy(PyObject *ob, PyObject *callback) - - Return a weak reference proxy object for the object *ob*. This will always - return a new reference, but is not guaranteed to create a new object; an - existing proxy object may be returned. The second parameter, *callback*, can - be a callable object that receives notification when *ob* is garbage - collected; it should accept a single parameter, which will be the weak - reference object itself. *callback* may also be ``None`` or ``NULL``. If *ob* - is not a weakly referencable object, or if *callback* is not callable, - ``None``, or ``NULL``, this will return ``NULL`` and raise :exc:`TypeError`. - - -.. c:function:: PyObject* PyWeakref_GetObject(PyObject *ref) - - Return the referenced object from a weak reference, *ref*. If the referent is - no longer live, returns ``Py_None``. - - .. note:: - - This function returns a :term:`borrowed reference` to the referenced object. - This means that you should always call :c:func:`Py_INCREF` on the object - except when it cannot be destroyed before the last usage of the borrowed - reference. - - -.. c:function:: PyObject* PyWeakref_GET_OBJECT(PyObject *ref) - - Similar to :c:func:`PyWeakref_GetObject`, but does no error checking. - - -.. c:function:: void PyObject_ClearWeakRefs(PyObject *object) - - This function is called by the :c:member:`~PyTypeObject.tp_dealloc` handler - to clear weak references. - - This iterates through the weak references for *object* and calls callbacks - for those references which have one. It returns when all callbacks have - been attempted. diff --git a/Doc/conf.py b/Doc/conf.py deleted file mode 100644 index 8c92799ecbbf3b..00000000000000 --- a/Doc/conf.py +++ /dev/null @@ -1,481 +0,0 @@ -# -# Python documentation build configuration file -# -# This file is execfile()d with the current directory set to its containing dir. -# -# The contents of this file are pickled, so don't put values in the namespace -# that aren't pickleable (module imports are okay, they're removed automatically). - -import sys, os, time -sys.path.append(os.path.abspath('tools/extensions')) -sys.path.append(os.path.abspath('includes')) - -# General configuration -# --------------------- - -extensions = [ - 'asdl_highlight', - 'c_annotations', - 'escape4chm', - 'glossary_search', - 'peg_highlight', - 'pyspecific', - 'sphinx.ext.coverage', - 'sphinx.ext.doctest', -] - -# Skip if downstream redistributors haven't installed it -try: - import sphinxext.opengraph -except ImportError: - pass -else: - extensions.append('sphinxext.opengraph') - - -doctest_global_setup = ''' -try: - import _tkinter -except ImportError: - _tkinter = None -''' - -manpages_url = 'https://manpages.debian.org/{path}' - -# General substitutions. -project = 'Python' -copyright = '2001-%s, Python Software Foundation' % time.strftime('%Y') - -# We look for the Include/patchlevel.h file in the current Python source tree -# and replace the values accordingly. -import patchlevel -version, release = patchlevel.get_version_info() - -# There are two options for replacing |today|: either, you set today to some -# non-false value, then it is used: -today = '' -# Else, today_fmt is used as the format for a strftime call. -today_fmt = '%B %d, %Y' - -# By default, highlight as Python 3. -highlight_language = 'python3' - -# Minimum version of sphinx required -needs_sphinx = '4.2' - -# Ignore any .rst files in the includes/ directory; -# they're embedded in pages but not rendered individually. -# Ignore any .rst files in the venv/ directory. -exclude_patterns = ['includes/*.rst', 'venv/*', 'README.rst'] -venvdir = os.getenv('VENVDIR') -if venvdir is not None: - exclude_patterns.append(venvdir + '/*') - -nitpick_ignore = [ - # Standard C functions - ('c:func', 'calloc'), - ('c:func', 'dlopen'), - ('c:func', 'exec'), - ('c:func', 'fcntl'), - ('c:func', 'fork'), - ('c:func', 'free'), - ('c:func', 'gmtime'), - ('c:func', 'localtime'), - ('c:func', 'main'), - ('c:func', 'malloc'), - ('c:func', 'printf'), - ('c:func', 'realloc'), - ('c:func', 'snprintf'), - ('c:func', 'sprintf'), - ('c:func', 'stat'), - ('c:func', 'system'), - ('c:func', 'vsnprintf'), - # Standard C types - ('c:type', 'FILE'), - ('c:type', 'int64_t'), - ('c:type', 'intmax_t'), - ('c:type', 'off_t'), - ('c:type', 'ptrdiff_t'), - ('c:type', 'siginfo_t'), - ('c:type', 'size_t'), - ('c:type', 'ssize_t'), - ('c:type', 'time_t'), - ('c:type', 'uint64_t'), - ('c:type', 'uintmax_t'), - ('c:type', 'uintptr_t'), - ('c:type', 'va_list'), - ('c:type', 'wchar_t'), - # Standard C structures - ('c:struct', 'in6_addr'), - ('c:struct', 'in_addr'), - ('c:struct', 'stat'), - ('c:struct', 'statvfs'), - # Standard C macros - ('c:macro', 'LLONG_MAX'), - ('c:macro', 'LLONG_MIN'), - ('c:macro', 'LONG_MAX'), - ('c:macro', 'LONG_MIN'), - # Standard C variables - ('c:data', 'errno'), - # Standard environment variables - ('envvar', 'BROWSER'), - ('envvar', 'COLUMNS'), - ('envvar', 'COMSPEC'), - ('envvar', 'DISPLAY'), - ('envvar', 'HOME'), - ('envvar', 'HOMEDRIVE'), - ('envvar', 'HOMEPATH'), - ('envvar', 'IDLESTARTUP'), - ('envvar', 'LANG'), - ('envvar', 'LANGUAGE'), - ('envvar', 'LC_ALL'), - ('envvar', 'LC_CTYPE'), - ('envvar', 'LC_COLLATE'), - ('envvar', 'LC_MESSAGES'), - ('envvar', 'LC_MONETARY'), - ('envvar', 'LC_NUMERIC'), - ('envvar', 'LC_TIME'), - ('envvar', 'LINES'), - ('envvar', 'LOGNAME'), - ('envvar', 'PAGER'), - ('envvar', 'PATH'), - ('envvar', 'PATHEXT'), - ('envvar', 'SOURCE_DATE_EPOCH'), - ('envvar', 'TEMP'), - ('envvar', 'TERM'), - ('envvar', 'TMP'), - ('envvar', 'TMPDIR'), - ('envvar', 'TZ'), - ('envvar', 'USER'), - ('envvar', 'USERNAME'), - ('envvar', 'USERPROFILE'), -] - -# Temporary undocumented names. -# In future this list must be empty. -nitpick_ignore += [ - # C API: Standard Python exception classes - ('c:data', 'PyExc_ArithmeticError'), - ('c:data', 'PyExc_AssertionError'), - ('c:data', 'PyExc_AttributeError'), - ('c:data', 'PyExc_BaseException'), - ('c:data', 'PyExc_BlockingIOError'), - ('c:data', 'PyExc_BrokenPipeError'), - ('c:data', 'PyExc_BufferError'), - ('c:data', 'PyExc_ChildProcessError'), - ('c:data', 'PyExc_ConnectionAbortedError'), - ('c:data', 'PyExc_ConnectionError'), - ('c:data', 'PyExc_ConnectionRefusedError'), - ('c:data', 'PyExc_ConnectionResetError'), - ('c:data', 'PyExc_EOFError'), - ('c:data', 'PyExc_Exception'), - ('c:data', 'PyExc_FileExistsError'), - ('c:data', 'PyExc_FileNotFoundError'), - ('c:data', 'PyExc_FloatingPointError'), - ('c:data', 'PyExc_GeneratorExit'), - ('c:data', 'PyExc_ImportError'), - ('c:data', 'PyExc_IndentationError'), - ('c:data', 'PyExc_IndexError'), - ('c:data', 'PyExc_InterruptedError'), - ('c:data', 'PyExc_IsADirectoryError'), - ('c:data', 'PyExc_KeyboardInterrupt'), - ('c:data', 'PyExc_KeyError'), - ('c:data', 'PyExc_LookupError'), - ('c:data', 'PyExc_MemoryError'), - ('c:data', 'PyExc_ModuleNotFoundError'), - ('c:data', 'PyExc_NameError'), - ('c:data', 'PyExc_NotADirectoryError'), - ('c:data', 'PyExc_NotImplementedError'), - ('c:data', 'PyExc_OSError'), - ('c:data', 'PyExc_OverflowError'), - ('c:data', 'PyExc_PermissionError'), - ('c:data', 'PyExc_ProcessLookupError'), - ('c:data', 'PyExc_RecursionError'), - ('c:data', 'PyExc_ReferenceError'), - ('c:data', 'PyExc_RuntimeError'), - ('c:data', 'PyExc_StopAsyncIteration'), - ('c:data', 'PyExc_StopIteration'), - ('c:data', 'PyExc_SyntaxError'), - ('c:data', 'PyExc_SystemError'), - ('c:data', 'PyExc_SystemExit'), - ('c:data', 'PyExc_TabError'), - ('c:data', 'PyExc_TimeoutError'), - ('c:data', 'PyExc_TypeError'), - ('c:data', 'PyExc_UnboundLocalError'), - ('c:data', 'PyExc_UnicodeDecodeError'), - ('c:data', 'PyExc_UnicodeEncodeError'), - ('c:data', 'PyExc_UnicodeError'), - ('c:data', 'PyExc_UnicodeTranslateError'), - ('c:data', 'PyExc_ValueError'), - ('c:data', 'PyExc_ZeroDivisionError'), - # C API: Standard Python warning classes - ('c:data', 'PyExc_BytesWarning'), - ('c:data', 'PyExc_DeprecationWarning'), - ('c:data', 'PyExc_FutureWarning'), - ('c:data', 'PyExc_ImportWarning'), - ('c:data', 'PyExc_PendingDeprecationWarning'), - ('c:data', 'PyExc_ResourceWarning'), - ('c:data', 'PyExc_RuntimeWarning'), - ('c:data', 'PyExc_SyntaxWarning'), - ('c:data', 'PyExc_UnicodeWarning'), - ('c:data', 'PyExc_UserWarning'), - ('c:data', 'PyExc_Warning'), - # Do not error nit-picky mode builds when _SubParsersAction.add_parser cannot - # be resolved, as the method is currently undocumented. For context, see - # https://github.com/python/cpython/pull/103289. - ('py:meth', '_SubParsersAction.add_parser'), -] - -# gh-106948: Copy standard C types declared in the "c:type" domain to the -# "c:identifier" domain, since "c:function" markup looks for types in the -# "c:identifier" domain. Use list() to not iterate on items which are being -# added -for role, name in list(nitpick_ignore): - if role == 'c:type': - nitpick_ignore.append(('c:identifier', name)) -del role, name - -# Disable Docutils smartquotes for several translations -smartquotes_excludes = { - 'languages': ['ja', 'fr', 'zh_TW', 'zh_CN'], 'builders': ['man', 'text'], -} - -# Avoid a warning with Sphinx >= 2.0 -master_doc = 'contents' - -# Allow translation of index directives -gettext_additional_targets = [ - 'index', -] - -# Options for HTML output -# ----------------------- - -# Use our custom theme. -html_theme = 'python_docs_theme' -html_theme_path = ['tools'] -html_theme_options = { - 'collapsiblesidebar': True, - 'issues_url': '/bugs.html', - 'license_url': '/license.html', - 'root_include_title': False # We use the version switcher instead. -} - -# Override stylesheet fingerprinting for Windows CHM htmlhelp to fix GH-91207 -# https://github.com/python/cpython/issues/91207 -if any('htmlhelp' in arg for arg in sys.argv): - html_style = 'pydoctheme.css' - print("\nWARNING: Windows CHM Help is no longer supported.") - print("It may be removed in the future\n") - -# Short title used e.g. for HTML tags. -html_short_title = '%s Documentation' % release - -# Deployment preview information -# (See .readthedocs.yml and https://docs.readthedocs.io/en/stable/reference/environment-variables.html) -repository_url = os.getenv("READTHEDOCS_GIT_CLONE_URL") -html_context = { - "is_deployment_preview": os.getenv("READTHEDOCS_VERSION_TYPE") == "external", - "repository_url": repository_url.removesuffix(".git") if repository_url else None, - "pr_id": os.getenv("READTHEDOCS_VERSION") -} - -# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, -# using the given strftime format. -html_last_updated_fmt = '%b %d, %Y' - -# Path to find HTML templates. -templates_path = ['tools/templates'] - -# Custom sidebar templates, filenames relative to this file. -html_sidebars = { - # Defaults taken from https://www.sphinx-doc.org/en/master/usage/configuration.html#confval-html_sidebars - # Removes the quick search block - '**': ['localtoc.html', 'relations.html', 'customsourcelink.html'], - 'index': ['indexsidebar.html'], -} - -# Additional templates that should be rendered to pages. -html_additional_pages = { - 'download': 'download.html', - 'index': 'indexcontent.html', -} - -# Output an OpenSearch description file. -html_use_opensearch = 'https://docs.python.org/' + version - -# Additional static files. -html_static_path = ['_static', 'tools/static'] - -# Output file base name for HTML help builder. -htmlhelp_basename = 'python' + release.replace('.', '') - -# Split the index -html_split_index = True - - -# Options for LaTeX output -# ------------------------ - -latex_engine = 'xelatex' - -# Get LaTeX to handle Unicode correctly -latex_elements = { -} - -# Additional stuff for the LaTeX preamble. -latex_elements['preamble'] = r''' -\authoraddress{ - \sphinxstrong{Python Software Foundation}\\ - Email: \sphinxemail{docs@python.org} -} -\let\Verbatim=\OriginalVerbatim -\let\endVerbatim=\endOriginalVerbatim -\setcounter{tocdepth}{2} -''' - -# The paper size ('letter' or 'a4'). -latex_elements['papersize'] = 'a4' - -# The font size ('10pt', '11pt' or '12pt'). -latex_elements['pointsize'] = '10pt' - -# Grouping the document tree into LaTeX files. List of tuples -# (source start file, target name, title, author, document class [howto/manual]). -_stdauthor = 'Guido van Rossum and the Python development team' -latex_documents = [ - ('c-api/index', 'c-api.tex', - 'The Python/C API', _stdauthor, 'manual'), - ('extending/index', 'extending.tex', - 'Extending and Embedding Python', _stdauthor, 'manual'), - ('installing/index', 'installing.tex', - 'Installing Python Modules', _stdauthor, 'manual'), - ('library/index', 'library.tex', - 'The Python Library Reference', _stdauthor, 'manual'), - ('reference/index', 'reference.tex', - 'The Python Language Reference', _stdauthor, 'manual'), - ('tutorial/index', 'tutorial.tex', - 'Python Tutorial', _stdauthor, 'manual'), - ('using/index', 'using.tex', - 'Python Setup and Usage', _stdauthor, 'manual'), - ('faq/index', 'faq.tex', - 'Python Frequently Asked Questions', _stdauthor, 'manual'), - ('whatsnew/' + version, 'whatsnew.tex', - 'What\'s New in Python', 'A. M. Kuchling', 'howto'), -] -# Collect all HOWTOs individually -latex_documents.extend(('howto/' + fn[:-4], 'howto-' + fn[:-4] + '.tex', - '', _stdauthor, 'howto') - for fn in os.listdir('howto') - if fn.endswith('.rst') and fn != 'index.rst') - -# Documents to append as an appendix to all manuals. -latex_appendices = ['glossary', 'about', 'license', 'copyright'] - -# Options for Epub output -# ----------------------- - -epub_author = 'Python Documentation Authors' -epub_publisher = 'Python Software Foundation' - -# Options for the coverage checker -# -------------------------------- - -# The coverage checker will ignore all modules/functions/classes whose names -# match any of the following regexes (using re.match). -coverage_ignore_modules = [ - r'[T|t][k|K]', - r'Tix', - r'distutils.*', -] - -coverage_ignore_functions = [ - 'test($|_)', -] - -coverage_ignore_classes = [ -] - -# Glob patterns for C source files for C API coverage, relative to this directory. -coverage_c_path = [ - '../Include/*.h', -] - -# Regexes to find C items in the source files. -coverage_c_regexes = { - 'cfunction': (r'^PyAPI_FUNC\(.*\)\s+([^_][\w_]+)'), - 'data': (r'^PyAPI_DATA\(.*\)\s+([^_][\w_]+)'), - 'macro': (r'^#define ([^_][\w_]+)\(.*\)[\s|\\]'), -} - -# The coverage checker will ignore all C items whose names match these regexes -# (using re.match) -- the keys must be the same as in coverage_c_regexes. -coverage_ignore_c_items = { -# 'cfunction': [...] -} - - -# Options for the link checker -# ---------------------------- - -linkcheck_allowed_redirects = { - # bpo-NNNN -> BPO -> GH Issues - r'https://bugs.python.org/issue\?@action=redirect&bpo=\d+': r'https://github.com/python/cpython/issues/\d+', - # GH-NNNN used to refer to pull requests - r'https://github.com/python/cpython/issues/\d+': r'https://github.com/python/cpython/pull/\d+', - # :source:`something` linking files in the repository - r'https://github.com/python/cpython/tree/.*': 'https://github.com/python/cpython/blob/.*', - # Intentional HTTP use at Misc/NEWS.d/3.5.0a1.rst - r'http://www.python.org/$': 'https://www.python.org/$', - # Used in license page, keep as is - r'https://www.zope.org/': r'https://www.zope.dev/', - # Microsoft's redirects to learn.microsoft.com - r'https://msdn.microsoft.com/.*': 'https://learn.microsoft.com/.*', - r'https://docs.microsoft.com/.*': 'https://learn.microsoft.com/.*', - r'https://go.microsoft.com/fwlink/\?LinkID=\d+': 'https://learn.microsoft.com/.*', - # Language redirects - r'https://toml.io': 'https://toml.io/en/', - r'https://www.redhat.com': 'https://www.redhat.com/en', - # Other redirects - r'https://www.boost.org/libs/.+': r'https://www.boost.org/doc/libs/\d_\d+_\d/.+', - r'https://support.microsoft.com/en-us/help/\d+': 'https://support.microsoft.com/en-us/topic/.+', - r'https://perf.wiki.kernel.org$': 'https://perf.wiki.kernel.org/index.php/Main_Page', - r'https://www.sqlite.org': 'https://www.sqlite.org/index.html', - r'https://mitpress.mit.edu/sicp$': 'https://mitpress.mit.edu/9780262510875/structure-and-interpretation-of-computer-programs/', - r'https://www.python.org/psf/': 'https://www.python.org/psf-landing/', -} - -linkcheck_anchors_ignore = [ - # ignore anchors that start with a '/', e.g. Wikipedia media files: - # https://en.wikipedia.org/wiki/Walrus#/media/File:Pacific_Walrus_-_Bull_(8247646168).jpg - r'\/.*', -] - -linkcheck_ignore = [ - # The crawler gets "Anchor not found" - r'https://developer.apple.com/documentation/.+?#.*', - r'https://devguide.python.org.+?/#.*', - r'https://github.com.+?#.*', - # Robot crawlers not allowed: "403 Client Error: Forbidden" - r'https://support.enthought.com/hc/.*', - # SSLError CertificateError, even though it is valid - r'https://unix.org/version2/whatsnew/lp64_wp.html', -] - - -# Options for extensions -# ---------------------- - -# Relative filename of the data files -refcount_file = 'data/refcounts.dat' -stable_abi_file = 'data/stable_abi.dat' - -# sphinxext-opengraph config -ogp_site_url = 'https://docs.python.org/3/' -ogp_site_name = 'Python documentation' -ogp_image = '_static/og-image.png' -ogp_custom_meta_tags = [ - '<meta property="og:image:width" content="200" />', - '<meta property="og:image:height" content="200" />', - '<meta name="theme-color" content="#3776ab" />', -] diff --git a/Doc/contents.rst b/Doc/contents.rst deleted file mode 100644 index 3eedede7ba49ab..00000000000000 --- a/Doc/contents.rst +++ /dev/null @@ -1,30 +0,0 @@ -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - Python Documentation contents -%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -.. toctree:: - - whatsnew/index.rst - tutorial/index.rst - using/index.rst - reference/index.rst - library/index.rst - extending/index.rst - c-api/index.rst - installing/index.rst - howto/index.rst - faq/index.rst - glossary.rst - - about.rst - bugs.rst - copyright.rst - license.rst - -.. to include legacy packaging docs in build - -.. toctree:: - :hidden: - - distutils/index.rst - install/index.rst diff --git a/Doc/copyright.rst b/Doc/copyright.rst deleted file mode 100644 index 9b71683155eebe..00000000000000 --- a/Doc/copyright.rst +++ /dev/null @@ -1,19 +0,0 @@ -********* -Copyright -********* - -Python and this documentation is: - -Copyright © 2001-2023 Python Software Foundation. All rights reserved. - -Copyright © 2000 BeOpen.com. All rights reserved. - -Copyright © 1995-2000 Corporation for National Research Initiatives. All rights -reserved. - -Copyright © 1991-1995 Stichting Mathematisch Centrum. All rights reserved. - -------- - -See :ref:`history-and-license` for complete license and permissions information. - diff --git a/Doc/data/refcounts.dat b/Doc/data/refcounts.dat deleted file mode 100644 index cbd22a3e23ece5..00000000000000 --- a/Doc/data/refcounts.dat +++ /dev/null @@ -1,2995 +0,0 @@ -# Created by Skip Montanaro <skip@mojam.com>. - -# Format: -# function ':' type ':' [param name] ':' [refcount effect] ':' [comment] -# If the param name slot is empty, that line corresponds to the function's -# return value, otherwise it's the type of the named parameter. - -# The first line of a function block gives type/refcount information for the -# function's return value. Successive lines with the same function name -# correspond to the function's parameter list and appear in the order the -# parameters appear in the function's prototype. - -# For readability, each function's lines are surrounded by a blank line. -# The blocks are sorted alphabetically by function name. - -# Refcount behavior is given for all PyObject* types: 0 (no change), +1 -# (increment) and -1 (decrement). A blank refcount field indicates the -# parameter or function value is not a PyObject* and is therefore not -# subject to reference counting. A special case for the value "null" -# (without quotes) is used for functions which return a PyObject* type but -# always return NULL. This is used by some of the PyErr_*() functions, in -# particular. - -# XXX NOTE: the 0/+1/-1 refcount information for arguments is -# confusing! Much more useful would be to indicate whether the -# function "steals" a reference to the argument or not. Take for -# example PyList_SetItem(list, i, item). This lists as a 0 change for -# both the list and the item arguments. However, in fact it steals a -# reference to the item argument! - -# The parameter names are as they appear in the API manual, not the source -# code. - -PyAnySet_Check:int::: -PyAnySet_Check:PyObject*:p:0: - -PyAnySet_CheckExact:int::: -PyAnySet_CheckExact:PyObject*:p:0: - -PyBool_Check:int::: -PyBool_Check:PyObject*:o:0: - -PyBool_FromLong:PyObject*::+1: -PyBool_FromLong:long:v:: - -PyBuffer_FillContiguousStrides:void::: -PyBuffer_FillContiguousStrides:int:ndims:: -PyBuffer_FillContiguousStrides:Py_ssize_t*:shape:: -PyBuffer_FillContiguousStrides:Py_ssize_t*:strides:: -PyBuffer_FillContiguousStrides:int:itemsize:: -PyBuffer_FillContiguousStrides:char:order:: - -PyBuffer_FillInfo:int::: -PyBuffer_FillInfo:Py_buffer*:view:: -PyBuffer_FillInfo:PyObject*:exporter:0: -PyBuffer_FillInfo:void*:buf:: -PyBuffer_FillInfo:Py_ssize_t:len:: -PyBuffer_FillInfo:int:readonly:: -PyBuffer_FillInfo:int:flags:: - -PyBuffer_IsContiguous:int::: -PyBuffer_IsContiguous:Py_buffer*:view:: -PyBuffer_IsContiguous:char:order:: - -PyBuffer_Release:void::: -PyBuffer_Release:Py_buffer*:view:: - -PyBuffer_ToContiguous:int::: -PyBuffer_ToContiguous:void*:buf:: -PyBuffer_ToContiguous:Py_buffer*:src:: -PyBuffer_ToContiguous:Py_ssize_t:len:: -PyBuffer_ToContiguous:char:order:: - -PyByteArray_AS_STRING:char*::: -PyByteArray_AS_STRING:PyObject*:bytearray:0: - -PyByteArray_AsString:char*::: -PyByteArray_AsString:PyObject*:bytearray:0: - -PyByteArray_Check:int::: -PyByteArray_Check:PyObject*:o:0: - -PyByteArray_CheckExact:int::: -PyByteArray_CheckExact:PyObject*:o:0: - -PyByteArray_Concat:PyObject*::+1: -PyByteArray_Concat:PyObject*:a:0: -PyByteArray_Concat:PyObject*:b:0: - -PyByteArray_FromObject:PyObject*::+1: -PyByteArray_FromObject:PyObject*:o:0: - -PyByteArray_FromStringAndSize:PyObject*::+1: -PyByteArray_FromStringAndSize:const char*:string:: -PyByteArray_FromStringAndSize:Py_ssize_t:len:: - -PyByteArray_GET_SIZE:Py_ssize_t::: -PyByteArray_GET_SIZE:PyObject*:bytearray:0: - -PyByteArray_Resize:int::: -PyByteArray_Resize:PyObject*:bytearray:0: -PyByteArray_Resize:Py_ssize_t:len:: - -PyByteArray_Size:Py_ssize_t::: -PyByteArray_Size:PyObject*:bytearray:0: - -PyBytes_AS_STRING:char*::: -PyBytes_AS_STRING:PyObject*:string:0: - -PyBytes_AsString:char*::: -PyBytes_AsString:PyObject*:o:0: - -PyBytes_AsStringAndSize:int::: -PyBytes_AsStringAndSize:PyObject*:obj:0: -PyBytes_AsStringAndSize:char**:buffer:: -PyBytes_AsStringAndSize:Py_ssize_t*:length:: - -PyBytes_Check:int::: -PyBytes_Check:PyObject*:o:0: - -PyBytes_CheckExact:int::: -PyBytes_CheckExact:PyObject*:o:0: - -PyBytes_Concat:void::: -PyBytes_Concat:PyObject**:bytes:0: -PyBytes_Concat:PyObject*:newpart:0: - -PyBytes_ConcatAndDel:void::: -PyBytes_ConcatAndDel:PyObject**:bytes:0: -PyBytes_ConcatAndDel:PyObject*:newpart:-1: - -PyBytes_FromString:PyObject*::+1: -PyBytes_FromString:const char*:v:: - -PyBytes_FromStringAndSize:PyObject*::+1: -PyBytes_FromStringAndSize:const char*:v:: -PyBytes_FromStringAndSize:Py_ssize_t:len:: - -PyBytes_FromFormat:PyObject*::+1: -PyBytes_FromFormat:const char*:format:: -PyBytes_FromFormat::...:: - -PyBytes_FromFormatV:PyObject*::+1: -PyBytes_FromFormatV:const char*:format:: -PyBytes_FromFormatV:va_list:vargs:: - -PyBytes_FromObject:PyObject*::+1: -PyBytes_FromObject:PyObject*:o:0: - -PyBytes_GET_SIZE:Py_ssize_t::: -PyBytes_GET_SIZE:PyObject*:o:0: - -PyBytes_Size:Py_ssize_t::: -PyBytes_Size:PyObject*:o:0: - -PyCapsule_CheckExact:int::: -PyCapsule_CheckExact:PyObject*:p:0: - -PyCapsule_GetContext:void *::: -PyCapsule_GetContext:PyObject*:self:0: - -PyCapsule_GetDestructor:void (*)(PyObject *)::: -PyCapsule_GetDestructor:PyObject*:self:0: - -PyCapsule_GetName:const char *::: -PyCapsule_GetName:PyObject*:self:0: - -PyCapsule_GetPointer:void*::: -PyCapsule_GetPointer:PyObject*:self:0: -PyCapsule_GetPointer:const char *:name:: - -PyCapsule_Import:void *::: -PyCapsule_Import:const char *:name:: -PyCapsule_Import:int:no_block:: - -PyCapsule_IsValid:int::: -PyCapsule_IsValid:PyObject*:capsule:0: -PyCapsule_IsValid:const char*:name:: - -PyCapsule_New:PyObject*::+1: -PyCapsule_New:void*:pointer:: -PyCapsule_New:const char *:name:: -PyCapsule_New::void (* destructor)(PyObject* ):: - -PyCapsule_SetContext:int::: -PyCapsule_SetContext:PyObject*:self:0: -PyCapsule_SetContext:void *:context:: - -PyCapsule_SetDestructor:int::: -PyCapsule_SetDestructor:PyObject*:self:0: -PyCapsule_SetDestructor:void (*)(PyObject *):destructor:: - -PyCapsule_SetName:int::: -PyCapsule_SetName:PyObject*:self:0: -PyCapsule_SetName:const char *:name:: - -PyCapsule_SetPointer:int::: -PyCapsule_SetPointer:PyObject*:self:0: -PyCapsule_SetPointer:void*:pointer:: - -PyCell_Check:int::: -PyCell_Check::ob:: - -PyCell_New:PyObject*::+1: -PyCell_New:PyObject*:ob:0: - -PyCell_GET:PyObject*::0: -PyCell_GET:PyObject*:ob:0: - -PyCell_Get:PyObject*::+1: -PyCell_Get:PyObject*:cell:0: - -PyCell_SET:void::: -PyCell_SET:PyObject*:cell:0: -PyCell_SET:PyObject*:value:0: - -PyCell_Set:int::: -PyCell_Set:PyObject*:cell:0: -PyCell_Set:PyObject*:value:0: - -PyCallIter_Check:int::: -PyCallIter_Check::op:: - -PyCallIter_New:PyObject*::+1: -PyCallIter_New:PyObject*:callable:+1: -PyCallIter_New:PyObject*:sentinel:+1: - -PyCallable_Check:int::: -PyCallable_Check:PyObject*:o:0: - -PyCode_Check:int::: -PyCode_Check:PyObject*:co:0: - -PyCode_GetNumFree:int::: -PyCode_GetNumFree:PyCodeObject*:co:0: - -PyCode_NewWithPosOnlyArgs:PyCodeObject*::+1: -PyCode_NewWithPosOnlyArgs:int:argcount:: -PyCode_NewWithPosOnlyArgs:int:posonlyargcount:: -PyCode_NewWithPosOnlyArgs:int:kwonlyargcount:: -PyCode_NewWithPosOnlyArgs:int:nlocals:: -PyCode_NewWithPosOnlyArgs:int:stacksize:: -PyCode_NewWithPosOnlyArgs:int:flags:: -PyCode_NewWithPosOnlyArgs:PyObject*:code:0: -PyCode_NewWithPosOnlyArgs:PyObject*:consts:0: -PyCode_NewWithPosOnlyArgs:PyObject*:names:0: -PyCode_NewWithPosOnlyArgs:PyObject*:varnames:0: -PyCode_NewWithPosOnlyArgs:PyObject*:freevars:0: -PyCode_NewWithPosOnlyArgs:PyObject*:cellvars:0: -PyCode_NewWithPosOnlyArgs:PyObject*:filename:0: -PyCode_NewWithPosOnlyArgs:PyObject*:name:0: -PyCode_NewWithPosOnlyArgs:int:firstlineno:: -PyCode_NewWithPosOnlyArgs:PyObject*:lnotab:0: - -PyCode_New:PyCodeObject*::+1: -PyCode_New:int:argcount:: -PyCode_New:int:kwonlyargcount:: -PyCode_New:int:nlocals:: -PyCode_New:int:stacksize:: -PyCode_New:int:flags:: -PyCode_New:PyObject*:code:0: -PyCode_New:PyObject*:consts:0: -PyCode_New:PyObject*:names:0: -PyCode_New:PyObject*:varnames:0: -PyCode_New:PyObject*:freevars:0: -PyCode_New:PyObject*:cellvars:0: -PyCode_New:PyObject*:filename:0: -PyCode_New:PyObject*:name:0: -PyCode_New:int:firstlineno:: -PyCode_New:PyObject*:lnotab:0: - -PyCode_NewEmpty:PyCodeObject*::+1: -PyCode_NewEmpty:const char*:filename:: -PyCode_NewEmpty:const char*:funcname:: -PyCode_NewEmpty:int:firstlineno:: - -PyCodec_Register:int::: -PyCodec_Register:PyObject*:search_function:+1: - -PyCodec_KnownEncoding:int::: -PyCodec_KnownEncoding:const char*:encoding:: - -PyCodec_Encode:PyObject*::+1: -PyCodec_Encode:PyObject*:object:0: -PyCodec_Encode:const char*:encoding:: -PyCodec_Encode:const char*:errors:: - -PyCodec_Decode:PyObject*::+1: -PyCodec_Decode:PyObject*:object:0: -PyCodec_Decode:const char*:encoding:: -PyCodec_Decode:const char*:errors:: - -PyCodec_Encoder:PyObject*::+1: -PyCodec_Encoder:const char*:encoding:: - -PyCodec_Decoder:PyObject*::+1: -PyCodec_Decoder:const char*:encoding:: - -PyCodec_IncrementalEncoder:PyObject*::+1: -PyCodec_IncrementalEncoder:const char*:encoding:: -PyCodec_IncrementalEncoder:const char*:errors:: - -PyCodec_IncrementalDecoder:PyObject*::+1: -PyCodec_IncrementalDecoder:const char*:encoding:: -PyCodec_IncrementalDecoder:const char*:errors:: - -PyCodec_StreamReader:PyObject*::+1: -PyCodec_StreamReader:const char*:encoding:: -PyCodec_StreamReader:PyObject*:stream:0: -PyCodec_StreamReader:const char*:errors:: - -PyCodec_StreamWriter:PyObject*::+1: -PyCodec_StreamWriter:const char*:encoding:: -PyCodec_StreamWriter:PyObject*:stream:0: -PyCodec_StreamWriter:const char*:errors:: - -PyCodec_RegisterError:int::: -PyCodec_RegisterError:const char*:name:: -PyCodec_RegisterError:PyObject*:error:+1: - -PyCodec_LookupError:PyObject*::+1: -PyCodec_LookupError:const char*:name:: - -PyCodec_StrictErrors:PyObject*::null: -PyCodec_StrictErrors:PyObject*:exc:0: - -PyCodec_IgnoreErrors:PyObject*::+1: -PyCodec_IgnoreErrors:PyObject*:exc:0: - -PyCodec_ReplaceErrors:PyObject*::+1: -PyCodec_ReplaceErrors:PyObject*:exc:0: - -PyCodec_XMLCharRefReplaceErrors:PyObject*::+1: -PyCodec_XMLCharRefReplaceErrors:PyObject*:exc:0: - -PyCodec_BackslashReplaceErrors:PyObject*::+1: -PyCodec_BackslashReplaceErrors:PyObject*:exc:0: - -PyCodec_NameReplaceErrors:PyObject*::+1: -PyCodec_NameReplaceErrors:PyObject*:exc:0: - -PyComplex_AsCComplex:Py_complex::: -PyComplex_AsCComplex:PyObject*:op:0: - -PyComplex_Check:int::: -PyComplex_Check:PyObject*:p:0: - -PyComplex_CheckExact:int::: -PyComplex_CheckExact:PyObject*:p:0: - -PyComplex_FromCComplex:PyObject*::+1: -PyComplex_FromCComplex::Py_complex v:: - -PyComplex_FromDoubles:PyObject*::+1: -PyComplex_FromDoubles::double real:: -PyComplex_FromDoubles::double imag:: - -PyComplex_ImagAsDouble:double::: -PyComplex_ImagAsDouble:PyObject*:op:0: - -PyComplex_RealAsDouble:double::: -PyComplex_RealAsDouble:PyObject*:op:0: - -PyContext_CheckExact:int::: -PyContext_CheckExact:PyObject*:o:0: - -PyContext_ClearFreeList:int::: - -PyContext_Copy:PyObject*::+1: -PyContext_Copy:PyObject*:ctx:0: - -PyContext_CopyCurrent:PyObject*::+1: - -PyContext_Enter:int::: -PyContext_Enter:PyObject*:ctx:+1: - -PyContext_Exit:int::: -PyContext_Exit:PyObject*:ctx:-1: - -PyContext_New:PyObject*::+1: - -PyContextToken_CheckExact:int::: -PyContextToken_CheckExact:PyObject*:o:0: - -PyContextVar_CheckExact:int::: -PyContextVar_CheckExact:PyObject*:o:0: - -PyContextVar_Get:int::: -PyContextVar_Get:PyObject*:var:0: -PyContextVar_Get:PyObject*:default_value:0: -PyContextVar_Get:PyObject**:value:+1:??? - -PyContextVar_New:PyObject*::+1: -PyContextVar_New:const char*:name:: -PyContextVar_New:PyObject*:def:+1: - -PyContextVar_Set:PyObject*::+1: -PyContextVar_Set:PyObject*:var:0: -PyContextVar_Set:PyObject*:value:+1: - -PyContextVar_Reset:int::: -PyContextVar_Reset:PyObject*:var:0: -PyContextVar_Reset:PyObject*:token:-1: - -PyDate_Check:int::: -PyDate_Check:PyObject*:ob:0: - -PyDate_CheckExact:int::: -PyDate_CheckExact:PyObject*:ob:0: - -PyDate_FromDate:PyObject*::+1: -PyDate_FromDate:int:year:: -PyDate_FromDate:int:month:: -PyDate_FromDate:int:day:: - -PyDate_FromTimestamp:PyObject*::+1: -PyDate_FromTimestamp:PyObject*:args:0: - -PyDateTime_Check:int::: -PyDateTime_Check:PyObject*:ob:0: - -PyDateTime_CheckExact:int::: -PyDateTime_CheckExact:PyObject*:ob:0: - -PyDateTime_FromDateAndTime:PyObject*::+1: -PyDateTime_FromDateAndTime:int:year:: -PyDateTime_FromDateAndTime:int:month:: -PyDateTime_FromDateAndTime:int:day:: -PyDateTime_FromDateAndTime:int:hour:: -PyDateTime_FromDateAndTime:int:minute:: -PyDateTime_FromDateAndTime:int:second:: -PyDateTime_FromDateAndTime:int:usecond:: - -PyDateTime_FromDateAndTimeAndFold:PyObject*::+1: -PyDateTime_FromDateAndTimeAndFold:int:year:: -PyDateTime_FromDateAndTimeAndFold:int:month:: -PyDateTime_FromDateAndTimeAndFold:int:day:: -PyDateTime_FromDateAndTimeAndFold:int:hour:: -PyDateTime_FromDateAndTimeAndFold:int:minute:: -PyDateTime_FromDateAndTimeAndFold:int:second:: -PyDateTime_FromDateAndTimeAndFold:int:usecond:: -PyDateTime_FromDateAndTimeAndFold:int:fold:: - -PyDateTime_FromTimestamp:PyObject*::+1: -PyDateTime_FromTimestamp:PyObject*:args:0: - -PyDelta_Check:int::: -PyDelta_Check:PyObject*:ob:0: - -PyDelta_CheckExact:int::: -PyDelta_CheckExact:PyObject*:ob:0: - -PyDelta_FromDSU:PyObject*::+1: -PyDelta_FromDSU:int:days:: -PyDelta_FromDSU:int:seconds:: -PyDelta_FromDSU:int:useconds:: - -PyTimeZone_FromOffset:PyObject*::+1: -PyTimeZone_FromOffset:PyObject*:offset:+1:Reference count not increased if offset is +00:00 - -PyTimeZone_FromOffsetAndName:PyObject*::+1: -PyTimeZone_FromOffsetAndName:PyObject*:offset:+1:Reference count not increased if offset is +00:00 and name == NULL -PyTimeZone_FromOffsetAndName:PyObject*:name:+1: - - -PyDescr_IsData:int::: -PyDescr_IsData:PyObject*:descr:0: - -PyDescr_NewClassMethod:PyObject*::+1: -PyDescr_NewClassMethod:PyTypeObject*:type:+1: -PyDescr_NewClassMethod:PyMethodDef*:method:: - -PyDescr_NewGetSet:PyObject*::+1: -PyDescr_NewGetSet:PyTypeObject*:type:+1: -PyDescr_NewGetSet:PyGetSetDef*:getset:: - -PyDescr_NewMember:PyObject*::+1: -PyDescr_NewMember:PyTypeObject*:type:+1: -PyDescr_NewMember:PyMemberDef*:member:: - -PyDescr_NewMethod:PyObject*::+1: -PyDescr_NewMethod:PyTypeObject*:type:+1: -PyDescr_NewMethod:PyMethodDef*:meth:: - -PyDescr_NewWrapper:PyObject*::+1: -PyDescr_NewWrapper:PyTypeObject*:type:+1: -PyDescr_NewWrapper:struct wrapperbase*:base:: -PyDescr_NewWrapper:void*:wrapped:: - -PyDict_Check:int::: -PyDict_Check:PyObject*:p:0: - -PyDict_CheckExact:int::: -PyDict_CheckExact:PyObject*:p:0: - -PyDict_Clear:void::: -PyDict_Clear:PyObject*:p:0: - -PyDict_Contains:int::: -PyDict_Contains:PyObject*:p:0: -PyDict_Contains:PyObject*:key:0: - -PyDict_DelItem:int::: -PyDict_DelItem:PyObject*:p:0: -PyDict_DelItem:PyObject*:key:0: - -PyDict_DelItemString:int::: -PyDict_DelItemString:PyObject*:p:0: -PyDict_DelItemString:const char*:key:: - -PyDict_GetItem:PyObject*::0: -PyDict_GetItem:PyObject*:p:0: -PyDict_GetItem:PyObject*:key:0: - -PyDict_GetItemWithError:PyObject*::0:0 -PyDict_GetItemWithError:PyObject*:p:0: -PyDict_GetItemWithError:PyObject*:key:0: - -PyDict_GetItemString:PyObject*::0: -PyDict_GetItemString:PyObject*:p:0: -PyDict_GetItemString:const char*:key:: - -PyDict_SetDefault:PyObject*::0: -PyDict_SetDefault:PyObject*:p:0: -PyDict_SetDefault:PyObject*:key:0:conditionally +1 if inserted into the dict -PyDict_SetDefault:PyObject*:default:0:conditionally +1 if inserted into the dict - -PyDict_Items:PyObject*::+1: -PyDict_Items:PyObject*:p:0: - -PyDict_Keys:PyObject*::+1: -PyDict_Keys:PyObject*:p:0: - -PyDict_New:PyObject*::+1: - -PyDict_Copy:PyObject*::+1: -PyDict_Copy:PyObject*:p:0: - -PyDict_Merge:int::: -PyDict_Merge:PyObject*:a:0: -PyDict_Merge:PyObject*:b:0: -PyDict_Merge:int:override:: - -PyDict_MergeFromSeq2:int::: -PyDict_MergeFromSeq2:PyObject*:a:0: -PyDict_MergeFromSeq2:PyObject*:seq2:0: -PyDict_MergeFromSeq2:int:override:: - -PyDict_Next:int::: -PyDict_Next:PyObject*:p:0: -PyDict_Next:Py_ssize_t:ppos:: -PyDict_Next:PyObject**:pkey:0: -PyDict_Next:PyObject**:pvalue:0: - -PyDict_SetItem:int::: -PyDict_SetItem:PyObject*:p:0: -PyDict_SetItem:PyObject*:key:+1: -PyDict_SetItem:PyObject*:val:+1: - -PyDict_SetItemString:int::: -PyDict_SetItemString:PyObject*:p:0: -PyDict_SetItemString:const char*:key:: -PyDict_SetItemString:PyObject*:val:+1: - -PyDict_Size:Py_ssize_t::: -PyDict_Size:PyObject*:p:0: - -PyDict_Update:int::: -PyDict_Update:PyObject*:a:0: -PyDict_Update:PyObject*:b:0: - -PyDict_Values:PyObject*::+1: -PyDict_Values:PyObject*:p:0: - -PyDictProxy_New:PyObject*::+1: -PyDictProxy_New:PyObject*:dict:0: - -PyErr_BadArgument:int::: - -PyErr_BadInternalCall:void::: - -PyErr_CheckSignals:int::: - -PyErr_Clear:void::: - -PyErr_ExceptionMatches:int::: -PyErr_ExceptionMatches:PyObject*:exc:0: - -PyErr_Fetch:void::: -PyErr_Fetch:PyObject**:ptype:0: -PyErr_Fetch:PyObject**:pvalue:0: -PyErr_Fetch:PyObject**:ptraceback:0: - -PyErr_Format:PyObject*::null: -PyErr_Format:PyObject*:exception:+1: -PyErr_Format:const char*:format:: -PyErr_Format::...:: - -PyErr_FormatV:PyObject*::null: -PyErr_FormatV:PyObject*:exception:+1: -PyErr_FormatV:const char*:format:: -PyErr_FormatV:va_list:vargs:: - -PyErr_GetExcInfo:void::: -PyErr_GetExcInfo:PyObject**:ptype:+1: -PyErr_GetExcInfo:PyObject**:pvalue:+1: -PyErr_GetExcInfo:PyObject**:ptraceback:+1: - -PyErr_GivenExceptionMatches:int::: -PyErr_GivenExceptionMatches:PyObject*:given:0: -PyErr_GivenExceptionMatches:PyObject*:exc:0: - -PyErr_NewException:PyObject*::+1: -PyErr_NewException:const char*:name:: -PyErr_NewException:PyObject*:base:0: -PyErr_NewException:PyObject*:dict:0: - -PyErr_NewExceptionWithDoc:PyObject*::+1: -PyErr_NewExceptionWithDoc:const char*:name:: -PyErr_NewExceptionWithDoc:const char*:doc:: -PyErr_NewExceptionWithDoc:PyObject*:base:0: -PyErr_NewExceptionWithDoc:PyObject*:dict:0: - -PyErr_NoMemory:PyObject*::null: - -PyErr_NormalizeException:void::: -PyErr_NormalizeException:PyObject**:exc::??? -PyErr_NormalizeException:PyObject**:val::??? -PyErr_NormalizeException:PyObject**:tb::??? - -PyErr_Occurred:PyObject*::0: - -PyErr_Print:void::: - -PyErr_PrintEx:void::: -PyErr_PrintEx:int:set_sys_last_vars:: - -PyErr_ResourceWarning:int::: -PyErr_ResourceWarning:PyObject*:source:0: -PyErr_ResourceWarning:Py_ssize_t:stack_level:: -PyErr_ResourceWarning:const char*:format:: -PyErr_ResourceWarning::...:: - -PyErr_Restore:void::: -PyErr_Restore:PyObject*:type:-1: -PyErr_Restore:PyObject*:value:-1: -PyErr_Restore:PyObject*:traceback:-1: - -PyErr_SetExcFromWindowsErr:PyObject*::null: -PyErr_SetExcFromWindowsErr:PyObject*:type:+1: -PyErr_SetExcFromWindowsErr:int:ierr:: - -PyErr_SetExcFromWindowsErrWithFilename:PyObject*::null: -PyErr_SetExcFromWindowsErrWithFilename:PyObject*:type:+1: -PyErr_SetExcFromWindowsErrWithFilename:int:ierr:: -PyErr_SetExcFromWindowsErrWithFilename:const char*:filename:: - -PyErr_SetExcFromWindowsErrWithFilenameObject:PyObject*::null: -PyErr_SetExcFromWindowsErrWithFilenameObject:PyObject*:type:+1: -PyErr_SetExcFromWindowsErrWithFilenameObject:int:ierr:: -PyErr_SetExcFromWindowsErrWithFilenameObject:PyObject*:filename:+1: - -PyErr_SetExcFromWindowsErrWithFilenameObjects:PyObject*::null: -PyErr_SetExcFromWindowsErrWithFilenameObjects:PyObject*:type:+1: -PyErr_SetExcFromWindowsErrWithFilenameObjects:int:ierr:: -PyErr_SetExcFromWindowsErrWithFilenameObjects:PyObject*:filename:+1: -PyErr_SetExcFromWindowsErrWithFilenameObjects:PyObject*:filename2:+1: - -PyErr_SetExcInfo:void::: -PyErr_SetExcInfo:PyObject*:type:0: -PyErr_SetExcInfo:PyObject*:value:0: -PyErr_SetExcInfo:PyObject*:traceback:0: - -PyErr_SetFromErrno:PyObject*::null: -PyErr_SetFromErrno:PyObject*:type:+1: - -PyErr_SetFromErrnoWithFilename:PyObject*::null: -PyErr_SetFromErrnoWithFilename:PyObject*:type:+1: -PyErr_SetFromErrnoWithFilename:const char*:filename:: - -PyErr_SetFromErrnoWithFilenameObject:PyObject*::null: -PyErr_SetFromErrnoWithFilenameObject:PyObject*:type:+1: -PyErr_SetFromErrnoWithFilenameObject:PyObject*:filenameObject:+1: - -PyErr_SetFromErrnoWithFilenameObjects:PyObject*::null: -PyErr_SetFromErrnoWithFilenameObjects:PyObject*:type:+1: -PyErr_SetFromErrnoWithFilenameObjects:PyObject*:filenameObject:+1: -PyErr_SetFromErrnoWithFilenameObjects:PyObject*:filenameObject2:+1: - -PyErr_SetFromWindowsErr:PyObject*::null: -PyErr_SetFromWindowsErr:int:ierr:: - -PyErr_SetFromWindowsErrWithFilename:PyObject*::null: -PyErr_SetFromWindowsErrWithFilename:int:ierr:: -PyErr_SetFromWindowsErrWithFilename:const char*:filename:: - -PyErr_SetImportError:PyObject*::null: -PyErr_SetImportError:PyObject*:msg:+1: -PyErr_SetImportError:PyObject*:name:+1: -PyErr_SetImportError:PyObject*:path:+1: - -PyErr_SetImportErrorSubclass:PyObject*::null: -PyErr_SetImportErrorSubclass:PyObject*:msg:+1: -PyErr_SetImportErrorSubclass:PyObject*:name:+1: -PyErr_SetImportErrorSubclass:PyObject*:path:+1: - -PyErr_SetInterrupt:void::: - -PyErr_SetNone:void::: -PyErr_SetNone:PyObject*:type:+1: - -PyErr_SetObject:void::: -PyErr_SetObject:PyObject*:type:+1: -PyErr_SetObject:PyObject*:value:+1: - -PyErr_SetString:void::: -PyErr_SetString:PyObject*:type:+1: -PyErr_SetString:const char*:message:: - -PyErr_SyntaxLocation:void::: -PyErr_SyntaxLocation:const char*:filename:: -PyErr_SyntaxLocation:int:lineno:: - -PyErr_SyntaxLocationEx:void::: -PyErr_SyntaxLocationEx:const char*:filename:: -PyErr_SyntaxLocationEx:int:lineno:: -PyErr_SyntaxLocationEx:int:col_offset:: - -PyErr_SyntaxLocationObject:void::: -PyErr_SyntaxLocationObject:PyObject*:filename:+1: -PyErr_SyntaxLocationObject:int:lineno:: -PyErr_SyntaxLocationObject:int:col_offset:: - -PyErr_WarnEx:int::: -PyErr_WarnEx:PyObject*:category:0: -PyErr_WarnEx:const char*:message:: -PyErr_WarnEx:Py_ssize_t:stack_level:: - -PyErr_WarnExplicit:int::: -PyErr_WarnExplicit:PyObject*:category:0: -PyErr_WarnExplicit:const char*:message:: -PyErr_WarnExplicit:const char*:filename:: -PyErr_WarnExplicit:int:lineno:: -PyErr_WarnExplicit:const char*:module:: -PyErr_WarnExplicit:PyObject*:registry:0: - -PyErr_WarnExplicitObject:int::: -PyErr_WarnExplicitObject:PyObject*:category:0: -PyErr_WarnExplicitObject:PyObject*:message:0: -PyErr_WarnExplicitObject:PyObject*:filename:0: -PyErr_WarnExplicitObject:int:lineno:: -PyErr_WarnExplicitObject:PyObject*:module:0: -PyErr_WarnExplicitObject:PyObject*:registry:0: - -PyErr_WarnFormat:int::: -PyErr_WarnFormat:PyObject*:category:0: -PyErr_WarnFormat:Py_ssize_t:stack_level:: -PyErr_WarnFormat:const char*:format:: -PyErr_WarnFormat::...:: - -PyErr_WriteUnraisable:void::: -PyErr_WriteUnraisable:PyObject*:obj:0: - -PyEval_AcquireLock:void::: - -PyEval_AcquireThread:void::: -PyEval_AcquireThread:PyThreadState*:tstate:: - -PyEval_GetBuiltins:PyObject*::0: - -PyEval_GetLocals:PyObject*::0: - -PyEval_GetGlobals:PyObject*::0: - -PyEval_GetFrame:PyObject*::0: - -PyEval_GetFuncDesc:const char*::: -PyEval_GetFuncDesc:PyObject*:func:0: - -PyEval_GetFuncName:const char*::: -PyEval_GetFuncName:PyObject*:func:0: - -PyEval_InitThreads:void::: - -PyEval_ReleaseLock:void::: - -PyEval_ReleaseThread:void::: -PyEval_ReleaseThread:PyThreadState*:tstate:: - -PyEval_RestoreThread:void::: -PyEval_RestoreThread:PyThreadState*:tstate:: - -PyEval_SaveThread:PyThreadState*::: - -PyEval_SetProfile:void::: -PyEval_SetProfile:Py_tracefunc:func:: -PyEval_SetProfile:PyObject*:obj:+1: - -PyEval_SetTrace:void::: -PyEval_SetTrace:Py_tracefunc:func:: -PyEval_SetTrace:PyObject*:obj:+1: - -PyEval_EvalCode:PyObject*::+1: -PyEval_EvalCode:PyObject*:co:0: -PyEval_EvalCode:PyObject*:globals:0: -PyEval_EvalCode:PyObject*:locals:0: - -PyEval_EvalCodeEx:PyObject*::+1: -PyEval_EvalCodeEx:PyObject*:co:0: -PyEval_EvalCodeEx:PyObject*:globals:0: -PyEval_EvalCodeEx:PyObject*:locals:0: -PyEval_EvalCodeEx:PyObject*const*:args:: -PyEval_EvalCodeEx:int:argcount:: -PyEval_EvalCodeEx:PyObject*const*:kws:: -PyEval_EvalCodeEx:int:kwcount:: -PyEval_EvalCodeEx:PyObject*const*:defs:: -PyEval_EvalCodeEx:int:defcount:: -PyEval_EvalCodeEx:PyObject*:kwdefs:0: -PyEval_EvalCodeEx:PyObject*:closure:0: - -PyEval_EvalFrame:PyObject*::+1: -PyEval_EvalFrame:PyFrameObject*:f:0: - -PyEval_EvalFrameEx:PyObject*::+1: -PyEval_EvalFrameEx:PyFrameObject*:f:0: -PyEval_EvalFrameEx:int:throwflag:: - -PyEval_MergeCompilerFlags:int::: -PyEval_MergeCompilerFlags:PyCompilerFlags*:cf:: - -PyException_GetCause:PyObject*::+1: -PyException_GetCause:PyObject*:ex:0: - -PyException_GetContext:PyObject*::+1: -PyException_GetContext:PyObject*:ex:0: - -PyException_GetTraceback:PyObject*::+1: -PyException_GetTraceback:PyObject*:ex:0: - -PyException_SetCause:void::: -PyException_SetCause:PyObject*:ex:0: -PyException_SetCause:PyObject*:cause:+1: - -PyException_SetContext:void::: -PyException_SetContext:PyObject*:ex:0: -PyException_SetContext:PyObject*:ctx:+1: - -PyException_SetTraceback:int::: -PyException_SetTraceback:PyObject*:ex:0: -PyException_SetTraceback:PyObject*:tb:+1: - -PyFile_FromFd:PyObject*::+1: -PyFile_FromFd:int:fd:: -PyFile_FromFd:const char*:name:: -PyFile_FromFd:const char*:mode:: -PyFile_FromFd:int:buffering:: -PyFile_FromFd:const char*:encoding:: -PyFile_FromFd:const char*:errors:: -PyFile_FromFd:const char*:newline:: -PyFile_FromFd:int:closefd:: - -PyFile_GetLine:PyObject*::+1: -PyFile_GetLine:PyObject*:p:0: -PyFile_GetLine:int:n:: - -PyFile_WriteObject:int::: -PyFile_WriteObject:PyObject*:obj:0: -PyFile_WriteObject:PyObject*:p:0: -PyFile_WriteObject:int:flags:: - -PyFile_WriteString:int::: -PyFile_WriteString:const char*:s:: -PyFile_WriteString:PyObject*:p:0: -PyFile_WriteString:int:flags:: - -PyFloat_AS_DOUBLE:double::: -PyFloat_AS_DOUBLE:PyObject*:pyfloat:0: - -PyFloat_AsDouble:double::: -PyFloat_AsDouble:PyObject*:pyfloat:0: - -PyFloat_Check:int::: -PyFloat_Check:PyObject*:p:0: - -PyFloat_CheckExact:int::: -PyFloat_CheckExact:PyObject*:p:0: - -PyFloat_FromDouble:PyObject*::+1: -PyFloat_FromDouble:double:v:: - -PyFloat_FromString:PyObject*::+1: -PyFloat_FromString:PyObject*:str:0: - -PyFloat_GetInfo:PyObject*::+1: -PyFloat_GetInfo::void:: - -PyFrozenSet_Check:int::: -PyFrozenSet_Check:PyObject*:p:0: - -PyFrozenSet_CheckExact:int::: -PyFrozenSet_CheckExact:PyObject*:p:0: - -PyFrozenSet_New:PyObject*::+1: -PyFrozenSet_New:PyObject*:iterable:0: - -PyFunction_Check:int::: -PyFunction_Check:PyObject*:o:0: - -PyFunction_GetAnnotations:PyObject*::0: -PyFunction_GetAnnotations:PyObject*:op:0: - -PyFunction_GetClosure:PyObject*::0: -PyFunction_GetClosure:PyObject*:op:0: - -PyFunction_GetCode:PyObject*::0: -PyFunction_GetCode:PyObject*:op:0: - -PyFunction_GetDefaults:PyObject*::0: -PyFunction_GetDefaults:PyObject*:op:0: - -PyFunction_GetGlobals:PyObject*::0: -PyFunction_GetGlobals:PyObject*:op:0: - -PyFunction_GetModule:PyObject*::0: -PyFunction_GetModule:PyObject*:op:0: - -PyFunction_New:PyObject*::+1: -PyFunction_New:PyObject*:code:+1: -PyFunction_New:PyObject*:globals:+1: - -PyFunction_NewWithQualName:PyObject*::+1: -PyFunction_NewWithQualName:PyObject*:code:+1: -PyFunction_NewWithQualName:PyObject*:globals:+1: -PyFunction_NewWithQualName:PyObject*:qualname:+1: - -PyFunction_SetAnnotations:int::: -PyFunction_SetAnnotations:PyObject*:op:0: -PyFunction_SetAnnotations:PyObject*:annotations:+1: - -PyFunction_SetClosure:int::: -PyFunction_SetClosure:PyObject*:op:0: -PyFunction_SetClosure:PyObject*:closure:+1: - -PyFunction_SetDefaults:int::: -PyFunction_SetDefaults:PyObject*:op:0: -PyFunction_SetDefaults:PyObject*:defaults:+1: - -PyGen_Check:int::: -PyGen_Check:PyObject*:ob:0: - -PyGen_CheckExact:int::: -PyGen_CheckExact:PyObject*:ob:0: - -PyGen_New:PyObject*::+1: -PyGen_New:PyFrameObject*:frame:0: - -PyGen_NewWithQualName:PyObject*::+1: -PyGen_NewWithQualName:PyFrameObject*:frame:0: -PyGen_NewWithQualName:PyObject*:name:0: -PyGen_NewWithQualName:PyObject*:qualname:0: - -PyCoro_CheckExact:int::: -PyCoro_CheckExact:PyObject*:ob:0: - -PyCoro_New:PyObject*::+1: -PyCoro_New:PyFrameObject*:frame:0: -PyCoro_New:PyObject*:name:0: -PyCoro_New:PyObject*:qualname:0: - -PyImport_AddModule:PyObject*::0:reference borrowed from sys.modules -PyImport_AddModule:const char*:name:: - -PyImport_AddModuleObject:PyObject*::0:reference borrowed from sys.modules -PyImport_AddModuleObject:PyObject*:name:0: - -PyImport_Cleanup:void::: - -PyImport_ExecCodeModule:PyObject*::+1: -PyImport_ExecCodeModule:const char*:name:: -PyImport_ExecCodeModule:PyObject*:co:0: - -PyImport_ExecCodeModuleEx:PyObject*::+1: -PyImport_ExecCodeModuleEx:const char*:name:: -PyImport_ExecCodeModuleEx:PyObject*:co:0: -PyImport_ExecCodeModuleEx:const char*:pathname:: - -PyImport_ExecCodeModuleObject:PyObject*::+1: -PyImport_ExecCodeModuleObject:const char*:name:: -PyImport_ExecCodeModuleObject:PyObject*:co:0: -PyImport_ExecCodeModuleObject:PyObject*:pathname:0: -PyImport_ExecCodeModuleObject:PyObject*:cpathname:0: - -PyImport_ExecCodeModuleWithPathnames:PyObject*::+1: -PyImport_ExecCodeModuleWithPathnames:const char*:name:: -PyImport_ExecCodeModuleWithPathnames:PyObject*:co:0: -PyImport_ExecCodeModuleWithPathnames:const char*:pathname:: -PyImport_ExecCodeModuleWithPathnames:const char*:cpathname:: - -PyImport_GetImporter:PyObject*::+1: -PyImport_GetImporter:PyObject*:path:0: - -PyImport_GetMagicNumber:long::: - -PyImport_GetModule:PyObject*::+1: -PyImport_GetModule:PyObject*:name:0: - -PyImport_GetModuleDict:PyObject*::0: - -PyImport_Import:PyObject*::+1: -PyImport_Import:PyObject*:name:0: - -PyImport_ImportFrozenModule:int::: -PyImport_ImportFrozenModule:const char*:name:: - -PyImport_ImportFrozenModuleObject:int::: -PyImport_ImportFrozenModuleObject:PyObject*:name:+1: - -PyImport_ImportModule:PyObject*::+1: -PyImport_ImportModule:const char*:name:: - -PyImport_ImportModuleEx:PyObject*::+1: -PyImport_ImportModuleEx:const char*:name:: -PyImport_ImportModuleEx:PyObject*:globals:0:??? -PyImport_ImportModuleEx:PyObject*:locals:0:??? -PyImport_ImportModuleEx:PyObject*:fromlist:0:??? - -PyImport_ImportModuleLevel:PyObject*::+1: -PyImport_ImportModuleLevel:const char*:name:: -PyImport_ImportModuleLevel:PyObject*:globals:0:??? -PyImport_ImportModuleLevel:PyObject*:locals:0:??? -PyImport_ImportModuleLevel:PyObject*:fromlist:0:??? -PyImport_ImportModuleLevel:int:level:: - -PyImport_ImportModuleLevelObject:PyObject*::+1: -PyImport_ImportModuleLevelObject:PyObject*:name:0: -PyImport_ImportModuleLevelObject:PyObject*:globals:0:??? -PyImport_ImportModuleLevelObject:PyObject*:locals:0:??? -PyImport_ImportModuleLevelObject:PyObject*:fromlist:0:??? -PyImport_ImportModuleLevelObject:int:level:: - -PyImport_ImportModuleNoBlock:PyObject*::+1: -PyImport_ImportModuleNoBlock:const char*:name:: - -PyImport_ReloadModule:PyObject*::+1: -PyImport_ReloadModule:PyObject*:m:0: - -PyIndex_Check:int::: -PyIndex_Check:PyObject*:o:0: - -PyInstanceMethod_Check:int::: -PyInstanceMethod_Check:PyObject*:o:0: - -PyInstanceMethod_Function:PyObject*::0: -PyInstanceMethod_Function:PyObject*:im:0: - -PyInstanceMethod_GET_FUNCTION:PyObject*::0: -PyInstanceMethod_GET_FUNCTION:PyObject*:im:0: - -PyInstanceMethod_New:PyObject*::+1: -PyInstanceMethod_New:PyObject*:func:0: - -PyInterpreterState_Clear:void::: -PyInterpreterState_Clear:PyInterpreterState*:interp:: - -PyInterpreterState_Delete:void::: -PyInterpreterState_Delete:PyInterpreterState*:interp:: - -PyInterpreterState_GetID:int64_t::: -PyInterpreterState_GetID:PyInterpreterState*:interp:: - -PyInterpreterState_New:PyInterpreterState*::: - -PyIter_Check:int::: -PyIter_Check:PyObject*:o:0: - -PyAIter_Check:int::: -PyAIter_Check:PyObject*:o:0: - -PyIter_Next:PyObject*::+1: -PyIter_Next:PyObject*:o:0: - -PyIter_Send:int::: -PyIter_Send:PyObject*:iter:0: -PyIter_Send:PyObject*:arg:0: -PyIter_Send:PyObject**:presult:+1: - -PyList_Append:int::: -PyList_Append:PyObject*:list:0: -PyList_Append:PyObject*:item:+1: - -PyList_AsTuple:PyObject*::+1: -PyList_AsTuple:PyObject*:list:0: - -PyList_Check:int::: -PyList_Check:PyObject*:p:0: - -PyList_CheckExact:int::: -PyList_CheckExact:PyObject*:p:0: - -PyList_GET_ITEM:PyObject*::0: -PyList_GET_ITEM:PyObject*:list:0: -PyList_GET_ITEM:Py_ssize_t:i:: - -PyList_GET_SIZE:Py_ssize_t::: -PyList_GET_SIZE:PyObject*:list:0: - -PyList_GetItem:PyObject*::0: -PyList_GetItem:PyObject*:list:0: -PyList_GetItem:Py_ssize_t:index:: - -PyList_GetSlice:PyObject*::+1: -PyList_GetSlice:PyObject*:list:0: -PyList_GetSlice:Py_ssize_t:low:: -PyList_GetSlice:Py_ssize_t:high:: - -PyList_Insert:int::: -PyList_Insert:PyObject*:list:0: -PyList_Insert:Py_ssize_t:index:: -PyList_Insert:PyObject*:item:+1: - -PyList_New:PyObject*::+1: -PyList_New:Py_ssize_t:len:: - -PyList_Reverse:int::: -PyList_Reverse:PyObject*:list:0: - -PyList_SET_ITEM:void::: -PyList_SET_ITEM:PyObject*:list:0: -PyList_SET_ITEM:Py_ssize_t:i:: -PyList_SET_ITEM:PyObject*:o:0: - -PyList_SetItem:int::: -PyList_SetItem:PyObject*:list:0: -PyList_SetItem:Py_ssize_t:index:: -PyList_SetItem:PyObject*:item:0: - -PyList_SetSlice:int::: -PyList_SetSlice:PyObject*:list:0: -PyList_SetSlice:Py_ssize_t:low:: -PyList_SetSlice:Py_ssize_t:high:: -PyList_SetSlice:PyObject*:itemlist:0:but increfs its elements? - -PyList_Size:Py_ssize_t::: -PyList_Size:PyObject*:list:0: - -PyList_Sort:int::: -PyList_Sort:PyObject*:list:0: - -PyLong_AsDouble:double::: -PyLong_AsDouble:PyObject*:pylong:0: - -PyLong_AsLong:long::: -PyLong_AsLong:PyObject*:pylong:0: - -PyLong_AsLongAndOverflow:long::: -PyLong_AsLongAndOverflow:PyObject*:obj:0: -PyLong_AsLongAndOverflow:int*:overflow:: - -PyLong_AsLongLong:long long::: -PyLong_AsLongLong:PyObject*:obj:0: - -PyLong_AsLongLongAndOverflow:long long::: -PyLong_AsLongLongAndOverflow:PyObject*:obj:0: -PyLong_AsLongLongAndOverflow:int*:overflow:: - -PyLong_AsSize_t:size_t::: -PyLong_AsSize_t:PyObject*:pylong:0: - -PyLong_AsSsize_t:Py_ssize_t::: -PyLong_AsSsize_t:PyObject*:pylong:0: - -PyLong_AsUnsignedLong:unsigned long::: -PyLong_AsUnsignedLong:PyObject*:pylong:0: - -PyLong_AsUnsignedLongLong:unsigned long long::: -PyLong_AsUnsignedLongLong:PyObject*:pylong:0: - -PyLong_AsUnsignedLongMask:unsigned long::: -PyLong_AsUnsignedLongMask:PyObject*:obj:0: - -PyLong_AsUnsignedLongLongMask:unsigned long long::: -PyLong_AsUnsignedLongLongMask:PyObject*:obj:0: - -PyLong_AsVoidPtr:void*::: -PyLong_AsVoidPtr:PyObject*:pylong:0: - -PyLong_Check:int::: -PyLong_Check:PyObject*:p:0: - -PyLong_CheckExact:int::: -PyLong_CheckExact:PyObject*:p:0: - -PyLong_FromDouble:PyObject*::+1: -PyLong_FromDouble:double:v:: - -PyLong_FromLong:PyObject*::+1: -PyLong_FromLong:long:v:: - -PyLong_FromLongLong:PyObject*::+1: -PyLong_FromLongLong:long long:v:: - -PyLong_FromUnsignedLongLong:PyObject*::+1: -PyLong_FromUnsignedLongLong:unsigned long long:v:: - -PyLong_FromSize_t:PyObject*::+1: -PyLong_FromSize_t:size_t:v:: - -PyLong_FromSsize_t:PyObject*::+1: -PyLong_FromSsize_t:Py_ssize_t:v:: - -PyLong_FromString:PyObject*::+1: -PyLong_FromString:const char*:str:: -PyLong_FromString:char**:pend:: -PyLong_FromString:int:base:: - -PyLong_FromUnicodeObject:PyObject*::+1: -PyLong_FromUnicodeObject:PyObject*:u:0: -PyLong_FromUnicodeObject:int:base:: - -PyLong_FromUnsignedLong:PyObject*::+1: -PyLong_FromUnsignedLong:unsignedlong:v:: - -PyLong_FromVoidPtr:PyObject*::+1: -PyLong_FromVoidPtr:void*:p:: - -PyMapping_Check:int::: -PyMapping_Check:PyObject*:o:0: - -PyMapping_DelItem:int::: -PyMapping_DelItem:PyObject*:o:0: -PyMapping_DelItem:PyObject*:key:0: - -PyMapping_DelItemString:int::: -PyMapping_DelItemString:PyObject*:o:0: -PyMapping_DelItemString:const char*:key:: - -PyMapping_GetItemString:PyObject*::+1: -PyMapping_GetItemString:PyObject*:o:0: -PyMapping_GetItemString:const char*:key:: - -PyMapping_HasKey:int::: -PyMapping_HasKey:PyObject*:o:0: -PyMapping_HasKey:PyObject*:key:: - -PyMapping_HasKeyString:int::: -PyMapping_HasKeyString:PyObject*:o:0: -PyMapping_HasKeyString:const char*:key:: - -PyMapping_Items:PyObject*::+1: -PyMapping_Items:PyObject*:o:0: - -PyMapping_Keys:PyObject*::+1: -PyMapping_Keys:PyObject*:o:0: - -PyMapping_Length:Py_ssize_t::: -PyMapping_Length:PyObject*:o:0: - -PyMapping_SetItemString:int::: -PyMapping_SetItemString:PyObject*:o:0: -PyMapping_SetItemString:const char*:key:: -PyMapping_SetItemString:PyObject*:v:+1: - -PyMapping_Size:Py_ssize_t::: -PyMapping_Size:PyObject*:o:0: - -PyMapping_Values:PyObject*::+1: -PyMapping_Values:PyObject*:o:0: - -PyMarshal_ReadLastObjectFromFile:PyObject*::+1: -PyMarshal_ReadLastObjectFromFile:FILE*:file:: - -PyMarshal_ReadObjectFromFile:PyObject*::+1: -PyMarshal_ReadObjectFromFile:FILE*:file:: - -PyMarshal_ReadObjectFromString:PyObject*::+1: -PyMarshal_ReadObjectFromString:const char*:string:: -PyMarshal_ReadObjectFromString:Py_ssize_t:len:: - -PyMarshal_WriteObjectToString:PyObject*::+1: -PyMarshal_WriteObjectToString:PyObject*:value:0: -PyMarshal_WriteObjectToString:int:version:: - -PyMemoryView_Check:int::: -PyMemoryView_Check:PyObject*:obj:0: - -PyMemoryView_FromBuffer:PyObject*::+1: -PyMemoryView_FromBuffer:Py_buffer*:view:: - -PyMemoryView_FromMemory:PyObject*::+1: -PyMemoryView_FromMemory:char*:mem:: -PyMemoryView_FromMemory:Py_ssize_t:size:: -PyMemoryView_FromMemory:int:flags:: - -PyMemoryView_FromObject:PyObject*::+1: -PyMemoryView_FromObject:PyObject*:obj:0: - -PyMemoryView_GET_BASE:Py_buffer*::: -PyMemoryView_GET_BASE:PyObject*:mview:0: - -PyMemoryView_GET_BUFFER:Py_buffer*::: -PyMemoryView_GET_BUFFER:PyObject*:mview:0: - -PyMemoryView_GetContiguous:PyObject*::+1: -PyMemoryView_GetContiguous:PyObject*:obj:0: -PyMemoryView_GetContiguous:int:buffertype:: -PyMemoryView_GetContiguous:char:order:: - -PyMethod_Check:int::: -PyMethod_Check:PyObject*:o:0: - -PyMethod_Function:PyObject*::0: -PyMethod_Function:PyObject*:im:0: - -PyMethod_GET_FUNCTION:PyObject*::0: -PyMethod_GET_FUNCTION:PyObject*:im:0: - -PyMethod_GET_SELF:PyObject*::0: -PyMethod_GET_SELF:PyObject*:im:0: - -PyMethod_New:PyObject*::+1: -PyMethod_New:PyObject*:func:0: -PyMethod_New:PyObject*:self:0: -PyMethod_New:PyObject*:class:0: - -PyMethod_Self:PyObject*::0: -PyMethod_Self:PyObject*:im:0: - -PyModule_AddFunctions:int::: -PyModule_AddFunctions:PyObject*:module:0: -PyModule_AddFunctions:PyMethodDef*:functions:: - -PyModule_AddIntConstant:int::: -PyModule_AddIntConstant:PyObject*:module:0: -PyModule_AddIntConstant:const char*:name:: -PyModule_AddIntConstant:long:value:: - -PyModule_AddIntMacro:int::: -PyModule_AddIntMacro:PyObject*:module:0: -PyModule_AddIntMacro::macro:: - -PyModule_AddObject:int::: -PyModule_AddObject:PyObject*:module:0: -PyModule_AddObject:const char*:name:: -PyModule_AddObject:PyObject*:value:+1: - -PyModule_AddStringConstant:int::: -PyModule_AddStringConstant:PyObject*:module:0: -PyModule_AddStringConstant:const char*:name:: -PyModule_AddStringConstant:const char*:value:: - -PyModule_AddStringMacro:int::: -PyModule_AddStringMacro:PyObject*:module:0: -PyModule_AddStringMacro::macro:: - -PyModule_Check:int::: -PyModule_Check:PyObject*:p:0: - -PyModule_CheckExact:int::: -PyModule_CheckExact:PyObject*:p:0: - -PyModule_Create:PyObject*::+1: -PyModule_Create:PyModuleDef*:def:: - -PyModule_Create2:PyObject*::+1: -PyModule_Create2:PyModuleDef*:def:: -PyModule_Create2:int:module_api_version:: - -PyModule_ExecDef:int::: -PyModule_ExecDef:PyObject*:module:0: -PyModule_ExecDef:PyModuleDef*:def:: - -PyModule_FromDefAndSpec:PyObject*::+1: -PyModule_FromDefAndSpec:PyModuleDef*:def:: -PyModule_FromDefAndSpec:PyObject*:spec:0: - -PyModule_FromDefAndSpec2:PyObject*::+1: -PyModule_FromDefAndSpec2:PyModuleDef*:def:: -PyModule_FromDefAndSpec2:PyObject*:spec:0: -PyModule_FromDefAndSpec2:int:module_api_version:: - -PyModule_GetDef:PyModuleDef*::0: -PyModule_GetDef:PyObject*:module:0: - -PyModule_GetDict:PyObject*::0: -PyModule_GetDict:PyObject*:module:0: - -PyModule_GetFilename:const char*::: -PyModule_GetFilename:PyObject*:module:0: - -PyModule_GetFilenameObject:PyObject*::+1: -PyModule_GetFilenameObject:PyObject*:module:0: - -PyModule_GetName:const char*::: -PyModule_GetName:PyObject*:module:0: - -PyModule_GetNameObject:PyObject*::+1: -PyModule_GetNameObject:PyObject*:module:0: - -PyModule_GetState:void*::: -PyModule_GetState:PyObject*:module:0: - -PyModule_New:PyObject*::+1: -PyModule_New::char* name:: - -PyModule_NewObject:PyObject*::+1: -PyModule_NewObject:PyObject*:name:+1: - -PyModule_SetDocString:int::: -PyModule_SetDocString:PyObject*:module:0: -PyModule_SetDocString:const char*:docstring:: - -PyModuleDef_Init:PyObject*::0: -PyModuleDef_Init:PyModuleDef*:def:0: - -PyNumber_Absolute:PyObject*::+1: -PyNumber_Absolute:PyObject*:o:0: - -PyNumber_Add:PyObject*::+1: -PyNumber_Add:PyObject*:o1:0: -PyNumber_Add:PyObject*:o2:0: - -PyNumber_And:PyObject*::+1: -PyNumber_And:PyObject*:o1:0: -PyNumber_And:PyObject*:o2:0: - -PyNumber_AsSsize_t:Py_ssize_t::: -PyNumber_AsSsize_t:PyObject*:o:0: -PyNumber_AsSsize_t:PyObject*:exc:0: - -PyNumber_Check:int::: -PyNumber_Check:PyObject*:o:0: - -PyNumber_Divmod:PyObject*::+1: -PyNumber_Divmod:PyObject*:o1:0: -PyNumber_Divmod:PyObject*:o2:0: - -PyNumber_Float:PyObject*::+1: -PyNumber_Float:PyObject*:o:0: - -PyNumber_FloorDivide:PyObject*::+1: -PyNumber_FloorDivide:PyObject*:v:0: -PyNumber_FloorDivide:PyObject*:w:0: - -PyNumber_Index:PyObject*::+1: -PyNumber_Index:PyObject*:o:0: - -PyNumber_InPlaceAdd:PyObject*::+1: -PyNumber_InPlaceAdd:PyObject*:v:0: -PyNumber_InPlaceAdd:PyObject*:w:0: - -PyNumber_InPlaceAnd:PyObject*::+1: -PyNumber_InPlaceAnd:PyObject*:v:0: -PyNumber_InPlaceAnd:PyObject*:w:0: - -PyNumber_InPlaceFloorDivide:PyObject*::+1: -PyNumber_InPlaceFloorDivide:PyObject*:v:0: -PyNumber_InPlaceFloorDivide:PyObject*:w:0: - -PyNumber_InPlaceLshift:PyObject*::+1: -PyNumber_InPlaceLshift:PyObject*:v:0: -PyNumber_InPlaceLshift:PyObject*:w:0: - -PyNumber_InPlaceMatrixMultiply:PyObject*::+1: -PyNumber_InPlaceMatrixMultiply:PyObject*:o1:0: -PyNumber_InPlaceMatrixMultiply:PyObject*:o2:0: - -PyNumber_InPlaceMultiply:PyObject*::+1: -PyNumber_InPlaceMultiply:PyObject*:v:0: -PyNumber_InPlaceMultiply:PyObject*:w:0: - -PyNumber_InPlaceOr:PyObject*::+1: -PyNumber_InPlaceOr:PyObject*:v:0: -PyNumber_InPlaceOr:PyObject*:w:0: - -PyNumber_InPlacePower:PyObject*::+1: -PyNumber_InPlacePower:PyObject*:v:0: -PyNumber_InPlacePower:PyObject*:w:0: -PyNumber_InPlacePower:PyObject*:z:0: - -PyNumber_InPlaceRemainder:PyObject*::+1: -PyNumber_InPlaceRemainder:PyObject*:v:0: -PyNumber_InPlaceRemainder:PyObject*:w:0: - -PyNumber_InPlaceRshift:PyObject*::+1: -PyNumber_InPlaceRshift:PyObject*:v:0: -PyNumber_InPlaceRshift:PyObject*:w:0: - -PyNumber_InPlaceSubtract:PyObject*::+1: -PyNumber_InPlaceSubtract:PyObject*:v:0: -PyNumber_InPlaceSubtract:PyObject*:w:0: - -PyNumber_InPlaceTrueDivide:PyObject*::+1: -PyNumber_InPlaceTrueDivide:PyObject*:v:0: -PyNumber_InPlaceTrueDivide:PyObject*:w:0: - -PyNumber_InPlaceXor:PyObject*::+1: -PyNumber_InPlaceXor:PyObject*:v:0: -PyNumber_InPlaceXor:PyObject*:w:0: - -PyNumber_Invert:PyObject*::+1: -PyNumber_Invert:PyObject*:o:0: - -PyNumber_Long:PyObject*::+1: -PyNumber_Long:PyObject*:o:0: - -PyNumber_Lshift:PyObject*::+1: -PyNumber_Lshift:PyObject*:o1:0: -PyNumber_Lshift:PyObject*:o2:0: - -PyNumber_MatrixMultiply:PyObject*::+1: -PyNumber_MatrixMultiply:PyObject*:o1:0: -PyNumber_MatrixMultiply:PyObject*:o2:0: - -PyNumber_Multiply:PyObject*::+1: -PyNumber_Multiply:PyObject*:o1:0: -PyNumber_Multiply:PyObject*:o2:0: - -PyNumber_Negative:PyObject*::+1: -PyNumber_Negative:PyObject*:o:0: - -PyNumber_Or:PyObject*::+1: -PyNumber_Or:PyObject*:o1:0: -PyNumber_Or:PyObject*:o2:0: - -PyNumber_Positive:PyObject*::+1: -PyNumber_Positive:PyObject*:o:0: - -PyNumber_Power:PyObject*::+1: -PyNumber_Power:PyObject*:o1:0: -PyNumber_Power:PyObject*:o2:0: -PyNumber_Power:PyObject*:o3:0: - -PyNumber_Remainder:PyObject*::+1: -PyNumber_Remainder:PyObject*:o1:0: -PyNumber_Remainder:PyObject*:o2:0: - -PyNumber_Rshift:PyObject*::+1: -PyNumber_Rshift:PyObject*:o1:0: -PyNumber_Rshift:PyObject*:o2:0: - -PyNumber_Subtract:PyObject*::+1: -PyNumber_Subtract:PyObject*:o1:0: -PyNumber_Subtract:PyObject*:o2:0: - -PyNumber_ToBase:PyObject*::+1: -PyNumber_ToBase:PyObject*:n:0: -PyNumber_ToBase:int:base:: - -PyNumber_TrueDivide:PyObject*::+1: -PyNumber_TrueDivide:PyObject*:v:0: -PyNumber_TrueDivide:PyObject*:w:0: - -PyNumber_Xor:PyObject*::+1: -PyNumber_Xor:PyObject*:o1:0: -PyNumber_Xor:PyObject*:o2:0: - -PyObject_AsFileDescriptor:int::: -PyObject_AsFileDescriptor:PyObject*:o:0: - -PyOS_AfterFork:void::: - -PyOS_AfterFork_Child:void::: - -PyOS_AfterFork_Parent:void::: - -PyOS_BeforeFork:void::: - -PyOS_FSPath:PyObject*::+1: -PyOS_FSPath:PyObject*:path:0: - -PyObject_ASCII:PyObject*::+1: -PyObject_ASCII:PyObject*:o:0: - -PyObject_AsCharBuffer:int::: -PyObject_AsCharBuffer:PyObject*:obj:0: -PyObject_AsCharBuffer:const char**:buffer:: -PyObject_AsCharBuffer:Py_ssize_t*:buffer_len:: - -PyObject_AsReadBuffer:int::: -PyObject_AsReadBuffer:PyObject*:obj:0: -PyObject_AsReadBuffer:const void**:buffer:: -PyObject_AsReadBuffer:Py_ssize_t*:buffer_len:: - -PyObject_AsWriteBuffer:int::: -PyObject_AsWriteBuffer:PyObject*:obj:0: -PyObject_AsWriteBuffer:void**:buffer:: -PyObject_AsWriteBuffer:Py_ssize_t*:buffer_len:: - -PyObject_Bytes:PyObject*::+1: -PyObject_Bytes:PyObject*:o:0: - -PyObject_Call:PyObject*::+1: -PyObject_Call:PyObject*:callable_object:0: -PyObject_Call:PyObject*:args:0: -PyObject_Call:PyObject*:kw:0: - -PyObject_CallFunction:PyObject*::+1: -PyObject_CallFunction:PyObject*:callable_object:0: -PyObject_CallFunction:const char*:format:: -PyObject_CallFunction::...:: - -PyObject_CallFunctionObjArgs:PyObject*::+1: -PyObject_CallFunctionObjArgs:PyObject*:callable:0: -PyObject_CallFunctionObjArgs::...:: - -PyObject_CallMethod:PyObject*::+1: -PyObject_CallMethod:PyObject*:o:0: -PyObject_CallMethod:const char*:m:: -PyObject_CallMethod:const char*:format:: -PyObject_CallMethod::...:: - -PyObject_CallMethodObjArgs:PyObject*::+1: -PyObject_CallMethodObjArgs:PyObject*:o:0: -PyObject_CallMethodObjArgs:PyObject*:name:0: -PyObject_CallMethodObjArgs::...:: - -PyObject_CallObject:PyObject*::+1: -PyObject_CallObject:PyObject*:callable_object:0: -PyObject_CallObject:PyObject*:args:0: - -PyObject_CheckBuffer:int::: -PyObject_CheckBuffer:PyObject*:obj:0: - -PyObject_CheckReadBuffer:int::: -PyObject_CheckReadBuffer:PyObject*:o:0: - -PyObject_DelAttr:int::: -PyObject_DelAttr:PyObject*:o:0: -PyObject_DelAttr:PyObject*:attr_name:0: - -PyObject_DelAttrString:int::: -PyObject_DelAttrString:PyObject*:o:0: -PyObject_DelAttrString:const char*:attr_name:: - -PyObject_DelItem:int::: -PyObject_DelItem:PyObject*:o:0: -PyObject_DelItem:PyObject*:key:0: - -PyObject_Dir:PyObject*::+1: -PyObject_Dir:PyObject*:o:0: - -PyObject_GC_Del:void::: -PyObject_GC_Del:void*:op:: - -PyObject_GC_New:TYPE*::+1: -PyObject_GC_New::TYPE:: -PyObject_GC_New:PyTypeObject*:type:0: - -PyObject_GC_NewVar:TYPE*::+1: -PyObject_GC_NewVar::TYPE:: -PyObject_GC_NewVar:PyTypeObject*:type:0: -PyObject_GC_NewVar:Py_ssize_t:size:: - -PyObject_GC_Resize:TYPE*::0: -PyObject_GC_Resize::TYPE:: -PyObject_GC_Resize:PyVarObject*:op:0: -PyObject_GC_Resize:Py_ssize_t:newsize:: - -PyObject_GC_Track:void::: -PyObject_GC_Track:PyObject*:op:0: - -PyObject_GC_UnTrack:void::: -PyObject_GC_UnTrack:void*:op:: - -PyObject_GenericGetAttr:PyObject*::+1: -PyObject_GenericGetAttr:PyObject*:o:0: -PyObject_GenericGetAttr:PyObject*:name:0: - -PyObject_GenericGetDict:PyObject*::+1: -PyObject_GenericGetDict:PyObject*:o:0: -PyObject_GenericGetDict:void*:context:: - -PyObject_GenericSetAttr:int::: -PyObject_GenericSetAttr:PyObject*:o:0: -PyObject_GenericSetAttr:PyObject*:name:0: -PyObject_GenericSetAttr:PyObject*:value:+1: - -PyObject_GenericSetDict:int::: -PyObject_GenericSetDict:PyObject*:o:0: -PyObject_GenericSetDict:PyObject*:value:+1: -PyObject_GenericSetDict:void*:context:: - -PyObject_GetAttr:PyObject*::+1: -PyObject_GetAttr:PyObject*:o:0: -PyObject_GetAttr:PyObject*:attr_name:0: - -PyObject_GetAttrString:PyObject*::+1: -PyObject_GetAttrString:PyObject*:o:0: -PyObject_GetAttrString:const char*:attr_name:: - -PyObject_GetBuffer:int::: -PyObject_GetBuffer:PyObject*:exporter:0: -PyObject_GetBuffer:Py_buffer*:view:: -PyObject_GetBuffer:int:flags:: - -PyObject_GetItem:PyObject*::+1: -PyObject_GetItem:PyObject*:o:0: -PyObject_GetItem:PyObject*:key:0: - -PyObject_GetIter:PyObject*::+1: -PyObject_GetIter:PyObject*:o:0: - -PyObject_GetAIter:PyObject*::+1: -PyObject_GetAIter:PyObject*:o:0: - -PyObject_HasAttr:int::: -PyObject_HasAttr:PyObject*:o:0: -PyObject_HasAttr:PyObject*:attr_name:0: - -PyObject_HasAttrString:int::: -PyObject_HasAttrString:PyObject*:o:0: -PyObject_HasAttrString:const char*:attr_name:: - -PyObject_Hash:int::: -PyObject_Hash:PyObject*:o:0: - -PyObject_HashNotImplemented:Py_hash_t::: -PyObject_HashNotImplemented:PyObject*:o:0: - -PyObject_IsInstance:int::: -PyObject_IsInstance:PyObject*:inst:0: -PyObject_IsInstance:PyObject*:cls:0: - -PyObject_IsSubclass:int::: -PyObject_IsSubclass:PyObject*:derived:0: -PyObject_IsSubclass:PyObject*:cls:0: - -PyObject_IsTrue:int::: -PyObject_IsTrue:PyObject*:o:0: - -PyObject_Init:PyObject*::0: -PyObject_Init:PyObject*:op:0: -PyObject_Init:PyTypeObject*:type:0: - -PyObject_InitVar:PyVarObject*::0: -PyObject_InitVar:PyVarObject*:op:0: - -PyObject_Length:Py_ssize_t::: -PyObject_Length:PyObject*:o:0: - -PyObject_LengthHint:Py_ssize_t::: -PyObject_LengthHint:PyObject*:o:0: -PyObject_LengthHint:Py_ssize_t:default:: - -PyObject_NEW:PyObject*::+1: -PyObject_NEW::TYPE:: -PyObject_NEW:PyTypeObject*:type:0: - -PyObject_New:PyObject*::+1: -PyObject_New::TYPE:: -PyObject_New:PyTypeObject*:type:0: - -PyObject_NEW_VAR:PyObject*::+1: -PyObject_NEW_VAR::TYPE:: -PyObject_NEW_VAR:PyTypeObject*:type:0: -PyObject_NEW_VAR:Py_ssize_t:size:: - -PyObject_NewVar:PyObject*::+1: -PyObject_NewVar::TYPE:: -PyObject_NewVar:PyTypeObject*:type:0: -PyObject_NewVar:Py_ssize_t:size:: - -PyObject_Not:int::: -PyObject_Not:PyObject*:o:0: - -PyObject_Print:int::: -PyObject_Print:PyObject*:o:0: -PyObject_Print:FILE*:fp:: -PyObject_Print:int:flags:: - -PyObject_Repr:PyObject*::+1: -PyObject_Repr:PyObject*:o:0: - -PyObject_RichCompare:PyObject*::+1: -PyObject_RichCompare:PyObject*:o1:0: -PyObject_RichCompare:PyObject*:o2:0: -PyObject_RichCompare:int:opid:: - -PyObject_RichCompareBool:int::: -PyObject_RichCompareBool:PyObject*:o1:0: -PyObject_RichCompareBool:PyObject*:o2:0: -PyObject_RichCompareBool:int:opid:: - -PyObject_SetAttr:int::: -PyObject_SetAttr:PyObject*:o:0: -PyObject_SetAttr:PyObject*:attr_name:0: -PyObject_SetAttr:PyObject*:v:+1: - -PyObject_SetAttrString:int::: -PyObject_SetAttrString:PyObject*:o:0: -PyObject_SetAttrString:const char*:attr_name:: -PyObject_SetAttrString:PyObject*:v:+1: - -PyObject_SetItem:int::: -PyObject_SetItem:PyObject*:o:0: -PyObject_SetItem:PyObject*:key:0: -PyObject_SetItem:PyObject*:v:+1: - -PyObject_Size:Py_ssize_t::: -PyObject_Size:PyObject*:o:0: - -PyObject_Str:PyObject*::+1: -PyObject_Str:PyObject*:o:0: - -PyObject_Type:PyObject*::+1: -PyObject_Type:PyObject*:o:0: - -PyObject_TypeCheck:int::: -PyObject_TypeCheck:PyObject*:o:0: -PyObject_TypeCheck:PyTypeObject*:type:0: - -PyRun_AnyFile:int::: -PyRun_AnyFile:FILE*:fp:: -PyRun_AnyFile:const char*:filename:: - -PyRun_AnyFileFlags:int::: -PyRun_AnyFileFlags:FILE*:fp:: -PyRun_AnyFileFlags:const char*:filename:: -PyRun_AnyFileFlags:PyCompilerFlags*:flags:: - -PyRun_AnyFileEx:int::: -PyRun_AnyFileEx:FILE*:fp:: -PyRun_AnyFileEx:const char*:filename:: -PyRun_AnyFileEx:int:closeit:: - -PyRun_AnyFileExFlags:int::: -PyRun_AnyFileExFlags:FILE*:fp:: -PyRun_AnyFileExFlags:const char*:filename:: -PyRun_AnyFileExFlags:int:closeit:: -PyRun_AnyFileExFlags:PyCompilerFlags*:flags:: - -PyRun_File:PyObject*::+1:??? -- same as eval_code2() -PyRun_File:FILE*:fp:: -PyRun_File:const char*:filename:: -PyRun_File:int:start:: -PyRun_File:PyObject*:globals:0: -PyRun_File:PyObject*:locals:0: - -PyRun_FileEx:PyObject*::+1:??? -- same as eval_code2() -PyRun_FileEx:FILE*:fp:: -PyRun_FileEx:const char*:filename:: -PyRun_FileEx:int:start:: -PyRun_FileEx:PyObject*:globals:0: -PyRun_FileEx:PyObject*:locals:0: -PyRun_FileEx:int:closeit:: - -PyRun_FileFlags:PyObject*::+1:??? -- same as eval_code2() -PyRun_FileFlags:FILE*:fp:: -PyRun_FileFlags:const char*:filename:: -PyRun_FileFlags:int:start:: -PyRun_FileFlags:PyObject*:globals:0: -PyRun_FileFlags:PyObject*:locals:0: -PyRun_FileFlags:PyCompilerFlags*:flags:: - -PyRun_FileExFlags:PyObject*::+1:??? -- same as eval_code2() -PyRun_FileExFlags:FILE*:fp:: -PyRun_FileExFlags:const char*:filename:: -PyRun_FileExFlags:int:start:: -PyRun_FileExFlags:PyObject*:globals:0: -PyRun_FileExFlags:PyObject*:locals:0: -PyRun_FileExFlags:int:closeit:: -PyRun_FileExFlags:PyCompilerFlags*:flags:: - -PyRun_InteractiveLoop:int::: -PyRun_InteractiveLoop:FILE*:fp:: -PyRun_InteractiveLoop:const char*:filename:: - -PyRun_InteractiveLoopFlags:int::: -PyRun_InteractiveLoopFlags:FILE*:fp:: -PyRun_InteractiveLoopFlags:const char*:filename:: -PyRun_InteractiveLoopFlags:PyCompilerFlags*:flags:: - -PyRun_InteractiveOne:int::: -PyRun_InteractiveOne:FILE*:fp:: -PyRun_InteractiveOne:const char*:filename:: - -PyRun_InteractiveOneFlags:int::: -PyRun_InteractiveOneFlags:FILE*:fp:: -PyRun_InteractiveOneFlags:const char*:filename:: -PyRun_InteractiveOneFlags:PyCompilerFlags*:flags:: - -PyRun_SimpleFile:int::: -PyRun_SimpleFile:FILE*:fp:: -PyRun_SimpleFile:const char*:filename:: - -PyRun_SimpleFileEx:int::: -PyRun_SimpleFileEx:FILE*:fp:: -PyRun_SimpleFileEx:const char*:filename:: -PyRun_SimpleFileEx:int:closeit:: - -PyRun_SimpleFileExFlags:int::: -PyRun_SimpleFileExFlags:FILE*:fp:: -PyRun_SimpleFileExFlags:const char*:filename:: -PyRun_SimpleFileExFlags:int:closeit:: -PyRun_SimpleFileExFlags:PyCompilerFlags*:flags:: - -PyRun_SimpleString:int::: -PyRun_SimpleString:const char*:command:: - -PyRun_SimpleStringFlags:int::: -PyRun_SimpleStringFlags:const char*:command:: -PyRun_SimpleStringFlags:PyCompilerFlags*:flags:: - -PyRun_String:PyObject*::+1:??? -- same as eval_code2() -PyRun_String:const char*:str:: -PyRun_String:int:start:: -PyRun_String:PyObject*:globals:0: -PyRun_String:PyObject*:locals:0: - -PyRun_StringFlags:PyObject*::+1:??? -- same as eval_code2() -PyRun_StringFlags:const char*:str:: -PyRun_StringFlags:int:start:: -PyRun_StringFlags:PyObject*:globals:0: -PyRun_StringFlags:PyObject*:locals:0: -PyRun_StringFlags:PyCompilerFlags*:flags:: - -PySeqIter_Check:int::: -PySeqIter_Check::op:: - -PySeqIter_New:PyObject*::+1: -PySeqIter_New:PyObject*:seq:: - -PySequence_Check:int::: -PySequence_Check:PyObject*:o:0: - -PySequence_Concat:PyObject*::+1: -PySequence_Concat:PyObject*:o1:0: -PySequence_Concat:PyObject*:o2:0: - -PySequence_Contains:int::: -PySequence_Contains:PyObject*:o:0: -PySequence_Contains:PyObject*:value:0: - -PySequence_Count:Py_ssize_t::: -PySequence_Count:PyObject*:o:0: -PySequence_Count:PyObject*:value:0: - -PySequence_DelItem:int::: -PySequence_DelItem:PyObject*:o:0: -PySequence_DelItem:Py_ssize_t:i:: - -PySequence_DelSlice:int::: -PySequence_DelSlice:PyObject*:o:0: -PySequence_DelSlice:Py_ssize_t:i1:: -PySequence_DelSlice:Py_ssize_t:i2:: - -PySequence_Fast:PyObject*::+1: -PySequence_Fast:PyObject*:v:0: -PySequence_Fast:const char*:m:: - -PySequence_Fast_GET_ITEM:PyObject*::0: -PySequence_Fast_GET_ITEM:PyObject*:o:0: -PySequence_Fast_GET_ITEM:Py_ssize_t:i:: - -PySequence_Fast_GET_SIZE:Py_ssize_t::: -PySequence_Fast_GET_SIZE:PyObject*:o:0: - -PySequence_Fast_ITEMS:PyObject**::: -PySequence_Fast_ITEMS:PyObject*:o:0: - -PySequence_GetItem:PyObject*::+1: -PySequence_GetItem:PyObject*:o:0: -PySequence_GetItem:Py_ssize_t:i:: - -PySequence_GetSlice:PyObject*::+1: -PySequence_GetSlice:PyObject*:o:0: -PySequence_GetSlice:Py_ssize_t:i1:: -PySequence_GetSlice:Py_ssize_t:i2:: - -PySequence_In:int::: -PySequence_In:PyObject*:o:0: -PySequence_In:PyObject*:value:0: - -PySequence_Index:Py_ssize_t::: -PySequence_Index:PyObject*:o:0: -PySequence_Index:PyObject*:value:0: - -PySequence_InPlaceConcat:PyObject*::+1: -PySequence_InPlaceConcat:PyObject*:s:0: -PySequence_InPlaceConcat:PyObject*:o:0: - -PySequence_InPlaceRepeat:PyObject*::+1: -PySequence_InPlaceRepeat:PyObject*:s:0: -PySequence_InPlaceRepeat:PyObject*:o:0: - -PySequence_ITEM:PyObject*::+1: -PySequence_ITEM:PyObject*:o:0: -PySequence_ITEM:Py_ssize_t:i:: - -PySequence_Repeat:PyObject*::+1: -PySequence_Repeat:PyObject*:o:0: -PySequence_Repeat:Py_ssize_t:count:: - -PySequence_SetItem:int::: -PySequence_SetItem:PyObject*:o:0: -PySequence_SetItem:Py_ssize_t:i:: -PySequence_SetItem:PyObject*:v:+1: - -PySequence_SetSlice:int::: -PySequence_SetSlice:PyObject*:o:0: -PySequence_SetSlice:Py_ssize_t:i1:: -PySequence_SetSlice:Py_ssize_t:i2:: -PySequence_SetSlice:PyObject*:v:0: - -PySequence_Size:Py_ssize_t::: -PySequence_Size:PyObject*:o:0: - -PySequence_List:PyObject*::+1: -PySequence_List:PyObject*:o:0: - -PySequence_Tuple:PyObject*::+1: -PySequence_Tuple:PyObject*:o:0: - -PySet_Add:int::: -PySet_Add:PyObject*:set:0: -PySet_Add:PyObject*:key:+1: - -PySet_Check:int::: -PySet_Check:PyObject*:p:0: - -PySet_Clear:int::: -PySet_Clear:PyObject*:set:0: - -PySet_Contains:int::: -PySet_Contains:PyObject*:anyset:0: -PySet_Contains:PyObject*:key:0: - -PySet_Discard:int::: -PySet_Discard:PyObject*:set:0: -PySet_Discard:PyObject*:key:-1:no effect if key not found - -PySet_GET_SIZE:Py_ssize_t::: -PySet_GET_SIZE:PyObject*:anyset:0: - -PySet_New:PyObject*::+1: -PySet_New:PyObject*:iterable:0: - -PySet_Pop:PyObject*::+1:or returns NULL and raises KeyError if set is empty -PySet_Pop:PyObject*:set:0: - -PySet_Size:Py_ssize_t::: -PySet_Size:PyObject*:anyset:0: - -PySignal_SetWakeupFd:int::: -PySignal_SetWakeupFd:int:fd:: - -PySlice_AdjustIndices:Py_ssize_t::: -PySlice_AdjustIndices:Py_ssize_t:length:: -PySlice_AdjustIndices:Py_ssize_t*:start:: -PySlice_AdjustIndices:Py_ssize_t*:stop:: -PySlice_AdjustIndices:Py_ssize_t*:step:: - -PySlice_Check:int::: -PySlice_Check:PyObject*:ob:0: - -PySlice_GetIndices:int::: -PySlice_GetIndices:PyObject*:slice:0: -PySlice_GetIndices:Py_ssize_t:length:: -PySlice_GetIndices:Py_ssize_t*:start:: -PySlice_GetIndices:Py_ssize_t*:stop:: -PySlice_GetIndices:Py_ssize_t*:step:: - -PySlice_GetIndicesEx:int::: -PySlice_GetIndicesEx:PyObject*:slice:0: -PySlice_GetIndicesEx:Py_ssize_t:length:: -PySlice_GetIndicesEx:Py_ssize_t*:start:: -PySlice_GetIndicesEx:Py_ssize_t*:stop:: -PySlice_GetIndicesEx:Py_ssize_t*:step:: -PySlice_GetIndicesEx:Py_ssize_t*:slicelength:: - -PySlice_New:PyObject*::+1: -PySlice_New:PyObject*:start:0: -PySlice_New:PyObject*:stop:0: -PySlice_New:PyObject*:step:0: - -PySlice_Unpack:int::: -PySlice_Unpack:PyObject*:slice:0: -PySlice_Unpack:Py_ssize_t*:start:: -PySlice_Unpack:Py_ssize_t*:stop:: -PySlice_Unpack:Py_ssize_t*:step:: - -PyState_AddModule:int::: -PyState_AddModule:PyObject*:module:+1: -PyState_AddModule:PyModuleDef*:def:: - -PyState_FindModule:PyObject*::0: -PyState_FindModule:PyModuleDef*:def:: - -PyState_RemoveModule:int::: -PyState_RemoveModule:PyModuleDef*:def:: - -PyStructSequence_GET_ITEM:PyObject*::0: -PyStructSequence_GET_ITEM:PyObject*:p:0: -PyStructSequence_GET_ITEM:Py_ssize_t:pos:: - -PyStructSequence_GetItem:PyObject*::0: -PyStructSequence_GetItem:PyObject*:p:0: -PyStructSequence_GetItem:Py_ssize_t:pos:: - -PyStructSequence_InitType:void::: -PyStructSequence_InitType:PyTypeObject*:type:+1: -PyStructSequence_InitType:PyStructSequence_Desc*:desc:: - -PyStructSequence_InitType2:int::: -PyStructSequence_InitType2:PyTypeObject*:type:+1: -PyStructSequence_InitType2:PyStructSequence_Desc*:desc:: - -PyStructSequence_New:PyObject*::+1: -PyStructSequence_New:PyTypeObject*:type:0: - -PyStructSequence_NewType:PyTypeObject*::+1: -PyStructSequence_NewType:PyStructSequence_Desc*:desc:: - -PyStructSequence_SET_ITEM:void::: -PyStructSequence_SET_ITEM:PyObject*:p:0: -PyStructSequence_SET_ITEM:Py_ssize_t*:pos:: -PyStructSequence_SET_ITEM:PyObject*:o:0: - -PyStructSequence_SetItem:void::: -PyStructSequence_SetItem:PyObject*:p:0: -PyStructSequence_SetItem:Py_ssize_t:pos:: -PyStructSequence_SetItem:PyObject*:o:0: - -PySys_AddWarnOption:void::: -PySys_AddWarnOption:const wchar_t*:s:: - -PySys_AddWarnOptionUnicode:void::: -PySys_AddWarnOptionUnicode:PyObject*:unicode:0: - -PySys_AddXOption:void::: -PySys_AddXOption:const wchar_t*:s:: - -PySys_FormatStderr:void::: -PySys_FormatStderr:const char*:format:: -PySys_FormatStderr::...:: - -PySys_FormatStdout:void::: -PySys_FormatStdout:const char*:format:: -PySys_FormatStdout::...:: - -PySys_GetObject:PyObject*::0: -PySys_GetObject:const char*:name:: - -PySys_GetXOptions:PyObject*::0: - -PySys_SetArgv:void::: -PySys_SetArgv:int:argc:: -PySys_SetArgv:wchar_t**:argv:: - -PySys_SetArgvEx:void::: -PySys_SetArgvEx:int:argc:: -PySys_SetArgvEx:wchar_t**:argv:: -PySys_SetArgvEx:int:updatepath:: - -PySys_SetObject:int::: -PySys_SetObject:const char*:name:: -PySys_SetObject:PyObject*:v:+1: - -PySys_ResetWarnOptions:void::: - -PySys_WriteStdout:void::: -PySys_WriteStdout:const char*:format:: -PySys_WriteStdout::...:: - -PySys_WriteStderr:void::: -PySys_WriteStderr:const char*:format:: -PySys_WriteStderr::...:: - -PyThreadState_Clear:void::: -PyThreadState_Clear:PyThreadState*:tstate:: - -PyThreadState_Delete:void::: -PyThreadState_Delete:PyThreadState*:tstate:: - -PyThreadState_Get:PyThreadState*::: - -PyThreadState_GetDict:PyObject*::0: - -PyThreadState_New:PyThreadState*::: -PyThreadState_New:PyInterpreterState*:interp:: - -PyThreadState_SetAsyncExc:int::: -PyThreadState_SetAsyncExc:unsigned long:id:: -PyThreadState_SetAsyncExc:PyObject*:exc:+1: - -PyThreadState_Swap:PyThreadState*::: -PyThreadState_Swap:PyThreadState*:tstate:: - -PyThread_tss_alloc:Py_tss_t*::: - -PyThread_tss_create:int::: -PyThread_tss_create:Py_tss_t*:key:: - -PyThread_tss_delete:void::: -PyThread_tss_delete:Py_tss_t*:key:: - -PyThread_tss_free:void::: -PyThread_tss_free:Py_tss_t*:key:: - -PyThread_tss_get:void*::: -PyThread_tss_get:Py_tss_t*:key:: - -PyThread_tss_is_created:int::: -PyThread_tss_is_created:Py_tss_t*:key:: - -PyThread_tss_set:int::: -PyThread_tss_set:Py_tss_t*:key:: -PyThread_tss_set:void*:value:: - -PyTime_Check:int::: -PyTime_Check:PyObject*:ob:0: - -PyTime_CheckExact:int::: -PyTime_CheckExact:PyObject*:ob:0: - -PyTime_FromTime:PyObject*::+1: -PyTime_FromTime:int:hour:: -PyTime_FromTime:int:minute:: -PyTime_FromTime:int:second:: -PyTime_FromTime:int:usecond:: - -PyTime_FromTimeAndFold:PyObject*::+1: -PyTime_FromTimeAndFold:int:hour:: -PyTime_FromTimeAndFold:int:minute:: -PyTime_FromTimeAndFold:int:second:: -PyTime_FromTimeAndFold:int:usecond:: -PyTime_FromTimeAndFold:int:fold:: - -PyTraceMalloc_Track:int::: -PyTraceMalloc_Track:unsigned int:domain:: -PyTraceMalloc_Track:uintptr_t:ptr:: -PyTraceMalloc_Track:size_t:size:: - -PyTraceMalloc_Untrack:int::: -PyTraceMalloc_Untrack:unsigned int:domain:: -PyTraceMalloc_Untrack:uintptr_t:ptr:: - -PyTuple_Check:int::: -PyTuple_Check:PyObject*:p:0: - -PyTuple_CheckExact:int::: -PyTuple_CheckExact:PyObject*:p:0: - -PyTuple_GET_ITEM:PyObject*::0: -PyTuple_GET_ITEM:PyObject*:p:0: -PyTuple_GET_ITEM:Py_ssize_t:pos:: - -PyTuple_GetItem:PyObject*::0: -PyTuple_GetItem:PyObject*:p:0: -PyTuple_GetItem:Py_ssize_t:pos:: - -PyTuple_GET_SIZE:Py_ssize_t::: -PyTuple_GET_SIZE:PyObject*:p:0: - -PyTuple_GetSlice:PyObject*::+1: -PyTuple_GetSlice:PyObject*:p:0: -PyTuple_GetSlice:Py_ssize_t:low:: -PyTuple_GetSlice:Py_ssize_t:high:: - -PyTuple_New:PyObject*::+1: -PyTuple_New:Py_ssize_t:len:: - -PyTuple_Pack:PyObject*::+1: -PyTuple_Pack:Py_ssize_t:len:: -PyTuple_Pack:PyObject*:...:0: - -PyTuple_SET_ITEM:void::: -PyTuple_SET_ITEM:PyObject*:p:0: -PyTuple_SET_ITEM:Py_ssize_t:pos:: -PyTuple_SET_ITEM:PyObject*:o:0: - -PyTuple_SetItem:int::: -PyTuple_SetItem:PyObject*:p:0: -PyTuple_SetItem:Py_ssize_t:pos:: -PyTuple_SetItem:PyObject*:o:0: - -PyTuple_Size:Py_ssize_t::: -PyTuple_Size:PyObject*:p:0: - -PyType_Check:int::: -PyType_Check:PyObject*:o:0: - -PyType_CheckExact:int::: -PyType_CheckExact:PyObject*:o:0: - -PyType_FromSpec:PyObject*::+1: -PyType_FromSpec:PyType_Spec*:spec:: - -PyType_FromModuleAndSpec:PyObject*::+1: -PyType_FromModuleAndSpec:PyObject*:module:+1: -PyType_FromModuleAndSpec:PyType_Spec*:spec:: -PyType_FromModuleAndSpec:PyObject*:bases:0: - -PyType_FromSpecWithBases:PyObject*::+1: -PyType_FromSpecWithBases:PyType_Spec*:spec:: -PyType_FromSpecWithBases:PyObject*:bases:0: - -PyType_GenericAlloc:PyObject*::+1: -PyType_GenericAlloc:PyObject*:type:0: -PyType_GenericAlloc:Py_ssize_t:nitems:: - -PyType_GenericNew:PyObject*::+1: -PyType_GenericNew:PyObject*:type:0: -PyType_GenericNew:PyObject*:args:0: -PyType_GenericNew:PyObject*:kwds:0: - -PyType_GetFlags:unsigned long::: -PyType_GetFlags:PyTypeObject*:type:0: - -PyType_GetName:PyObject*::+1: -PyType_GetName:PyTypeObject*:type:0: - -PyType_GetQualName:PyObject*::+1: -PyType_GetQualName:PyTypeObject*:type:0: - -PyType_GetSlot:void*::: -PyType_GetSlot:PyTypeObject*:type:0: -PyType_GetSlot:int:slot:: - -PyType_HasFeature:int::: -PyType_HasFeature:PyTypeObject*:o:0: -PyType_HasFeature:int:feature:: - -PyType_IS_GC:int::: -PyType_IS_GC:PyTypeObject*:o:0: - -PyType_IsSubtype:int::: -PyType_IsSubtype:PyTypeObject*:a:0: -PyType_IsSubtype:PyTypeObject*:b:0: - -PyType_Modified:void::: -PyType_Modified:PyTypeObject*:type:0: - -PyType_Ready:int::: -PyType_Ready:PyTypeObject*:type:0: - -PyUnicode_1BYTE_DATA:Py_UCS1*::: -PyUnicode_1BYTE_DATA:PyObject*:o:0: - -PyUnicode_Check:int::: -PyUnicode_Check:PyObject*:o:0: - -PyUnicode_CheckExact:int::: -PyUnicode_CheckExact:PyObject*:o:0: - -PyUnicode_DATA:void*::: -PyUnicode_DATA:PyObject*:o:0: - -PyUnicode_GET_LENGTH:Py_ssize_t::: -PyUnicode_GET_LENGTH:PyObject*:o:0: - -PyUnicode_GET_SIZE:Py_ssize_t::: -PyUnicode_GET_SIZE:PyObject*:o:0: - -PyUnicode_GET_DATA_SIZE:Py_ssize_t::: -PyUnicode_GET_DATA_SIZE:PyObject*:o:0: - -PyUnicode_KIND:int::: -PyUnicode_KIND:PyObject*:o:0: - -PyUnicode_MAX_CHAR_VALUE:::: -PyUnicode_MAX_CHAR_VALUE:PyObject*:o:0: - -PyUnicode_AS_UNICODE:Py_UNICODE*::: -PyUnicode_AS_UNICODE:PyObject*:o:0: - -PyUnicode_AS_DATA:const char*::: -PyUnicode_AS_DATA:PyObject*:o:0: - -Py_UNICODE_ISALNUM:int::: -Py_UNICODE_ISALNUM:Py_UNICODE:ch:: - -Py_UNICODE_ISALPHA:int::: -Py_UNICODE_ISALPHA:Py_UNICODE:ch:: - -Py_UNICODE_ISSPACE:int::: -Py_UNICODE_ISSPACE:Py_UNICODE:ch:: - -Py_UNICODE_ISLOWER:int::: -Py_UNICODE_ISLOWER:Py_UNICODE:ch:: - -Py_UNICODE_ISUPPER:int::: -Py_UNICODE_ISUPPER:Py_UNICODE:ch:: - -Py_UNICODE_ISTITLE:int::: -Py_UNICODE_ISTITLE:Py_UNICODE:ch:: - -Py_UNICODE_ISLINEBREAK:int::: -Py_UNICODE_ISLINEBREAK:Py_UNICODE:ch:: - -Py_UNICODE_ISDECIMAL:int::: -Py_UNICODE_ISDECIMAL:Py_UNICODE:ch:: - -Py_UNICODE_ISDIGIT:int::: -Py_UNICODE_ISDIGIT:Py_UNICODE:ch:: - -Py_UNICODE_ISNUMERIC:int::: -Py_UNICODE_ISNUMERIC:Py_UNICODE:ch:: - -Py_UNICODE_ISPRINTABLE:int::: -Py_UNICODE_ISPRINTABLE:Py_UNICODE:ch:: - -Py_UNICODE_TOLOWER:Py_UNICODE::: -Py_UNICODE_TOLOWER:Py_UNICODE:ch:: - -Py_UNICODE_TOUPPER:Py_UNICODE::: -Py_UNICODE_TOUPPER:Py_UNICODE:ch:: - -Py_UNICODE_TOTITLE:Py_UNICODE::: -Py_UNICODE_TOTITLE:Py_UNICODE:ch:: - -Py_UNICODE_TODECIMAL:int::: -Py_UNICODE_TODECIMAL:Py_UNICODE:ch:: - -Py_UNICODE_TODIGIT:int::: -Py_UNICODE_TODIGIT:Py_UNICODE:ch:: - -Py_UNICODE_TONUMERIC:double::: -Py_UNICODE_TONUMERIC:Py_UNICODE:ch:: - -PyUnicode_FromUnicode:PyObject*::+1: -PyUnicode_FromUnicode:const Py_UNICODE*:u:: -PyUnicode_FromUnicode:Py_ssize_t:size:: - -PyUnicode_AsUnicode:Py_UNICODE*::: -PyUnicode_AsUnicode:PyObject*:unicode:0: - -PyUnicode_AsUnicodeAndSize:Py_UNICODE*::: -PyUnicode_AsUnicodeAndSize:PyObject*:unicode:0: -PyUnicode_AsUnicodeAndSize:Py_ssize_t*:size:: - -PyUnicode_GetSize:Py_ssize_t::: -PyUnicode_GetSize:PyObject*:unicode:0: - -PyUnicode_FromObject:PyObject*::+1: -PyUnicode_FromObject:PyObject*:obj:0: - -PyUnicode_FromEncodedObject:PyObject*::+1: -PyUnicode_FromEncodedObject:PyObject*:obj:0: -PyUnicode_FromEncodedObject:const char*:encoding:: -PyUnicode_FromEncodedObject:const char*:errors:: - -PyUnicode_FromWideChar:PyObject*::+1: -PyUnicode_FromWideChar:const wchar_t*:w:: -PyUnicode_FromWideChar:Py_ssize_t:size:: - -PyUnicode_AsWideChar:Py_ssize_t::: -PyUnicode_AsWideChar:PyObject*:*unicode:0: -PyUnicode_AsWideChar:wchar_t*:w:: -PyUnicode_AsWideChar:Py_ssize_t:size:: - -PyUnicode_AsWideCharString:wchar_t*::: -PyUnicode_AsWideCharString:PyObject*:unicode:0: -PyUnicode_AsWideCharString:Py_ssize_t*:size:: - -PyUnicode_Decode:PyObject*::+1: -PyUnicode_Decode:const char*:s:: -PyUnicode_Decode:Py_ssize_t:size:: -PyUnicode_Decode:const char*:encoding:: -PyUnicode_Decode:const char*:errors:: - -PyUnicode_DecodeUTF16Stateful:PyObject*::+1: -PyUnicode_DecodeUTF16Stateful:const char*:s:: -PyUnicode_DecodeUTF16Stateful:Py_ssize_t:size:: -PyUnicode_DecodeUTF16Stateful:const char*:errors:: -PyUnicode_DecodeUTF16Stateful:int*:byteorder:: -PyUnicode_DecodeUTF16Stateful:Py_ssize_t*:consumed:: - -PyUnicode_DecodeUTF8Stateful:PyObject*::+1: -PyUnicode_DecodeUTF8Stateful:const char*:s:: -PyUnicode_DecodeUTF8Stateful:Py_ssize_t:size:: -PyUnicode_DecodeUTF8Stateful:const char*:errors:: -PyUnicode_DecodeUTF8Stateful:Py_ssize_t*:consumed:: - -PyUnicode_AsEncodedString:PyObject*::+1: -PyUnicode_AsEncodedString:PyObject*:unicode:0: -PyUnicode_AsEncodedString:const char*:encoding:: -PyUnicode_AsEncodedString:const char*:errors:: - -PyUnicode_DecodeUTF7:PyObject*::+1: -PyUnicode_DecodeUTF7:const char*:s:: -PyUnicode_DecodeUTF7:Py_ssize_t:size:: -PyUnicode_DecodeUTF7:const char*:errors:: - -PyUnicode_DecodeUTF7Stateful:PyObject*::+1: -PyUnicode_DecodeUTF7Stateful:const char*:s:: -PyUnicode_DecodeUTF7Stateful:Py_ssize_t:size:: -PyUnicode_DecodeUTF7Stateful:const char*:errors:: -PyUnicode_DecodeUTF7Stateful:Py_ssize_t*:consumed:: - -PyUnicode_DecodeUTF8:PyObject*::+1: -PyUnicode_DecodeUTF8:const char*:s:: -PyUnicode_DecodeUTF8:Py_ssize_t:size:: -PyUnicode_DecodeUTF8:const char*:errors:: - -PyUnicode_AsUTF8String:PyObject*::+1: -PyUnicode_AsUTF8String:PyObject*:unicode:0: - -PyUnicode_AsUTF8AndSize:const char*::: -PyUnicode_AsUTF8AndSize:PyObject*:unicode:0: -PyUnicode_AsUTF8AndSize:Py_ssize_t*:size:0: - -PyUnicode_AsUTF8:const char*::: -PyUnicode_AsUTF8:PyObject*:unicode:0: - -PyUnicode_DecodeUTF16:PyObject*::+1: -PyUnicode_DecodeUTF16:const char*:s:: -PyUnicode_DecodeUTF16:Py_ssize_t:size:: -PyUnicode_DecodeUTF16:const char*:errors:: -PyUnicode_DecodeUTF16:int*:byteorder:: - -PyUnicode_AsUTF16String:PyObject*::+1: -PyUnicode_AsUTF16String:PyObject*:unicode:0: - -PyUnicode_DecodeUTF32:PyObject*::+1: -PyUnicode_DecodeUTF32:const char*:s:: -PyUnicode_DecodeUTF32:Py_ssize_t:size:: -PyUnicode_DecodeUTF32:const char*:errors:: -PyUnicode_DecodeUTF32:int*:byteorder:: - -PyUnicode_DecodeUTF32Stateful:PyObject*::+1: -PyUnicode_DecodeUTF32Stateful:const char*:s:: -PyUnicode_DecodeUTF32Stateful:Py_ssize_t:size:: -PyUnicode_DecodeUTF32Stateful:const char*:errors:: -PyUnicode_DecodeUTF32Stateful:int*:byteorder:: -PyUnicode_DecodeUTF32Stateful:Py_ssize_t*:consumed:: - -PyUnicode_AsUTF32String:PyObject*::+1: -PyUnicode_AsUTF32String:PyObject*:unicode:0: - -PyUnicode_DecodeUnicodeEscape:PyObject*::+1: -PyUnicode_DecodeUnicodeEscape:const char*:s:: -PyUnicode_DecodeUnicodeEscape:Py_ssize_t:size:: -PyUnicode_DecodeUnicodeEscape:const char*:errors:: - -PyUnicode_AsUnicodeEscapeString:PyObject*::+1: -PyUnicode_AsUnicodeEscapeString:PyObject*:unicode:0: - -PyUnicode_DecodeRawUnicodeEscape:PyObject*::+1: -PyUnicode_DecodeRawUnicodeEscape:const char*:s:: -PyUnicode_DecodeRawUnicodeEscape:Py_ssize_t:size:: -PyUnicode_DecodeRawUnicodeEscape:const char*:errors:: - -PyUnicode_AsRawUnicodeEscapeString:PyObject*::+1: -PyUnicode_AsRawUnicodeEscapeString:PyObject*:unicode:0: - -PyUnicode_DecodeLatin1:PyObject*::+1: -PyUnicode_DecodeLatin1:const char*:s:: -PyUnicode_DecodeLatin1:Py_ssize_t:size:: -PyUnicode_DecodeLatin1:const char*:errors:: - -PyUnicode_AsLatin1String:PyObject*::+1: -PyUnicode_AsLatin1String:PyObject*:unicode:0: - -PyUnicode_DecodeASCII:PyObject*::+1: -PyUnicode_DecodeASCII:const char*:s:: -PyUnicode_DecodeASCII:Py_ssize_t:size:: -PyUnicode_DecodeASCII:const char*:errors:: - -PyUnicode_AsASCIIString:PyObject*::+1: -PyUnicode_AsASCIIString:PyObject*:unicode:0: - -PyUnicode_DecodeCharmap:PyObject*::+1: -PyUnicode_DecodeCharmap:const char*:s:: -PyUnicode_DecodeCharmap:Py_ssize_t:size:: -PyUnicode_DecodeCharmap:PyObject*:mapping:0: -PyUnicode_DecodeCharmap:const char*:errors:: - -PyUnicode_AsCharmapString:PyObject*::+1: -PyUnicode_AsCharmapString:PyObject*:unicode:0: -PyUnicode_AsCharmapString:PyObject*:mapping:0: - -PyUnicode_DecodeMBCS:PyObject*::+1: -PyUnicode_DecodeMBCS:const char*:s:: -PyUnicode_DecodeMBCS:Py_ssize_t:size:: -PyUnicode_DecodeMBCS:const char*:errors:: - -PyUnicode_DecodeMBCSStateful:PyObject*::+1: -PyUnicode_DecodeMBCSStateful:const char*:s:: -PyUnicode_DecodeMBCSStateful:Py_ssize_t:size:: -PyUnicode_DecodeMBCSStateful:const char*:errors:: -PyUnicode_DecodeMBCSStateful:Py_ssize_t*:consumed:: - -PyUnicode_EncodeCodePage:PyObject*::+1: -PyUnicode_EncodeCodePage:int:code_page:: -PyUnicode_EncodeCodePage:PyObject*:unicode:0: -PyUnicode_EncodeCodePage:const char*:errors:: - -PyUnicode_AsMBCSString:PyObject*::+1: -PyUnicode_AsMBCSString:PyObject*:unicode:0: - -PyUnicode_Concat:PyObject*::+1: -PyUnicode_Concat:PyObject*:left:0: -PyUnicode_Concat:PyObject*:right:0: - -PyUnicode_Split:PyObject*::+1: -PyUnicode_Split:PyObject*:left:0: -PyUnicode_Split:PyObject*:right:0: -PyUnicode_Split:Py_ssize_t:maxsplit:: - -PyUnicode_Splitlines:PyObject*::+1: -PyUnicode_Splitlines:PyObject*:s:0: -PyUnicode_Splitlines:int:keepend:: - -PyUnicode_Translate:PyObject*::+1: -PyUnicode_Translate:PyObject*:str:0: -PyUnicode_Translate:PyObject*:table:0: -PyUnicode_Translate:const char*:errors:: - -PyUnicode_Join:PyObject*::+1: -PyUnicode_Join:PyObject*:separator:0: -PyUnicode_Join:PyObject*:seq:0: - -PyUnicode_Tailmatch:Py_ssize_t::: -PyUnicode_Tailmatch:PyObject*:str:0: -PyUnicode_Tailmatch:PyObject*:substr:0: -PyUnicode_Tailmatch:Py_ssize_t:start:: -PyUnicode_Tailmatch:Py_ssize_t:end:: -PyUnicode_Tailmatch:int:direction:: - -PyUnicode_Find:Py_ssize_t::: -PyUnicode_Find:PyObject*:str:0: -PyUnicode_Find:PyObject*:substr:0: -PyUnicode_Find:Py_ssize_t:start:: -PyUnicode_Find:Py_ssize_t:end:: -PyUnicode_Find:int:direction:: - -PyUnicode_FindChar:Py_ssize_t::: -PyUnicode_FindChar:PyObject*:str:0: -PyUnicode_FindChar:Py_UCS4:ch:: -PyUnicode_FindChar:Py_ssize_t:start:: -PyUnicode_FindChar:Py_ssize_t:end:: -PyUnicode_FindChar:int:direction:: - -PyUnicode_Count:Py_ssize_t::: -PyUnicode_Count:PyObject*:str:0: -PyUnicode_Count:PyObject*:substr:0: -PyUnicode_Count:Py_ssize_t:start:: -PyUnicode_Count:Py_ssize_t:end:: - -PyUnicode_Replace:PyObject*::+1: -PyUnicode_Replace:PyObject*:str:0: -PyUnicode_Replace:PyObject*:substr:0: -PyUnicode_Replace:PyObject*:replstr:0: -PyUnicode_Replace:Py_ssize_t:maxcount:: - -PyUnicode_Compare:int::: -PyUnicode_Compare:PyObject*:left:0: -PyUnicode_Compare:PyObject*:right:0: - -PyUnicode_CompareWithASCIIString:int::: -PyUnicode_CompareWithASCIIString:PyObject*:uni:0: -PyUnicode_CompareWithASCIIString:const char*:string:: - -PyUnicode_RichCompare:PyObject*::+1: -PyUnicode_RichCompare:PyObject*:left:0: -PyUnicode_RichCompare:PyObject*:right:0: -PyUnicode_RichCompare:int:op:: - -PyUnicode_Format:PyObject*::+1: -PyUnicode_Format:PyObject*:format:0: -PyUnicode_Format:PyObject*:args:0: - -PyUnicode_Contains:int::: -PyUnicode_Contains:PyObject*:container:0: -PyUnicode_Contains:PyObject*:element:0: - -PyUnicode_InternInPlace:void::: -PyUnicode_InternInPlace:PyObject**:string:+1: - -PyUnicode_InternFromString:PyObject*::+1: -PyUnicode_InternFromString:const char*:v:: - -PyUnicode_New:PyObject*::+1: -PyUnicode_New:Py_ssize_t:size:: -PyUnicode_New:Py_UCS4:maxchar:: - -PyUnicode_FromKindAndData:PyObject*::+1: -PyUnicode_FromKindAndData:int:kind:: -PyUnicode_FromKindAndData:const void*:buffer:: -PyUnicode_FromKindAndData:Py_ssize_t:size:: - -PyUnicode_FromStringAndSize:PyObject*::+1: -PyUnicode_FromStringAndSize:const char*:u:: -PyUnicode_FromStringAndSize:Py_ssize_t:size:: - -PyUnicode_FromString:PyObject*::+1: -PyUnicode_FromString:const char*:u:: - -PyUnicode_FromFormat:PyObject*::+1: -PyUnicode_FromFormat:const char*:format:: -PyUnicode_FromFormat::...:: - -PyUnicode_FromFormatV:PyObject*::+1: -PyUnicode_FromFormatV:const char*:format:: -PyUnicode_FromFormatV:va_list:args:: - -PyUnicode_GetLength:Py_ssize_t::: -PyUnicode_GetLength:PyObject*:unicode:0: - -PyUnicode_CopyCharacters:Py_ssize_t::: -PyUnicode_CopyCharacters:PyObject*:to:0: -PyUnicode_CopyCharacters:Py_ssize_t:to_start:: -PyUnicode_CopyCharacters:PyObject*:from:0: -PyUnicode_CopyCharacters:Py_ssize_t:from_start:: -PyUnicode_CopyCharacters:Py_ssize_t:how_many:: - -PyUnicode_Fill:Py_ssize_t::: -PyUnicode_Fill:PyObject*:unicode:0: -PyUnicode_Fill:Py_ssize_t:start:: -PyUnicode_Fill:Py_ssize_t:length:: -PyUnicode_Fill:Py_UCS4:fill_char:: - -PyUnicode_READ:Py_UCS4::: -PyUnicode_READ:int:kind:: -PyUnicode_READ:void*:data:: -PyUnicode_READ:Py_ssize_t:index:: - -PyUnicode_READ_CHAR:Py_UCS4::: -PyUnicode_READ_CHAR:PyObject*:o:0: -PyUnicode_READ_CHAR:Py_ssize_t:index:: - -PyUnicode_ReadChar:Py_UCS4::: -PyUnicode_ReadChar:PyObject*:unicode:0: -PyUnicode_ReadChar:Py_ssize_t:index:: - -PyUnicode_WRITE:void::: -PyUnicode_WRITE:int:kind:: -PyUnicode_WRITE:void*:data:: -PyUnicode_WRITE:Py_ssize_t:index:: -PyUnicode_WRITE:Py_UCS4:value:: - -PyUnicode_WriteChar:int::: -PyUnicode_WriteChar:PyObject*:unicode:0: -PyUnicode_WriteChar:Py_ssize_t:index:: -PyUnicode_WriteChar:Py_UCS4:character:: - -PyUnicode_READY:int::: -PyUnicode_READY:PyObject*:o:0: - -PyUnicode_Substring:PyObject*::+1: -PyUnicode_Substring:PyObject*:str:0: -PyUnicode_Substring:Py_ssize_t:start:: -PyUnicode_Substring:Py_ssize_t:end:: - -PyUnicode_AsUCS4:Py_UCS4*::: -PyUnicode_AsUCS4:PyObject*:u:0: -PyUnicode_AsUCS4:Py_UCS4*:buffer:: -PyUnicode_AsUCS4:Py_ssize_t:buflen:: -PyUnicode_AsUCS4:int:copy_null:: - -PyUnicode_AsUCS4Copy:Py_UCS4*::: -PyUnicode_AsUCS4Copy:PyObject*:u:0: - -PyUnicode_DecodeLocaleAndSize:PyObject*::+1: -PyUnicode_DecodeLocaleAndSize:const char*:str:: -PyUnicode_DecodeLocaleAndSize:Py_ssize_t:len:: -PyUnicode_DecodeLocaleAndSize:const char*:errors:: - -PyUnicode_DecodeLocale:PyObject*::+1: -PyUnicode_DecodeLocale:const char*:str:: -PyUnicode_DecodeLocale:const char*:errors:: - -PyUnicode_EncodeLocale:PyObject*::+1: -PyUnicode_EncodeLocale:PyObject*:unicode:0: -PyUnicode_EncodeLocale:const char*:errors:: - -PyUnicode_FSConverter:int::: -PyUnicode_FSConverter:PyObject*:obj:0: -PyUnicode_FSConverter:void*:result:: - -PyUnicode_FSDecoder:int::: -PyUnicode_FSDecoder:PyObject*:obj:0: -PyUnicode_FSDecoder:void*:result:: - -PyUnicode_DecodeFSDefaultAndSize:PyObject*::+1: -PyUnicode_DecodeFSDefaultAndSize:const char*:s:: -PyUnicode_DecodeFSDefaultAndSize:Py_ssize_t:size:: - -PyUnicode_DecodeFSDefault:PyObject*::+1: -PyUnicode_DecodeFSDefault:const char*:s:: - -PyUnicode_EncodeFSDefault:PyObject*::+1: -PyUnicode_EncodeFSDefault:PyObject*:unicode:0: - -PyUnicodeDecodeError_Create:PyObject*::+1: -PyUnicodeDecodeError_Create:const char*:encoding:: -PyUnicodeDecodeError_Create:const char*:object:: -PyUnicodeDecodeError_Create:Py_ssize_t:length:: -PyUnicodeDecodeError_Create:Py_ssize_t:start:: -PyUnicodeDecodeError_Create:Py_ssize_t:end:: -PyUnicodeDecodeError_Create:const char*:reason:: - -PyUnicodeDecodeError_GetEncoding:PyObject*::+1: -PyUnicodeDecodeError_GetEncoding:PyObject*:exc:0: - -PyUnicodeDecodeError_GetEnd:Py_ssize_t::: -PyUnicodeDecodeError_GetEnd:PyObject*:exc:0: -PyUnicodeDecodeError_GetEnd:Py_ssize_t*:end:: - -PyUnicodeDecodeError_GetObject:PyObject*::+1: -PyUnicodeDecodeError_GetObject:PyObject*:exc:0: - -PyUnicodeDecodeError_GetReason:PyObject*::+1: -PyUnicodeDecodeError_GetReason:PyObject*:exc:0: - -PyUnicodeDecodeError_GetStart:Py_ssize_t::: -PyUnicodeDecodeError_GetStart:PyObject*:exc:0: -PyUnicodeDecodeError_GetStart:Py_ssize_t*:start:: - -PyUnicodeDecodeError_SetEnd:int::: -PyUnicodeDecodeError_SetEnd:PyObject*:exc:0: -PyUnicodeDecodeError_SetEnd:Py_ssize_t:end:: - -PyUnicodeDecodeError_SetReason:int::: -PyUnicodeDecodeError_SetReason:PyObject*:exc:0: -PyUnicodeDecodeError_SetReason:const char*:reason:: - -PyUnicodeDecodeError_SetStart:int::: -PyUnicodeDecodeError_SetStart:PyObject*:exc:0: -PyUnicodeDecodeError_SetStart:Py_ssize_t:start:: - -PyWeakref_Check:int::: -PyWeakref_Check:PyObject*:ob:: - -PyWeakref_CheckProxy:int::: -PyWeakref_CheckProxy:PyObject*:ob:: - -PyWeakref_CheckRef:int::: -PyWeakref_CheckRef:PyObject*:ob:: - -PyWeakref_GET_OBJECT:PyObject*::0: -PyWeakref_GET_OBJECT:PyObject*:ref:0: - -PyWeakref_GetObject:PyObject*::0: -PyWeakref_GetObject:PyObject*:ref:0: - -PyWeakref_NewProxy:PyObject*::+1: -PyWeakref_NewProxy:PyObject*:ob:0: -PyWeakref_NewProxy:PyObject*:callback:0: - -PyWeakref_NewRef:PyObject*::+1: -PyWeakref_NewRef:PyObject*:ob:0: -PyWeakref_NewRef:PyObject*:callback:0: - -PyWrapper_New:PyObject*::+1: -PyWrapper_New:PyObject*:d:0: -PyWrapper_New:PyObject*:self:0: - -Py_AtExit:int::: -Py_AtExit:void (*)():func:: - -Py_BuildValue:PyObject*::+1: -Py_BuildValue:const char*:format:: -Py_BuildValue::...:: - -Py_VaBuildValue:PyObject*::+1: -Py_VaBuildValue:const char*:format:: -Py_VaBuildValue:va_list:vargs:: - -Py_CLEAR:void::: -Py_CLEAR:PyObject*:o:-1: - -Py_CompileString:PyObject*::+1: -Py_CompileString:const char*:str:: -Py_CompileString:const char*:filename:: -Py_CompileString:int:start:: - -Py_CompileStringExFlags:PyObject*::+1: -Py_CompileStringExFlags:const char*:str:: -Py_CompileStringExFlags:const char*:filename:: -Py_CompileStringExFlags:int:start:: -Py_CompileStringExFlags:PyCompilerFlags*:flags:: -Py_CompileStringExFlags:int:optimize:: - -Py_CompileStringFlags:PyObject*::+1: -Py_CompileStringFlags:const char*:str:: -Py_CompileStringFlags:const char*:filename:: -Py_CompileStringFlags:int:start:: -Py_CompileStringFlags:PyCompilerFlags*:flags:: - -Py_CompileStringObject:PyObject*::+1: -Py_CompileStringObject:const char*:str:: -Py_CompileStringObject:PyObject*:filename:0: -Py_CompileStringObject:int:start:: -Py_CompileStringObject:PyCompilerFlags*:flags:: -Py_CompileStringObject:int:optimize:: - -Py_DECREF:void::: -Py_DECREF:PyObject*:o:-1: - -Py_EndInterpreter:void::: -Py_EndInterpreter:PyThreadState*:tstate:: - -Py_Exit:void::: -Py_Exit:int:status:: - -Py_FatalError:void::: -Py_FatalError:const char*:message:: - -Py_FdIsInteractive:int::: -Py_FdIsInteractive:FILE*:fp:: -Py_FdIsInteractive:const char*:filename:: - -Py_Finalize:void::: - -Py_GetBuildInfo:const char*::: - -Py_GetCompiler:const char*::: - -Py_GetCopyright:const char*::: - -Py_GetExecPrefix:wchar_t*::: - -Py_GetPath:wchar_t*::: - -Py_GetPlatform:const char*::: - -Py_GetPrefix:wchar_t*::: - -Py_GetProgramFullPath:wchar_t*::: - -Py_GetProgramName:wchar_t*::: - -Py_GetVersion:const char*::: - -Py_INCREF:void::: -Py_INCREF:PyObject*:o:+1: - -Py_NewRef:void::: -Py_NewRef:PyObject*:o:+1: - -Py_Initialize:void::: - -Py_IsInitialized:int::: - -Py_NewInterpreter:PyThreadState*::: - -Py_ReprEnter:int::: -Py_ReprEnter:PyObject*:object:+1: - -Py_ReprLeave:void::: -Py_ReprLeave:PyObject*:object:-1: - -Py_SetProgramName:void::: -Py_SetProgramName:const wchar_t*:name:: - -Py_XDECREF:void::: -Py_XDECREF:PyObject*:o:-1:if o is not NULL - -Py_XINCREF:void::: -Py_XINCREF:PyObject*:o:+1:if o is not NULL - -Py_XNewRef:void::: -Py_XNewRef:PyObject*:o:+1:if o is not NULL - -_PyImport_Fini:void::: - -_PyObject_New:PyObject*::+1: -_PyObject_New:PyTypeObject*:type:0: - -_PyObject_NewVar:PyVarObject*::+1: -_PyObject_NewVar:PyTypeObject*:type:0: -_PyObject_NewVar:Py_ssize_t:size:: - -_PyBytes_Resize:int::: -_PyBytes_Resize:PyObject**:bytes:0: -_PyBytes_Resize:Py_ssize_t:newsize:: - -_PyTuple_Resize:int::: -_PyTuple_Resize:PyObject**:p:0: -_PyTuple_Resize:Py_ssize_t:new:: - -_Py_c_diff:Py_complex::: -_Py_c_diff:Py_complex:left:: -_Py_c_diff:Py_complex:right:: - -_Py_c_neg:Py_complex::: -_Py_c_neg:Py_complex:complex:: - -_Py_c_pow:Py_complex::: -_Py_c_pow:Py_complex:num:: -_Py_c_pow:Py_complex:exp:: - -_Py_c_prod:Py_complex::: -_Py_c_prod:Py_complex:left:: -_Py_c_prod:Py_complex:right:: - -_Py_c_quot:Py_complex::: -_Py_c_quot:Py_complex:dividend:: -_Py_c_quot:Py_complex:divisor:: - -_Py_c_sum:Py_complex::: -_Py_c_sum:Py_complex:left:: -_Py_c_sum:Py_complex:right:: diff --git a/Doc/distributing/index.rst b/Doc/distributing/index.rst deleted file mode 100644 index 8622ec60c0f2f6..00000000000000 --- a/Doc/distributing/index.rst +++ /dev/null @@ -1,20 +0,0 @@ -:orphan: - -.. This page is retained solely for existing links to /distributing/index.html. - Direct readers to the PPUG instead. - -.. _distributing-index: -.. _publishing-python-packages: - -############################### - Distributing Python Modules -############################### - -.. note:: - - Information and guidance on distributing Python modules and packages - has been moved to the `Python Packaging User Guide`_, - and the tutorial on `packaging Python projects`_. - - .. _Python Packaging User Guide: https://packaging.python.org/ - .. _packaging Python projects: https://packaging.python.org/en/latest/tutorials/packaging-projects/ diff --git a/Doc/distutils/_setuptools_disclaimer.rst b/Doc/distutils/_setuptools_disclaimer.rst deleted file mode 100644 index cc75858326d44d..00000000000000 --- a/Doc/distutils/_setuptools_disclaimer.rst +++ /dev/null @@ -1,5 +0,0 @@ -.. note:: - - This document is being retained solely until the ``setuptools`` documentation - at https://setuptools.readthedocs.io/en/latest/setuptools.html - independently covers all of the relevant information currently included here. diff --git a/Doc/distutils/apiref.rst b/Doc/distutils/apiref.rst deleted file mode 100644 index 56bed7d63f1e90..00000000000000 --- a/Doc/distutils/apiref.rst +++ /dev/null @@ -1,2041 +0,0 @@ -.. _api-reference: - -************* -API Reference -************* - -.. seealso:: - - `New and changed setup.py arguments in setuptools`_ - The ``setuptools`` project adds new capabilities to the ``setup`` function - and other APIs, makes the API consistent across different Python versions, - and is hence recommended over using ``distutils`` directly. - -.. _New and changed setup.py arguments in setuptools: https://web.archive.org/web/20210614192516/https://setuptools.pypa.io/en/stable/userguide/keywords.html - -.. include:: ./_setuptools_disclaimer.rst - -:mod:`distutils.core` --- Core Distutils functionality -====================================================== - -.. module:: distutils.core - :synopsis: The core Distutils functionality - - -The :mod:`distutils.core` module is the only module that needs to be installed -to use the Distutils. It provides the :func:`setup` (which is called from the -setup script). Indirectly provides the :class:`distutils.dist.Distribution` and -:class:`distutils.cmd.Command` class. - - -.. function:: setup(arguments) - - The basic do-everything function that does most everything you could ever ask - for from a Distutils method. - - The setup function takes a large number of arguments. These are laid out in the - following table. - - .. tabularcolumns:: |l|L|L| - - +--------------------+--------------------------------+-------------------------------------------------------------+ - | argument name | value | type | - +====================+================================+=============================================================+ - | *name* | The name of the package | a string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *version* | The version number of the | a string | - | | package; see | | - | | :mod:`distutils.version` | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *description* | A single line describing the | a string | - | | package | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *long_description* | Longer description of the | a string | - | | package | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *author* | The name of the package author | a string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *author_email* | The email address of the | a string | - | | package author | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *maintainer* | The name of the current | a string | - | | maintainer, if different from | | - | | the author. Note that if | | - | | the maintainer is provided, | | - | | distutils will use it as the | | - | | author in :file:`PKG-INFO` | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *maintainer_email* | The email address of the | a string | - | | current maintainer, if | | - | | different from the author | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *url* | A URL for the package | a string | - | | (homepage) | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *download_url* | A URL to download the package | a string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *packages* | A list of Python packages that | a list of strings | - | | distutils will manipulate | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *py_modules* | A list of Python modules that | a list of strings | - | | distutils will manipulate | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *scripts* | A list of standalone script | a list of strings | - | | files to be built and | | - | | installed | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *ext_modules* | A list of Python extensions to | a list of instances of | - | | be built | :class:`distutils.core.Extension` | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *classifiers* | A list of categories for the | a list of strings; valid classifiers are listed on `PyPI | - | | package | <https://pypi.org/classifiers>`_. | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *distclass* | the :class:`Distribution` | a subclass of | - | | class to use | :class:`distutils.core.Distribution` | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *script_name* | The name of the setup.py | a string | - | | script - defaults to | | - | | ``sys.argv[0]`` | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *script_args* | Arguments to supply to the | a list of strings | - | | setup script | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *options* | default options for the setup | a dictionary | - | | script | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *license* | The license for the package | a string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *keywords* | Descriptive meta-data, see | a list of strings or a comma-separated string | - | | :pep:`314` | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *platforms* | | a list of strings or a comma-separated string | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *cmdclass* | A mapping of command names to | a dictionary | - | | :class:`Command` subclasses | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *data_files* | A list of data files to | a list | - | | install | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - | *package_dir* | A mapping of package to | a dictionary | - | | directory names | | - +--------------------+--------------------------------+-------------------------------------------------------------+ - - - -.. function:: run_setup(script_name[, script_args=None, stop_after='run']) - - Run a setup script in a somewhat controlled environment, and return the - :class:`distutils.dist.Distribution` instance that drives things. This is - useful if you need to find out the distribution meta-data (passed as keyword - args from *script* to :func:`setup`), or the contents of the config files or - command-line. - - *script_name* is a file that will be read and run with :func:`exec`. ``sys.argv[0]`` - will be replaced with *script* for the duration of the call. *script_args* is a - list of strings; if supplied, ``sys.argv[1:]`` will be replaced by *script_args* - for the duration of the call. - - *stop_after* tells :func:`setup` when to stop processing; possible values: - - .. tabularcolumns:: |l|L| - - +---------------+---------------------------------------------+ - | value | description | - +===============+=============================================+ - | *init* | Stop after the :class:`Distribution` | - | | instance has been created and populated | - | | with the keyword arguments to :func:`setup` | - +---------------+---------------------------------------------+ - | *config* | Stop after config files have been parsed | - | | (and their data stored in the | - | | :class:`Distribution` instance) | - +---------------+---------------------------------------------+ - | *commandline* | Stop after the command-line | - | | (``sys.argv[1:]`` or *script_args*) have | - | | been parsed (and the data stored in the | - | | :class:`Distribution` instance.) | - +---------------+---------------------------------------------+ - | *run* | Stop after all commands have been run (the | - | | same as if :func:`setup` had been called | - | | in the usual way). This is the default | - | | value. | - +---------------+---------------------------------------------+ - -In addition, the :mod:`distutils.core` module exposed a number of classes that -live elsewhere. - -* :class:`~distutils.extension.Extension` from :mod:`distutils.extension` - -* :class:`~distutils.cmd.Command` from :mod:`distutils.cmd` - -* :class:`~distutils.dist.Distribution` from :mod:`distutils.dist` - -A short description of each of these follows, but see the relevant module for -the full reference. - - -.. class:: Extension - - The Extension class describes a single C or C++ extension module in a setup - script. It accepts the following keyword arguments in its constructor: - - .. tabularcolumns:: |l|L|l| - - +------------------------+--------------------------------+---------------------------+ - | argument name | value | type | - +========================+================================+===========================+ - | *name* | the full name of the | a string | - | | extension, including any | | - | | packages --- ie. *not* a | | - | | filename or pathname, but | | - | | Python dotted name | | - +------------------------+--------------------------------+---------------------------+ - | *sources* | list of source filenames, | a list of strings | - | | relative to the distribution | | - | | root (where the setup script | | - | | lives), in Unix form | | - | | (slash-separated) for | | - | | portability. | | - | | Source files may be C, C++, | | - | | SWIG (.i), platform-specific | | - | | resource files, or whatever | | - | | else is recognized by the | | - | | :command:`build_ext` command | | - | | as source for a Python | | - | | extension. | | - +------------------------+--------------------------------+---------------------------+ - | *include_dirs* | list of directories to search | a list of strings | - | | for C/C++ header files (in | | - | | Unix form for portability) | | - +------------------------+--------------------------------+---------------------------+ - | *define_macros* | list of macros to define; each | a list of tuples | - | | macro is defined using a | | - | | 2-tuple ``(name, value)``, | | - | | where *value* is | | - | | either the string to define it | | - | | to or ``None`` to define it | | - | | without a particular value | | - | | (equivalent of ``#define FOO`` | | - | | in source or :option:`!-DFOO` | | - | | on Unix C compiler command | | - | | line) | | - +------------------------+--------------------------------+---------------------------+ - | *undef_macros* | list of macros to undefine | a list of strings | - | | explicitly | | - +------------------------+--------------------------------+---------------------------+ - | *library_dirs* | list of directories to search | a list of strings | - | | for C/C++ libraries at link | | - | | time | | - +------------------------+--------------------------------+---------------------------+ - | *libraries* | list of library names (not | a list of strings | - | | filenames or paths) to link | | - | | against | | - +------------------------+--------------------------------+---------------------------+ - | *runtime_library_dirs* | list of directories to search | a list of strings | - | | for C/C++ libraries at run | | - | | time (for shared extensions, | | - | | this is when the extension is | | - | | loaded) | | - +------------------------+--------------------------------+---------------------------+ - | *extra_objects* | list of extra files to link | a list of strings | - | | with (eg. object files not | | - | | implied by 'sources', static | | - | | library that must be | | - | | explicitly specified, binary | | - | | resource files, etc.) | | - +------------------------+--------------------------------+---------------------------+ - | *extra_compile_args* | any extra platform- and | a list of strings | - | | compiler-specific information | | - | | to use when compiling the | | - | | source files in 'sources'. For | | - | | platforms and compilers where | | - | | a command line makes sense, | | - | | this is typically a list of | | - | | command-line arguments, but | | - | | for other platforms it could | | - | | be anything. | | - +------------------------+--------------------------------+---------------------------+ - | *extra_link_args* | any extra platform- and | a list of strings | - | | compiler-specific information | | - | | to use when linking object | | - | | files together to create the | | - | | extension (or to create a new | | - | | static Python interpreter). | | - | | Similar interpretation as for | | - | | 'extra_compile_args'. | | - +------------------------+--------------------------------+---------------------------+ - | *export_symbols* | list of symbols to be exported | a list of strings | - | | from a shared extension. Not | | - | | used on all platforms, and not | | - | | generally necessary for Python | | - | | extensions, which typically | | - | | export exactly one symbol: | | - | | ``init`` + extension_name. | | - +------------------------+--------------------------------+---------------------------+ - | *depends* | list of files that the | a list of strings | - | | extension depends on | | - +------------------------+--------------------------------+---------------------------+ - | *language* | extension language (i.e. | a string | - | | ``'c'``, ``'c++'``, | | - | | ``'objc'``). Will be detected | | - | | from the source extensions if | | - | | not provided. | | - +------------------------+--------------------------------+---------------------------+ - | *optional* | specifies that a build failure | a boolean | - | | in the extension should not | | - | | abort the build process, but | | - | | simply skip the extension. | | - +------------------------+--------------------------------+---------------------------+ - - .. versionchanged:: 3.8 - - On Unix, C extensions are no longer linked to libpython except on - Android and Cygwin. - - -.. class:: Distribution - - A :class:`Distribution` describes how to build, install and package up a Python - software package. - - See the :func:`setup` function for a list of keyword arguments accepted by the - Distribution constructor. :func:`setup` creates a Distribution instance. - - .. versionchanged:: 3.7 - :class:`~distutils.core.Distribution` now warns if ``classifiers``, - ``keywords`` and ``platforms`` fields are not specified as a list or - a string. - -.. class:: Command - - A :class:`Command` class (or rather, an instance of one of its subclasses) - implement a single distutils command. - - -:mod:`distutils.ccompiler` --- CCompiler base class -=================================================== - -.. module:: distutils.ccompiler - :synopsis: Abstract CCompiler class - - -This module provides the abstract base class for the :class:`CCompiler` -classes. A :class:`CCompiler` instance can be used for all the compile and -link steps needed to build a single project. Methods are provided to set -options for the compiler --- macro definitions, include directories, link path, -libraries and the like. - -This module provides the following functions. - - -.. function:: gen_lib_options(compiler, library_dirs, runtime_library_dirs, libraries) - - Generate linker options for searching library directories and linking with - specific libraries. *libraries* and *library_dirs* are, respectively, lists of - library names (not filenames!) and search directories. Returns a list of - command-line options suitable for use with some compiler (depending on the two - format strings passed in). - - -.. function:: gen_preprocess_options(macros, include_dirs) - - Generate C pre-processor options (:option:`!-D`, :option:`!-U`, :option:`!-I`) as - used by at least two types of compilers: the typical Unix compiler and Visual - C++. *macros* is the usual thing, a list of 1- or 2-tuples, where ``(name,)`` - means undefine (:option:`!-U`) macro *name*, and ``(name, value)`` means define - (:option:`!-D`) macro *name* to *value*. *include_dirs* is just a list of - directory names to be added to the header file search path (:option:`!-I`). - Returns a list of command-line options suitable for either Unix compilers or - Visual C++. - - -.. function:: get_default_compiler(osname, platform) - - Determine the default compiler to use for the given platform. - - *osname* should be one of the standard Python OS names (i.e. the ones returned - by ``os.name``) and *platform* the common value returned by ``sys.platform`` for - the platform in question. - - The default values are ``os.name`` and ``sys.platform`` in case the parameters - are not given. - - -.. function:: new_compiler(plat=None, compiler=None, verbose=0, dry_run=0, force=0) - - Factory function to generate an instance of some CCompiler subclass for the - supplied platform/compiler combination. *plat* defaults to ``os.name`` (eg. - ``'posix'``, ``'nt'``), and *compiler* defaults to the default compiler for - that platform. Currently only ``'posix'`` and ``'nt'`` are supported, and the - default compilers are "traditional Unix interface" (:class:`UnixCCompiler` - class) and Visual C++ (:class:`MSVCCompiler` class). Note that it's perfectly - possible to ask for a Unix compiler object under Windows, and a Microsoft - compiler object under Unix---if you supply a value for *compiler*, *plat* is - ignored. - - .. % Is the posix/nt only thing still true? macOS seems to work, and - .. % returns a UnixCCompiler instance. How to document this... hmm. - - -.. function:: show_compilers() - - Print list of available compilers (used by the :option:`!--help-compiler` options - to :command:`build`, :command:`build_ext`, :command:`build_clib`). - - -.. class:: CCompiler([verbose=0, dry_run=0, force=0]) - - The abstract base class :class:`CCompiler` defines the interface that must be - implemented by real compiler classes. The class also has some utility methods - used by several compiler classes. - - The basic idea behind a compiler abstraction class is that each instance can be - used for all the compile/link steps in building a single project. Thus, - attributes common to all of those compile and link steps --- include - directories, macros to define, libraries to link against, etc. --- are - attributes of the compiler instance. To allow for variability in how individual - files are treated, most of those attributes may be varied on a per-compilation - or per-link basis. - - The constructor for each subclass creates an instance of the Compiler object. - Flags are *verbose* (show verbose output), *dry_run* (don't actually execute the - steps) and *force* (rebuild everything, regardless of dependencies). All of - these flags default to ``0`` (off). Note that you probably don't want to - instantiate :class:`CCompiler` or one of its subclasses directly - use the - :func:`distutils.CCompiler.new_compiler` factory function instead. - - The following methods allow you to manually alter compiler options for the - instance of the Compiler class. - - - .. method:: CCompiler.add_include_dir(dir) - - Add *dir* to the list of directories that will be searched for header files. - The compiler is instructed to search directories in the order in which they are - supplied by successive calls to :meth:`add_include_dir`. - - - .. method:: CCompiler.set_include_dirs(dirs) - - Set the list of directories that will be searched to *dirs* (a list of strings). - Overrides any preceding calls to :meth:`add_include_dir`; subsequent calls to - :meth:`add_include_dir` add to the list passed to :meth:`set_include_dirs`. - This does not affect any list of standard include directories that the compiler - may search by default. - - - .. method:: CCompiler.add_library(libname) - - Add *libname* to the list of libraries that will be included in all links driven - by this compiler object. Note that *libname* should \*not\* be the name of a - file containing a library, but the name of the library itself: the actual - filename will be inferred by the linker, the compiler, or the compiler class - (depending on the platform). - - The linker will be instructed to link against libraries in the order they were - supplied to :meth:`add_library` and/or :meth:`set_libraries`. It is perfectly - valid to duplicate library names; the linker will be instructed to link against - libraries as many times as they are mentioned. - - - .. method:: CCompiler.set_libraries(libnames) - - Set the list of libraries to be included in all links driven by this compiler - object to *libnames* (a list of strings). This does not affect any standard - system libraries that the linker may include by default. - - - .. method:: CCompiler.add_library_dir(dir) - - Add *dir* to the list of directories that will be searched for libraries - specified to :meth:`add_library` and :meth:`set_libraries`. The linker will be - instructed to search for libraries in the order they are supplied to - :meth:`add_library_dir` and/or :meth:`set_library_dirs`. - - - .. method:: CCompiler.set_library_dirs(dirs) - - Set the list of library search directories to *dirs* (a list of strings). This - does not affect any standard library search path that the linker may search by - default. - - - .. method:: CCompiler.add_runtime_library_dir(dir) - - Add *dir* to the list of directories that will be searched for shared libraries - at runtime. - - - .. method:: CCompiler.set_runtime_library_dirs(dirs) - - Set the list of directories to search for shared libraries at runtime to *dirs* - (a list of strings). This does not affect any standard search path that the - runtime linker may search by default. - - - .. method:: CCompiler.define_macro(name[, value=None]) - - Define a preprocessor macro for all compilations driven by this compiler object. - The optional parameter *value* should be a string; if it is not supplied, then - the macro will be defined without an explicit value and the exact outcome - depends on the compiler used. - - .. XXX true? does ANSI say anything about this? - - - .. method:: CCompiler.undefine_macro(name) - - Undefine a preprocessor macro for all compilations driven by this compiler - object. If the same macro is defined by :meth:`define_macro` and - undefined by :meth:`undefine_macro` the last call takes precedence - (including multiple redefinitions or undefinitions). If the macro is - redefined/undefined on a per-compilation basis (ie. in the call to - :meth:`compile`), then that takes precedence. - - - .. method:: CCompiler.add_link_object(object) - - Add *object* to the list of object files (or analogues, such as explicitly named - library files or the output of "resource compilers") to be included in every - link driven by this compiler object. - - - .. method:: CCompiler.set_link_objects(objects) - - Set the list of object files (or analogues) to be included in every link to - *objects*. This does not affect any standard object files that the linker may - include by default (such as system libraries). - - The following methods implement methods for autodetection of compiler options, - providing some functionality similar to GNU :program:`autoconf`. - - - .. method:: CCompiler.detect_language(sources) - - Detect the language of a given file, or list of files. Uses the instance - attributes :attr:`language_map` (a dictionary), and :attr:`language_order` (a - list) to do the job. - - - .. method:: CCompiler.find_library_file(dirs, lib[, debug=0]) - - Search the specified list of directories for a static or shared library file - *lib* and return the full path to that file. If *debug* is true, look for a - debugging version (if that makes sense on the current platform). Return - ``None`` if *lib* wasn't found in any of the specified directories. - - - .. method:: CCompiler.has_function(funcname [, includes=None, include_dirs=None, libraries=None, library_dirs=None]) - - Return a boolean indicating whether *funcname* is supported on the current - platform. The optional arguments can be used to augment the compilation - environment by providing additional include files and paths and libraries and - paths. - - - .. method:: CCompiler.library_dir_option(dir) - - Return the compiler option to add *dir* to the list of directories searched for - libraries. - - - .. method:: CCompiler.library_option(lib) - - Return the compiler option to add *lib* to the list of libraries linked into the - shared library or executable. - - - .. method:: CCompiler.runtime_library_dir_option(dir) - - Return the compiler option to add *dir* to the list of directories searched for - runtime libraries. - - - .. method:: CCompiler.set_executables(**args) - - Define the executables (and options for them) that will be run to perform the - various stages of compilation. The exact set of executables that may be - specified here depends on the compiler class (via the 'executables' class - attribute), but most will have: - - +--------------+------------------------------------------+ - | attribute | description | - +==============+==========================================+ - | *compiler* | the C/C++ compiler | - +--------------+------------------------------------------+ - | *linker_so* | linker used to create shared objects and | - | | libraries | - +--------------+------------------------------------------+ - | *linker_exe* | linker used to create binary executables | - +--------------+------------------------------------------+ - | *archiver* | static library creator | - +--------------+------------------------------------------+ - - On platforms with a command-line (Unix, DOS/Windows), each of these is a string - that will be split into executable name and (optional) list of arguments. - (Splitting the string is done similarly to how Unix shells operate: words are - delimited by spaces, but quotes and backslashes can override this. See - :func:`distutils.util.split_quoted`.) - - The following methods invoke stages in the build process. - - - .. method:: CCompiler.compile(sources[, output_dir=None, macros=None, include_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, depends=None]) - - Compile one or more source files. Generates object files (e.g. transforms a - :file:`.c` file to a :file:`.o` file.) - - *sources* must be a list of filenames, most likely C/C++ files, but in reality - anything that can be handled by a particular compiler and compiler class (eg. - :class:`MSVCCompiler` can handle resource files in *sources*). Return a list of - object filenames, one per source filename in *sources*. Depending on the - implementation, not all source files will necessarily be compiled, but all - corresponding object filenames will be returned. - - If *output_dir* is given, object files will be put under it, while retaining - their original path component. That is, :file:`foo/bar.c` normally compiles to - :file:`foo/bar.o` (for a Unix implementation); if *output_dir* is *build*, then - it would compile to :file:`build/foo/bar.o`. - - *macros*, if given, must be a list of macro definitions. A macro definition is - either a ``(name, value)`` 2-tuple or a ``(name,)`` 1-tuple. The former defines - a macro; if the value is ``None``, the macro is defined without an explicit - value. The 1-tuple case undefines a macro. Later - definitions/redefinitions/undefinitions take precedence. - - *include_dirs*, if given, must be a list of strings, the directories to add to - the default include file search path for this compilation only. - - *debug* is a boolean; if true, the compiler will be instructed to output debug - symbols in (or alongside) the object file(s). - - *extra_preargs* and *extra_postargs* are implementation-dependent. On platforms - that have the notion of a command-line (e.g. Unix, DOS/Windows), they are most - likely lists of strings: extra command-line arguments to prepend/append to the - compiler command line. On other platforms, consult the implementation class - documentation. In any event, they are intended as an escape hatch for those - occasions when the abstract compiler framework doesn't cut the mustard. - - *depends*, if given, is a list of filenames that all targets depend on. If a - source file is older than any file in depends, then the source file will be - recompiled. This supports dependency tracking, but only at a coarse - granularity. - - Raises :exc:`CompileError` on failure. - - - .. method:: CCompiler.create_static_lib(objects, output_libname[, output_dir=None, debug=0, target_lang=None]) - - Link a bunch of stuff together to create a static library file. The "bunch of - stuff" consists of the list of object files supplied as *objects*, the extra - object files supplied to :meth:`add_link_object` and/or - :meth:`set_link_objects`, the libraries supplied to :meth:`add_library` and/or - :meth:`set_libraries`, and the libraries supplied as *libraries* (if any). - - *output_libname* should be a library name, not a filename; the filename will be - inferred from the library name. *output_dir* is the directory where the library - file will be put. - - .. XXX defaults to what? - - *debug* is a boolean; if true, debugging information will be included in the - library (note that on most platforms, it is the compile step where this matters: - the *debug* flag is included here just for consistency). - - *target_lang* is the target language for which the given objects are being - compiled. This allows specific linkage time treatment of certain languages. - - Raises :exc:`LibError` on failure. - - - .. method:: CCompiler.link(target_desc, objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) - - Link a bunch of stuff together to create an executable or shared library file. - - The "bunch of stuff" consists of the list of object files supplied as *objects*. - *output_filename* should be a filename. If *output_dir* is supplied, - *output_filename* is relative to it (i.e. *output_filename* can provide - directory components if needed). - - *libraries* is a list of libraries to link against. These are library names, - not filenames, since they're translated into filenames in a platform-specific - way (eg. *foo* becomes :file:`libfoo.a` on Unix and :file:`foo.lib` on - DOS/Windows). However, they can include a directory component, which means the - linker will look in that specific directory rather than searching all the normal - locations. - - *library_dirs*, if supplied, should be a list of directories to search for - libraries that were specified as bare library names (ie. no directory - component). These are on top of the system default and those supplied to - :meth:`add_library_dir` and/or :meth:`set_library_dirs`. *runtime_library_dirs* - is a list of directories that will be embedded into the shared library and used - to search for other shared libraries that \*it\* depends on at run-time. (This - may only be relevant on Unix.) - - *export_symbols* is a list of symbols that the shared library will export. - (This appears to be relevant only on Windows.) - - *debug* is as for :meth:`compile` and :meth:`create_static_lib`, with the - slight distinction that it actually matters on most platforms (as opposed to - :meth:`create_static_lib`, which includes a *debug* flag mostly for form's - sake). - - *extra_preargs* and *extra_postargs* are as for :meth:`compile` (except of - course that they supply command-line arguments for the particular linker being - used). - - *target_lang* is the target language for which the given objects are being - compiled. This allows specific linkage time treatment of certain languages. - - Raises :exc:`LinkError` on failure. - - - .. method:: CCompiler.link_executable(objects, output_progname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, debug=0, extra_preargs=None, extra_postargs=None, target_lang=None]) - - Link an executable. *output_progname* is the name of the file executable, while - *objects* are a list of object filenames to link in. Other arguments are as for - the :meth:`link` method. - - - .. method:: CCompiler.link_shared_lib(objects, output_libname[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) - - Link a shared library. *output_libname* is the name of the output library, - while *objects* is a list of object filenames to link in. Other arguments are - as for the :meth:`link` method. - - - .. method:: CCompiler.link_shared_object(objects, output_filename[, output_dir=None, libraries=None, library_dirs=None, runtime_library_dirs=None, export_symbols=None, debug=0, extra_preargs=None, extra_postargs=None, build_temp=None, target_lang=None]) - - Link a shared object. *output_filename* is the name of the shared object that - will be created, while *objects* is a list of object filenames to link in. - Other arguments are as for the :meth:`link` method. - - - .. method:: CCompiler.preprocess(source[, output_file=None, macros=None, include_dirs=None, extra_preargs=None, extra_postargs=None]) - - Preprocess a single C/C++ source file, named in *source*. Output will be written - to file named *output_file*, or *stdout* if *output_file* not supplied. - *macros* is a list of macro definitions as for :meth:`compile`, which will - augment the macros set with :meth:`define_macro` and :meth:`undefine_macro`. - *include_dirs* is a list of directory names that will be added to the default - list, in the same way as :meth:`add_include_dir`. - - Raises :exc:`PreprocessError` on failure. - - The following utility methods are defined by the :class:`CCompiler` class, for - use by the various concrete subclasses. - - - .. method:: CCompiler.executable_filename(basename[, strip_dir=0, output_dir='']) - - Returns the filename of the executable for the given *basename*. Typically for - non-Windows platforms this is the same as the basename, while Windows will get - a :file:`.exe` added. - - - .. method:: CCompiler.library_filename(libname[, lib_type='static', strip_dir=0, output_dir='']) - - Returns the filename for the given library name on the current platform. On Unix - a library with *lib_type* of ``'static'`` will typically be of the form - :file:`liblibname.a`, while a *lib_type* of ``'dynamic'`` will be of the form - :file:`liblibname.so`. - - - .. method:: CCompiler.object_filenames(source_filenames[, strip_dir=0, output_dir='']) - - Returns the name of the object files for the given source files. - *source_filenames* should be a list of filenames. - - - .. method:: CCompiler.shared_object_filename(basename[, strip_dir=0, output_dir='']) - - Returns the name of a shared object file for the given file name *basename*. - - - .. method:: CCompiler.execute(func, args[, msg=None, level=1]) - - Invokes :func:`distutils.util.execute`. This method invokes a Python function - *func* with the given arguments *args*, after logging and taking into account - the *dry_run* flag. - - - .. method:: CCompiler.spawn(cmd) - - Invokes :func:`distutils.util.spawn`. This invokes an external process to run - the given command. - - - .. method:: CCompiler.mkpath(name[, mode=511]) - - Invokes :func:`distutils.dir_util.mkpath`. This creates a directory and any - missing ancestor directories. - - - .. method:: CCompiler.move_file(src, dst) - - Invokes :meth:`distutils.file_util.move_file`. Renames *src* to *dst*. - - - .. method:: CCompiler.announce(msg[, level=1]) - - Write a message using :func:`distutils.log.debug`. - - - .. method:: CCompiler.warn(msg) - - Write a warning message *msg* to standard error. - - - .. method:: CCompiler.debug_print(msg) - - If the *debug* flag is set on this :class:`CCompiler` instance, print *msg* to - standard output, otherwise do nothing. - -.. % \subsection{Compiler-specific modules} -.. % -.. % The following modules implement concrete subclasses of the abstract -.. % \class{CCompiler} class. They should not be instantiated directly, but should -.. % be created using \function{distutils.ccompiler.new_compiler()} factory -.. % function. - - -:mod:`distutils.unixccompiler` --- Unix C Compiler -================================================== - -.. module:: distutils.unixccompiler - :synopsis: UNIX C Compiler - - -This module provides the :class:`UnixCCompiler` class, a subclass of -:class:`CCompiler` that handles the typical Unix-style command-line C compiler: - -* macros defined with :option:`!-Dname[=value]` - -* macros undefined with :option:`!-Uname` - -* include search directories specified with :option:`!-Idir` - -* libraries specified with :option:`!-llib` - -* library search directories specified with :option:`!-Ldir` - -* compile handled by :program:`cc` (or similar) executable with :option:`!-c` - option: compiles :file:`.c` to :file:`.o` - -* link static library handled by :program:`ar` command (possibly with - :program:`ranlib`) - -* link shared library handled by :program:`cc` :option:`!-shared` - - -:mod:`distutils.msvccompiler` --- Microsoft Compiler -==================================================== - -.. module:: distutils.msvccompiler - :synopsis: Microsoft Compiler - -.. XXX: This is *waaaaay* out of date! - -This module provides :class:`MSVCCompiler`, an implementation of the abstract -:class:`CCompiler` class for Microsoft Visual Studio. Typically, extension -modules need to be compiled with the same compiler that was used to compile -Python. For Python 2.3 and earlier, the compiler was Visual Studio 6. For Python -2.4 and 2.5, the compiler is Visual Studio .NET 2003. - -:class:`MSVCCompiler` will normally choose the right compiler, linker etc. on -its own. To override this choice, the environment variables *DISTUTILS_USE_SDK* -and *MSSdk* must be both set. *MSSdk* indicates that the current environment has -been setup by the SDK's ``SetEnv.Cmd`` script, or that the environment variables -had been registered when the SDK was installed; *DISTUTILS_USE_SDK* indicates -that the distutils user has made an explicit choice to override the compiler -selection by :class:`MSVCCompiler`. - - -:mod:`distutils.bcppcompiler` --- Borland Compiler -================================================== - -.. module:: distutils.bcppcompiler - - -This module provides :class:`BorlandCCompiler`, a subclass of the abstract -:class:`CCompiler` class for the Borland C++ compiler. - - -:mod:`distutils.cygwincompiler` --- Cygwin Compiler -=================================================== - -.. module:: distutils.cygwinccompiler - - -This module provides the :class:`CygwinCCompiler` class, a subclass of -:class:`UnixCCompiler` that handles the Cygwin port of the GNU C compiler to -Windows. It also contains the Mingw32CCompiler class which handles the mingw32 -port of GCC (same as cygwin in no-cygwin mode). - - -:mod:`distutils.archive_util` --- Archiving utilities -====================================================== - -.. module:: distutils.archive_util - :synopsis: Utility functions for creating archive files (tarballs, zip files, ...) - - -This module provides a few functions for creating archive files, such as -tarballs or zipfiles. - - -.. function:: make_archive(base_name, format[, root_dir=None, base_dir=None, verbose=0, dry_run=0]) - - Create an archive file (eg. ``zip`` or ``tar``). *base_name* is the name of - the file to create, minus any format-specific extension; *format* is the - archive format: one of ``zip``, ``tar``, ``gztar``, ``bztar``, ``xztar``, or - ``ztar``. *root_dir* is a directory that will be the root directory of the - archive; ie. we typically ``chdir`` into *root_dir* before creating the - archive. *base_dir* is the directory where we start archiving from; ie. - *base_dir* will be the common prefix of all files and directories in the - archive. *root_dir* and *base_dir* both default to the current directory. - Returns the name of the archive file. - - .. versionchanged:: 3.5 - Added support for the ``xztar`` format. - - -.. function:: make_tarball(base_name, base_dir[, compress='gzip', verbose=0, dry_run=0]) - - 'Create an (optional compressed) archive as a tar file from all files in and - under *base_dir*. *compress* must be ``'gzip'`` (the default), - ``'bzip2'``, ``'xz'``, ``'compress'``, or ``None``. For the ``'compress'`` - method the compression utility named by :program:`compress` must be on the - default program search path, so this is probably Unix-specific. The output - tar file will be named :file:`base_dir.tar`, possibly plus the appropriate - compression extension (``.gz``, ``.bz2``, ``.xz`` or ``.Z``). Return the - output filename. - - .. versionchanged:: 3.5 - Added support for the ``xz`` compression. - - -.. function:: make_zipfile(base_name, base_dir[, verbose=0, dry_run=0]) - - Create a zip file from all files in and under *base_dir*. The output zip file - will be named *base_name* + :file:`.zip`. Uses either the :mod:`zipfile` Python - module (if available) or the InfoZIP :file:`zip` utility (if installed and - found on the default search path). If neither tool is available, raises - :exc:`DistutilsExecError`. Returns the name of the output zip file. - - -:mod:`distutils.dep_util` --- Dependency checking -================================================= - -.. module:: distutils.dep_util - :synopsis: Utility functions for simple dependency checking - - -This module provides functions for performing simple, timestamp-based -dependency of files and groups of files; also, functions based entirely on such -timestamp dependency analysis. - - -.. function:: newer(source, target) - - Return true if *source* exists and is more recently modified than *target*, or - if *source* exists and *target* doesn't. Return false if both exist and *target* - is the same age or newer than *source*. Raise :exc:`DistutilsFileError` if - *source* does not exist. - - -.. function:: newer_pairwise(sources, targets) - - Walk two filename lists in parallel, testing if each source is newer than its - corresponding target. Return a pair of lists (*sources*, *targets*) where - source is newer than target, according to the semantics of :func:`newer`. - - .. % % equivalent to a listcomp... - - -.. function:: newer_group(sources, target[, missing='error']) - - Return true if *target* is out-of-date with respect to any file listed in - *sources*. In other words, if *target* exists and is newer than every file in - *sources*, return false; otherwise return true. *missing* controls what we do - when a source file is missing; the default (``'error'``) is to blow up with an - :exc:`OSError` from inside :func:`os.stat`; if it is ``'ignore'``, we silently - drop any missing source files; if it is ``'newer'``, any missing source files - make us assume that *target* is out-of-date (this is handy in "dry-run" mode: - it'll make you pretend to carry out commands that wouldn't work because inputs - are missing, but that doesn't matter because you're not actually going to run - the commands). - - -:mod:`distutils.dir_util` --- Directory tree operations -======================================================= - -.. module:: distutils.dir_util - :synopsis: Utility functions for operating on directories and directory trees - - -This module provides functions for operating on directories and trees of -directories. - - -.. function:: mkpath(name[, mode=0o777, verbose=0, dry_run=0]) - - Create a directory and any missing ancestor directories. If the directory - already exists (or if *name* is the empty string, which means the current - directory, which of course exists), then do nothing. Raise - :exc:`DistutilsFileError` if unable to create some directory along the way (eg. - some sub-path exists, but is a file rather than a directory). If *verbose* is - true, print a one-line summary of each mkdir to stdout. Return the list of - directories actually created. - - -.. function:: create_tree(base_dir, files[, mode=0o777, verbose=0, dry_run=0]) - - Create all the empty directories under *base_dir* needed to put *files* there. - *base_dir* is just the name of a directory which doesn't necessarily exist - yet; *files* is a list of filenames to be interpreted relative to *base_dir*. - *base_dir* + the directory portion of every file in *files* will be created if - it doesn't already exist. *mode*, *verbose* and *dry_run* flags are as for - :func:`mkpath`. - - -.. function:: copy_tree(src, dst[, preserve_mode=1, preserve_times=1, preserve_symlinks=0, update=0, verbose=0, dry_run=0]) - - Copy an entire directory tree *src* to a new location *dst*. Both *src* and - *dst* must be directory names. If *src* is not a directory, raise - :exc:`DistutilsFileError`. If *dst* does not exist, it is created with - :func:`mkpath`. The end result of the copy is that every file in *src* is - copied to *dst*, and directories under *src* are recursively copied to *dst*. - Return the list of files that were copied or might have been copied, using their - output name. The return value is unaffected by *update* or *dry_run*: it is - simply the list of all files under *src*, with the names changed to be under - *dst*. - - *preserve_mode* and *preserve_times* are the same as for - :func:`distutils.file_util.copy_file`; note that they only apply to - regular files, not to - directories. If *preserve_symlinks* is true, symlinks will be copied as - symlinks (on platforms that support them!); otherwise (the default), the - destination of the symlink will be copied. *update* and *verbose* are the same - as for :func:`copy_file`. - - Files in *src* that begin with :file:`.nfs` are skipped (more information on - these files is available in answer D2 of the `NFS FAQ page - <https://nfs.sourceforge.net/#section_d>`_). - - .. versionchanged:: 3.3.1 - NFS files are ignored. - -.. function:: remove_tree(directory[, verbose=0, dry_run=0]) - - Recursively remove *directory* and all files and directories underneath it. Any - errors are ignored (apart from being reported to ``sys.stdout`` if *verbose* is - true). - - -:mod:`distutils.file_util` --- Single file operations -===================================================== - -.. module:: distutils.file_util - :synopsis: Utility functions for operating on single files - - -This module contains some utility functions for operating on individual files. - - -.. function:: copy_file(src, dst[, preserve_mode=1, preserve_times=1, update=0, link=None, verbose=0, dry_run=0]) - - Copy file *src* to *dst*. If *dst* is a directory, then *src* is copied there - with the same name; otherwise, it must be a filename. (If the file exists, it - will be ruthlessly clobbered.) If *preserve_mode* is true (the default), the - file's mode (type and permission bits, or whatever is analogous on the - current platform) is copied. If *preserve_times* is true (the default), the - last-modified and last-access times are copied as well. If *update* is true, - *src* will only be copied if *dst* does not exist, or if *dst* does exist but - is older than *src*. - - *link* allows you to make hard links (using :func:`os.link`) or symbolic links - (using :func:`os.symlink`) instead of copying: set it to ``'hard'`` or - ``'sym'``; if it is ``None`` (the default), files are copied. Don't set *link* - on systems that don't support it: :func:`copy_file` doesn't check if hard or - symbolic linking is available. It uses :func:`_copy_file_contents` to copy file - contents. - - Return a tuple ``(dest_name, copied)``: *dest_name* is the actual name of the - output file, and *copied* is true if the file was copied (or would have been - copied, if *dry_run* true). - - .. % XXX if the destination file already exists, we clobber it if - .. % copying, but blow up if linking. Hmmm. And I don't know what - .. % macostools.copyfile() does. Should definitely be consistent, and - .. % should probably blow up if destination exists and we would be - .. % changing it (ie. it's not already a hard/soft link to src OR - .. % (not update) and (src newer than dst)). - - -.. function:: move_file(src, dst[, verbose, dry_run]) - - Move file *src* to *dst*. If *dst* is a directory, the file will be moved into - it with the same name; otherwise, *src* is just renamed to *dst*. Returns the - new full name of the file. - - .. warning:: - - Handles cross-device moves on Unix using :func:`copy_file`. What about - other systems? - - -.. function:: write_file(filename, contents) - - Create a file called *filename* and write *contents* (a sequence of strings - without line terminators) to it. - - -:mod:`distutils.util` --- Miscellaneous other utility functions -=============================================================== - -.. module:: distutils.util - :synopsis: Miscellaneous other utility functions - - -This module contains other assorted bits and pieces that don't fit into any -other utility module. - - -.. function:: get_platform() - - Return a string that identifies the current platform. This is used mainly to - distinguish platform-specific build directories and platform-specific built - distributions. Typically includes the OS name and version and the - architecture (as supplied by 'os.uname()'), although the exact information - included depends on the OS; e.g., on Linux, the kernel version isn't - particularly important. - - Examples of returned values: - - * ``linux-i586`` - * ``linux-alpha`` - * ``solaris-2.6-sun4u`` - - For non-POSIX platforms, currently just returns ``sys.platform``. - - For macOS systems the OS version reflects the minimal version on which - binaries will run (that is, the value of ``MACOSX_DEPLOYMENT_TARGET`` - during the build of Python), not the OS version of the current system. - - For universal binary builds on macOS the architecture value reflects - the universal binary status instead of the architecture of the current - processor. For 32-bit universal binaries the architecture is ``fat``, - for 64-bit universal binaries the architecture is ``fat64``, and - for 4-way universal binaries the architecture is ``universal``. Starting - from Python 2.7 and Python 3.2 the architecture ``fat3`` is used for - a 3-way universal build (ppc, i386, x86_64) and ``intel`` is used for - a universal build with the i386 and x86_64 architectures - - Examples of returned values on macOS: - - * ``macosx-10.3-ppc`` - - * ``macosx-10.3-fat`` - - * ``macosx-10.5-universal`` - - * ``macosx-10.6-intel`` - - For AIX, Python 3.9 and later return a string starting with "aix", followed - by additional fields (separated by ``'-'``) that represent the combined - values of AIX Version, Release and Technology Level (first field), Build Date - (second field), and bit-size (third field). Python 3.8 and earlier returned - only a single additional field with the AIX Version and Release. - - Examples of returned values on AIX: - - * ``aix-5307-0747-32`` # 32-bit build on AIX ``oslevel -s``: 5300-07-00-0000 - - * ``aix-7105-1731-64`` # 64-bit build on AIX ``oslevel -s``: 7100-05-01-1731 - - * ``aix-7.2`` # Legacy form reported in Python 3.8 and earlier - - .. versionchanged:: 3.9 - The AIX platform string format now also includes the technology level, - build date, and ABI bit-size. - - -.. function:: convert_path(pathname) - - Return 'pathname' as a name that will work on the native filesystem, i.e. split - it on '/' and put it back together again using the current directory separator. - Needed because filenames in the setup script are always supplied in Unix style, - and have to be converted to the local convention before we can actually use them - in the filesystem. Raises :exc:`ValueError` on non-Unix-ish systems if - *pathname* either starts or ends with a slash. - - -.. function:: change_root(new_root, pathname) - - Return *pathname* with *new_root* prepended. If *pathname* is relative, this is - equivalent to ``os.path.join(new_root,pathname)`` Otherwise, it requires making - *pathname* relative and then joining the two, which is tricky on DOS/Windows. - - -.. function:: check_environ() - - Ensure that 'os.environ' has all the environment variables we guarantee that - users can use in config files, command-line options, etc. Currently this - includes: - - * :envvar:`HOME` - user's home directory (Unix only) - * :envvar:`PLAT` - description of the current platform, including hardware and - OS (see :func:`get_platform`) - - -.. function:: subst_vars(s, local_vars) - - Perform shell/Perl-style variable substitution on *s*. Every occurrence of - ``$`` followed by a name is considered a variable, and variable is substituted - by the value found in the *local_vars* dictionary, or in ``os.environ`` if it's - not in *local_vars*. *os.environ* is first checked/augmented to guarantee that - it contains certain values: see :func:`check_environ`. Raise :exc:`ValueError` - for any variables not found in either *local_vars* or ``os.environ``. - - Note that this is not a full-fledged string interpolation function. A valid - ``$variable`` can consist only of upper and lower case letters, numbers and an - underscore. No { } or ( ) style quoting is available. - - -.. function:: split_quoted(s) - - Split a string up according to Unix shell-like rules for quotes and backslashes. - In short: words are delimited by spaces, as long as those spaces are not escaped - by a backslash, or inside a quoted string. Single and double quotes are - equivalent, and the quote characters can be backslash-escaped. The backslash is - stripped from any two-character escape sequence, leaving only the escaped - character. The quote characters are stripped from any quoted string. Returns a - list of words. - - .. % Should probably be moved into the standard library. - - -.. function:: execute(func, args[, msg=None, verbose=0, dry_run=0]) - - Perform some action that affects the outside world (for instance, writing to the - filesystem). Such actions are special because they are disabled by the - *dry_run* flag. This method takes care of all that bureaucracy for you; all - you have to do is supply the function to call and an argument tuple for it (to - embody the "external action" being performed), and an optional message to print. - - -.. function:: strtobool(val) - - Convert a string representation of truth to true (1) or false (0). - - True values are ``y``, ``yes``, ``t``, ``true``, ``on`` and ``1``; false values - are ``n``, ``no``, ``f``, ``false``, ``off`` and ``0``. Raises - :exc:`ValueError` if *val* is anything else. - - -.. function:: byte_compile(py_files[, optimize=0, force=0, prefix=None, base_dir=None, verbose=1, dry_run=0, direct=None]) - - Byte-compile a collection of Python source files to :file:`.pyc` files in a - :file:`__pycache__` subdirectory (see :pep:`3147` and :pep:`488`). - *py_files* is a list of files to compile; any files that don't end in - :file:`.py` are silently skipped. *optimize* must be one of the following: - - * ``0`` - don't optimize - * ``1`` - normal optimization (like ``python -O``) - * ``2`` - extra optimization (like ``python -OO``) - - If *force* is true, all files are recompiled regardless of timestamps. - - The source filename encoded in each :term:`bytecode` file defaults to the filenames - listed in *py_files*; you can modify these with *prefix* and *basedir*. - *prefix* is a string that will be stripped off of each source filename, and - *base_dir* is a directory name that will be prepended (after *prefix* is - stripped). You can supply either or both (or neither) of *prefix* and - *base_dir*, as you wish. - - If *dry_run* is true, doesn't actually do anything that would affect the - filesystem. - - Byte-compilation is either done directly in this interpreter process with the - standard :mod:`py_compile` module, or indirectly by writing a temporary script - and executing it. Normally, you should let :func:`byte_compile` figure out to - use direct compilation or not (see the source for details). The *direct* flag - is used by the script generated in indirect mode; unless you know what you're - doing, leave it set to ``None``. - - .. versionchanged:: 3.2.3 - Create ``.pyc`` files with an :func:`import magic tag - <imp.get_tag>` in their name, in a :file:`__pycache__` subdirectory - instead of files without tag in the current directory. - - .. versionchanged:: 3.5 - Create ``.pyc`` files according to :pep:`488`. - - -.. function:: rfc822_escape(header) - - Return a version of *header* escaped for inclusion in an :rfc:`822` header, by - ensuring there are 8 spaces space after each newline. Note that it does no other - modification of the string. - - .. % this _can_ be replaced - -.. % \subsection{Distutils objects} - - -:mod:`distutils.dist` --- The Distribution class -================================================ - -.. module:: distutils.dist - :synopsis: Provides the Distribution class, which represents the module distribution being - built/installed/distributed - - -This module provides the :class:`~distutils.core.Distribution` class, which -represents the module distribution being built/installed/distributed. - - -:mod:`distutils.extension` --- The Extension class -================================================== - -.. module:: distutils.extension - :synopsis: Provides the Extension class, used to describe C/C++ extension modules in setup - scripts - - -This module provides the :class:`Extension` class, used to describe C/C++ -extension modules in setup scripts. - -.. % \subsection{Ungrouped modules} -.. % The following haven't been moved into a more appropriate section yet. - - -:mod:`distutils.debug` --- Distutils debug mode -=============================================== - -.. module:: distutils.debug - :synopsis: Provides the debug flag for distutils - - -This module provides the DEBUG flag. - - -:mod:`distutils.errors` --- Distutils exceptions -================================================ - -.. module:: distutils.errors - :synopsis: Provides standard distutils exceptions - - -Provides exceptions used by the Distutils modules. Note that Distutils modules -may raise standard exceptions; in particular, SystemExit is usually raised for -errors that are obviously the end-user's fault (eg. bad command-line arguments). - -This module is safe to use in ``from ... import *`` mode; it only exports -symbols whose names start with ``Distutils`` and end with ``Error``. - - -:mod:`distutils.fancy_getopt` --- Wrapper around the standard getopt module -=========================================================================== - -.. module:: distutils.fancy_getopt - :synopsis: Additional getopt functionality - - -This module provides a wrapper around the standard :mod:`getopt` module that -provides the following additional features: - -* short and long options are tied together - -* options have help strings, so :func:`fancy_getopt` could potentially create a - complete usage summary - -* options set attributes of a passed-in object - -* boolean options can have "negative aliases" --- eg. if :option:`!--quiet` is - the "negative alias" of :option:`!--verbose`, then :option:`!--quiet` on the - command line sets *verbose* to false. - -.. function:: fancy_getopt(options, negative_opt, object, args) - - Wrapper function. *options* is a list of ``(long_option, short_option, - help_string)`` 3-tuples as described in the constructor for - :class:`FancyGetopt`. *negative_opt* should be a dictionary mapping option names - to option names, both the key and value should be in the *options* list. - *object* is an object which will be used to store values (see the :meth:`getopt` - method of the :class:`FancyGetopt` class). *args* is the argument list. Will use - ``sys.argv[1:]`` if you pass ``None`` as *args*. - - -.. function:: wrap_text(text, width) - - Wraps *text* to less than *width* wide. - - -.. class:: FancyGetopt([option_table=None]) - - The option_table is a list of 3-tuples: ``(long_option, short_option, - help_string)`` - - If an option takes an argument, its *long_option* should have ``'='`` appended; - *short_option* should just be a single character, no ``':'`` in any case. - *short_option* should be ``None`` if a *long_option* doesn't have a - corresponding *short_option*. All option tuples must have long options. - -The :class:`FancyGetopt` class provides the following methods: - - -.. method:: FancyGetopt.getopt([args=None, object=None]) - - Parse command-line options in args. Store as attributes on *object*. - - If *args* is ``None`` or not supplied, uses ``sys.argv[1:]``. If *object* is - ``None`` or not supplied, creates a new :class:`OptionDummy` instance, stores - option values there, and returns a tuple ``(args, object)``. If *object* is - supplied, it is modified in place and :func:`getopt` just returns *args*; in - both cases, the returned *args* is a modified copy of the passed-in *args* list, - which is left untouched. - - .. % and args returned are? - - -.. method:: FancyGetopt.get_option_order() - - Returns the list of ``(option, value)`` tuples processed by the previous run of - :meth:`getopt` Raises :exc:`RuntimeError` if :meth:`getopt` hasn't been called - yet. - - -.. method:: FancyGetopt.generate_help([header=None]) - - Generate help text (a list of strings, one per suggested line of output) from - the option table for this :class:`FancyGetopt` object. - - If supplied, prints the supplied *header* at the top of the help. - - -:mod:`distutils.filelist` --- The FileList class -================================================ - -.. module:: distutils.filelist - :synopsis: The FileList class, used for poking about the file system and - building lists of files. - - -This module provides the :class:`FileList` class, used for poking about the -filesystem and building lists of files. - - -:mod:`distutils.log` --- Simple :pep:`282`-style logging -======================================================== - -.. module:: distutils.log - :synopsis: A simple logging mechanism, :pep:`282`-style - - -:mod:`distutils.spawn` --- Spawn a sub-process -============================================== - -.. module:: distutils.spawn - :synopsis: Provides the spawn() function - - -This module provides the :func:`spawn` function, a front-end to various -platform-specific functions for launching another program in a sub-process. -Also provides :func:`find_executable` to search the path for a given executable -name. - - -:mod:`distutils.sysconfig` --- System configuration information -=============================================================== - -.. module:: distutils.sysconfig - :synopsis: Low-level access to configuration information of the Python interpreter. -.. deprecated:: 3.10 - :mod:`distutils.sysconfig` has been merged into :mod:`sysconfig`. -.. moduleauthor:: Fred L. Drake, Jr. <fdrake@acm.org> -.. moduleauthor:: Greg Ward <gward@python.net> -.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org> - - -The :mod:`distutils.sysconfig` module provides access to Python's low-level -configuration information. The specific configuration variables available -depend heavily on the platform and configuration. The specific variables depend -on the build process for the specific version of Python being run; the variables -are those found in the :file:`Makefile` and configuration header that are -installed with Python on Unix systems. The configuration header is called -:file:`pyconfig.h` for Python versions starting with 2.2, and :file:`config.h` -for earlier versions of Python. - -Some additional functions are provided which perform some useful manipulations -for other parts of the :mod:`distutils` package. - - -.. data:: PREFIX - - The result of ``os.path.normpath(sys.prefix)``. - - -.. data:: EXEC_PREFIX - - The result of ``os.path.normpath(sys.exec_prefix)``. - - -.. function:: get_config_var(name) - - Return the value of a single variable. This is equivalent to - ``get_config_vars().get(name)``. - - -.. function:: get_config_vars(...) - - Return a set of variable definitions. If there are no arguments, this returns a - dictionary mapping names of configuration variables to values. If arguments are - provided, they should be strings, and the return value will be a sequence giving - the associated values. If a given name does not have a corresponding value, - ``None`` will be included for that variable. - - -.. function:: get_config_h_filename() - - Return the full path name of the configuration header. For Unix, this will be - the header generated by the :program:`configure` script; for other platforms the - header will have been supplied directly by the Python source distribution. The - file is a platform-specific text file. - - -.. function:: get_makefile_filename() - - Return the full path name of the :file:`Makefile` used to build Python. For - Unix, this will be a file generated by the :program:`configure` script; the - meaning for other platforms will vary. The file is a platform-specific text - file, if it exists. This function is only useful on POSIX platforms. - -The following functions are deprecated together with this module and they -have no direct replacement. - - -.. function:: get_python_inc([plat_specific[, prefix]]) - - Return the directory for either the general or platform-dependent C include - files. If *plat_specific* is true, the platform-dependent include directory is - returned; if false or omitted, the platform-independent directory is returned. - If *prefix* is given, it is used as either the prefix instead of - :const:`PREFIX`, or as the exec-prefix instead of :const:`EXEC_PREFIX` if - *plat_specific* is true. - - -.. function:: get_python_lib([plat_specific[, standard_lib[, prefix]]]) - - Return the directory for either the general or platform-dependent library - installation. If *plat_specific* is true, the platform-dependent include - directory is returned; if false or omitted, the platform-independent directory - is returned. If *prefix* is given, it is used as either the prefix instead of - :const:`PREFIX`, or as the exec-prefix instead of :const:`EXEC_PREFIX` if - *plat_specific* is true. If *standard_lib* is true, the directory for the - standard library is returned rather than the directory for the installation of - third-party extensions. - -The following function is only intended for use within the :mod:`distutils` -package. - - -.. function:: customize_compiler(compiler) - - Do any platform-specific customization of a - :class:`distutils.ccompiler.CCompiler` instance. - - This function is only needed on Unix at this time, but should be called - consistently to support forward-compatibility. It inserts the information that - varies across Unix flavors and is stored in Python's :file:`Makefile`. This - information includes the selected compiler, compiler and linker options, and the - extension used by the linker for shared objects. - -This function is even more special-purpose, and should only be used from -Python's own build procedures. - - -.. function:: set_python_build() - - Inform the :mod:`distutils.sysconfig` module that it is being used as part of - the build process for Python. This changes a lot of relative locations for - files, allowing them to be located in the build area rather than in an installed - Python. - - -:mod:`distutils.text_file` --- The TextFile class -================================================= - -.. module:: distutils.text_file - :synopsis: Provides the TextFile class, a simple interface to text files - - -This module provides the :class:`TextFile` class, which gives an interface to -text files that (optionally) takes care of stripping comments, ignoring blank -lines, and joining lines with backslashes. - - -.. class:: TextFile([filename=None, file=None, **options]) - - This class provides a file-like object that takes care of all the things you - commonly want to do when processing a text file that has some line-by-line - syntax: strip comments (as long as ``#`` is your comment character), skip blank - lines, join adjacent lines by escaping the newline (ie. backslash at end of - line), strip leading and/or trailing whitespace. All of these are optional and - independently controllable. - - The class provides a :meth:`warn` method so you can generate warning messages - that report physical line number, even if the logical line in question spans - multiple physical lines. Also provides :meth:`unreadline` for implementing - line-at-a-time lookahead. - - :class:`TextFile` instances are create with either *filename*, *file*, or both. - :exc:`RuntimeError` is raised if both are ``None``. *filename* should be a - string, and *file* a file object (or something that provides :meth:`readline` - and :meth:`close` methods). It is recommended that you supply at least - *filename*, so that :class:`TextFile` can include it in warning messages. If - *file* is not supplied, :class:`TextFile` creates its own using the - :func:`open` built-in function. - - The options are all boolean, and affect the values returned by :meth:`readline` - - .. tabularcolumns:: |l|L|l| - - +------------------+--------------------------------+---------+ - | option name | description | default | - +==================+================================+=========+ - | *strip_comments* | strip from ``'#'`` to | true | - | | end-of-line, as well as any | | - | | whitespace leading up to the | | - | | ``'#'``\ ---unless it is | | - | | escaped by a backslash | | - +------------------+--------------------------------+---------+ - | *lstrip_ws* | strip leading whitespace from | false | - | | each line before returning it | | - +------------------+--------------------------------+---------+ - | *rstrip_ws* | strip trailing whitespace | true | - | | (including line terminator!) | | - | | from each line before | | - | | returning it. | | - +------------------+--------------------------------+---------+ - | *skip_blanks* | skip lines that are empty | true | - | | \*after\* stripping comments | | - | | and whitespace. (If both | | - | | lstrip_ws and rstrip_ws are | | - | | false, then some lines may | | - | | consist of solely whitespace: | | - | | these will \*not\* be skipped, | | - | | even if *skip_blanks* is | | - | | true.) | | - +------------------+--------------------------------+---------+ - | *join_lines* | if a backslash is the last | false | - | | non-newline character on a | | - | | line after stripping comments | | - | | and whitespace, join the | | - | | following line to it to form | | - | | one logical line; if N | | - | | consecutive lines end with a | | - | | backslash, then N+1 physical | | - | | lines will be joined to form | | - | | one logical line. | | - +------------------+--------------------------------+---------+ - | *collapse_join* | strip leading whitespace from | false | - | | lines that are joined to their | | - | | predecessor; only matters if | | - | | ``(join_lines and not | | - | | lstrip_ws)`` | | - +------------------+--------------------------------+---------+ - - Note that since *rstrip_ws* can strip the trailing newline, the semantics of - :meth:`readline` must differ from those of the built-in file object's - :meth:`readline` method! In particular, :meth:`readline` returns ``None`` for - end-of-file: an empty string might just be a blank line (or an all-whitespace - line), if *rstrip_ws* is true but *skip_blanks* is not. - - - .. method:: TextFile.open(filename) - - Open a new file *filename*. This overrides any *file* or *filename* - constructor arguments. - - - .. method:: TextFile.close() - - Close the current file and forget everything we know about it (including the - filename and the current line number). - - - .. method:: TextFile.warn(msg[,line=None]) - - Print (to stderr) a warning message tied to the current logical line in the - current file. If the current logical line in the file spans multiple physical - lines, the warning refers to the whole range, such as ``"lines 3-5"``. If - *line* is supplied, it overrides the current line number; it may be a list or - tuple to indicate a range of physical lines, or an integer for a single - physical line. - - - .. method:: TextFile.readline() - - Read and return a single logical line from the current file (or from an internal - buffer if lines have previously been "unread" with :meth:`unreadline`). If the - *join_lines* option is true, this may involve reading multiple physical lines - concatenated into a single string. Updates the current line number, so calling - :meth:`warn` after :meth:`readline` emits a warning about the physical line(s) - just read. Returns ``None`` on end-of-file, since the empty string can occur - if *rstrip_ws* is true but *strip_blanks* is not. - - - .. method:: TextFile.readlines() - - Read and return the list of all logical lines remaining in the current file. - This updates the current line number to the last line of the file. - - - .. method:: TextFile.unreadline(line) - - Push *line* (a string) onto an internal buffer that will be checked by future - :meth:`readline` calls. Handy for implementing a parser with line-at-a-time - lookahead. Note that lines that are "unread" with :meth:`unreadline` are not - subsequently re-cleansed (whitespace stripped, or whatever) when read with - :meth:`readline`. If multiple calls are made to :meth:`unreadline` before a call - to :meth:`readline`, the lines will be returned most in most recent first order. - - -:mod:`distutils.version` --- Version number classes -=================================================== - -.. module:: distutils.version - :synopsis: Implements classes that represent module version numbers. - - -.. % todo -.. % \section{Distutils Commands} -.. % -.. % This part of Distutils implements the various Distutils commands, such -.. % as \code{build}, \code{install} \&c. Each command is implemented as a -.. % separate module, with the command name as the name of the module. - - -:mod:`distutils.cmd` --- Abstract base class for Distutils commands -=================================================================== - -.. module:: distutils.cmd - :synopsis: Provides the abstract base class :class:`~distutils.cmd.Command`. This class - is subclassed by the modules in the distutils.command subpackage. - - -This module supplies the abstract base class :class:`Command`. - - -.. class:: Command(dist) - - Abstract base class for defining command classes, the "worker bees" of the - Distutils. A useful analogy for command classes is to think of them as - subroutines with local variables called *options*. The options are declared - in :meth:`initialize_options` and defined (given their final values) in - :meth:`finalize_options`, both of which must be defined by every command - class. The distinction between the two is necessary because option values - might come from the outside world (command line, config file, ...), and any - options dependent on other options must be computed after these outside - influences have been processed --- hence :meth:`finalize_options`. The body - of the subroutine, where it does all its work based on the values of its - options, is the :meth:`run` method, which must also be implemented by every - command class. - - The class constructor takes a single argument *dist*, a - :class:`~distutils.core.Distribution` instance. - - -Creating a new Distutils command -================================ - -This section outlines the steps to create a new Distutils command. - -A new command lives in a module in the :mod:`distutils.command` package. There -is a sample template in that directory called :file:`command_template`. Copy -this file to a new module with the same name as the new command you're -implementing. This module should implement a class with the same name as the -module (and the command). So, for instance, to create the command -``peel_banana`` (so that users can run ``setup.py peel_banana``), you'd copy -:file:`command_template` to :file:`distutils/command/peel_banana.py`, then edit -it so that it's implementing the class :class:`peel_banana`, a subclass of -:class:`distutils.cmd.Command`. - -Subclasses of :class:`Command` must define the following methods. - -.. method:: Command.initialize_options() - - Set default values for all the options that this command supports. Note that - these defaults may be overridden by other commands, by the setup script, by - config files, or by the command-line. Thus, this is not the place to code - dependencies between options; generally, :meth:`initialize_options` - implementations are just a bunch of ``self.foo = None`` assignments. - - -.. method:: Command.finalize_options() - - Set final values for all the options that this command supports. This is - always called as late as possible, ie. after any option assignments from the - command-line or from other commands have been done. Thus, this is the place - to code option dependencies: if *foo* depends on *bar*, then it is safe to - set *foo* from *bar* as long as *foo* still has the same value it was - assigned in :meth:`initialize_options`. - - -.. method:: Command.run() - - A command's raison d'etre: carry out the action it exists to perform, controlled - by the options initialized in :meth:`initialize_options`, customized by other - commands, the setup script, the command-line, and config files, and finalized in - :meth:`finalize_options`. All terminal output and filesystem interaction should - be done by :meth:`run`. - - -.. attribute:: Command.sub_commands - - *sub_commands* formalizes the notion of a "family" of commands, - e.g. ``install`` as the parent with sub-commands ``install_lib``, - ``install_headers``, etc. The parent of a family of commands defines - *sub_commands* as a class attribute; it's a list of 2-tuples ``(command_name, - predicate)``, with *command_name* a string and *predicate* a function, a - string or ``None``. *predicate* is a method of the parent command that - determines whether the corresponding command is applicable in the current - situation. (E.g. ``install_headers`` is only applicable if we have any C - header files to install.) If *predicate* is ``None``, that command is always - applicable. - - *sub_commands* is usually defined at the *end* of a class, because - predicates can be methods of the class, so they must already have been - defined. The canonical example is the :command:`install` command. - - -:mod:`distutils.command` --- Individual Distutils commands -========================================================== - -.. module:: distutils.command - :synopsis: Contains one module for each standard Distutils command. - - -.. % \subsubsection{Individual Distutils commands} -.. % todo - - -:mod:`distutils.command.bdist` --- Build a binary installer -=========================================================== - -.. module:: distutils.command.bdist - :synopsis: Build a binary installer for a package - - -.. % todo - - -:mod:`distutils.command.bdist_packager` --- Abstract base class for packagers -============================================================================= - -.. module:: distutils.command.bdist_packager - :synopsis: Abstract base class for packagers - - -.. % todo - - -:mod:`distutils.command.bdist_dumb` --- Build a "dumb" installer -================================================================ - -.. module:: distutils.command.bdist_dumb - :synopsis: Build a "dumb" installer - a simple archive of files - - -.. % todo - - -:mod:`distutils.command.bdist_rpm` --- Build a binary distribution as a Redhat RPM and SRPM -=========================================================================================== - -.. module:: distutils.command.bdist_rpm - :synopsis: Build a binary distribution as a Redhat RPM and SRPM - - -.. % todo - - -:mod:`distutils.command.sdist` --- Build a source distribution -============================================================== - -.. module:: distutils.command.sdist - :synopsis: Build a source distribution - - -.. % todo - - -:mod:`distutils.command.build` --- Build all files of a package -=============================================================== - -.. module:: distutils.command.build - :synopsis: Build all files of a package - - -.. % todo - - -:mod:`distutils.command.build_clib` --- Build any C libraries in a package -========================================================================== - -.. module:: distutils.command.build_clib - :synopsis: Build any C libraries in a package - - -.. % todo - - -:mod:`distutils.command.build_ext` --- Build any extensions in a package -======================================================================== - -.. module:: distutils.command.build_ext - :synopsis: Build any extensions in a package - - -.. % todo - - -:mod:`distutils.command.build_py` --- Build the .py/.pyc files of a package -=========================================================================== - -.. module:: distutils.command.build_py - :synopsis: Build the .py/.pyc files of a package - - -.. class:: build_py - -.. class:: build_py_2to3 - - Alternative implementation of build_py which also runs the - 2to3 conversion library on each .py file that is going to be - installed. To use this in a setup.py file for a distribution - that is designed to run with both Python 2.x and 3.x, add:: - - try: - from distutils.command.build_py import build_py_2to3 as build_py - except ImportError: - from distutils.command.build_py import build_py - - to your setup.py, and later:: - - cmdclass = {'build_py': build_py} - - to the invocation of setup(). - - -:mod:`distutils.command.build_scripts` --- Build the scripts of a package -========================================================================= - -.. module:: distutils.command.build_scripts - :synopsis: Build the scripts of a package - - -.. % todo - - -:mod:`distutils.command.clean` --- Clean a package build area -============================================================= - -.. module:: distutils.command.clean - :synopsis: Clean a package build area - -This command removes the temporary files created by :command:`build` -and its subcommands, like intermediary compiled object files. With -the ``--all`` option, the complete build directory will be removed. - -Extension modules built :ref:`in place <distutils-build-ext-inplace>` -will not be cleaned, as they are not in the build directory. - - -:mod:`distutils.command.config` --- Perform package configuration -================================================================= - -.. module:: distutils.command.config - :synopsis: Perform package configuration - - -.. % todo - - -:mod:`distutils.command.install` --- Install a package -====================================================== - -.. module:: distutils.command.install - :synopsis: Install a package - - -.. % todo - - -:mod:`distutils.command.install_data` --- Install data files from a package -=========================================================================== - -.. module:: distutils.command.install_data - :synopsis: Install data files from a package - - -.. % todo - - -:mod:`distutils.command.install_headers` --- Install C/C++ header files from a package -====================================================================================== - -.. module:: distutils.command.install_headers - :synopsis: Install C/C++ header files from a package - - -.. % todo - - -:mod:`distutils.command.install_lib` --- Install library files from a package -============================================================================= - -.. module:: distutils.command.install_lib - :synopsis: Install library files from a package - - -.. % todo - - -:mod:`distutils.command.install_scripts` --- Install script files from a package -================================================================================ - -.. module:: distutils.command.install_scripts - :synopsis: Install script files from a package - - -.. % todo - - -:mod:`distutils.command.register` --- Register a module with the Python Package Index -===================================================================================== - -.. module:: distutils.command.register - :synopsis: Register a module with the Python Package Index - - -The ``register`` command registers the package with the Python Package Index. -This is described in more detail in :pep:`301`. - -.. % todo - - -:mod:`distutils.command.check` --- Check the meta-data of a package -=================================================================== - -.. module:: distutils.command.check - :synopsis: Check the meta-data of a package - - -The ``check`` command performs some tests on the meta-data of a package. -For example, it verifies that all required meta-data are provided as -the arguments passed to the :func:`setup` function. - -.. % todo diff --git a/Doc/distutils/builtdist.rst b/Doc/distutils/builtdist.rst deleted file mode 100644 index c1d9ea533dab65..00000000000000 --- a/Doc/distutils/builtdist.rst +++ /dev/null @@ -1,392 +0,0 @@ -.. _built-dist: - -**************************** -Creating Built Distributions -**************************** - -.. include:: ./_setuptools_disclaimer.rst - -A "built distribution" is what you're probably used to thinking of either as a -"binary package" or an "installer" (depending on your background). It's not -necessarily binary, though, because it might contain only Python source code -and/or byte-code; and we don't call it a package, because that word is already -spoken for in Python. (And "installer" is a term specific to the world of -mainstream desktop systems.) - -A built distribution is how you make life as easy as possible for installers of -your module distribution: for users of RPM-based Linux systems, it's a binary -RPM; for Windows users, it's an executable installer; for Debian-based Linux -users, it's a Debian package; and so forth. Obviously, no one person will be -able to create built distributions for every platform under the sun, so the -Distutils are designed to enable module developers to concentrate on their -specialty---writing code and creating source distributions---while an -intermediary species called *packagers* springs up to turn source distributions -into built distributions for as many platforms as there are packagers. - -Of course, the module developer could be their own packager; or the packager could -be a volunteer "out there" somewhere who has access to a platform which the -original developer does not; or it could be software periodically grabbing new -source distributions and turning them into built distributions for as many -platforms as the software has access to. Regardless of who they are, a packager -uses the setup script and the :command:`bdist` command family to generate built -distributions. - -As a simple example, if I run the following command in the Distutils source -tree:: - - python setup.py bdist - -then the Distutils builds my module distribution (the Distutils itself in this -case), does a "fake" installation (also in the :file:`build` directory), and -creates the default type of built distribution for my platform. The default -format for built distributions is a "dumb" tar file on Unix, and a simple -executable installer on Windows. (That tar file is considered "dumb" because it -has to be unpacked in a specific location to work.) - -Thus, the above command on a Unix system creates -:file:`Distutils-1.0.{plat}.tar.gz`; unpacking this tarball from the right place -installs the Distutils just as though you had downloaded the source distribution -and run ``python setup.py install``. (The "right place" is either the root of -the filesystem or Python's :file:`{prefix}` directory, depending on the options -given to the :command:`bdist_dumb` command; the default is to make dumb -distributions relative to :file:`{prefix}`.) - -Obviously, for pure Python distributions, this isn't any simpler than just -running ``python setup.py install``\ ---but for non-pure distributions, which -include extensions that would need to be compiled, it can mean the difference -between someone being able to use your extensions or not. And creating "smart" -built distributions, such as an RPM package or an executable installer for -Windows, is far more convenient for users even if your distribution doesn't -include any extensions. - -The :command:`bdist` command has a :option:`!--formats` option, similar to the -:command:`sdist` command, which you can use to select the types of built -distribution to generate: for example, :: - - python setup.py bdist --format=zip - -would, when run on a Unix system, create -:file:`Distutils-1.0.{plat}.zip`\ ---again, this archive would be unpacked -from the root directory to install the Distutils. - -The available formats for built distributions are: - -+-------------+------------------------------+---------+ -| Format | Description | Notes | -+=============+==============================+=========+ -| ``gztar`` | gzipped tar file | \(1) | -| | (:file:`.tar.gz`) | | -+-------------+------------------------------+---------+ -| ``bztar`` | bzipped tar file | | -| | (:file:`.tar.bz2`) | | -+-------------+------------------------------+---------+ -| ``xztar`` | xzipped tar file | | -| | (:file:`.tar.xz`) | | -+-------------+------------------------------+---------+ -| ``ztar`` | compressed tar file | \(3) | -| | (:file:`.tar.Z`) | | -+-------------+------------------------------+---------+ -| ``tar`` | tar file (:file:`.tar`) | | -+-------------+------------------------------+---------+ -| ``zip`` | zip file (:file:`.zip`) | (2),(4) | -+-------------+------------------------------+---------+ -| ``rpm`` | RPM | \(5) | -+-------------+------------------------------+---------+ -| ``pkgtool`` | Solaris :program:`pkgtool` | | -+-------------+------------------------------+---------+ -| ``sdux`` | HP-UX :program:`swinstall` | | -+-------------+------------------------------+---------+ -| ``msi`` | Microsoft Installer. | | -+-------------+------------------------------+---------+ - -.. versionchanged:: 3.5 - Added support for the ``xztar`` format. - - -Notes: - -(1) - default on Unix - -(2) - default on Windows - -(3) - requires external :program:`compress` utility. - -(4) - requires either external :program:`zip` utility or :mod:`zipfile` module (part - of the standard Python library since Python 1.6) - -(5) - requires external :program:`rpm` utility, version 3.0.4 or better (use ``rpm - --version`` to find out which version you have) - -You don't have to use the :command:`bdist` command with the :option:`!--formats` -option; you can also use the command that directly implements the format you're -interested in. Some of these :command:`bdist` "sub-commands" actually generate -several similar formats; for instance, the :command:`bdist_dumb` command -generates all the "dumb" archive formats (``tar``, ``gztar``, ``bztar``, -``xztar``, ``ztar``, and ``zip``), and :command:`bdist_rpm` generates both -binary and source RPMs. The :command:`bdist` sub-commands, and the formats -generated by each, are: - -+--------------------------+-------------------------------------+ -| Command | Formats | -+==========================+=====================================+ -| :command:`bdist_dumb` | tar, gztar, bztar, xztar, ztar, zip | -+--------------------------+-------------------------------------+ -| :command:`bdist_rpm` | rpm, srpm | -+--------------------------+-------------------------------------+ - -The following sections give details on the individual :command:`bdist_\*` -commands. - - -.. .. _creating-dumb: - -.. Creating dumb built distributions -.. ================================= - -.. XXX Need to document absolute vs. prefix-relative packages here, but first - I have to implement it! - - -.. _creating-rpms: - -Creating RPM packages -===================== - -The RPM format is used by many popular Linux distributions, including Red Hat, -SuSE, and Mandrake. If one of these (or any of the other RPM-based Linux -distributions) is your usual environment, creating RPM packages for other users -of that same distribution is trivial. Depending on the complexity of your module -distribution and differences between Linux distributions, you may also be able -to create RPMs that work on different RPM-based distributions. - -The usual way to create an RPM of your module distribution is to run the -:command:`bdist_rpm` command:: - - python setup.py bdist_rpm - -or the :command:`bdist` command with the :option:`!--format` option:: - - python setup.py bdist --formats=rpm - -The former allows you to specify RPM-specific options; the latter allows you to -easily specify multiple formats in one run. If you need to do both, you can -explicitly specify multiple :command:`bdist_\*` commands and their options:: - - python setup.py bdist_rpm --packager="John Doe <jdoe@example.org>" - -Creating RPM packages is driven by a :file:`.spec` file, much as using the -Distutils is driven by the setup script. To make your life easier, the -:command:`bdist_rpm` command normally creates a :file:`.spec` file based on the -information you supply in the setup script, on the command line, and in any -Distutils configuration files. Various options and sections in the -:file:`.spec` file are derived from options in the setup script as follows: - -+------------------------------------------+----------------------------------------------+ -| RPM :file:`.spec` file option or section | Distutils setup script option | -+==========================================+==============================================+ -| Name | ``name`` | -+------------------------------------------+----------------------------------------------+ -| Summary (in preamble) | ``description`` | -+------------------------------------------+----------------------------------------------+ -| Version | ``version`` | -+------------------------------------------+----------------------------------------------+ -| Vendor | ``author`` and ``author_email``, | -| | or --- & ``maintainer`` and | -| | ``maintainer_email`` | -+------------------------------------------+----------------------------------------------+ -| Copyright | ``license`` | -+------------------------------------------+----------------------------------------------+ -| Url | ``url`` | -+------------------------------------------+----------------------------------------------+ -| %description (section) | ``long_description`` | -+------------------------------------------+----------------------------------------------+ - -Additionally, there are many options in :file:`.spec` files that don't have -corresponding options in the setup script. Most of these are handled through -options to the :command:`bdist_rpm` command as follows: - -+-------------------------------+-----------------------------+-------------------------+ -| RPM :file:`.spec` file option | :command:`bdist_rpm` option | default value | -| or section | | | -+===============================+=============================+=========================+ -| Release | ``release`` | "1" | -+-------------------------------+-----------------------------+-------------------------+ -| Group | ``group`` | "Development/Libraries" | -+-------------------------------+-----------------------------+-------------------------+ -| Vendor | ``vendor`` | (see above) | -+-------------------------------+-----------------------------+-------------------------+ -| Packager | ``packager`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Provides | ``provides`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Requires | ``requires`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Conflicts | ``conflicts`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Obsoletes | ``obsoletes`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Distribution | ``distribution_name`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| BuildRequires | ``build_requires`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ -| Icon | ``icon`` | (none) | -+-------------------------------+-----------------------------+-------------------------+ - -Obviously, supplying even a few of these options on the command-line would be -tedious and error-prone, so it's usually best to put them in the setup -configuration file, :file:`setup.cfg`\ ---see section :ref:`setup-config`. If -you distribute or package many Python module distributions, you might want to -put options that apply to all of them in your personal Distutils configuration -file (:file:`~/.pydistutils.cfg`). If you want to temporarily disable -this file, you can pass the :option:`!--no-user-cfg` option to :file:`setup.py`. - -There are three steps to building a binary RPM package, all of which are -handled automatically by the Distutils: - -#. create a :file:`.spec` file, which describes the package (analogous to the - Distutils setup script; in fact, much of the information in the setup script - winds up in the :file:`.spec` file) - -#. create the source RPM - -#. create the "binary" RPM (which may or may not contain binary code, depending - on whether your module distribution contains Python extensions) - -Normally, RPM bundles the last two steps together; when you use the Distutils, -all three steps are typically bundled together. - -If you wish, you can separate these three steps. You can use the -:option:`!--spec-only` option to make :command:`bdist_rpm` just create the -:file:`.spec` file and exit; in this case, the :file:`.spec` file will be -written to the "distribution directory"---normally :file:`dist/`, but -customizable with the :option:`!--dist-dir` option. (Normally, the :file:`.spec` -file winds up deep in the "build tree," in a temporary directory created by -:command:`bdist_rpm`.) - -.. % \XXX{this isn't implemented yet---is it needed?!} -.. % You can also specify a custom \file{.spec} file with the -.. % \longprogramopt{spec-file} option; used in conjunction with -.. % \longprogramopt{spec-only}, this gives you an opportunity to customize -.. % the \file{.spec} file manually: -.. % -.. % \ begin{verbatim} -.. % > python setup.py bdist_rpm --spec-only -.. % # ...edit dist/FooBar-1.0.spec -.. % > python setup.py bdist_rpm --spec-file=dist/FooBar-1.0.spec -.. % \ end{verbatim} -.. % -.. % (Although a better way to do this is probably to override the standard -.. % \command{bdist\_rpm} command with one that writes whatever else you want -.. % to the \file{.spec} file.) - - -.. _cross-compile-windows: - -Cross-compiling on Windows -========================== - -Starting with Python 2.6, distutils is capable of cross-compiling between -Windows platforms. In practice, this means that with the correct tools -installed, you can use a 32bit version of Windows to create 64bit extensions -and vice-versa. - -To build for an alternate platform, specify the :option:`!--plat-name` option -to the build command. Valid values are currently 'win32', and 'win-amd64'. -For example, on a 32bit version of Windows, you could execute:: - - python setup.py build --plat-name=win-amd64 - -to build a 64bit version of your extension. - -would create a 64bit installation executable on your 32bit version of Windows. - -To cross-compile, you must download the Python source code and cross-compile -Python itself for the platform you are targeting - it is not possible from a -binary installation of Python (as the .lib etc file for other platforms are -not included.) In practice, this means the user of a 32 bit operating -system will need to use Visual Studio 2008 to open the -:file:`PCbuild/PCbuild.sln` solution in the Python source tree and build the -"x64" configuration of the 'pythoncore' project before cross-compiling -extensions is possible. - -Note that by default, Visual Studio 2008 does not install 64bit compilers or -tools. You may need to reexecute the Visual Studio setup process and select -these tools (using Control Panel->[Add/Remove] Programs is a convenient way to -check or modify your existing install.) - -.. _postinstallation-script: - -The Postinstallation script ---------------------------- - -Starting with Python 2.3, a postinstallation script can be specified with the -:option:`!--install-script` option. The basename of the script must be -specified, and the script filename must also be listed in the scripts argument -to the setup function. - -This script will be run at installation time on the target system after all the -files have been copied, with ``argv[1]`` set to :option:`!-install`, and again at -uninstallation time before the files are removed with ``argv[1]`` set to -:option:`!-remove`. - -The installation script runs embedded in the windows installer, every output -(``sys.stdout``, ``sys.stderr``) is redirected into a buffer and will be -displayed in the GUI after the script has finished. - -Some functions especially useful in this context are available as additional -built-in functions in the installation script. - - -.. function:: directory_created(path) - file_created(path) - - These functions should be called when a directory or file is created by the - postinstall script at installation time. It will register *path* with the - uninstaller, so that it will be removed when the distribution is uninstalled. - To be safe, directories are only removed if they are empty. - - -.. function:: get_special_folder_path(csidl_string) - - This function can be used to retrieve special folder locations on Windows like - the Start Menu or the Desktop. It returns the full path to the folder. - *csidl_string* must be one of the following strings:: - - "CSIDL_APPDATA" - - "CSIDL_COMMON_STARTMENU" - "CSIDL_STARTMENU" - - "CSIDL_COMMON_DESKTOPDIRECTORY" - "CSIDL_DESKTOPDIRECTORY" - - "CSIDL_COMMON_STARTUP" - "CSIDL_STARTUP" - - "CSIDL_COMMON_PROGRAMS" - "CSIDL_PROGRAMS" - - "CSIDL_FONTS" - - If the folder cannot be retrieved, :exc:`OSError` is raised. - - Which folders are available depends on the exact Windows version, and probably - also the configuration. For details refer to Microsoft's documentation of the - :c:func:`SHGetSpecialFolderPath` function. - - -.. function:: create_shortcut(target, description, filename[, arguments[, workdir[, iconpath[, iconindex]]]]) - - This function creates a shortcut. *target* is the path to the program to be - started by the shortcut. *description* is the description of the shortcut. - *filename* is the title of the shortcut that the user will see. *arguments* - specifies the command line arguments, if any. *workdir* is the working directory - for the program. *iconpath* is the file containing the icon for the shortcut, - and *iconindex* is the index of the icon in the file *iconpath*. Again, for - details consult the Microsoft documentation for the :class:`IShellLink` - interface. diff --git a/Doc/distutils/commandref.rst b/Doc/distutils/commandref.rst deleted file mode 100644 index 3e247e68d3a05f..00000000000000 --- a/Doc/distutils/commandref.rst +++ /dev/null @@ -1,105 +0,0 @@ -.. _reference: - -***************** -Command Reference -***************** - -.. include:: ./_setuptools_disclaimer.rst - -.. % \section{Building modules: the \protect\command{build} command family} -.. % \label{build-cmds} -.. % \subsubsection{\protect\command{build}} -.. % \label{build-cmd} -.. % \subsubsection{\protect\command{build\_py}} -.. % \label{build-py-cmd} -.. % \subsubsection{\protect\command{build\_ext}} -.. % \label{build-ext-cmd} -.. % \subsubsection{\protect\command{build\_clib}} -.. % \label{build-clib-cmd} - - -.. _install-cmd: - -Installing modules: the :command:`install` command family -========================================================= - -The install command ensures that the build commands have been run and then runs -the subcommands :command:`install_lib`, :command:`install_data` and -:command:`install_scripts`. - -.. % \subsubsection{\protect\command{install\_lib}} -.. % \label{install-lib-cmd} - - -.. _install-data-cmd: - -:command:`install_data` ------------------------ - -This command installs all data files provided with the distribution. - - -.. _install-scripts-cmd: - -:command:`install_scripts` --------------------------- - -This command installs all (Python) scripts in the distribution. - -.. % \subsection{Cleaning up: the \protect\command{clean} command} -.. % \label{clean-cmd} - - -.. _sdist-cmd: - -Creating a source distribution: the :command:`sdist` command -============================================================ - -.. XXX fragment moved down from above: needs context! - -The manifest template commands are: - -+-------------------------------------------+-----------------------------------------------+ -| Command | Description | -+===========================================+===============================================+ -| :command:`include pat1 pat2 ...` | include all files matching any of the listed | -| | patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`exclude pat1 pat2 ...` | exclude all files matching any of the listed | -| | patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`recursive-include dir pat1 pat2 | include all files under *dir* matching any of | -| ...` | the listed patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`recursive-exclude dir pat1 pat2 | exclude all files under *dir* matching any of | -| ...` | the listed patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`global-include pat1 pat2 ...` | include all files anywhere in the source tree | -| | matching --- & any of the listed patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`global-exclude pat1 pat2 ...` | exclude all files anywhere in the source tree | -| | matching --- & any of the listed patterns | -+-------------------------------------------+-----------------------------------------------+ -| :command:`prune dir` | exclude all files under *dir* | -+-------------------------------------------+-----------------------------------------------+ -| :command:`graft dir` | include all files under *dir* | -+-------------------------------------------+-----------------------------------------------+ - -The patterns here are Unix-style "glob" patterns: ``*`` matches any sequence of -regular filename characters, ``?`` matches any single regular filename -character, and ``[range]`` matches any of the characters in *range* (e.g., -``a-z``, ``a-zA-Z``, ``a-f0-9_.``). The definition of "regular filename -character" is platform-specific: on Unix it is anything except slash; on Windows -anything except backslash or colon. - -.. XXX Windows support not there yet - -.. % \section{Creating a built distribution: the -.. % \protect\command{bdist} command family} -.. % \label{bdist-cmds} - -.. % \subsection{\protect\command{bdist}} -.. % \subsection{\protect\command{bdist\_dumb}} -.. % \subsection{\protect\command{bdist\_rpm}} - - diff --git a/Doc/distutils/configfile.rst b/Doc/distutils/configfile.rst deleted file mode 100644 index 2a5c8329e318b7..00000000000000 --- a/Doc/distutils/configfile.rst +++ /dev/null @@ -1,144 +0,0 @@ -.. _setup-config: - -************************************ -Writing the Setup Configuration File -************************************ - -.. include:: ./_setuptools_disclaimer.rst - -Often, it's not possible to write down everything needed to build a distribution -*a priori*: you may need to get some information from the user, or from the -user's system, in order to proceed. As long as that information is fairly -simple---a list of directories to search for C header files or libraries, for -example---then providing a configuration file, :file:`setup.cfg`, for users to -edit is a cheap and easy way to solicit it. Configuration files also let you -provide default values for any command option, which the installer can then -override either on the command-line or by editing the config file. - -The setup configuration file is a useful middle-ground between the setup -script---which, ideally, would be opaque to installers [#]_---and the command-line to -the setup script, which is outside of your control and entirely up to the -installer. In fact, :file:`setup.cfg` (and any other Distutils configuration -files present on the target system) are processed after the contents of the -setup script, but before the command-line. This has several useful -consequences: - -.. % (If you have more advanced needs, such as determining which extensions -.. % to build based on what capabilities are present on the target system, -.. % then you need the Distutils ``auto-configuration'' facility. This -.. % started to appear in Distutils 0.9 but, as of this writing, isn't mature -.. % or stable enough yet for real-world use.) - -* installers can override some of what you put in :file:`setup.py` by editing - :file:`setup.cfg` - -* you can provide non-standard defaults for options that are not easily set in - :file:`setup.py` - -* installers can override anything in :file:`setup.cfg` using the command-line - options to :file:`setup.py` - -The basic syntax of the configuration file is simple: - -.. code-block:: ini - - [command] - option=value - ... - -where *command* is one of the Distutils commands (e.g. :command:`build_py`, -:command:`install`), and *option* is one of the options that command supports. -Any number of options can be supplied for each command, and any number of -command sections can be included in the file. Blank lines are ignored, as are -comments, which run from a ``'#'`` character until the end of the line. Long -option values can be split across multiple lines simply by indenting the -continuation lines. - -You can find out the list of options supported by a particular command with the -universal :option:`!--help` option, e.g. - -.. code-block:: shell-session - - $ python setup.py --help build_ext - [...] - Options for 'build_ext' command: - --build-lib (-b) directory for compiled extension modules - --build-temp (-t) directory for temporary files (build by-products) - --inplace (-i) ignore build-lib and put compiled extensions into the - source directory alongside your pure Python modules - --include-dirs (-I) list of directories to search for header files - --define (-D) C preprocessor macros to define - --undef (-U) C preprocessor macros to undefine - --swig-opts list of SWIG command line options - [...] - -Note that an option spelled :option:`!--foo-bar` on the command-line is spelled -``foo_bar`` in configuration files. - -.. _distutils-build-ext-inplace: - -For example, say you want your extensions to be built "in-place"---that is, you -have an extension :mod:`pkg.ext`, and you want the compiled extension file -(:file:`ext.so` on Unix, say) to be put in the same source directory as your -pure Python modules :mod:`pkg.mod1` and :mod:`pkg.mod2`. You can always use the -:option:`!--inplace` option on the command-line to ensure this: - -.. code-block:: sh - - python setup.py build_ext --inplace - -But this requires that you always specify the :command:`build_ext` command -explicitly, and remember to provide :option:`!--inplace`. An easier way is to -"set and forget" this option, by encoding it in :file:`setup.cfg`, the -configuration file for this distribution: - -.. code-block:: ini - - [build_ext] - inplace=1 - -This will affect all builds of this module distribution, whether or not you -explicitly specify :command:`build_ext`. If you include :file:`setup.cfg` in -your source distribution, it will also affect end-user builds---which is -probably a bad idea for this option, since always building extensions in-place -would break installation of the module distribution. In certain peculiar cases, -though, modules are built right in their installation directory, so this is -conceivably a useful ability. (Distributing extensions that expect to be built -in their installation directory is almost always a bad idea, though.) - -Another example: certain commands take a lot of options that don't change from -run to run; for example, :command:`bdist_rpm` needs to know everything required -to generate a "spec" file for creating an RPM distribution. Some of this -information comes from the setup script, and some is automatically generated by -the Distutils (such as the list of files installed). But some of it has to be -supplied as options to :command:`bdist_rpm`, which would be very tedious to do -on the command-line for every run. Hence, here is a snippet from the Distutils' -own :file:`setup.cfg`: - -.. code-block:: ini - - [bdist_rpm] - release = 1 - packager = Greg Ward <gward@python.net> - doc_files = CHANGES.txt - README.txt - USAGE.txt - doc/ - examples/ - -Note that the ``doc_files`` option is simply a whitespace-separated string -split across multiple lines for readability. - - -.. seealso:: - - :ref:`inst-config-syntax` in "Installing Python Modules" - More information on the configuration files is available in the manual for - system administrators. - - -.. rubric:: Footnotes - -.. [#] This ideal probably won't be achieved until auto-configuration is fully - supported by the Distutils. - diff --git a/Doc/distutils/examples.rst b/Doc/distutils/examples.rst deleted file mode 100644 index 49e4b6e4b984de..00000000000000 --- a/Doc/distutils/examples.rst +++ /dev/null @@ -1,340 +0,0 @@ -.. _distutils_examples: - -****************** -Distutils Examples -****************** - -.. include:: ./_setuptools_disclaimer.rst - -This chapter provides a number of basic examples to help get started with -distutils. Additional information about using distutils can be found in the -Distutils Cookbook. - - -.. seealso:: - - `Distutils Cookbook <https://wiki.python.org/moin/Distutils/Cookbook>`_ - Collection of recipes showing how to achieve more control over distutils. - - -.. _pure-mod: - -Pure Python distribution (by module) -==================================== - -If you're just distributing a couple of modules, especially if they don't live -in a particular package, you can specify them individually using the -``py_modules`` option in the setup script. - -In the simplest case, you'll have two files to worry about: a setup script and -the single module you're distributing, :file:`foo.py` in this example:: - - <root>/ - setup.py - foo.py - -(In all diagrams in this section, *<root>* will refer to the distribution root -directory.) A minimal setup script to describe this situation would be:: - - from distutils.core import setup - setup(name='foo', - version='1.0', - py_modules=['foo'], - ) - -Note that the name of the distribution is specified independently with the -``name`` option, and there's no rule that says it has to be the same as -the name of the sole module in the distribution (although that's probably a good -convention to follow). However, the distribution name is used to generate -filenames, so you should stick to letters, digits, underscores, and hyphens. - -Since ``py_modules`` is a list, you can of course specify multiple -modules, eg. if you're distributing modules :mod:`foo` and :mod:`bar`, your -setup might look like this:: - - <root>/ - setup.py - foo.py - bar.py - -and the setup script might be :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - py_modules=['foo', 'bar'], - ) - -You can put module source files into another directory, but if you have enough -modules to do that, it's probably easier to specify modules by package rather -than listing them individually. - - -.. _pure-pkg: - -Pure Python distribution (by package) -===================================== - -If you have more than a couple of modules to distribute, especially if they are -in multiple packages, it's probably easier to specify whole packages rather than -individual modules. This works even if your modules are not in a package; you -can just tell the Distutils to process modules from the root package, and that -works the same as any other package (except that you don't have to have an -:file:`__init__.py` file). - -The setup script from the last example could also be written as :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - packages=[''], - ) - -(The empty string stands for the root package.) - -If those two files are moved into a subdirectory, but remain in the root -package, e.g.:: - - <root>/ - setup.py - src/ foo.py - bar.py - -then you would still specify the root package, but you have to tell the -Distutils where source files in the root package live:: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - package_dir={'': 'src'}, - packages=[''], - ) - -More typically, though, you will want to distribute multiple modules in the same -package (or in sub-packages). For example, if the :mod:`foo` and :mod:`bar` -modules belong in package :mod:`foobar`, one way to layout your source tree is -:: - - <root>/ - setup.py - foobar/ - __init__.py - foo.py - bar.py - -This is in fact the default layout expected by the Distutils, and the one that -requires the least work to describe in your setup script:: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - packages=['foobar'], - ) - -If you want to put modules in directories not named for their package, then you -need to use the ``package_dir`` option again. For example, if the -:file:`src` directory holds modules in the :mod:`foobar` package:: - - <root>/ - setup.py - src/ - __init__.py - foo.py - bar.py - -an appropriate setup script would be :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - package_dir={'foobar': 'src'}, - packages=['foobar'], - ) - -Or, you might put modules from your main package right in the distribution -root:: - - <root>/ - setup.py - __init__.py - foo.py - bar.py - -in which case your setup script would be :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - package_dir={'foobar': ''}, - packages=['foobar'], - ) - -(The empty string also stands for the current directory.) - -If you have sub-packages, they must be explicitly listed in ``packages``, -but any entries in ``package_dir`` automatically extend to sub-packages. -(In other words, the Distutils does *not* scan your source tree, trying to -figure out which directories correspond to Python packages by looking for -:file:`__init__.py` files.) Thus, if the default layout grows a sub-package:: - - <root>/ - setup.py - foobar/ - __init__.py - foo.py - bar.py - subfoo/ - __init__.py - blah.py - -then the corresponding setup script would be :: - - from distutils.core import setup - setup(name='foobar', - version='1.0', - packages=['foobar', 'foobar.subfoo'], - ) - - -.. _single-ext: - -Single extension module -======================= - -Extension modules are specified using the ``ext_modules`` option. -``package_dir`` has no effect on where extension source files are found; -it only affects the source for pure Python modules. The simplest case, a -single extension module in a single C source file, is:: - - <root>/ - setup.py - foo.c - -If the :mod:`foo` extension belongs in the root package, the setup script for -this could be :: - - from distutils.core import setup - from distutils.extension import Extension - setup(name='foobar', - version='1.0', - ext_modules=[Extension('foo', ['foo.c'])], - ) - -If the extension actually belongs in a package, say :mod:`foopkg`, then - -With exactly the same source tree layout, this extension can be put in the -:mod:`foopkg` package simply by changing the name of the extension:: - - from distutils.core import setup - from distutils.extension import Extension - setup(name='foobar', - version='1.0', - ext_modules=[Extension('foopkg.foo', ['foo.c'])], - ) - -Checking a package -================== - -The ``check`` command allows you to verify if your package meta-data -meet the minimum requirements to build a distribution. - -To run it, just call it using your :file:`setup.py` script. If something is -missing, ``check`` will display a warning. - -Let's take an example with a simple script:: - - from distutils.core import setup - - setup(name='foobar') - -Running the ``check`` command will display some warnings: - -.. code-block:: shell-session - - $ python setup.py check - running check - warning: check: missing required meta-data: version, url - warning: check: missing meta-data: either (author and author_email) or - (maintainer and maintainer_email) should be supplied - - -If you use the reStructuredText syntax in the ``long_description`` field and -`docutils`_ is installed you can check if the syntax is fine with the -``check`` command, using the ``restructuredtext`` option. - -For example, if the :file:`setup.py` script is changed like this:: - - from distutils.core import setup - - desc = """\ - My description - ============== - - This is the description of the ``foobar`` package. - """ - - setup(name='foobar', version='1', author='tarek', - author_email='tarek@ziade.org', - url='http://example.com', long_description=desc) - -Where the long description is broken, ``check`` will be able to detect it -by using the :mod:`docutils` parser: - -.. code-block:: shell-session - - $ python setup.py check --restructuredtext - running check - warning: check: Title underline too short. (line 2) - warning: check: Could not finish the parsing. - -Reading the metadata -===================== - -The :func:`distutils.core.setup` function provides a command-line interface -that allows you to query the metadata fields of a project through the -``setup.py`` script of a given project: - -.. code-block:: shell-session - - $ python setup.py --name - distribute - -This call reads the ``name`` metadata by running the -:func:`distutils.core.setup` function. Although, when a source or binary -distribution is created with Distutils, the metadata fields are written -in a static file called :file:`PKG-INFO`. When a Distutils-based project is -installed in Python, the :file:`PKG-INFO` file is copied alongside the modules -and packages of the distribution under :file:`NAME-VERSION-pyX.X.egg-info`, -where ``NAME`` is the name of the project, ``VERSION`` its version as defined -in the Metadata, and ``pyX.X`` the major and minor version of Python like -``2.7`` or ``3.2``. - -You can read back this static file, by using the -:class:`distutils.dist.DistributionMetadata` class and its -:func:`read_pkg_file` method:: - - >>> from distutils.dist import DistributionMetadata - >>> metadata = DistributionMetadata() - >>> metadata.read_pkg_file(open('distribute-0.6.8-py2.7.egg-info')) - >>> metadata.name - 'distribute' - >>> metadata.version - '0.6.8' - >>> metadata.description - 'Easily download, build, install, upgrade, and uninstall Python packages' - -Notice that the class can also be instantiated with a metadata file path to -loads its values:: - - >>> pkg_info_path = 'distribute-0.6.8-py2.7.egg-info' - >>> DistributionMetadata(pkg_info_path).name - 'distribute' - - -.. % \section{Multiple extension modules} -.. % \label{multiple-ext} - -.. % \section{Putting it all together} - - -.. _docutils: https://docutils.sourceforge.io diff --git a/Doc/distutils/extending.rst b/Doc/distutils/extending.rst deleted file mode 100644 index 1075e81779a7ba..00000000000000 --- a/Doc/distutils/extending.rst +++ /dev/null @@ -1,98 +0,0 @@ -.. _extending-distutils: - -******************* -Extending Distutils -******************* - -.. include:: ./_setuptools_disclaimer.rst - -Distutils can be extended in various ways. Most extensions take the form of new -commands or replacements for existing commands. New commands may be written to -support new types of platform-specific packaging, for example, while -replacements for existing commands may be made to modify details of how the -command operates on a package. - -Most extensions of the distutils are made within :file:`setup.py` scripts that -want to modify existing commands; many simply add a few file extensions that -should be copied into packages in addition to :file:`.py` files as a -convenience. - -Most distutils command implementations are subclasses of the -:class:`distutils.cmd.Command` class. New commands may directly inherit from -:class:`Command`, while replacements often derive from :class:`Command` -indirectly, directly subclassing the command they are replacing. Commands are -required to derive from :class:`Command`. - -.. % \section{Extending existing commands} -.. % \label{extend-existing} - -.. % \section{Writing new commands} -.. % \label{new-commands} -.. % \XXX{Would an uninstall command be a good example here?} - - -Integrating new commands -======================== - -There are different ways to integrate new command implementations into -distutils. The most difficult is to lobby for the inclusion of the new features -in distutils itself, and wait for (and require) a version of Python that -provides that support. This is really hard for many reasons. - -The most common, and possibly the most reasonable for most needs, is to include -the new implementations with your :file:`setup.py` script, and cause the -:func:`distutils.core.setup` function use them:: - - from distutils.command.build_py import build_py as _build_py - from distutils.core import setup - - class build_py(_build_py): - """Specialized Python source builder.""" - - # implement whatever needs to be different... - - setup(cmdclass={'build_py': build_py}, - ...) - -This approach is most valuable if the new implementations must be used to use a -particular package, as everyone interested in the package will need to have the -new command implementation. - -Beginning with Python 2.4, a third option is available, intended to allow new -commands to be added which can support existing :file:`setup.py` scripts without -requiring modifications to the Python installation. This is expected to allow -third-party extensions to provide support for additional packaging systems, but -the commands can be used for anything distutils commands can be used for. A new -configuration option, ``command_packages`` (command-line option -:option:`!--command-packages`), can be used to specify additional packages to be -searched for modules implementing commands. Like all distutils options, this -can be specified on the command line or in a configuration file. This option -can only be set in the ``[global]`` section of a configuration file, or before -any commands on the command line. If set in a configuration file, it can be -overridden from the command line; setting it to an empty string on the command -line causes the default to be used. This should never be set in a configuration -file provided with a package. - -This new option can be used to add any number of packages to the list of -packages searched for command implementations; multiple package names should be -separated by commas. When not specified, the search is only performed in the -:mod:`distutils.command` package. When :file:`setup.py` is run with the option -``--command-packages distcmds,buildcmds``, however, the packages -:mod:`distutils.command`, :mod:`distcmds`, and :mod:`buildcmds` will be searched -in that order. New commands are expected to be implemented in modules of the -same name as the command by classes sharing the same name. Given the example -command line option above, the command :command:`bdist_openpkg` could be -implemented by the class :class:`distcmds.bdist_openpkg.bdist_openpkg` or -:class:`buildcmds.bdist_openpkg.bdist_openpkg`. - - -Adding new distribution types -============================= - -Commands that create distributions (files in the :file:`dist/` directory) need -to add ``(command, filename)`` pairs to ``self.distribution.dist_files`` so that -:command:`upload` can upload it to PyPI. The *filename* in the pair contains no -path information, only the name of the file itself. In dry-run mode, pairs -should still be added to represent what would have been created. - - diff --git a/Doc/distutils/index.rst b/Doc/distutils/index.rst deleted file mode 100644 index 2ccddc38b5f26f..00000000000000 --- a/Doc/distutils/index.rst +++ /dev/null @@ -1,49 +0,0 @@ -.. _distutils-index: - -############################################## - Distributing Python Modules (Legacy version) -############################################## - -:Authors: Greg Ward, Anthony Baxter -:Email: distutils-sig@python.org - -.. seealso:: - - :ref:`distributing-index` - The up to date module distribution documentations - -.. note:: - - The entire ``distutils`` package has been deprecated and will be - removed in Python 3.12. This documentation is retained as a - reference only, and will be removed with the package. See the - :ref:`What's New <distutils-deprecated>` entry for more information. - -.. include:: ./_setuptools_disclaimer.rst - -.. note:: - - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section <https://packaging.python.org/guides/tool-recommendations/>`__ - in the Python Packaging User Guide for more information. - -This document describes the Python Distribution Utilities ("Distutils") from -the module developer's point of view, describing the underlying capabilities -that ``setuptools`` builds on to allow Python developers to make Python modules -and extensions readily available to a wider audience. - -.. toctree:: - :maxdepth: 2 - :numbered: - - introduction.rst - setupscript.rst - configfile.rst - sourcedist.rst - builtdist.rst - examples.rst - extending.rst - commandref.rst - apiref.rst diff --git a/Doc/distutils/introduction.rst b/Doc/distutils/introduction.rst deleted file mode 100644 index 87ed178e52bd45..00000000000000 --- a/Doc/distutils/introduction.rst +++ /dev/null @@ -1,203 +0,0 @@ -.. _distutils-intro: - -**************************** -An Introduction to Distutils -**************************** - -.. include:: ./_setuptools_disclaimer.rst - -This document covers using the Distutils to distribute your Python modules, -concentrating on the role of developer/distributor: if you're looking for -information on installing Python modules, you should refer to the -:ref:`install-index` chapter. - - -.. _distutils-concepts: - -Concepts & Terminology -====================== - -Using the Distutils is quite simple, both for module developers and for -users/administrators installing third-party modules. As a developer, your -responsibilities (apart from writing solid, well-documented and well-tested -code, of course!) are: - -* write a setup script (:file:`setup.py` by convention) - -* (optional) write a setup configuration file - -* create a source distribution - -* (optional) create one or more built (binary) distributions - -Each of these tasks is covered in this document. - -Not all module developers have access to a multitude of platforms, so it's not -always feasible to expect them to create a multitude of built distributions. It -is hoped that a class of intermediaries, called *packagers*, will arise to -address this need. Packagers will take source distributions released by module -developers, build them on one or more platforms, and release the resulting built -distributions. Thus, users on the most popular platforms will be able to -install most popular Python module distributions in the most natural way for -their platform, without having to run a single setup script or compile a line of -code. - - -.. _distutils-simple-example: - -A Simple Example -================ - -The setup script is usually quite simple, although since it's written in Python, -there are no arbitrary limits to what you can do with it, though you should be -careful about putting arbitrarily expensive operations in your setup script. -Unlike, say, Autoconf-style configure scripts, the setup script may be run -multiple times in the course of building and installing your module -distribution. - -If all you want to do is distribute a module called :mod:`foo`, contained in a -file :file:`foo.py`, then your setup script can be as simple as this:: - - from distutils.core import setup - setup(name='foo', - version='1.0', - py_modules=['foo'], - ) - -Some observations: - -* most information that you supply to the Distutils is supplied as keyword - arguments to the :func:`setup` function - -* those keyword arguments fall into two categories: package metadata (name, - version number) and information about what's in the package (a list of pure - Python modules, in this case) - -* modules are specified by module name, not filename (the same will hold true - for packages and extensions) - -* it's recommended that you supply a little more metadata, in particular your - name, email address and a URL for the project (see section :ref:`setup-script` - for an example) - -To create a source distribution for this module, you would create a setup -script, :file:`setup.py`, containing the above code, and run this command from a -terminal:: - - python setup.py sdist - -For Windows, open a command prompt window (:menuselection:`Start --> -Accessories`) and change the command to:: - - setup.py sdist - -:command:`sdist` will create an archive file (e.g., tarball on Unix, ZIP file on Windows) -containing your setup script :file:`setup.py`, and your module :file:`foo.py`. -The archive file will be named :file:`foo-1.0.tar.gz` (or :file:`.zip`), and -will unpack into a directory :file:`foo-1.0`. - -If an end-user wishes to install your :mod:`foo` module, all they have to do is -download :file:`foo-1.0.tar.gz` (or :file:`.zip`), unpack it, and---from the -:file:`foo-1.0` directory---run :: - - python setup.py install - -which will ultimately copy :file:`foo.py` to the appropriate directory for -third-party modules in their Python installation. - -This simple example demonstrates some fundamental concepts of the Distutils. -First, both developers and installers have the same basic user interface, i.e. -the setup script. The difference is which Distutils *commands* they use: the -:command:`sdist` command is almost exclusively for module developers, while -:command:`install` is more often for installers (although most developers will -want to install their own code occasionally). - -Other useful built distribution formats are RPM, implemented by the -:command:`bdist_rpm` command, Solaris :program:`pkgtool` -(:command:`bdist_pkgtool`), and HP-UX :program:`swinstall` -(:command:`bdist_sdux`). For example, the following command will create an RPM -file called :file:`foo-1.0.noarch.rpm`:: - - python setup.py bdist_rpm - -(The :command:`bdist_rpm` command uses the :command:`rpm` executable, therefore -this has to be run on an RPM-based system such as Red Hat Linux, SuSE Linux, or -Mandrake Linux.) - -You can find out what distribution formats are available at any time by running -:: - - python setup.py bdist --help-formats - - -.. _python-terms: - -General Python terminology -========================== - -If you're reading this document, you probably have a good idea of what modules, -extensions, and so forth are. Nevertheless, just to be sure that everyone is -operating from a common starting point, we offer the following glossary of -common Python terms: - -module - the basic unit of code reusability in Python: a block of code imported by some - other code. Three types of modules concern us here: pure Python modules, - extension modules, and packages. - -pure Python module - a module written in Python and contained in a single :file:`.py` file (and - possibly associated :file:`.pyc` files). Sometimes referred to as a - "pure module." - -extension module - a module written in the low-level language of the Python implementation: C/C++ - for Python, Java for Jython. Typically contained in a single dynamically - loadable pre-compiled file, e.g. a shared object (:file:`.so`) file for Python - extensions on Unix, a DLL (given the :file:`.pyd` extension) for Python - extensions on Windows, or a Java class file for Jython extensions. (Note that - currently, the Distutils only handles C/C++ extensions for Python.) - -package - a module that contains other modules; typically contained in a directory in the - filesystem and distinguished from other directories by the presence of a file - :file:`__init__.py`. - -root package - the root of the hierarchy of packages. (This isn't really a package, since it - doesn't have an :file:`__init__.py` file. But we have to call it something.) - The vast majority of the standard library is in the root package, as are many - small, standalone third-party modules that don't belong to a larger module - collection. Unlike regular packages, modules in the root package can be found in - many directories: in fact, every directory listed in ``sys.path`` contributes - modules to the root package. - - -.. _distutils-term: - -Distutils-specific terminology -============================== - -The following terms apply more specifically to the domain of distributing Python -modules using the Distutils: - -module distribution - a collection of Python modules distributed together as a single downloadable - resource and meant to be installed *en masse*. Examples of some well-known - module distributions are NumPy, SciPy, Pillow, - or mxBase. (This would be called a *package*, except that term is - already taken in the Python context: a single module distribution may contain - zero, one, or many Python packages.) - -pure module distribution - a module distribution that contains only pure Python modules and packages. - Sometimes referred to as a "pure distribution." - -non-pure module distribution - a module distribution that contains at least one extension module. Sometimes - referred to as a "non-pure distribution." - -distribution root - the top-level directory of your source tree (or source distribution); the - directory where :file:`setup.py` exists. Generally :file:`setup.py` will be - run from this directory. diff --git a/Doc/distutils/packageindex.rst b/Doc/distutils/packageindex.rst deleted file mode 100644 index ccb9a598b2b7a2..00000000000000 --- a/Doc/distutils/packageindex.rst +++ /dev/null @@ -1,16 +0,0 @@ -:orphan: - -.. _package-index: - -******************************* -The Python Package Index (PyPI) -******************************* - -The `Python Package Index (PyPI)`_ stores metadata describing distributions -packaged with distutils and other publishing tools, as well the distribution -archives themselves. - -References to up to date PyPI documentation can be found at -:ref:`publishing-python-packages`. - -.. _Python Package Index (PyPI): https://pypi.org diff --git a/Doc/distutils/setupscript.rst b/Doc/distutils/setupscript.rst deleted file mode 100644 index 8635c911622b2f..00000000000000 --- a/Doc/distutils/setupscript.rst +++ /dev/null @@ -1,713 +0,0 @@ -.. _setup-script: - -************************ -Writing the Setup Script -************************ - -.. include:: ./_setuptools_disclaimer.rst - -The setup script is the centre of all activity in building, distributing, and -installing modules using the Distutils. The main purpose of the setup script is -to describe your module distribution to the Distutils, so that the various -commands that operate on your modules do the right thing. As we saw in section -:ref:`distutils-simple-example` above, the setup script consists mainly of a call to -:func:`setup`, and most information supplied to the Distutils by the module -developer is supplied as keyword arguments to :func:`setup`. - -Here's a slightly more involved example, which we'll follow for the next couple -of sections: the Distutils' own setup script. (Keep in mind that although the -Distutils are included with Python 1.6 and later, they also have an independent -existence so that Python 1.5.2 users can use them to install other module -distributions. The Distutils' own setup script, shown here, is used to install -the package into Python 1.5.2.) :: - - #!/usr/bin/env python - - from distutils.core import setup - - setup(name='Distutils', - version='1.0', - description='Python Distribution Utilities', - author='Greg Ward', - author_email='gward@python.net', - url='https://www.python.org/sigs/distutils-sig/', - packages=['distutils', 'distutils.command'], - ) - -There are only two differences between this and the trivial one-file -distribution presented in section :ref:`distutils-simple-example`: more metadata, and the -specification of pure Python modules by package, rather than by module. This is -important since the Distutils consist of a couple of dozen modules split into -(so far) two packages; an explicit list of every module would be tedious to -generate and difficult to maintain. For more information on the additional -meta-data, see section :ref:`meta-data`. - -Note that any pathnames (files or directories) supplied in the setup script -should be written using the Unix convention, i.e. slash-separated. The -Distutils will take care of converting this platform-neutral representation into -whatever is appropriate on your current platform before actually using the -pathname. This makes your setup script portable across operating systems, which -of course is one of the major goals of the Distutils. In this spirit, all -pathnames in this document are slash-separated. - -This, of course, only applies to pathnames given to Distutils functions. If -you, for example, use standard Python functions such as :func:`glob.glob` or -:func:`os.listdir` to specify files, you should be careful to write portable -code instead of hardcoding path separators:: - - glob.glob(os.path.join('mydir', 'subdir', '*.html')) - os.listdir(os.path.join('mydir', 'subdir')) - - -.. _listing-packages: - -Listing whole packages -====================== - -The ``packages`` option tells the Distutils to process (build, distribute, -install, etc.) all pure Python modules found in each package mentioned in the -``packages`` list. In order to do this, of course, there has to be a -correspondence between package names and directories in the filesystem. The -default correspondence is the most obvious one, i.e. package :mod:`distutils` is -found in the directory :file:`distutils` relative to the distribution root. -Thus, when you say ``packages = ['foo']`` in your setup script, you are -promising that the Distutils will find a file :file:`foo/__init__.py` (which -might be spelled differently on your system, but you get the idea) relative to -the directory where your setup script lives. If you break this promise, the -Distutils will issue a warning but still process the broken package anyway. - -If you use a different convention to lay out your source directory, that's no -problem: you just have to supply the ``package_dir`` option to tell the -Distutils about your convention. For example, say you keep all Python source -under :file:`lib`, so that modules in the "root package" (i.e., not in any -package at all) are in :file:`lib`, modules in the :mod:`foo` package are in -:file:`lib/foo`, and so forth. Then you would put :: - - package_dir = {'': 'lib'} - -in your setup script. The keys to this dictionary are package names, and an -empty package name stands for the root package. The values are directory names -relative to your distribution root. In this case, when you say ``packages = -['foo']``, you are promising that the file :file:`lib/foo/__init__.py` exists. - -Another possible convention is to put the :mod:`foo` package right in -:file:`lib`, the :mod:`foo.bar` package in :file:`lib/bar`, etc. This would be -written in the setup script as :: - - package_dir = {'foo': 'lib'} - -A ``package: dir`` entry in the ``package_dir`` dictionary implicitly -applies to all packages below *package*, so the :mod:`foo.bar` case is -automatically handled here. In this example, having ``packages = ['foo', -'foo.bar']`` tells the Distutils to look for :file:`lib/__init__.py` and -:file:`lib/bar/__init__.py`. (Keep in mind that although ``package_dir`` -applies recursively, you must explicitly list all packages in -``packages``: the Distutils will *not* recursively scan your source tree -looking for any directory with an :file:`__init__.py` file.) - - -.. _listing-modules: - -Listing individual modules -========================== - -For a small module distribution, you might prefer to list all modules rather -than listing packages---especially the case of a single module that goes in the -"root package" (i.e., no package at all). This simplest case was shown in -section :ref:`distutils-simple-example`; here is a slightly more involved example:: - - py_modules = ['mod1', 'pkg.mod2'] - -This describes two modules, one of them in the "root" package, the other in the -:mod:`pkg` package. Again, the default package/directory layout implies that -these two modules can be found in :file:`mod1.py` and :file:`pkg/mod2.py`, and -that :file:`pkg/__init__.py` exists as well. And again, you can override the -package/directory correspondence using the ``package_dir`` option. - - -.. _describing-extensions: - -Describing extension modules -============================ - -Just as writing Python extension modules is a bit more complicated than writing -pure Python modules, describing them to the Distutils is a bit more complicated. -Unlike pure modules, it's not enough just to list modules or packages and expect -the Distutils to go out and find the right files; you have to specify the -extension name, source file(s), and any compile/link requirements (include -directories, libraries to link with, etc.). - -.. XXX read over this section - -All of this is done through another keyword argument to :func:`setup`, the -``ext_modules`` option. ``ext_modules`` is just a list of -:class:`~distutils.core.Extension` instances, each of which describes a -single extension module. -Suppose your distribution includes a single extension, called :mod:`foo` and -implemented by :file:`foo.c`. If no additional instructions to the -compiler/linker are needed, describing this extension is quite simple:: - - Extension('foo', ['foo.c']) - -The :class:`Extension` class can be imported from :mod:`distutils.core` along -with :func:`setup`. Thus, the setup script for a module distribution that -contains only this one extension and nothing else might be:: - - from distutils.core import setup, Extension - setup(name='foo', - version='1.0', - ext_modules=[Extension('foo', ['foo.c'])], - ) - -The :class:`Extension` class (actually, the underlying extension-building -machinery implemented by the :command:`build_ext` command) supports a great deal -of flexibility in describing Python extensions, which is explained in the -following sections. - - -Extension names and packages ----------------------------- - -The first argument to the :class:`~distutils.core.Extension` constructor is -always the name of the extension, including any package names. For example, :: - - Extension('foo', ['src/foo1.c', 'src/foo2.c']) - -describes an extension that lives in the root package, while :: - - Extension('pkg.foo', ['src/foo1.c', 'src/foo2.c']) - -describes the same extension in the :mod:`pkg` package. The source files and -resulting object code are identical in both cases; the only difference is where -in the filesystem (and therefore where in Python's namespace hierarchy) the -resulting extension lives. - -If you have a number of extensions all in the same package (or all under the -same base package), use the ``ext_package`` keyword argument to -:func:`setup`. For example, :: - - setup(..., - ext_package='pkg', - ext_modules=[Extension('foo', ['foo.c']), - Extension('subpkg.bar', ['bar.c'])], - ) - -will compile :file:`foo.c` to the extension :mod:`pkg.foo`, and :file:`bar.c` to -:mod:`pkg.subpkg.bar`. - - -Extension source files ----------------------- - -The second argument to the :class:`~distutils.core.Extension` constructor is -a list of source -files. Since the Distutils currently only support C, C++, and Objective-C -extensions, these are normally C/C++/Objective-C source files. (Be sure to use -appropriate extensions to distinguish C++ source files: :file:`.cc` and -:file:`.cpp` seem to be recognized by both Unix and Windows compilers.) - -However, you can also include SWIG interface (:file:`.i`) files in the list; the -:command:`build_ext` command knows how to deal with SWIG extensions: it will run -SWIG on the interface file and compile the resulting C/C++ file into your -extension. - -.. XXX SWIG support is rough around the edges and largely untested! - -This warning notwithstanding, options to SWIG can be currently passed like -this:: - - setup(..., - ext_modules=[Extension('_foo', ['foo.i'], - swig_opts=['-modern', '-I../include'])], - py_modules=['foo'], - ) - -Or on the commandline like this:: - - > python setup.py build_ext --swig-opts="-modern -I../include" - -On some platforms, you can include non-source files that are processed by the -compiler and included in your extension. Currently, this just means Windows -message text (:file:`.mc`) files and resource definition (:file:`.rc`) files for -Visual C++. These will be compiled to binary resource (:file:`.res`) files and -linked into the executable. - - -Preprocessor options --------------------- - -Three optional arguments to :class:`~distutils.core.Extension` will help if -you need to specify include directories to search or preprocessor macros to -define/undefine: ``include_dirs``, ``define_macros``, and ``undef_macros``. - -For example, if your extension requires header files in the :file:`include` -directory under your distribution root, use the ``include_dirs`` option:: - - Extension('foo', ['foo.c'], include_dirs=['include']) - -You can specify absolute directories there; if you know that your extension will -only be built on Unix systems with X11R6 installed to :file:`/usr`, you can get -away with :: - - Extension('foo', ['foo.c'], include_dirs=['/usr/include/X11']) - -You should avoid this sort of non-portable usage if you plan to distribute your -code: it's probably better to write C code like :: - - #include <X11/Xlib.h> - -If you need to include header files from some other Python extension, you can -take advantage of the fact that header files are installed in a consistent way -by the Distutils :command:`install_headers` command. For example, the Numerical -Python header files are installed (on a standard Unix installation) to -:file:`/usr/local/include/python1.5/Numerical`. (The exact location will differ -according to your platform and Python installation.) Since the Python include -directory---\ :file:`/usr/local/include/python1.5` in this case---is always -included in the search path when building Python extensions, the best approach -is to write C code like :: - - #include <Numerical/arrayobject.h> - -If you must put the :file:`Numerical` include directory right into your header -search path, though, you can find that directory using the Distutils -:mod:`distutils.sysconfig` module:: - - from distutils.sysconfig import get_python_inc - incdir = os.path.join(get_python_inc(plat_specific=1), 'Numerical') - setup(..., - Extension(..., include_dirs=[incdir]), - ) - -Even though this is quite portable---it will work on any Python installation, -regardless of platform---it's probably easier to just write your C code in the -sensible way. - -You can define and undefine pre-processor macros with the ``define_macros`` and -``undef_macros`` options. ``define_macros`` takes a list of ``(name, value)`` -tuples, where ``name`` is the name of the macro to define (a string) and -``value`` is its value: either a string or ``None``. (Defining a macro ``FOO`` -to ``None`` is the equivalent of a bare ``#define FOO`` in your C source: with -most compilers, this sets ``FOO`` to the string ``1``.) ``undef_macros`` is -just a list of macros to undefine. - -For example:: - - Extension(..., - define_macros=[('NDEBUG', '1'), - ('HAVE_STRFTIME', None)], - undef_macros=['HAVE_FOO', 'HAVE_BAR']) - -is the equivalent of having this at the top of every C source file:: - - #define NDEBUG 1 - #define HAVE_STRFTIME - #undef HAVE_FOO - #undef HAVE_BAR - - -Library options ---------------- - -You can also specify the libraries to link against when building your extension, -and the directories to search for those libraries. The ``libraries`` option is -a list of libraries to link against, ``library_dirs`` is a list of directories -to search for libraries at link-time, and ``runtime_library_dirs`` is a list of -directories to search for shared (dynamically loaded) libraries at run-time. - -For example, if you need to link against libraries known to be in the standard -library search path on target systems :: - - Extension(..., - libraries=['gdbm', 'readline']) - -If you need to link with libraries in a non-standard location, you'll have to -include the location in ``library_dirs``:: - - Extension(..., - library_dirs=['/usr/X11R6/lib'], - libraries=['X11', 'Xt']) - -(Again, this sort of non-portable construct should be avoided if you intend to -distribute your code.) - -.. XXX Should mention clib libraries here or somewhere else! - - -Other options -------------- - -There are still some other options which can be used to handle special cases. - -The ``optional`` option is a boolean; if it is true, -a build failure in the extension will not abort the build process, but -instead simply not install the failing extension. - -The ``extra_objects`` option is a list of object files to be passed to the -linker. These files must not have extensions, as the default extension for the -compiler is used. - -``extra_compile_args`` and ``extra_link_args`` can be used to -specify additional command line options for the respective compiler and linker -command lines. - -``export_symbols`` is only useful on Windows. It can contain a list of -symbols (functions or variables) to be exported. This option is not needed when -building compiled extensions: Distutils will automatically add ``initmodule`` -to the list of exported symbols. - -The ``depends`` option is a list of files that the extension depends on -(for example header files). The build command will call the compiler on the -sources to rebuild extension if any on this files has been modified since the -previous build. - -Relationships between Distributions and Packages -================================================ - -A distribution may relate to packages in three specific ways: - -#. It can require packages or modules. - -#. It can provide packages or modules. - -#. It can obsolete packages or modules. - -These relationships can be specified using keyword arguments to the -:func:`distutils.core.setup` function. - -Dependencies on other Python modules and packages can be specified by supplying -the *requires* keyword argument to :func:`setup`. The value must be a list of -strings. Each string specifies a package that is required, and optionally what -versions are sufficient. - -To specify that any version of a module or package is required, the string -should consist entirely of the module or package name. Examples include -``'mymodule'`` and ``'xml.parsers.expat'``. - -If specific versions are required, a sequence of qualifiers can be supplied in -parentheses. Each qualifier may consist of a comparison operator and a version -number. The accepted comparison operators are:: - - < > == - <= >= != - -These can be combined by using multiple qualifiers separated by commas (and -optional whitespace). In this case, all of the qualifiers must be matched; a -logical AND is used to combine the evaluations. - -Let's look at a bunch of examples: - -+-------------------------+----------------------------------------------+ -| Requires Expression | Explanation | -+=========================+==============================================+ -| ``==1.0`` | Only version ``1.0`` is compatible | -+-------------------------+----------------------------------------------+ -| ``>1.0, !=1.5.1, <2.0`` | Any version after ``1.0`` and before ``2.0`` | -| | is compatible, except ``1.5.1`` | -+-------------------------+----------------------------------------------+ - -Now that we can specify dependencies, we also need to be able to specify what we -provide that other distributions can require. This is done using the *provides* -keyword argument to :func:`setup`. The value for this keyword is a list of -strings, each of which names a Python module or package, and optionally -identifies the version. If the version is not specified, it is assumed to match -that of the distribution. - -Some examples: - -+---------------------+----------------------------------------------+ -| Provides Expression | Explanation | -+=====================+==============================================+ -| ``mypkg`` | Provide ``mypkg``, using the distribution | -| | version | -+---------------------+----------------------------------------------+ -| ``mypkg (1.1)`` | Provide ``mypkg`` version 1.1, regardless of | -| | the distribution version | -+---------------------+----------------------------------------------+ - -A package can declare that it obsoletes other packages using the *obsoletes* -keyword argument. The value for this is similar to that of the *requires* -keyword: a list of strings giving module or package specifiers. Each specifier -consists of a module or package name optionally followed by one or more version -qualifiers. Version qualifiers are given in parentheses after the module or -package name. - -The versions identified by the qualifiers are those that are obsoleted by the -distribution being described. If no qualifiers are given, all versions of the -named module or package are understood to be obsoleted. - -.. _distutils-installing-scripts: - -Installing Scripts -================== - -So far we have been dealing with pure and non-pure Python modules, which are -usually not run by themselves but imported by scripts. - -Scripts are files containing Python source code, intended to be started from the -command line. Scripts don't require Distutils to do anything very complicated. -The only clever feature is that if the first line of the script starts with -``#!`` and contains the word "python", the Distutils will adjust the first line -to refer to the current interpreter location. By default, it is replaced with -the current interpreter location. The :option:`!--executable` (or :option:`!-e`) -option will allow the interpreter path to be explicitly overridden. - -The ``scripts`` option simply is a list of files to be handled in this -way. From the PyXML setup script:: - - setup(..., - scripts=['scripts/xmlproc_parse', 'scripts/xmlproc_val'] - ) - -.. versionchanged:: 3.1 - All the scripts will also be added to the ``MANIFEST`` file if no template is - provided. See :ref:`manifest`. - - -.. _distutils-installing-package-data: - -Installing Package Data -======================= - -Often, additional files need to be installed into a package. These files are -often data that's closely related to the package's implementation, or text files -containing documentation that might be of interest to programmers using the -package. These files are called :dfn:`package data`. - -Package data can be added to packages using the ``package_data`` keyword -argument to the :func:`setup` function. The value must be a mapping from -package name to a list of relative path names that should be copied into the -package. The paths are interpreted as relative to the directory containing the -package (information from the ``package_dir`` mapping is used if appropriate); -that is, the files are expected to be part of the package in the source -directories. They may contain glob patterns as well. - -The path names may contain directory portions; any necessary directories will be -created in the installation. - -For example, if a package should contain a subdirectory with several data files, -the files can be arranged like this in the source tree:: - - setup.py - src/ - mypkg/ - __init__.py - module.py - data/ - tables.dat - spoons.dat - forks.dat - -The corresponding call to :func:`setup` might be:: - - setup(..., - packages=['mypkg'], - package_dir={'mypkg': 'src/mypkg'}, - package_data={'mypkg': ['data/*.dat']}, - ) - - -.. versionchanged:: 3.1 - All the files that match ``package_data`` will be added to the ``MANIFEST`` - file if no template is provided. See :ref:`manifest`. - - -.. _distutils-additional-files: - -Installing Additional Files -=========================== - -The ``data_files`` option can be used to specify additional files needed -by the module distribution: configuration files, message catalogs, data files, -anything which doesn't fit in the previous categories. - -``data_files`` specifies a sequence of (*directory*, *files*) pairs in the -following way:: - - setup(..., - data_files=[('bitmaps', ['bm/b1.gif', 'bm/b2.gif']), - ('config', ['cfg/data.cfg'])], - ) - -Each (*directory*, *files*) pair in the sequence specifies the installation -directory and the files to install there. - -Each file name in *files* is interpreted relative to the :file:`setup.py` -script at the top of the package source distribution. Note that you can -specify the directory where the data files will be installed, but you cannot -rename the data files themselves. - -The *directory* should be a relative path. It is interpreted relative to the -installation prefix (Python's ``sys.prefix`` for system installations; -``site.USER_BASE`` for user installations). Distutils allows *directory* to be -an absolute installation path, but this is discouraged since it is -incompatible with the wheel packaging format. No directory information from -*files* is used to determine the final location of the installed file; only -the name of the file is used. - -You can specify the ``data_files`` options as a simple sequence of files -without specifying a target directory, but this is not recommended, and the -:command:`install` command will print a warning in this case. To install data -files directly in the target directory, an empty string should be given as the -directory. - -.. versionchanged:: 3.1 - All the files that match ``data_files`` will be added to the ``MANIFEST`` - file if no template is provided. See :ref:`manifest`. - - -.. _meta-data: - -Additional meta-data -==================== - -The setup script may include additional meta-data beyond the name and version. -This information includes: - -+----------------------+---------------------------+-----------------+--------+ -| Meta-Data | Description | Value | Notes | -+======================+===========================+=================+========+ -| ``name`` | name of the package | short string | \(1) | -+----------------------+---------------------------+-----------------+--------+ -| ``version`` | version of this release | short string | (1)(2) | -+----------------------+---------------------------+-----------------+--------+ -| ``author`` | package author's name | short string | \(3) | -+----------------------+---------------------------+-----------------+--------+ -| ``author_email`` | email address of the | email address | \(3) | -| | package author | | | -+----------------------+---------------------------+-----------------+--------+ -| ``maintainer`` | package maintainer's name | short string | \(3) | -+----------------------+---------------------------+-----------------+--------+ -| ``maintainer_email`` | email address of the | email address | \(3) | -| | package maintainer | | | -+----------------------+---------------------------+-----------------+--------+ -| ``url`` | home page for the package | URL | \(1) | -+----------------------+---------------------------+-----------------+--------+ -| ``description`` | short, summary | short string | | -| | description of the | | | -| | package | | | -+----------------------+---------------------------+-----------------+--------+ -| ``long_description`` | longer description of the | long string | \(4) | -| | package | | | -+----------------------+---------------------------+-----------------+--------+ -| ``download_url`` | location where the | URL | | -| | package may be downloaded | | | -+----------------------+---------------------------+-----------------+--------+ -| ``classifiers`` | a list of classifiers | list of strings | (6)(7) | -+----------------------+---------------------------+-----------------+--------+ -| ``platforms`` | a list of platforms | list of strings | (6)(8) | -+----------------------+---------------------------+-----------------+--------+ -| ``keywords`` | a list of keywords | list of strings | (6)(8) | -+----------------------+---------------------------+-----------------+--------+ -| ``license`` | license for the package | short string | \(5) | -+----------------------+---------------------------+-----------------+--------+ - -Notes: - -(1) - These fields are required. - -(2) - It is recommended that versions take the form *major.minor[.patch[.sub]]*. - -(3) - Either the author or the maintainer must be identified. If maintainer is - provided, distutils lists it as the author in :file:`PKG-INFO`. - -(4) - The ``long_description`` field is used by PyPI when you publish a package, - to build its project page. - -(5) - The ``license`` field is a text indicating the license covering the - package where the license is not a selection from the "License" Trove - classifiers. See the ``Classifier`` field. Notice that - there's a ``licence`` distribution option which is deprecated but still - acts as an alias for ``license``. - -(6) - This field must be a list. - -(7) - The valid classifiers are listed on - `PyPI <https://pypi.org/classifiers>`_. - -(8) - To preserve backward compatibility, this field also accepts a string. If - you pass a comma-separated string ``'foo, bar'``, it will be converted to - ``['foo', 'bar']``, Otherwise, it will be converted to a list of one - string. - -'short string' - A single line of text, not more than 200 characters. - -'long string' - Multiple lines of plain text in reStructuredText format (see - https://docutils.sourceforge.io/). - -'list of strings' - See below. - -Encoding the version information is an art in itself. Python packages generally -adhere to the version format *major.minor[.patch][sub]*. The major number is 0 -for initial, experimental releases of software. It is incremented for releases -that represent major milestones in a package. The minor number is incremented -when important new features are added to the package. The patch number -increments when bug-fix releases are made. Additional trailing version -information is sometimes used to indicate sub-releases. These are -"a1,a2,...,aN" (for alpha releases, where functionality and API may change), -"b1,b2,...,bN" (for beta releases, which only fix bugs) and "pr1,pr2,...,prN" -(for final pre-release release testing). Some examples: - -0.1.0 - the first, experimental release of a package - -1.0.1a2 - the second alpha release of the first patch version of 1.0 - -``classifiers`` must be specified in a list:: - - setup(..., - classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Console', - 'Environment :: Web Environment', - 'Intended Audience :: End Users/Desktop', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: Python Software Foundation License', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: Microsoft :: Windows', - 'Operating System :: POSIX', - 'Programming Language :: Python', - 'Topic :: Communications :: Email', - 'Topic :: Office/Business', - 'Topic :: Software Development :: Bug Tracking', - ], - ) - -.. versionchanged:: 3.7 - :class:`~distutils.core.setup` now warns when ``classifiers``, ``keywords`` - or ``platforms`` fields are not specified as a list or a string. - -.. _debug-setup-script: - -Debugging the setup script -========================== - -Sometimes things go wrong, and the setup script doesn't do what the developer -wants. - -Distutils catches any exceptions when running the setup script, and print a -simple error message before the script is terminated. The motivation for this -behaviour is to not confuse administrators who don't know much about Python and -are trying to install a package. If they get a big long traceback from deep -inside the guts of Distutils, they may think the package or the Python -installation is broken because they don't read all the way down to the bottom -and see that it's a permission problem. - -On the other hand, this doesn't help the developer to find the cause of the -failure. For this purpose, the :envvar:`DISTUTILS_DEBUG` environment variable can be set -to anything except an empty string, and distutils will now print detailed -information about what it is doing, dump the full traceback when an exception -occurs, and print the whole command line when an external program (like a C -compiler) fails. diff --git a/Doc/distutils/sourcedist.rst b/Doc/distutils/sourcedist.rst deleted file mode 100644 index b55d01157f3ee1..00000000000000 --- a/Doc/distutils/sourcedist.rst +++ /dev/null @@ -1,245 +0,0 @@ -.. _source-dist: - -****************************** -Creating a Source Distribution -****************************** - -.. include:: ./_setuptools_disclaimer.rst - -As shown in section :ref:`distutils-simple-example`, you use the :command:`sdist` command -to create a source distribution. In the simplest case, :: - - python setup.py sdist - -(assuming you haven't specified any :command:`sdist` options in the setup script -or config file), :command:`sdist` creates the archive of the default format for -the current platform. The default format is a gzip'ed tar file -(:file:`.tar.gz`) on Unix, and ZIP file on Windows. - -You can specify as many formats as you like using the :option:`!--formats` -option, for example:: - - python setup.py sdist --formats=gztar,zip - -to create a gzipped tarball and a zip file. The available formats are: - -+-----------+-------------------------+-------------+ -| Format | Description | Notes | -+===========+=========================+=============+ -| ``zip`` | zip file (:file:`.zip`) | (1),(3) | -+-----------+-------------------------+-------------+ -| ``gztar`` | gzip'ed tar file | \(2) | -| | (:file:`.tar.gz`) | | -+-----------+-------------------------+-------------+ -| ``bztar`` | bzip2'ed tar file | \(5) | -| | (:file:`.tar.bz2`) | | -+-----------+-------------------------+-------------+ -| ``xztar`` | xz'ed tar file | \(5) | -| | (:file:`.tar.xz`) | | -+-----------+-------------------------+-------------+ -| ``ztar`` | compressed tar file | (4),(5) | -| | (:file:`.tar.Z`) | | -+-----------+-------------------------+-------------+ -| ``tar`` | tar file (:file:`.tar`) | \(5) | -+-----------+-------------------------+-------------+ - -.. versionchanged:: 3.5 - Added support for the ``xztar`` format. - -Notes: - -(1) - default on Windows - -(2) - default on Unix - -(3) - requires either external :program:`zip` utility or :mod:`zipfile` module (part - of the standard Python library since Python 1.6) - -(4) - requires the :program:`compress` program. Notice that this format is now - pending for deprecation and will be removed in the future versions of Python. -(5) - deprecated by `PEP 527 <https://peps.python.org/pep-0527/>`_; - `PyPI <https://pypi.org>`_ only accepts ``.zip`` and ``.tar.gz`` files. - -When using any ``tar`` format (``gztar``, ``bztar``, ``xztar``, ``ztar`` or -``tar``), under Unix you can specify the ``owner`` and ``group`` names -that will be set for each member of the archive. - -For example, if you want all files of the archive to be owned by root:: - - python setup.py sdist --owner=root --group=root - - -.. _manifest: - -Specifying the files to distribute -================================== - -If you don't supply an explicit list of files (or instructions on how to -generate one), the :command:`sdist` command puts a minimal default set into the -source distribution: - -* all Python source files implied by the ``py_modules`` and - ``packages`` options - -* all C source files mentioned in the ``ext_modules`` or - ``libraries`` options - - .. XXX getting C library sources currently broken---no - :meth:`get_source_files` method in :file:`build_clib.py`! - -* scripts identified by the ``scripts`` option - See :ref:`distutils-installing-scripts`. - -* anything that looks like a test script: :file:`test/test\*.py` (currently, the - Distutils don't do anything with test scripts except include them in source - distributions, but in the future there will be a standard for testing Python - module distributions) - -* Any of the standard README files (:file:`README`, :file:`README.txt`, - or :file:`README.rst`), :file:`setup.py` (or whatever you called your setup - script), and :file:`setup.cfg`. - -* all files that matches the ``package_data`` metadata. - See :ref:`distutils-installing-package-data`. - -* all files that matches the ``data_files`` metadata. - See :ref:`distutils-additional-files`. - -Sometimes this is enough, but usually you will want to specify additional files -to distribute. The typical way to do this is to write a *manifest template*, -called :file:`MANIFEST.in` by default. The manifest template is just a list of -instructions for how to generate your manifest file, :file:`MANIFEST`, which is -the exact list of files to include in your source distribution. The -:command:`sdist` command processes this template and generates a manifest based -on its instructions and what it finds in the filesystem. - -If you prefer to roll your own manifest file, the format is simple: one filename -per line, regular files (or symlinks to them) only. If you do supply your own -:file:`MANIFEST`, you must specify everything: the default set of files -described above does not apply in this case. - -.. versionchanged:: 3.1 - An existing generated :file:`MANIFEST` will be regenerated without - :command:`sdist` comparing its modification time to the one of - :file:`MANIFEST.in` or :file:`setup.py`. - -.. versionchanged:: 3.1.3 - :file:`MANIFEST` files start with a comment indicating they are generated. - Files without this comment are not overwritten or removed. - -.. versionchanged:: 3.2.2 - :command:`sdist` will read a :file:`MANIFEST` file if no :file:`MANIFEST.in` - exists, like it used to do. - -.. versionchanged:: 3.7 - :file:`README.rst` is now included in the list of distutils standard READMEs. - - -The manifest template has one command per line, where each command specifies a -set of files to include or exclude from the source distribution. For an -example, again we turn to the Distutils' own manifest template: - -.. code-block:: none - - include *.txt - recursive-include examples *.txt *.py - prune examples/sample?/build - -The meanings should be fairly clear: include all files in the distribution root -matching :file:`\*.txt`, all files anywhere under the :file:`examples` directory -matching :file:`\*.txt` or :file:`\*.py`, and exclude all directories matching -:file:`examples/sample?/build`. All of this is done *after* the standard -include set, so you can exclude files from the standard set with explicit -instructions in the manifest template. (Or, you can use the -:option:`!--no-defaults` option to disable the standard set entirely.) There are -several other commands available in the manifest template mini-language; see -section :ref:`sdist-cmd`. - -The order of commands in the manifest template matters: initially, we have the -list of default files as described above, and each command in the template adds -to or removes from that list of files. Once we have fully processed the -manifest template, we remove files that should not be included in the source -distribution: - -* all files in the Distutils "build" tree (default :file:`build/`) - -* all files in directories named :file:`RCS`, :file:`CVS`, :file:`.svn`, - :file:`.hg`, :file:`.git`, :file:`.bzr` or :file:`_darcs` - -Now we have our complete list of files, which is written to the manifest for -future reference, and then used to build the source distribution archive(s). - -You can disable the default set of included files with the -:option:`!--no-defaults` option, and you can disable the standard exclude set -with :option:`!--no-prune`. - -Following the Distutils' own manifest template, let's trace how the -:command:`sdist` command builds the list of files to include in the Distutils -source distribution: - -#. include all Python source files in the :file:`distutils` and - :file:`distutils/command` subdirectories (because packages corresponding to - those two directories were mentioned in the ``packages`` option in the - setup script---see section :ref:`setup-script`) - -#. include :file:`README.txt`, :file:`setup.py`, and :file:`setup.cfg` (standard - files) - -#. include :file:`test/test\*.py` (standard files) - -#. include :file:`\*.txt` in the distribution root (this will find - :file:`README.txt` a second time, but such redundancies are weeded out later) - -#. include anything matching :file:`\*.txt` or :file:`\*.py` in the sub-tree - under :file:`examples`, - -#. exclude all files in the sub-trees starting at directories matching - :file:`examples/sample?/build`\ ---this may exclude files included by the - previous two steps, so it's important that the ``prune`` command in the manifest - template comes after the ``recursive-include`` command - -#. exclude the entire :file:`build` tree, and any :file:`RCS`, :file:`CVS`, - :file:`.svn`, :file:`.hg`, :file:`.git`, :file:`.bzr` and :file:`_darcs` - directories - -Just like in the setup script, file and directory names in the manifest template -should always be slash-separated; the Distutils will take care of converting -them to the standard representation on your platform. That way, the manifest -template is portable across operating systems. - - -.. _manifest-options: - -Manifest-related options -======================== - -The normal course of operations for the :command:`sdist` command is as follows: - -* if the manifest file (:file:`MANIFEST` by default) exists and the first line - does not have a comment indicating it is generated from :file:`MANIFEST.in`, - then it is used as is, unaltered - -* if the manifest file doesn't exist or has been previously automatically - generated, read :file:`MANIFEST.in` and create the manifest - -* if neither :file:`MANIFEST` nor :file:`MANIFEST.in` exist, create a manifest - with just the default file set - -* use the list of files now in :file:`MANIFEST` (either just generated or read - in) to create the source distribution archive(s) - -There are a couple of options that modify this behaviour. First, use the -:option:`!--no-defaults` and :option:`!--no-prune` to disable the standard -"include" and "exclude" sets. - -Second, you might just want to (re)generate the manifest, but not create a source -distribution:: - - python setup.py sdist --manifest-only - -:option:`!-o` is a shortcut for :option:`!--manifest-only`. diff --git a/Doc/distutils/uploading.rst b/Doc/distutils/uploading.rst deleted file mode 100644 index 4c391cab072ea6..00000000000000 --- a/Doc/distutils/uploading.rst +++ /dev/null @@ -1,8 +0,0 @@ -:orphan: - -*************************************** -Uploading Packages to the Package Index -*************************************** - -References to up to date PyPI documentation can be found at -:ref:`publishing-python-packages`. diff --git a/Doc/extending/building.rst b/Doc/extending/building.rst deleted file mode 100644 index 53817074ab1ee5..00000000000000 --- a/Doc/extending/building.rst +++ /dev/null @@ -1,166 +0,0 @@ -.. highlight:: c - -.. _building: - -***************************** -Building C and C++ Extensions -***************************** - -A C extension for CPython is a shared library (e.g. a ``.so`` file on Linux, -``.pyd`` on Windows), which exports an *initialization function*. - -To be importable, the shared library must be available on :envvar:`PYTHONPATH`, -and must be named after the module name, with an appropriate extension. -When using distutils, the correct filename is generated automatically. - -The initialization function has the signature: - -.. c:function:: PyObject* PyInit_modulename(void) - -It returns either a fully initialized module, or a :c:type:`PyModuleDef` -instance. See :ref:`initializing-modules` for details. - -.. highlight:: python - -For modules with ASCII-only names, the function must be named -``PyInit_<modulename>``, with ``<modulename>`` replaced by the name of the -module. When using :ref:`multi-phase-initialization`, non-ASCII module names -are allowed. In this case, the initialization function name is -``PyInitU_<modulename>``, with ``<modulename>`` encoded using Python's -*punycode* encoding with hyphens replaced by underscores. In Python:: - - def initfunc_name(name): - try: - suffix = b'_' + name.encode('ascii') - except UnicodeEncodeError: - suffix = b'U_' + name.encode('punycode').replace(b'-', b'_') - return b'PyInit' + suffix - -It is possible to export multiple modules from a single shared library by -defining multiple initialization functions. However, importing them requires -using symbolic links or a custom importer, because by default only the -function corresponding to the filename is found. -See the *"Multiple modules in one library"* section in :pep:`489` for details. - - -.. highlight:: c - -Building C and C++ Extensions with distutils -============================================ - -.. sectionauthor:: Martin v. Löwis <martin@v.loewis.de> - -Extension modules can be built using distutils, which is included in Python. -Since distutils also supports creation of binary packages, users don't -necessarily need a compiler and distutils to install the extension. - -A distutils package contains a driver script, :file:`setup.py`. This is a plain -Python file, which, in the most simple case, could look like this: - -.. code-block:: python3 - - from distutils.core import setup, Extension - - module1 = Extension('demo', - sources = ['demo.c']) - - setup (name = 'PackageName', - version = '1.0', - description = 'This is a demo package', - ext_modules = [module1]) - - -With this :file:`setup.py`, and a file :file:`demo.c`, running :: - - python setup.py build - -will compile :file:`demo.c`, and produce an extension module named ``demo`` in -the :file:`build` directory. Depending on the system, the module file will end -up in a subdirectory :file:`build/lib.system`, and may have a name like -:file:`demo.so` or :file:`demo.pyd`. - -In the :file:`setup.py`, all execution is performed by calling the ``setup`` -function. This takes a variable number of keyword arguments, of which the -example above uses only a subset. Specifically, the example specifies -meta-information to build packages, and it specifies the contents of the -package. Normally, a package will contain additional modules, like Python -source modules, documentation, subpackages, etc. Please refer to the distutils -documentation in :ref:`distutils-index` to learn more about the features of -distutils; this section explains building extension modules only. - -It is common to pre-compute arguments to :func:`setup`, to better structure the -driver script. In the example above, the ``ext_modules`` argument to -:func:`~distutils.core.setup` is a list of extension modules, each of which is -an instance of -the :class:`~distutils.extension.Extension`. In the example, the instance -defines an extension named ``demo`` which is build by compiling a single source -file, :file:`demo.c`. - -In many cases, building an extension is more complex, since additional -preprocessor defines and libraries may be needed. This is demonstrated in the -example below. - -.. code-block:: python3 - - from distutils.core import setup, Extension - - module1 = Extension('demo', - define_macros = [('MAJOR_VERSION', '1'), - ('MINOR_VERSION', '0')], - include_dirs = ['/usr/local/include'], - libraries = ['tcl83'], - library_dirs = ['/usr/local/lib'], - sources = ['demo.c']) - - setup (name = 'PackageName', - version = '1.0', - description = 'This is a demo package', - author = 'Martin v. Loewis', - author_email = 'martin@v.loewis.de', - url = 'https://docs.python.org/extending/building', - long_description = ''' - This is really just a demo package. - ''', - ext_modules = [module1]) - - -In this example, :func:`~distutils.core.setup` is called with additional -meta-information, which -is recommended when distribution packages have to be built. For the extension -itself, it specifies preprocessor defines, include directories, library -directories, and libraries. Depending on the compiler, distutils passes this -information in different ways to the compiler. For example, on Unix, this may -result in the compilation commands :: - - gcc -DNDEBUG -g -O3 -Wall -Wstrict-prototypes -fPIC -DMAJOR_VERSION=1 -DMINOR_VERSION=0 -I/usr/local/include -I/usr/local/include/python2.2 -c demo.c -o build/temp.linux-i686-2.2/demo.o - - gcc -shared build/temp.linux-i686-2.2/demo.o -L/usr/local/lib -ltcl83 -o build/lib.linux-i686-2.2/demo.so - -These lines are for demonstration purposes only; distutils users should trust -that distutils gets the invocations right. - - -.. _distributing: - -Distributing your extension modules -=================================== - -When an extension has been successfully built, there are three ways to use it. - -End-users will typically want to install the module, they do so by running :: - - python setup.py install - -Module maintainers should produce source packages; to do so, they run :: - - python setup.py sdist - -In some cases, additional files need to be included in a source distribution; -this is done through a :file:`MANIFEST.in` file; see :ref:`manifest` for details. - -If the source distribution has been built successfully, maintainers can also -create binary distributions. Depending on the platform, one of the following -commands can be used to do so. :: - - python setup.py bdist_rpm - python setup.py bdist_dumb diff --git a/Doc/extending/embedding.rst b/Doc/extending/embedding.rst deleted file mode 100644 index 4c7c7ec98a6c95..00000000000000 --- a/Doc/extending/embedding.rst +++ /dev/null @@ -1,336 +0,0 @@ -.. highlight:: c - - -.. _embedding: - -*************************************** -Embedding Python in Another Application -*************************************** - -The previous chapters discussed how to extend Python, that is, how to extend the -functionality of Python by attaching a library of C functions to it. It is also -possible to do it the other way around: enrich your C/C++ application by -embedding Python in it. Embedding provides your application with the ability to -implement some of the functionality of your application in Python rather than C -or C++. This can be used for many purposes; one example would be to allow users -to tailor the application to their needs by writing some scripts in Python. You -can also use it yourself if some of the functionality can be written in Python -more easily. - -Embedding Python is similar to extending it, but not quite. The difference is -that when you extend Python, the main program of the application is still the -Python interpreter, while if you embed Python, the main program may have nothing -to do with Python --- instead, some parts of the application occasionally call -the Python interpreter to run some Python code. - -So if you are embedding Python, you are providing your own main program. One of -the things this main program has to do is initialize the Python interpreter. At -the very least, you have to call the function :c:func:`Py_Initialize`. There are -optional calls to pass command line arguments to Python. Then later you can -call the interpreter from any part of the application. - -There are several different ways to call the interpreter: you can pass a string -containing Python statements to :c:func:`PyRun_SimpleString`, or you can pass a -stdio file pointer and a file name (for identification in error messages only) -to :c:func:`PyRun_SimpleFile`. You can also call the lower-level operations -described in the previous chapters to construct and use Python objects. - - -.. seealso:: - - :ref:`c-api-index` - The details of Python's C interface are given in this manual. A great deal of - necessary information can be found here. - - -.. _high-level-embedding: - -Very High Level Embedding -========================= - -The simplest form of embedding Python is the use of the very high level -interface. This interface is intended to execute a Python script without needing -to interact with the application directly. This can for example be used to -perform some operation on a file. :: - - #define PY_SSIZE_T_CLEAN - #include <Python.h> - - int - main(int argc, char *argv[]) - { - wchar_t *program = Py_DecodeLocale(argv[0], NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); - exit(1); - } - Py_SetProgramName(program); /* optional but recommended */ - Py_Initialize(); - PyRun_SimpleString("from time import time,ctime\n" - "print('Today is', ctime(time()))\n"); - if (Py_FinalizeEx() < 0) { - exit(120); - } - PyMem_RawFree(program); - return 0; - } - -The :c:func:`Py_SetProgramName` function should be called before -:c:func:`Py_Initialize` to inform the interpreter about paths to Python run-time -libraries. Next, the Python interpreter is initialized with -:c:func:`Py_Initialize`, followed by the execution of a hard-coded Python script -that prints the date and time. Afterwards, the :c:func:`Py_FinalizeEx` call shuts -the interpreter down, followed by the end of the program. In a real program, -you may want to get the Python script from another source, perhaps a text-editor -routine, a file, or a database. Getting the Python code from a file can better -be done by using the :c:func:`PyRun_SimpleFile` function, which saves you the -trouble of allocating memory space and loading the file contents. - - -.. _lower-level-embedding: - -Beyond Very High Level Embedding: An overview -============================================= - -The high level interface gives you the ability to execute arbitrary pieces of -Python code from your application, but exchanging data values is quite -cumbersome to say the least. If you want that, you should use lower level calls. -At the cost of having to write more C code, you can achieve almost anything. - -It should be noted that extending Python and embedding Python is quite the same -activity, despite the different intent. Most topics discussed in the previous -chapters are still valid. To show this, consider what the extension code from -Python to C really does: - -#. Convert data values from Python to C, - -#. Perform a function call to a C routine using the converted values, and - -#. Convert the data values from the call from C to Python. - -When embedding Python, the interface code does: - -#. Convert data values from C to Python, - -#. Perform a function call to a Python interface routine using the converted - values, and - -#. Convert the data values from the call from Python to C. - -As you can see, the data conversion steps are simply swapped to accommodate the -different direction of the cross-language transfer. The only difference is the -routine that you call between both data conversions. When extending, you call a -C routine, when embedding, you call a Python routine. - -This chapter will not discuss how to convert data from Python to C and vice -versa. Also, proper use of references and dealing with errors is assumed to be -understood. Since these aspects do not differ from extending the interpreter, -you can refer to earlier chapters for the required information. - - -.. _pure-embedding: - -Pure Embedding -============== - -The first program aims to execute a function in a Python script. Like in the -section about the very high level interface, the Python interpreter does not -directly interact with the application (but that will change in the next -section). - -The code to run a function defined in a Python script is: - -.. literalinclude:: ../includes/run-func.c - - -This code loads a Python script using ``argv[1]``, and calls the function named -in ``argv[2]``. Its integer arguments are the other values of the ``argv`` -array. If you :ref:`compile and link <compiling>` this program (let's call -the finished executable :program:`call`), and use it to execute a Python -script, such as: - -.. code-block:: python - - def multiply(a,b): - print("Will compute", a, "times", b) - c = 0 - for i in range(0, a): - c = c + b - return c - -then the result should be: - -.. code-block:: shell-session - - $ call multiply multiply 3 2 - Will compute 3 times 2 - Result of call: 6 - -Although the program is quite large for its functionality, most of the code is -for data conversion between Python and C, and for error reporting. The -interesting part with respect to embedding Python starts with :: - - Py_Initialize(); - pName = PyUnicode_DecodeFSDefault(argv[1]); - /* Error checking of pName left out */ - pModule = PyImport_Import(pName); - -After initializing the interpreter, the script is loaded using -:c:func:`PyImport_Import`. This routine needs a Python string as its argument, -which is constructed using the :c:func:`PyUnicode_FromString` data conversion -routine. :: - - pFunc = PyObject_GetAttrString(pModule, argv[2]); - /* pFunc is a new reference */ - - if (pFunc && PyCallable_Check(pFunc)) { - ... - } - Py_XDECREF(pFunc); - -Once the script is loaded, the name we're looking for is retrieved using -:c:func:`PyObject_GetAttrString`. If the name exists, and the object returned is -callable, you can safely assume that it is a function. The program then -proceeds by constructing a tuple of arguments as normal. The call to the Python -function is then made with:: - - pValue = PyObject_CallObject(pFunc, pArgs); - -Upon return of the function, ``pValue`` is either ``NULL`` or it contains a -reference to the return value of the function. Be sure to release the reference -after examining the value. - - -.. _extending-with-embedding: - -Extending Embedded Python -========================= - -Until now, the embedded Python interpreter had no access to functionality from -the application itself. The Python API allows this by extending the embedded -interpreter. That is, the embedded interpreter gets extended with routines -provided by the application. While it sounds complex, it is not so bad. Simply -forget for a while that the application starts the Python interpreter. Instead, -consider the application to be a set of subroutines, and write some glue code -that gives Python access to those routines, just like you would write a normal -Python extension. For example:: - - static int numargs=0; - - /* Return the number of arguments of the application command line */ - static PyObject* - emb_numargs(PyObject *self, PyObject *args) - { - if(!PyArg_ParseTuple(args, ":numargs")) - return NULL; - return PyLong_FromLong(numargs); - } - - static PyMethodDef EmbMethods[] = { - {"numargs", emb_numargs, METH_VARARGS, - "Return the number of arguments received by the process."}, - {NULL, NULL, 0, NULL} - }; - - static PyModuleDef EmbModule = { - PyModuleDef_HEAD_INIT, "emb", NULL, -1, EmbMethods, - NULL, NULL, NULL, NULL - }; - - static PyObject* - PyInit_emb(void) - { - return PyModule_Create(&EmbModule); - } - -Insert the above code just above the :c:func:`main` function. Also, insert the -following two statements before the call to :c:func:`Py_Initialize`:: - - numargs = argc; - PyImport_AppendInittab("emb", &PyInit_emb); - -These two lines initialize the ``numargs`` variable, and make the -:func:`!emb.numargs` function accessible to the embedded Python interpreter. -With these extensions, the Python script can do things like - -.. code-block:: python - - import emb - print("Number of arguments", emb.numargs()) - -In a real application, the methods will expose an API of the application to -Python. - -.. TODO: threads, code examples do not really behave well if errors happen - (what to watch out for) - - -.. _embeddingincplusplus: - -Embedding Python in C++ -======================= - -It is also possible to embed Python in a C++ program; precisely how this is done -will depend on the details of the C++ system used; in general you will need to -write the main program in C++, and use the C++ compiler to compile and link your -program. There is no need to recompile Python itself using C++. - - -.. _compiling: - -Compiling and Linking under Unix-like systems -============================================= - -It is not necessarily trivial to find the right flags to pass to your -compiler (and linker) in order to embed the Python interpreter into your -application, particularly because Python needs to load library modules -implemented as C dynamic extensions (:file:`.so` files) linked against -it. - -To find out the required compiler and linker flags, you can execute the -:file:`python{X.Y}-config` script which is generated as part of the -installation process (a :file:`python3-config` script may also be -available). This script has several options, of which the following will -be directly useful to you: - -* ``pythonX.Y-config --cflags`` will give you the recommended flags when - compiling: - - .. code-block:: shell-session - - $ /opt/bin/python3.11-config --cflags - -I/opt/include/python3.11 -I/opt/include/python3.11 -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall - -* ``pythonX.Y-config --ldflags --embed`` will give you the recommended flags - when linking: - - .. code-block:: shell-session - - $ /opt/bin/python3.11-config --ldflags --embed - -L/opt/lib/python3.11/config-3.11-x86_64-linux-gnu -L/opt/lib -lpython3.11 -lpthread -ldl -lutil -lm - -.. note:: - To avoid confusion between several Python installations (and especially - between the system Python and your own compiled Python), it is recommended - that you use the absolute path to :file:`python{X.Y}-config`, as in the above - example. - -If this procedure doesn't work for you (it is not guaranteed to work for -all Unix-like platforms; however, we welcome :ref:`bug reports <reporting-bugs>`) -you will have to read your system's documentation about dynamic linking and/or -examine Python's :file:`Makefile` (use :func:`sysconfig.get_makefile_filename` -to find its location) and compilation -options. In this case, the :mod:`sysconfig` module is a useful tool to -programmatically extract the configuration values that you will want to -combine together. For example: - -.. code-block:: pycon - - >>> import sysconfig - >>> sysconfig.get_config_var('LIBS') - '-lpthread -ldl -lutil' - >>> sysconfig.get_config_var('LINKFORSHARED') - '-Xlinker -export-dynamic' - - -.. XXX similar documentation for Windows missing diff --git a/Doc/extending/extending.rst b/Doc/extending/extending.rst deleted file mode 100644 index 68f8e0c6674657..00000000000000 --- a/Doc/extending/extending.rst +++ /dev/null @@ -1,1376 +0,0 @@ -.. highlight:: c - - -.. _extending-intro: - -****************************** -Extending Python with C or C++ -****************************** - -It is quite easy to add new built-in modules to Python, if you know how to -program in C. Such :dfn:`extension modules` can do two things that can't be -done directly in Python: they can implement new built-in object types, and they -can call C library functions and system calls. - -To support extensions, the Python API (Application Programmers Interface) -defines a set of functions, macros and variables that provide access to most -aspects of the Python run-time system. The Python API is incorporated in a C -source file by including the header ``"Python.h"``. - -The compilation of an extension module depends on its intended use as well as on -your system setup; details are given in later chapters. - -.. note:: - - The C extension interface is specific to CPython, and extension modules do - not work on other Python implementations. In many cases, it is possible to - avoid writing C extensions and preserve portability to other implementations. - For example, if your use case is calling C library functions or system calls, - you should consider using the :mod:`ctypes` module or the `cffi - <https://cffi.readthedocs.io/>`_ library rather than writing - custom C code. - These modules let you write Python code to interface with C code and are more - portable between implementations of Python than writing and compiling a C - extension module. - - -.. _extending-simpleexample: - -A Simple Example -================ - -Let's create an extension module called ``spam`` (the favorite food of Monty -Python fans...) and let's say we want to create a Python interface to the C -library function :c:func:`system` [#]_. This function takes a null-terminated -character string as argument and returns an integer. We want this function to -be callable from Python as follows: - -.. code-block:: pycon - - >>> import spam - >>> status = spam.system("ls -l") - -Begin by creating a file :file:`spammodule.c`. (Historically, if a module is -called ``spam``, the C file containing its implementation is called -:file:`spammodule.c`; if the module name is very long, like ``spammify``, the -module name can be just :file:`spammify.c`.) - -The first two lines of our file can be:: - - #define PY_SSIZE_T_CLEAN - #include <Python.h> - -which pulls in the Python API (you can add a comment describing the purpose of -the module and a copyright notice if you like). - -.. note:: - - Since Python may define some pre-processor definitions which affect the standard - headers on some systems, you *must* include :file:`Python.h` before any standard - headers are included. - - It is recommended to always define ``PY_SSIZE_T_CLEAN`` before including - ``Python.h``. See :ref:`parsetuple` for a description of this macro. - -All user-visible symbols defined by :file:`Python.h` have a prefix of ``Py`` or -``PY``, except those defined in standard header files. For convenience, and -since they are used extensively by the Python interpreter, ``"Python.h"`` -includes a few standard header files: ``<stdio.h>``, ``<string.h>``, -``<errno.h>``, and ``<stdlib.h>``. If the latter header file does not exist on -your system, it declares the functions :c:func:`malloc`, :c:func:`free` and -:c:func:`realloc` directly. - -The next thing we add to our module file is the C function that will be called -when the Python expression ``spam.system(string)`` is evaluated (we'll see -shortly how it ends up being called):: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - return PyLong_FromLong(sts); - } - -There is a straightforward translation from the argument list in Python (for -example, the single expression ``"ls -l"``) to the arguments passed to the C -function. The C function always has two arguments, conventionally named *self* -and *args*. - -The *self* argument points to the module object for module-level functions; -for a method it would point to the object instance. - -The *args* argument will be a pointer to a Python tuple object containing the -arguments. Each item of the tuple corresponds to an argument in the call's -argument list. The arguments are Python objects --- in order to do anything -with them in our C function we have to convert them to C values. The function -:c:func:`PyArg_ParseTuple` in the Python API checks the argument types and -converts them to C values. It uses a template string to determine the required -types of the arguments as well as the types of the C variables into which to -store the converted values. More about this later. - -:c:func:`PyArg_ParseTuple` returns true (nonzero) if all arguments have the right -type and its components have been stored in the variables whose addresses are -passed. It returns false (zero) if an invalid argument list was passed. In the -latter case it also raises an appropriate exception so the calling function can -return ``NULL`` immediately (as we saw in the example). - - -.. _extending-errors: - -Intermezzo: Errors and Exceptions -================================= - -An important convention throughout the Python interpreter is the following: when -a function fails, it should set an exception condition and return an error value -(usually ``-1`` or a ``NULL`` pointer). Exception information is stored in -three members of the interpreter's thread state. These are ``NULL`` if -there is no exception. Otherwise they are the C equivalents of the members -of the Python tuple returned by :meth:`sys.exc_info`. These are the -exception type, exception instance, and a traceback object. It is important -to know about them to understand how errors are passed around. - -The Python API defines a number of functions to set various types of exceptions. - -The most common one is :c:func:`PyErr_SetString`. Its arguments are an exception -object and a C string. The exception object is usually a predefined object like -:c:data:`PyExc_ZeroDivisionError`. The C string indicates the cause of the error -and is converted to a Python string object and stored as the "associated value" -of the exception. - -Another useful function is :c:func:`PyErr_SetFromErrno`, which only takes an -exception argument and constructs the associated value by inspection of the -global variable :c:data:`errno`. The most general function is -:c:func:`PyErr_SetObject`, which takes two object arguments, the exception and -its associated value. You don't need to :c:func:`Py_INCREF` the objects passed -to any of these functions. - -You can test non-destructively whether an exception has been set with -:c:func:`PyErr_Occurred`. This returns the current exception object, or ``NULL`` -if no exception has occurred. You normally don't need to call -:c:func:`PyErr_Occurred` to see whether an error occurred in a function call, -since you should be able to tell from the return value. - -When a function *f* that calls another function *g* detects that the latter -fails, *f* should itself return an error value (usually ``NULL`` or ``-1``). It -should *not* call one of the ``PyErr_*`` functions --- one has already -been called by *g*. *f*'s caller is then supposed to also return an error -indication to *its* caller, again *without* calling ``PyErr_*``, and so on ---- the most detailed cause of the error was already reported by the function -that first detected it. Once the error reaches the Python interpreter's main -loop, this aborts the currently executing Python code and tries to find an -exception handler specified by the Python programmer. - -(There are situations where a module can actually give a more detailed error -message by calling another ``PyErr_*`` function, and in such cases it is -fine to do so. As a general rule, however, this is not necessary, and can cause -information about the cause of the error to be lost: most operations can fail -for a variety of reasons.) - -To ignore an exception set by a function call that failed, the exception -condition must be cleared explicitly by calling :c:func:`PyErr_Clear`. The only -time C code should call :c:func:`PyErr_Clear` is if it doesn't want to pass the -error on to the interpreter but wants to handle it completely by itself -(possibly by trying something else, or pretending nothing went wrong). - -Every failing :c:func:`malloc` call must be turned into an exception --- the -direct caller of :c:func:`malloc` (or :c:func:`realloc`) must call -:c:func:`PyErr_NoMemory` and return a failure indicator itself. All the -object-creating functions (for example, :c:func:`PyLong_FromLong`) already do -this, so this note is only relevant to those who call :c:func:`malloc` directly. - -Also note that, with the important exception of :c:func:`PyArg_ParseTuple` and -friends, functions that return an integer status usually return a positive value -or zero for success and ``-1`` for failure, like Unix system calls. - -Finally, be careful to clean up garbage (by making :c:func:`Py_XDECREF` or -:c:func:`Py_DECREF` calls for objects you have already created) when you return -an error indicator! - -The choice of which exception to raise is entirely yours. There are predeclared -C objects corresponding to all built-in Python exceptions, such as -:c:data:`PyExc_ZeroDivisionError`, which you can use directly. Of course, you -should choose exceptions wisely --- don't use :c:data:`PyExc_TypeError` to mean -that a file couldn't be opened (that should probably be :c:data:`PyExc_OSError`). -If something's wrong with the argument list, the :c:func:`PyArg_ParseTuple` -function usually raises :c:data:`PyExc_TypeError`. If you have an argument whose -value must be in a particular range or must satisfy other conditions, -:c:data:`PyExc_ValueError` is appropriate. - -You can also define a new exception that is unique to your module. For this, you -usually declare a static object variable at the beginning of your file:: - - static PyObject *SpamError; - -and initialize it in your module's initialization function (:c:func:`!PyInit_spam`) -with an exception object:: - - PyMODINIT_FUNC - PyInit_spam(void) - { - PyObject *m; - - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; - - SpamError = PyErr_NewException("spam.error", NULL, NULL); - Py_XINCREF(SpamError); - if (PyModule_AddObject(m, "error", SpamError) < 0) { - Py_XDECREF(SpamError); - Py_CLEAR(SpamError); - Py_DECREF(m); - return NULL; - } - - return m; - } - -Note that the Python name for the exception object is :exc:`!spam.error`. The -:c:func:`PyErr_NewException` function may create a class with the base class -being :exc:`Exception` (unless another class is passed in instead of ``NULL``), -described in :ref:`bltin-exceptions`. - -Note also that the :c:data:`!SpamError` variable retains a reference to the newly -created exception class; this is intentional! Since the exception could be -removed from the module by external code, an owned reference to the class is -needed to ensure that it will not be discarded, causing :c:data:`!SpamError` to -become a dangling pointer. Should it become a dangling pointer, C code which -raises the exception could cause a core dump or other unintended side effects. - -We discuss the use of :c:macro:`PyMODINIT_FUNC` as a function return type later in this -sample. - -The :exc:`!spam.error` exception can be raised in your extension module using a -call to :c:func:`PyErr_SetString` as shown below:: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - if (sts < 0) { - PyErr_SetString(SpamError, "System command failed"); - return NULL; - } - return PyLong_FromLong(sts); - } - - -.. _backtoexample: - -Back to the Example -=================== - -Going back to our example function, you should now be able to understand this -statement:: - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - -It returns ``NULL`` (the error indicator for functions returning object pointers) -if an error is detected in the argument list, relying on the exception set by -:c:func:`PyArg_ParseTuple`. Otherwise the string value of the argument has been -copied to the local variable :c:data:`!command`. This is a pointer assignment and -you are not supposed to modify the string to which it points (so in Standard C, -the variable :c:data:`!command` should properly be declared as ``const char -*command``). - -The next statement is a call to the Unix function :c:func:`system`, passing it -the string we just got from :c:func:`PyArg_ParseTuple`:: - - sts = system(command); - -Our :func:`!spam.system` function must return the value of :c:data:`!sts` as a -Python object. This is done using the function :c:func:`PyLong_FromLong`. :: - - return PyLong_FromLong(sts); - -In this case, it will return an integer object. (Yes, even integers are objects -on the heap in Python!) - -If you have a C function that returns no useful argument (a function returning -:c:expr:`void`), the corresponding Python function must return ``None``. You -need this idiom to do so (which is implemented by the :c:macro:`Py_RETURN_NONE` -macro):: - - Py_INCREF(Py_None); - return Py_None; - -:c:data:`Py_None` is the C name for the special Python object ``None``. It is a -genuine Python object rather than a ``NULL`` pointer, which means "error" in most -contexts, as we have seen. - - -.. _methodtable: - -The Module's Method Table and Initialization Function -===================================================== - -I promised to show how :c:func:`!spam_system` is called from Python programs. -First, we need to list its name and address in a "method table":: - - static PyMethodDef SpamMethods[] = { - ... - {"system", spam_system, METH_VARARGS, - "Execute a shell command."}, - ... - {NULL, NULL, 0, NULL} /* Sentinel */ - }; - -Note the third entry (``METH_VARARGS``). This is a flag telling the interpreter -the calling convention to be used for the C function. It should normally always -be ``METH_VARARGS`` or ``METH_VARARGS | METH_KEYWORDS``; a value of ``0`` means -that an obsolete variant of :c:func:`PyArg_ParseTuple` is used. - -When using only ``METH_VARARGS``, the function should expect the Python-level -parameters to be passed in as a tuple acceptable for parsing via -:c:func:`PyArg_ParseTuple`; more information on this function is provided below. - -The :c:macro:`METH_KEYWORDS` bit may be set in the third field if keyword -arguments should be passed to the function. In this case, the C function should -accept a third ``PyObject *`` parameter which will be a dictionary of keywords. -Use :c:func:`PyArg_ParseTupleAndKeywords` to parse the arguments to such a -function. - -The method table must be referenced in the module definition structure:: - - static struct PyModuleDef spammodule = { - PyModuleDef_HEAD_INIT, - "spam", /* name of module */ - spam_doc, /* module documentation, may be NULL */ - -1, /* size of per-interpreter state of the module, - or -1 if the module keeps state in global variables. */ - SpamMethods - }; - -This structure, in turn, must be passed to the interpreter in the module's -initialization function. The initialization function must be named -:c:func:`!PyInit_name`, where *name* is the name of the module, and should be the -only non-\ ``static`` item defined in the module file:: - - PyMODINIT_FUNC - PyInit_spam(void) - { - return PyModule_Create(&spammodule); - } - -Note that :c:macro:`PyMODINIT_FUNC` declares the function as ``PyObject *`` return type, -declares any special linkage declarations required by the platform, and for C++ -declares the function as ``extern "C"``. - -When the Python program imports module :mod:`!spam` for the first time, -:c:func:`!PyInit_spam` is called. (See below for comments about embedding Python.) -It calls :c:func:`PyModule_Create`, which returns a module object, and -inserts built-in function objects into the newly created module based upon the -table (an array of :c:type:`PyMethodDef` structures) found in the module definition. -:c:func:`PyModule_Create` returns a pointer to the module object -that it creates. It may abort with a fatal error for -certain errors, or return ``NULL`` if the module could not be initialized -satisfactorily. The init function must return the module object to its caller, -so that it then gets inserted into ``sys.modules``. - -When embedding Python, the :c:func:`!PyInit_spam` function is not called -automatically unless there's an entry in the :c:data:`PyImport_Inittab` table. -To add the module to the initialization table, use :c:func:`PyImport_AppendInittab`, -optionally followed by an import of the module:: - - int - main(int argc, char *argv[]) - { - wchar_t *program = Py_DecodeLocale(argv[0], NULL); - if (program == NULL) { - fprintf(stderr, "Fatal error: cannot decode argv[0]\n"); - exit(1); - } - - /* Add a built-in module, before Py_Initialize */ - if (PyImport_AppendInittab("spam", PyInit_spam) == -1) { - fprintf(stderr, "Error: could not extend in-built modules table\n"); - exit(1); - } - - /* Pass argv[0] to the Python interpreter */ - Py_SetProgramName(program); - - /* Initialize the Python interpreter. Required. - If this step fails, it will be a fatal error. */ - Py_Initialize(); - - /* Optionally import the module; alternatively, - import can be deferred until the embedded script - imports it. */ - PyObject *pmodule = PyImport_ImportModule("spam"); - if (!pmodule) { - PyErr_Print(); - fprintf(stderr, "Error: could not import module 'spam'\n"); - } - - ... - - PyMem_RawFree(program); - return 0; - } - -.. note:: - - Removing entries from ``sys.modules`` or importing compiled modules into - multiple interpreters within a process (or following a :c:func:`fork` without an - intervening :c:func:`exec`) can create problems for some extension modules. - Extension module authors should exercise caution when initializing internal data - structures. - -A more substantial example module is included in the Python source distribution -as :file:`Modules/xxmodule.c`. This file may be used as a template or simply -read as an example. - -.. note:: - - Unlike our ``spam`` example, ``xxmodule`` uses *multi-phase initialization* - (new in Python 3.5), where a PyModuleDef structure is returned from - ``PyInit_spam``, and creation of the module is left to the import machinery. - For details on multi-phase initialization, see :PEP:`489`. - - -.. _compilation: - -Compilation and Linkage -======================= - -There are two more things to do before you can use your new extension: compiling -and linking it with the Python system. If you use dynamic loading, the details -may depend on the style of dynamic loading your system uses; see the chapters -about building extension modules (chapter :ref:`building`) and additional -information that pertains only to building on Windows (chapter -:ref:`building-on-windows`) for more information about this. - -If you can't use dynamic loading, or if you want to make your module a permanent -part of the Python interpreter, you will have to change the configuration setup -and rebuild the interpreter. Luckily, this is very simple on Unix: just place -your file (:file:`spammodule.c` for example) in the :file:`Modules/` directory -of an unpacked source distribution, add a line to the file -:file:`Modules/Setup.local` describing your file: - -.. code-block:: sh - - spam spammodule.o - -and rebuild the interpreter by running :program:`make` in the toplevel -directory. You can also run :program:`make` in the :file:`Modules/` -subdirectory, but then you must first rebuild :file:`Makefile` there by running -':program:`make` Makefile'. (This is necessary each time you change the -:file:`Setup` file.) - -If your module requires additional libraries to link with, these can be listed -on the line in the configuration file as well, for instance: - -.. code-block:: sh - - spam spammodule.o -lX11 - - -.. _callingpython: - -Calling Python Functions from C -=============================== - -So far we have concentrated on making C functions callable from Python. The -reverse is also useful: calling Python functions from C. This is especially the -case for libraries that support so-called "callback" functions. If a C -interface makes use of callbacks, the equivalent Python often needs to provide a -callback mechanism to the Python programmer; the implementation will require -calling the Python callback functions from a C callback. Other uses are also -imaginable. - -Fortunately, the Python interpreter is easily called recursively, and there is a -standard interface to call a Python function. (I won't dwell on how to call the -Python parser with a particular string as input --- if you're interested, have a -look at the implementation of the :option:`-c` command line option in -:file:`Modules/main.c` from the Python source code.) - -Calling a Python function is easy. First, the Python program must somehow pass -you the Python function object. You should provide a function (or some other -interface) to do this. When this function is called, save a pointer to the -Python function object (be careful to :c:func:`Py_INCREF` it!) in a global -variable --- or wherever you see fit. For example, the following function might -be part of a module definition:: - - static PyObject *my_callback = NULL; - - static PyObject * - my_set_callback(PyObject *dummy, PyObject *args) - { - PyObject *result = NULL; - PyObject *temp; - - if (PyArg_ParseTuple(args, "O:set_callback", &temp)) { - if (!PyCallable_Check(temp)) { - PyErr_SetString(PyExc_TypeError, "parameter must be callable"); - return NULL; - } - Py_XINCREF(temp); /* Add a reference to new callback */ - Py_XDECREF(my_callback); /* Dispose of previous callback */ - my_callback = temp; /* Remember new callback */ - /* Boilerplate to return "None" */ - Py_INCREF(Py_None); - result = Py_None; - } - return result; - } - -This function must be registered with the interpreter using the -:c:macro:`METH_VARARGS` flag; this is described in section :ref:`methodtable`. The -:c:func:`PyArg_ParseTuple` function and its arguments are documented in section -:ref:`parsetuple`. - -The macros :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF` increment/decrement the -reference count of an object and are safe in the presence of ``NULL`` pointers -(but note that *temp* will not be ``NULL`` in this context). More info on them -in section :ref:`refcounts`. - -.. index:: single: PyObject_CallObject() - -Later, when it is time to call the function, you call the C function -:c:func:`PyObject_CallObject`. This function has two arguments, both pointers to -arbitrary Python objects: the Python function, and the argument list. The -argument list must always be a tuple object, whose length is the number of -arguments. To call the Python function with no arguments, pass in ``NULL``, or -an empty tuple; to call it with one argument, pass a singleton tuple. -:c:func:`Py_BuildValue` returns a tuple when its format string consists of zero -or more format codes between parentheses. For example:: - - int arg; - PyObject *arglist; - PyObject *result; - ... - arg = 123; - ... - /* Time to call the callback */ - arglist = Py_BuildValue("(i)", arg); - result = PyObject_CallObject(my_callback, arglist); - Py_DECREF(arglist); - -:c:func:`PyObject_CallObject` returns a Python object pointer: this is the return -value of the Python function. :c:func:`PyObject_CallObject` is -"reference-count-neutral" with respect to its arguments. In the example a new -tuple was created to serve as the argument list, which is -:c:func:`Py_DECREF`\ -ed immediately after the :c:func:`PyObject_CallObject` -call. - -The return value of :c:func:`PyObject_CallObject` is "new": either it is a brand -new object, or it is an existing object whose reference count has been -incremented. So, unless you want to save it in a global variable, you should -somehow :c:func:`Py_DECREF` the result, even (especially!) if you are not -interested in its value. - -Before you do this, however, it is important to check that the return value -isn't ``NULL``. If it is, the Python function terminated by raising an exception. -If the C code that called :c:func:`PyObject_CallObject` is called from Python, it -should now return an error indication to its Python caller, so the interpreter -can print a stack trace, or the calling Python code can handle the exception. -If this is not possible or desirable, the exception should be cleared by calling -:c:func:`PyErr_Clear`. For example:: - - if (result == NULL) - return NULL; /* Pass error back */ - ...use result... - Py_DECREF(result); - -Depending on the desired interface to the Python callback function, you may also -have to provide an argument list to :c:func:`PyObject_CallObject`. In some cases -the argument list is also provided by the Python program, through the same -interface that specified the callback function. It can then be saved and used -in the same manner as the function object. In other cases, you may have to -construct a new tuple to pass as the argument list. The simplest way to do this -is to call :c:func:`Py_BuildValue`. For example, if you want to pass an integral -event code, you might use the following code:: - - PyObject *arglist; - ... - arglist = Py_BuildValue("(l)", eventcode); - result = PyObject_CallObject(my_callback, arglist); - Py_DECREF(arglist); - if (result == NULL) - return NULL; /* Pass error back */ - /* Here maybe use the result */ - Py_DECREF(result); - -Note the placement of ``Py_DECREF(arglist)`` immediately after the call, before -the error check! Also note that strictly speaking this code is not complete: -:c:func:`Py_BuildValue` may run out of memory, and this should be checked. - -You may also call a function with keyword arguments by using -:c:func:`PyObject_Call`, which supports arguments and keyword arguments. As in -the above example, we use :c:func:`Py_BuildValue` to construct the dictionary. :: - - PyObject *dict; - ... - dict = Py_BuildValue("{s:i}", "name", val); - result = PyObject_Call(my_callback, NULL, dict); - Py_DECREF(dict); - if (result == NULL) - return NULL; /* Pass error back */ - /* Here maybe use the result */ - Py_DECREF(result); - - -.. _parsetuple: - -Extracting Parameters in Extension Functions -============================================ - -.. index:: single: PyArg_ParseTuple() - -The :c:func:`PyArg_ParseTuple` function is declared as follows:: - - int PyArg_ParseTuple(PyObject *arg, const char *format, ...); - -The *arg* argument must be a tuple object containing an argument list passed -from Python to a C function. The *format* argument must be a format string, -whose syntax is explained in :ref:`arg-parsing` in the Python/C API Reference -Manual. The remaining arguments must be addresses of variables whose type is -determined by the format string. - -Note that while :c:func:`PyArg_ParseTuple` checks that the Python arguments have -the required types, it cannot check the validity of the addresses of C variables -passed to the call: if you make mistakes there, your code will probably crash or -at least overwrite random bits in memory. So be careful! - -Note that any Python object references which are provided to the caller are -*borrowed* references; do not decrement their reference count! - -Some example calls:: - - #define PY_SSIZE_T_CLEAN /* Make "s#" use Py_ssize_t rather than int. */ - #include <Python.h> - -:: - - int ok; - int i, j; - long k, l; - const char *s; - Py_ssize_t size; - - ok = PyArg_ParseTuple(args, ""); /* No arguments */ - /* Python call: f() */ - -:: - - ok = PyArg_ParseTuple(args, "s", &s); /* A string */ - /* Possible Python call: f('whoops!') */ - -:: - - ok = PyArg_ParseTuple(args, "lls", &k, &l, &s); /* Two longs and a string */ - /* Possible Python call: f(1, 2, 'three') */ - -:: - - ok = PyArg_ParseTuple(args, "(ii)s#", &i, &j, &s, &size); - /* A pair of ints and a string, whose size is also returned */ - /* Possible Python call: f((1, 2), 'three') */ - -:: - - { - const char *file; - const char *mode = "r"; - int bufsize = 0; - ok = PyArg_ParseTuple(args, "s|si", &file, &mode, &bufsize); - /* A string, and optionally another string and an integer */ - /* Possible Python calls: - f('spam') - f('spam', 'w') - f('spam', 'wb', 100000) */ - } - -:: - - { - int left, top, right, bottom, h, v; - ok = PyArg_ParseTuple(args, "((ii)(ii))(ii)", - &left, &top, &right, &bottom, &h, &v); - /* A rectangle and a point */ - /* Possible Python call: - f(((0, 0), (400, 300)), (10, 10)) */ - } - -:: - - { - Py_complex c; - ok = PyArg_ParseTuple(args, "D:myfunction", &c); - /* a complex, also providing a function name for errors */ - /* Possible Python call: myfunction(1+2j) */ - } - - -.. _parsetupleandkeywords: - -Keyword Parameters for Extension Functions -========================================== - -.. index:: single: PyArg_ParseTupleAndKeywords() - -The :c:func:`PyArg_ParseTupleAndKeywords` function is declared as follows:: - - int PyArg_ParseTupleAndKeywords(PyObject *arg, PyObject *kwdict, - const char *format, char *kwlist[], ...); - -The *arg* and *format* parameters are identical to those of the -:c:func:`PyArg_ParseTuple` function. The *kwdict* parameter is the dictionary of -keywords received as the third parameter from the Python runtime. The *kwlist* -parameter is a ``NULL``-terminated list of strings which identify the parameters; -the names are matched with the type information from *format* from left to -right. On success, :c:func:`PyArg_ParseTupleAndKeywords` returns true, otherwise -it returns false and raises an appropriate exception. - -.. note:: - - Nested tuples cannot be parsed when using keyword arguments! Keyword parameters - passed in which are not present in the *kwlist* will cause :exc:`TypeError` to - be raised. - -.. index:: single: Philbrick, Geoff - -Here is an example module which uses keywords, based on an example by Geoff -Philbrick (philbrick@hks.com):: - - #define PY_SSIZE_T_CLEAN /* Make "s#" use Py_ssize_t rather than int. */ - #include <Python.h> - - static PyObject * - keywdarg_parrot(PyObject *self, PyObject *args, PyObject *keywds) - { - int voltage; - const char *state = "a stiff"; - const char *action = "voom"; - const char *type = "Norwegian Blue"; - - static char *kwlist[] = {"voltage", "state", "action", "type", NULL}; - - if (!PyArg_ParseTupleAndKeywords(args, keywds, "i|sss", kwlist, - &voltage, &state, &action, &type)) - return NULL; - - printf("-- This parrot wouldn't %s if you put %i Volts through it.\n", - action, voltage); - printf("-- Lovely plumage, the %s -- It's %s!\n", type, state); - - Py_RETURN_NONE; - } - - static PyMethodDef keywdarg_methods[] = { - /* The cast of the function is necessary since PyCFunction values - * only take two PyObject* parameters, and keywdarg_parrot() takes - * three. - */ - {"parrot", (PyCFunction)(void(*)(void))keywdarg_parrot, METH_VARARGS | METH_KEYWORDS, - "Print a lovely skit to standard output."}, - {NULL, NULL, 0, NULL} /* sentinel */ - }; - - static struct PyModuleDef keywdargmodule = { - PyModuleDef_HEAD_INIT, - "keywdarg", - NULL, - -1, - keywdarg_methods - }; - - PyMODINIT_FUNC - PyInit_keywdarg(void) - { - return PyModule_Create(&keywdargmodule); - } - - -.. _buildvalue: - -Building Arbitrary Values -========================= - -This function is the counterpart to :c:func:`PyArg_ParseTuple`. It is declared -as follows:: - - PyObject *Py_BuildValue(const char *format, ...); - -It recognizes a set of format units similar to the ones recognized by -:c:func:`PyArg_ParseTuple`, but the arguments (which are input to the function, -not output) must not be pointers, just values. It returns a new Python object, -suitable for returning from a C function called from Python. - -One difference with :c:func:`PyArg_ParseTuple`: while the latter requires its -first argument to be a tuple (since Python argument lists are always represented -as tuples internally), :c:func:`Py_BuildValue` does not always build a tuple. It -builds a tuple only if its format string contains two or more format units. If -the format string is empty, it returns ``None``; if it contains exactly one -format unit, it returns whatever object is described by that format unit. To -force it to return a tuple of size 0 or one, parenthesize the format string. - -Examples (to the left the call, to the right the resulting Python value): - -.. code-block:: none - - Py_BuildValue("") None - Py_BuildValue("i", 123) 123 - Py_BuildValue("iii", 123, 456, 789) (123, 456, 789) - Py_BuildValue("s", "hello") 'hello' - Py_BuildValue("y", "hello") b'hello' - Py_BuildValue("ss", "hello", "world") ('hello', 'world') - Py_BuildValue("s#", "hello", 4) 'hell' - Py_BuildValue("y#", "hello", 4) b'hell' - Py_BuildValue("()") () - Py_BuildValue("(i)", 123) (123,) - Py_BuildValue("(ii)", 123, 456) (123, 456) - Py_BuildValue("(i,i)", 123, 456) (123, 456) - Py_BuildValue("[i,i]", 123, 456) [123, 456] - Py_BuildValue("{s:i,s:i}", - "abc", 123, "def", 456) {'abc': 123, 'def': 456} - Py_BuildValue("((ii)(ii)) (ii)", - 1, 2, 3, 4, 5, 6) (((1, 2), (3, 4)), (5, 6)) - - -.. _refcounts: - -Reference Counts -================ - -In languages like C or C++, the programmer is responsible for dynamic allocation -and deallocation of memory on the heap. In C, this is done using the functions -:c:func:`malloc` and :c:func:`free`. In C++, the operators ``new`` and -``delete`` are used with essentially the same meaning and we'll restrict -the following discussion to the C case. - -Every block of memory allocated with :c:func:`malloc` should eventually be -returned to the pool of available memory by exactly one call to :c:func:`free`. -It is important to call :c:func:`free` at the right time. If a block's address -is forgotten but :c:func:`free` is not called for it, the memory it occupies -cannot be reused until the program terminates. This is called a :dfn:`memory -leak`. On the other hand, if a program calls :c:func:`free` for a block and then -continues to use the block, it creates a conflict with re-use of the block -through another :c:func:`malloc` call. This is called :dfn:`using freed memory`. -It has the same bad consequences as referencing uninitialized data --- core -dumps, wrong results, mysterious crashes. - -Common causes of memory leaks are unusual paths through the code. For instance, -a function may allocate a block of memory, do some calculation, and then free -the block again. Now a change in the requirements for the function may add a -test to the calculation that detects an error condition and can return -prematurely from the function. It's easy to forget to free the allocated memory -block when taking this premature exit, especially when it is added later to the -code. Such leaks, once introduced, often go undetected for a long time: the -error exit is taken only in a small fraction of all calls, and most modern -machines have plenty of virtual memory, so the leak only becomes apparent in a -long-running process that uses the leaking function frequently. Therefore, it's -important to prevent leaks from happening by having a coding convention or -strategy that minimizes this kind of errors. - -Since Python makes heavy use of :c:func:`malloc` and :c:func:`free`, it needs a -strategy to avoid memory leaks as well as the use of freed memory. The chosen -method is called :dfn:`reference counting`. The principle is simple: every -object contains a counter, which is incremented when a reference to the object -is stored somewhere, and which is decremented when a reference to it is deleted. -When the counter reaches zero, the last reference to the object has been deleted -and the object is freed. - -An alternative strategy is called :dfn:`automatic garbage collection`. -(Sometimes, reference counting is also referred to as a garbage collection -strategy, hence my use of "automatic" to distinguish the two.) The big -advantage of automatic garbage collection is that the user doesn't need to call -:c:func:`free` explicitly. (Another claimed advantage is an improvement in speed -or memory usage --- this is no hard fact however.) The disadvantage is that for -C, there is no truly portable automatic garbage collector, while reference -counting can be implemented portably (as long as the functions :c:func:`malloc` -and :c:func:`free` are available --- which the C Standard guarantees). Maybe some -day a sufficiently portable automatic garbage collector will be available for C. -Until then, we'll have to live with reference counts. - -While Python uses the traditional reference counting implementation, it also -offers a cycle detector that works to detect reference cycles. This allows -applications to not worry about creating direct or indirect circular references; -these are the weakness of garbage collection implemented using only reference -counting. Reference cycles consist of objects which contain (possibly indirect) -references to themselves, so that each object in the cycle has a reference count -which is non-zero. Typical reference counting implementations are not able to -reclaim the memory belonging to any objects in a reference cycle, or referenced -from the objects in the cycle, even though there are no further references to -the cycle itself. - -The cycle detector is able to detect garbage cycles and can reclaim them. -The :mod:`gc` module exposes a way to run the detector (the -:func:`~gc.collect` function), as well as configuration -interfaces and the ability to disable the detector at runtime. - - -.. _refcountsinpython: - -Reference Counting in Python ----------------------------- - -There are two macros, ``Py_INCREF(x)`` and ``Py_DECREF(x)``, which handle the -incrementing and decrementing of the reference count. :c:func:`Py_DECREF` also -frees the object when the count reaches zero. For flexibility, it doesn't call -:c:func:`free` directly --- rather, it makes a call through a function pointer in -the object's :dfn:`type object`. For this purpose (and others), every object -also contains a pointer to its type object. - -The big question now remains: when to use ``Py_INCREF(x)`` and ``Py_DECREF(x)``? -Let's first introduce some terms. Nobody "owns" an object; however, you can -:dfn:`own a reference` to an object. An object's reference count is now defined -as the number of owned references to it. The owner of a reference is -responsible for calling :c:func:`Py_DECREF` when the reference is no longer -needed. Ownership of a reference can be transferred. There are three ways to -dispose of an owned reference: pass it on, store it, or call :c:func:`Py_DECREF`. -Forgetting to dispose of an owned reference creates a memory leak. - -It is also possible to :dfn:`borrow` [#]_ a reference to an object. The -borrower of a reference should not call :c:func:`Py_DECREF`. The borrower must -not hold on to the object longer than the owner from which it was borrowed. -Using a borrowed reference after the owner has disposed of it risks using freed -memory and should be avoided completely [#]_. - -The advantage of borrowing over owning a reference is that you don't need to -take care of disposing of the reference on all possible paths through the code ---- in other words, with a borrowed reference you don't run the risk of leaking -when a premature exit is taken. The disadvantage of borrowing over owning is -that there are some subtle situations where in seemingly correct code a borrowed -reference can be used after the owner from which it was borrowed has in fact -disposed of it. - -A borrowed reference can be changed into an owned reference by calling -:c:func:`Py_INCREF`. This does not affect the status of the owner from which the -reference was borrowed --- it creates a new owned reference, and gives full -owner responsibilities (the new owner must dispose of the reference properly, as -well as the previous owner). - - -.. _ownershiprules: - -Ownership Rules ---------------- - -Whenever an object reference is passed into or out of a function, it is part of -the function's interface specification whether ownership is transferred with the -reference or not. - -Most functions that return a reference to an object pass on ownership with the -reference. In particular, all functions whose function it is to create a new -object, such as :c:func:`PyLong_FromLong` and :c:func:`Py_BuildValue`, pass -ownership to the receiver. Even if the object is not actually new, you still -receive ownership of a new reference to that object. For instance, -:c:func:`PyLong_FromLong` maintains a cache of popular values and can return a -reference to a cached item. - -Many functions that extract objects from other objects also transfer ownership -with the reference, for instance :c:func:`PyObject_GetAttrString`. The picture -is less clear, here, however, since a few common routines are exceptions: -:c:func:`PyTuple_GetItem`, :c:func:`PyList_GetItem`, :c:func:`PyDict_GetItem`, and -:c:func:`PyDict_GetItemString` all return references that you borrow from the -tuple, list or dictionary. - -The function :c:func:`PyImport_AddModule` also returns a borrowed reference, even -though it may actually create the object it returns: this is possible because an -owned reference to the object is stored in ``sys.modules``. - -When you pass an object reference into another function, in general, the -function borrows the reference from you --- if it needs to store it, it will use -:c:func:`Py_INCREF` to become an independent owner. There are exactly two -important exceptions to this rule: :c:func:`PyTuple_SetItem` and -:c:func:`PyList_SetItem`. These functions take over ownership of the item passed -to them --- even if they fail! (Note that :c:func:`PyDict_SetItem` and friends -don't take over ownership --- they are "normal.") - -When a C function is called from Python, it borrows references to its arguments -from the caller. The caller owns a reference to the object, so the borrowed -reference's lifetime is guaranteed until the function returns. Only when such a -borrowed reference must be stored or passed on, it must be turned into an owned -reference by calling :c:func:`Py_INCREF`. - -The object reference returned from a C function that is called from Python must -be an owned reference --- ownership is transferred from the function to its -caller. - - -.. _thinice: - -Thin Ice --------- - -There are a few situations where seemingly harmless use of a borrowed reference -can lead to problems. These all have to do with implicit invocations of the -interpreter, which can cause the owner of a reference to dispose of it. - -The first and most important case to know about is using :c:func:`Py_DECREF` on -an unrelated object while borrowing a reference to a list item. For instance:: - - void - bug(PyObject *list) - { - PyObject *item = PyList_GetItem(list, 0); - - PyList_SetItem(list, 1, PyLong_FromLong(0L)); - PyObject_Print(item, stdout, 0); /* BUG! */ - } - -This function first borrows a reference to ``list[0]``, then replaces -``list[1]`` with the value ``0``, and finally prints the borrowed reference. -Looks harmless, right? But it's not! - -Let's follow the control flow into :c:func:`PyList_SetItem`. The list owns -references to all its items, so when item 1 is replaced, it has to dispose of -the original item 1. Now let's suppose the original item 1 was an instance of a -user-defined class, and let's further suppose that the class defined a -:meth:`!__del__` method. If this class instance has a reference count of 1, -disposing of it will call its :meth:`!__del__` method. - -Since it is written in Python, the :meth:`!__del__` method can execute arbitrary -Python code. Could it perhaps do something to invalidate the reference to -``item`` in :c:func:`!bug`? You bet! Assuming that the list passed into -:c:func:`!bug` is accessible to the :meth:`!__del__` method, it could execute a -statement to the effect of ``del list[0]``, and assuming this was the last -reference to that object, it would free the memory associated with it, thereby -invalidating ``item``. - -The solution, once you know the source of the problem, is easy: temporarily -increment the reference count. The correct version of the function reads:: - - void - no_bug(PyObject *list) - { - PyObject *item = PyList_GetItem(list, 0); - - Py_INCREF(item); - PyList_SetItem(list, 1, PyLong_FromLong(0L)); - PyObject_Print(item, stdout, 0); - Py_DECREF(item); - } - -This is a true story. An older version of Python contained variants of this bug -and someone spent a considerable amount of time in a C debugger to figure out -why his :meth:`!__del__` methods would fail... - -The second case of problems with a borrowed reference is a variant involving -threads. Normally, multiple threads in the Python interpreter can't get in each -other's way, because there is a global lock protecting Python's entire object -space. However, it is possible to temporarily release this lock using the macro -:c:macro:`Py_BEGIN_ALLOW_THREADS`, and to re-acquire it using -:c:macro:`Py_END_ALLOW_THREADS`. This is common around blocking I/O calls, to -let other threads use the processor while waiting for the I/O to complete. -Obviously, the following function has the same problem as the previous one:: - - void - bug(PyObject *list) - { - PyObject *item = PyList_GetItem(list, 0); - Py_BEGIN_ALLOW_THREADS - ...some blocking I/O call... - Py_END_ALLOW_THREADS - PyObject_Print(item, stdout, 0); /* BUG! */ - } - - -.. _nullpointers: - -NULL Pointers -------------- - -In general, functions that take object references as arguments do not expect you -to pass them ``NULL`` pointers, and will dump core (or cause later core dumps) if -you do so. Functions that return object references generally return ``NULL`` only -to indicate that an exception occurred. The reason for not testing for ``NULL`` -arguments is that functions often pass the objects they receive on to other -function --- if each function were to test for ``NULL``, there would be a lot of -redundant tests and the code would run more slowly. - -It is better to test for ``NULL`` only at the "source:" when a pointer that may be -``NULL`` is received, for example, from :c:func:`malloc` or from a function that -may raise an exception. - -The macros :c:func:`Py_INCREF` and :c:func:`Py_DECREF` do not check for ``NULL`` -pointers --- however, their variants :c:func:`Py_XINCREF` and :c:func:`Py_XDECREF` -do. - -The macros for checking for a particular object type (``Pytype_Check()``) don't -check for ``NULL`` pointers --- again, there is much code that calls several of -these in a row to test an object against various different expected types, and -this would generate redundant tests. There are no variants with ``NULL`` -checking. - -The C function calling mechanism guarantees that the argument list passed to C -functions (``args`` in the examples) is never ``NULL`` --- in fact it guarantees -that it is always a tuple [#]_. - -It is a severe error to ever let a ``NULL`` pointer "escape" to the Python user. - -.. Frank Stajano: - A pedagogically buggy example, along the lines of the previous listing, would - be helpful here -- showing in more concrete terms what sort of actions could - cause the problem. I can't very well imagine it from the description. - - -.. _cplusplus: - -Writing Extensions in C++ -========================= - -It is possible to write extension modules in C++. Some restrictions apply. If -the main program (the Python interpreter) is compiled and linked by the C -compiler, global or static objects with constructors cannot be used. This is -not a problem if the main program is linked by the C++ compiler. Functions that -will be called by the Python interpreter (in particular, module initialization -functions) have to be declared using ``extern "C"``. It is unnecessary to -enclose the Python header files in ``extern "C" {...}`` --- they use this form -already if the symbol ``__cplusplus`` is defined (all recent C++ compilers -define this symbol). - - -.. _using-capsules: - -Providing a C API for an Extension Module -========================================= - -.. sectionauthor:: Konrad Hinsen <hinsen@cnrs-orleans.fr> - - -Many extension modules just provide new functions and types to be used from -Python, but sometimes the code in an extension module can be useful for other -extension modules. For example, an extension module could implement a type -"collection" which works like lists without order. Just like the standard Python -list type has a C API which permits extension modules to create and manipulate -lists, this new collection type should have a set of C functions for direct -manipulation from other extension modules. - -At first sight this seems easy: just write the functions (without declaring them -``static``, of course), provide an appropriate header file, and document -the C API. And in fact this would work if all extension modules were always -linked statically with the Python interpreter. When modules are used as shared -libraries, however, the symbols defined in one module may not be visible to -another module. The details of visibility depend on the operating system; some -systems use one global namespace for the Python interpreter and all extension -modules (Windows, for example), whereas others require an explicit list of -imported symbols at module link time (AIX is one example), or offer a choice of -different strategies (most Unices). And even if symbols are globally visible, -the module whose functions one wishes to call might not have been loaded yet! - -Portability therefore requires not to make any assumptions about symbol -visibility. This means that all symbols in extension modules should be declared -``static``, except for the module's initialization function, in order to -avoid name clashes with other extension modules (as discussed in section -:ref:`methodtable`). And it means that symbols that *should* be accessible from -other extension modules must be exported in a different way. - -Python provides a special mechanism to pass C-level information (pointers) from -one extension module to another one: Capsules. A Capsule is a Python data type -which stores a pointer (:c:expr:`void \*`). Capsules can only be created and -accessed via their C API, but they can be passed around like any other Python -object. In particular, they can be assigned to a name in an extension module's -namespace. Other extension modules can then import this module, retrieve the -value of this name, and then retrieve the pointer from the Capsule. - -There are many ways in which Capsules can be used to export the C API of an -extension module. Each function could get its own Capsule, or all C API pointers -could be stored in an array whose address is published in a Capsule. And the -various tasks of storing and retrieving the pointers can be distributed in -different ways between the module providing the code and the client modules. - -Whichever method you choose, it's important to name your Capsules properly. -The function :c:func:`PyCapsule_New` takes a name parameter -(:c:expr:`const char \*`); you're permitted to pass in a ``NULL`` name, but -we strongly encourage you to specify a name. Properly named Capsules provide -a degree of runtime type-safety; there is no feasible way to tell one unnamed -Capsule from another. - -In particular, Capsules used to expose C APIs should be given a name following -this convention:: - - modulename.attributename - -The convenience function :c:func:`PyCapsule_Import` makes it easy to -load a C API provided via a Capsule, but only if the Capsule's name -matches this convention. This behavior gives C API users a high degree -of certainty that the Capsule they load contains the correct C API. - -The following example demonstrates an approach that puts most of the burden on -the writer of the exporting module, which is appropriate for commonly used -library modules. It stores all C API pointers (just one in the example!) in an -array of :c:expr:`void` pointers which becomes the value of a Capsule. The header -file corresponding to the module provides a macro that takes care of importing -the module and retrieving its C API pointers; client modules only have to call -this macro before accessing the C API. - -The exporting module is a modification of the :mod:`!spam` module from section -:ref:`extending-simpleexample`. The function :func:`!spam.system` does not call -the C library function :c:func:`system` directly, but a function -:c:func:`!PySpam_System`, which would of course do something more complicated in -reality (such as adding "spam" to every command). This function -:c:func:`!PySpam_System` is also exported to other extension modules. - -The function :c:func:`!PySpam_System` is a plain C function, declared -``static`` like everything else:: - - static int - PySpam_System(const char *command) - { - return system(command); - } - -The function :c:func:`!spam_system` is modified in a trivial way:: - - static PyObject * - spam_system(PyObject *self, PyObject *args) - { - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = PySpam_System(command); - return PyLong_FromLong(sts); - } - -In the beginning of the module, right after the line :: - - #include <Python.h> - -two more lines must be added:: - - #define SPAM_MODULE - #include "spammodule.h" - -The ``#define`` is used to tell the header file that it is being included in the -exporting module, not a client module. Finally, the module's initialization -function must take care of initializing the C API pointer array:: - - PyMODINIT_FUNC - PyInit_spam(void) - { - PyObject *m; - static void *PySpam_API[PySpam_API_pointers]; - PyObject *c_api_object; - - m = PyModule_Create(&spammodule); - if (m == NULL) - return NULL; - - /* Initialize the C API pointer array */ - PySpam_API[PySpam_System_NUM] = (void *)PySpam_System; - - /* Create a Capsule containing the API pointer array's address */ - c_api_object = PyCapsule_New((void *)PySpam_API, "spam._C_API", NULL); - - if (PyModule_AddObject(m, "_C_API", c_api_object) < 0) { - Py_XDECREF(c_api_object); - Py_DECREF(m); - return NULL; - } - - return m; - } - -Note that ``PySpam_API`` is declared ``static``; otherwise the pointer -array would disappear when :c:func:`!PyInit_spam` terminates! - -The bulk of the work is in the header file :file:`spammodule.h`, which looks -like this:: - - #ifndef Py_SPAMMODULE_H - #define Py_SPAMMODULE_H - #ifdef __cplusplus - extern "C" { - #endif - - /* Header file for spammodule */ - - /* C API functions */ - #define PySpam_System_NUM 0 - #define PySpam_System_RETURN int - #define PySpam_System_PROTO (const char *command) - - /* Total number of C API pointers */ - #define PySpam_API_pointers 1 - - - #ifdef SPAM_MODULE - /* This section is used when compiling spammodule.c */ - - static PySpam_System_RETURN PySpam_System PySpam_System_PROTO; - - #else - /* This section is used in modules that use spammodule's API */ - - static void **PySpam_API; - - #define PySpam_System \ - (*(PySpam_System_RETURN (*)PySpam_System_PROTO) PySpam_API[PySpam_System_NUM]) - - /* Return -1 on error, 0 on success. - * PyCapsule_Import will set an exception if there's an error. - */ - static int - import_spam(void) - { - PySpam_API = (void **)PyCapsule_Import("spam._C_API", 0); - return (PySpam_API != NULL) ? 0 : -1; - } - - #endif - - #ifdef __cplusplus - } - #endif - - #endif /* !defined(Py_SPAMMODULE_H) */ - -All that a client module must do in order to have access to the function -:c:func:`!PySpam_System` is to call the function (or rather macro) -:c:func:`!import_spam` in its initialization function:: - - PyMODINIT_FUNC - PyInit_client(void) - { - PyObject *m; - - m = PyModule_Create(&clientmodule); - if (m == NULL) - return NULL; - if (import_spam() < 0) - return NULL; - /* additional initialization can happen here */ - return m; - } - -The main disadvantage of this approach is that the file :file:`spammodule.h` is -rather complicated. However, the basic structure is the same for each function -that is exported, so it has to be learned only once. - -Finally it should be mentioned that Capsules offer additional functionality, -which is especially useful for memory allocation and deallocation of the pointer -stored in a Capsule. The details are described in the Python/C API Reference -Manual in the section :ref:`capsules` and in the implementation of Capsules (files -:file:`Include/pycapsule.h` and :file:`Objects/pycapsule.c` in the Python source -code distribution). - -.. rubric:: Footnotes - -.. [#] An interface for this function already exists in the standard module :mod:`os` - --- it was chosen as a simple and straightforward example. - -.. [#] The metaphor of "borrowing" a reference is not completely correct: the owner - still has a copy of the reference. - -.. [#] Checking that the reference count is at least 1 **does not work** --- the - reference count itself could be in freed memory and may thus be reused for - another object! - -.. [#] These guarantees don't hold when you use the "old" style calling convention --- - this is still found in much existing code. diff --git a/Doc/extending/index.rst b/Doc/extending/index.rst deleted file mode 100644 index 01b4df6d44acff..00000000000000 --- a/Doc/extending/index.rst +++ /dev/null @@ -1,74 +0,0 @@ -.. _extending-index: - -################################################## - Extending and Embedding the Python Interpreter -################################################## - -This document describes how to write modules in C or C++ to extend the Python -interpreter with new modules. Those modules can not only define new functions -but also new object types and their methods. The document also describes how -to embed the Python interpreter in another application, for use as an extension -language. Finally, it shows how to compile and link extension modules so that -they can be loaded dynamically (at run time) into the interpreter, if the -underlying operating system supports this feature. - -This document assumes basic knowledge about Python. For an informal -introduction to the language, see :ref:`tutorial-index`. :ref:`reference-index` -gives a more formal definition of the language. :ref:`library-index` documents -the existing object types, functions and modules (both built-in and written in -Python) that give the language its wide application range. - -For a detailed description of the whole Python/C API, see the separate -:ref:`c-api-index`. - - -Recommended third party tools -============================= - -This guide only covers the basic tools for creating extensions provided -as part of this version of CPython. Third party tools like -`Cython <https://cython.org/>`_, `cffi <https://cffi.readthedocs.io>`_, -`SWIG <https://www.swig.org>`_ and `Numba <https://numba.pydata.org/>`_ -offer both simpler and more sophisticated approaches to creating C and C++ -extensions for Python. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions <https://packaging.python.org/guides/packaging-binary-extensions/>`_ - The Python Packaging User Guide not only covers several available - tools that simplify the creation of binary extensions, but also - discusses the various reasons why creating an extension module may be - desirable in the first place. - - -Creating extensions without third party tools -============================================= - -This section of the guide covers creating C and C++ extensions without -assistance from third party tools. It is intended primarily for creators -of those tools, rather than being a recommended way to create your own -C extensions. - -.. toctree:: - :maxdepth: 2 - :numbered: - - extending.rst - newtypes_tutorial.rst - newtypes.rst - building.rst - windows.rst - -Embedding the CPython runtime in a larger application -===================================================== - -Sometimes, rather than creating an extension that runs inside the Python -interpreter as the main application, it is desirable to instead embed -the CPython runtime inside a larger application. This section covers -some of the details involved in doing that successfully. - -.. toctree:: - :maxdepth: 2 - :numbered: - - embedding.rst diff --git a/Doc/extending/newtypes.rst b/Doc/extending/newtypes.rst deleted file mode 100644 index 8ea76aec8290db..00000000000000 --- a/Doc/extending/newtypes.rst +++ /dev/null @@ -1,638 +0,0 @@ -.. highlight:: c - -.. _new-types-topics: - -***************************************** -Defining Extension Types: Assorted Topics -***************************************** - -.. _dnt-type-methods: - -This section aims to give a quick fly-by on the various type methods you can -implement and what they do. - -Here is the definition of :c:type:`PyTypeObject`, with some fields only used in -:ref:`debug builds <debug-build>` omitted: - -.. literalinclude:: ../includes/typestruct.h - - -Now that's a *lot* of methods. Don't worry too much though -- if you have -a type you want to define, the chances are very good that you will only -implement a handful of these. - -As you probably expect by now, we're going to go over this and give more -information about the various handlers. We won't go in the order they are -defined in the structure, because there is a lot of historical baggage that -impacts the ordering of the fields. It's often easiest to find an example -that includes the fields you need and then change the values to suit your new -type. :: - - const char *tp_name; /* For printing */ - -The name of the type -- as mentioned in the previous chapter, this will appear in -various places, almost entirely for diagnostic purposes. Try to choose something -that will be helpful in such a situation! :: - - Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ - -These fields tell the runtime how much memory to allocate when new objects of -this type are created. Python has some built-in support for variable length -structures (think: strings, tuples) which is where the :c:member:`~PyTypeObject.tp_itemsize` field -comes in. This will be dealt with later. :: - - const char *tp_doc; - -Here you can put a string (or its address) that you want returned when the -Python script references ``obj.__doc__`` to retrieve the doc string. - -Now we come to the basic type methods -- the ones most extension types will -implement. - - -Finalization and De-allocation ------------------------------- - -.. index:: - single: object; deallocation - single: deallocation, object - single: object; finalization - single: finalization, of objects - -:: - - destructor tp_dealloc; - -This function is called when the reference count of the instance of your type is -reduced to zero and the Python interpreter wants to reclaim it. If your type -has memory to free or other clean-up to perform, you can put it here. The -object itself needs to be freed here as well. Here is an example of this -function:: - - static void - newdatatype_dealloc(newdatatypeobject *obj) - { - free(obj->obj_UnderlyingDatatypePtr); - Py_TYPE(obj)->tp_free((PyObject *)obj); - } - -If your type supports garbage collection, the destructor should call -:c:func:`PyObject_GC_UnTrack` before clearing any member fields:: - - static void - newdatatype_dealloc(newdatatypeobject *obj) - { - PyObject_GC_UnTrack(obj); - Py_CLEAR(obj->other_obj); - ... - Py_TYPE(obj)->tp_free((PyObject *)obj); - } - -.. index:: - single: PyErr_Fetch() - single: PyErr_Restore() - -One important requirement of the deallocator function is that it leaves any -pending exceptions alone. This is important since deallocators are frequently -called as the interpreter unwinds the Python stack; when the stack is unwound -due to an exception (rather than normal returns), nothing is done to protect the -deallocators from seeing that an exception has already been set. Any actions -which a deallocator performs which may cause additional Python code to be -executed may detect that an exception has been set. This can lead to misleading -errors from the interpreter. The proper way to protect against this is to save -a pending exception before performing the unsafe action, and restoring it when -done. This can be done using the :c:func:`PyErr_Fetch` and -:c:func:`PyErr_Restore` functions:: - - static void - my_dealloc(PyObject *obj) - { - MyObject *self = (MyObject *) obj; - PyObject *cbresult; - - if (self->my_callback != NULL) { - PyObject *err_type, *err_value, *err_traceback; - - /* This saves the current exception state */ - PyErr_Fetch(&err_type, &err_value, &err_traceback); - - cbresult = PyObject_CallNoArgs(self->my_callback); - if (cbresult == NULL) - PyErr_WriteUnraisable(self->my_callback); - else - Py_DECREF(cbresult); - - /* This restores the saved exception state */ - PyErr_Restore(err_type, err_value, err_traceback); - - Py_DECREF(self->my_callback); - } - Py_TYPE(obj)->tp_free((PyObject*)self); - } - -.. note:: - There are limitations to what you can safely do in a deallocator function. - First, if your type supports garbage collection (using :c:member:`~PyTypeObject.tp_traverse` - and/or :c:member:`~PyTypeObject.tp_clear`), some of the object's members can have been - cleared or finalized by the time :c:member:`~PyTypeObject.tp_dealloc` is called. Second, in - :c:member:`~PyTypeObject.tp_dealloc`, your object is in an unstable state: its reference - count is equal to zero. Any call to a non-trivial object or API (as in the - example above) might end up calling :c:member:`~PyTypeObject.tp_dealloc` again, causing a - double free and a crash. - - Starting with Python 3.4, it is recommended not to put any complex - finalization code in :c:member:`~PyTypeObject.tp_dealloc`, and instead use the new - :c:member:`~PyTypeObject.tp_finalize` type method. - - .. seealso:: - :pep:`442` explains the new finalization scheme. - -.. index:: - single: string; object representation - pair: built-in function; repr - -Object Presentation -------------------- - -In Python, there are two ways to generate a textual representation of an object: -the :func:`repr` function, and the :func:`str` function. (The :func:`print` -function just calls :func:`str`.) These handlers are both optional. - -:: - - reprfunc tp_repr; - reprfunc tp_str; - -The :c:member:`~PyTypeObject.tp_repr` handler should return a string object containing a -representation of the instance for which it is called. Here is a simple -example:: - - static PyObject * - newdatatype_repr(newdatatypeobject * obj) - { - return PyUnicode_FromFormat("Repr-ified_newdatatype{{size:%d}}", - obj->obj_UnderlyingDatatypePtr->size); - } - -If no :c:member:`~PyTypeObject.tp_repr` handler is specified, the interpreter will supply a -representation that uses the type's :c:member:`~PyTypeObject.tp_name` and a uniquely identifying -value for the object. - -The :c:member:`~PyTypeObject.tp_str` handler is to :func:`str` what the :c:member:`~PyTypeObject.tp_repr` handler -described above is to :func:`repr`; that is, it is called when Python code calls -:func:`str` on an instance of your object. Its implementation is very similar -to the :c:member:`~PyTypeObject.tp_repr` function, but the resulting string is intended for human -consumption. If :c:member:`~PyTypeObject.tp_str` is not specified, the :c:member:`~PyTypeObject.tp_repr` handler is -used instead. - -Here is a simple example:: - - static PyObject * - newdatatype_str(newdatatypeobject * obj) - { - return PyUnicode_FromFormat("Stringified_newdatatype{{size:%d}}", - obj->obj_UnderlyingDatatypePtr->size); - } - - - -Attribute Management --------------------- - -For every object which can support attributes, the corresponding type must -provide the functions that control how the attributes are resolved. There needs -to be a function which can retrieve attributes (if any are defined), and another -to set attributes (if setting attributes is allowed). Removing an attribute is -a special case, for which the new value passed to the handler is ``NULL``. - -Python supports two pairs of attribute handlers; a type that supports attributes -only needs to implement the functions for one pair. The difference is that one -pair takes the name of the attribute as a :c:expr:`char\*`, while the other -accepts a :c:expr:`PyObject*`. Each type can use whichever pair makes more -sense for the implementation's convenience. :: - - getattrfunc tp_getattr; /* char * version */ - setattrfunc tp_setattr; - /* ... */ - getattrofunc tp_getattro; /* PyObject * version */ - setattrofunc tp_setattro; - -If accessing attributes of an object is always a simple operation (this will be -explained shortly), there are generic implementations which can be used to -provide the :c:expr:`PyObject*` version of the attribute management functions. -The actual need for type-specific attribute handlers almost completely -disappeared starting with Python 2.2, though there are many examples which have -not been updated to use some of the new generic mechanism that is available. - - -.. _generic-attribute-management: - -Generic Attribute Management -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Most extension types only use *simple* attributes. So, what makes the -attributes simple? There are only a couple of conditions that must be met: - -#. The name of the attributes must be known when :c:func:`PyType_Ready` is - called. - -#. No special processing is needed to record that an attribute was looked up or - set, nor do actions need to be taken based on the value. - -Note that this list does not place any restrictions on the values of the -attributes, when the values are computed, or how relevant data is stored. - -When :c:func:`PyType_Ready` is called, it uses three tables referenced by the -type object to create :term:`descriptor`\s which are placed in the dictionary of the -type object. Each descriptor controls access to one attribute of the instance -object. Each of the tables is optional; if all three are ``NULL``, instances of -the type will only have attributes that are inherited from their base type, and -should leave the :c:member:`~PyTypeObject.tp_getattro` and :c:member:`~PyTypeObject.tp_setattro` fields ``NULL`` as -well, allowing the base type to handle attributes. - -The tables are declared as three fields of the type object:: - - struct PyMethodDef *tp_methods; - struct PyMemberDef *tp_members; - struct PyGetSetDef *tp_getset; - -If :c:member:`~PyTypeObject.tp_methods` is not ``NULL``, it must refer to an array of -:c:type:`PyMethodDef` structures. Each entry in the table is an instance of this -structure:: - - typedef struct PyMethodDef { - const char *ml_name; /* method name */ - PyCFunction ml_meth; /* implementation function */ - int ml_flags; /* flags */ - const char *ml_doc; /* docstring */ - } PyMethodDef; - -One entry should be defined for each method provided by the type; no entries are -needed for methods inherited from a base type. One additional entry is needed -at the end; it is a sentinel that marks the end of the array. The -:c:member:`~PyMethodDef.ml_name` field of the sentinel must be ``NULL``. - -The second table is used to define attributes which map directly to data stored -in the instance. A variety of primitive C types are supported, and access may -be read-only or read-write. The structures in the table are defined as:: - - typedef struct PyMemberDef { - const char *name; - int type; - int offset; - int flags; - const char *doc; - } PyMemberDef; - -For each entry in the table, a :term:`descriptor` will be constructed and added to the -type which will be able to extract a value from the instance structure. The -:c:member:`~PyMemberDef.type` field should contain one of the type codes defined in the -:file:`structmember.h` header; the value will be used to determine how to -convert Python values to and from C values. The :c:member:`~PyMemberDef.flags` field is used to -store flags which control how the attribute can be accessed. - -The following flag constants are defined in :file:`structmember.h`; they may be -combined using bitwise-OR. - -+---------------------------+----------------------------------------------+ -| Constant | Meaning | -+===========================+==============================================+ -| :const:`READONLY` | Never writable. | -+---------------------------+----------------------------------------------+ -| :const:`PY_AUDIT_READ` | Emit an ``object.__getattr__`` | -| | :ref:`audit events <audit-events>` before | -| | reading. | -+---------------------------+----------------------------------------------+ - -.. versionchanged:: 3.10 - :c:macro:`RESTRICTED`, :c:macro:`READ_RESTRICTED` and :c:macro:`WRITE_RESTRICTED` - are deprecated. However, :c:macro:`READ_RESTRICTED` is an alias for - :c:macro:`PY_AUDIT_READ`, so fields that specify either :c:macro:`RESTRICTED` - or :c:macro:`READ_RESTRICTED` will also raise an audit event. - -.. index:: - single: READONLY - single: READ_RESTRICTED - single: WRITE_RESTRICTED - single: RESTRICTED - single: PY_AUDIT_READ - -An interesting advantage of using the :c:member:`~PyTypeObject.tp_members` table to build -descriptors that are used at runtime is that any attribute defined this way can -have an associated doc string simply by providing the text in the table. An -application can use the introspection API to retrieve the descriptor from the -class object, and get the doc string using its :attr:`__doc__` attribute. - -As with the :c:member:`~PyTypeObject.tp_methods` table, a sentinel entry with a :c:member:`~PyMethodDef.ml_name` value -of ``NULL`` is required. - -.. XXX Descriptors need to be explained in more detail somewhere, but not here. - - Descriptor objects have two handler functions which correspond to the - \member{tp_getattro} and \member{tp_setattro} handlers. The - \method{__get__()} handler is a function which is passed the descriptor, - instance, and type objects, and returns the value of the attribute, or it - returns \NULL{} and sets an exception. The \method{__set__()} handler is - passed the descriptor, instance, type, and new value; - - -Type-specific Attribute Management -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -For simplicity, only the :c:expr:`char\*` version will be demonstrated here; the -type of the name parameter is the only difference between the :c:expr:`char\*` -and :c:expr:`PyObject*` flavors of the interface. This example effectively does -the same thing as the generic example above, but does not use the generic -support added in Python 2.2. It explains how the handler functions are -called, so that if you do need to extend their functionality, you'll understand -what needs to be done. - -The :c:member:`~PyTypeObject.tp_getattr` handler is called when the object requires an attribute -look-up. It is called in the same situations where the :meth:`~object.__getattr__` -method of a class would be called. - -Here is an example:: - - static PyObject * - newdatatype_getattr(newdatatypeobject *obj, char *name) - { - if (strcmp(name, "data") == 0) - { - return PyLong_FromLong(obj->data); - } - - PyErr_Format(PyExc_AttributeError, - "'%.50s' object has no attribute '%.400s'", - tp->tp_name, name); - return NULL; - } - -The :c:member:`~PyTypeObject.tp_setattr` handler is called when the :meth:`~object.__setattr__` or -:meth:`~object.__delattr__` method of a class instance would be called. When an -attribute should be deleted, the third parameter will be ``NULL``. Here is an -example that simply raises an exception; if this were really all you wanted, the -:c:member:`~PyTypeObject.tp_setattr` handler should be set to ``NULL``. :: - - static int - newdatatype_setattr(newdatatypeobject *obj, char *name, PyObject *v) - { - PyErr_Format(PyExc_RuntimeError, "Read-only attribute: %s", name); - return -1; - } - -Object Comparison ------------------ - -:: - - richcmpfunc tp_richcompare; - -The :c:member:`~PyTypeObject.tp_richcompare` handler is called when comparisons are needed. It is -analogous to the :ref:`rich comparison methods <richcmpfuncs>`, like -:meth:`!__lt__`, and also called by :c:func:`PyObject_RichCompare` and -:c:func:`PyObject_RichCompareBool`. - -This function is called with two Python objects and the operator as arguments, -where the operator is one of ``Py_EQ``, ``Py_NE``, ``Py_LE``, ``Py_GE``, -``Py_LT`` or ``Py_GT``. It should compare the two objects with respect to the -specified operator and return ``Py_True`` or ``Py_False`` if the comparison is -successful, ``Py_NotImplemented`` to indicate that comparison is not -implemented and the other object's comparison method should be tried, or ``NULL`` -if an exception was set. - -Here is a sample implementation, for a datatype that is considered equal if the -size of an internal pointer is equal:: - - static PyObject * - newdatatype_richcmp(PyObject *obj1, PyObject *obj2, int op) - { - PyObject *result; - int c, size1, size2; - - /* code to make sure that both arguments are of type - newdatatype omitted */ - - size1 = obj1->obj_UnderlyingDatatypePtr->size; - size2 = obj2->obj_UnderlyingDatatypePtr->size; - - switch (op) { - case Py_LT: c = size1 < size2; break; - case Py_LE: c = size1 <= size2; break; - case Py_EQ: c = size1 == size2; break; - case Py_NE: c = size1 != size2; break; - case Py_GT: c = size1 > size2; break; - case Py_GE: c = size1 >= size2; break; - } - result = c ? Py_True : Py_False; - Py_INCREF(result); - return result; - } - - -Abstract Protocol Support -------------------------- - -Python supports a variety of *abstract* 'protocols;' the specific interfaces -provided to use these interfaces are documented in :ref:`abstract`. - - -A number of these abstract interfaces were defined early in the development of -the Python implementation. In particular, the number, mapping, and sequence -protocols have been part of Python since the beginning. Other protocols have -been added over time. For protocols which depend on several handler routines -from the type implementation, the older protocols have been defined as optional -blocks of handlers referenced by the type object. For newer protocols there are -additional slots in the main type object, with a flag bit being set to indicate -that the slots are present and should be checked by the interpreter. (The flag -bit does not indicate that the slot values are non-``NULL``. The flag may be set -to indicate the presence of a slot, but a slot may still be unfilled.) :: - - PyNumberMethods *tp_as_number; - PySequenceMethods *tp_as_sequence; - PyMappingMethods *tp_as_mapping; - -If you wish your object to be able to act like a number, a sequence, or a -mapping object, then you place the address of a structure that implements the C -type :c:type:`PyNumberMethods`, :c:type:`PySequenceMethods`, or -:c:type:`PyMappingMethods`, respectively. It is up to you to fill in this -structure with appropriate values. You can find examples of the use of each of -these in the :file:`Objects` directory of the Python source distribution. :: - - hashfunc tp_hash; - -This function, if you choose to provide it, should return a hash number for an -instance of your data type. Here is a simple example:: - - static Py_hash_t - newdatatype_hash(newdatatypeobject *obj) - { - Py_hash_t result; - result = obj->some_size + 32767 * obj->some_number; - if (result == -1) - result = -2; - return result; - } - -:c:type:`Py_hash_t` is a signed integer type with a platform-varying width. -Returning ``-1`` from :c:member:`~PyTypeObject.tp_hash` indicates an error, -which is why you should be careful to avoid returning it when hash computation -is successful, as seen above. - -:: - - ternaryfunc tp_call; - -This function is called when an instance of your data type is "called", for -example, if ``obj1`` is an instance of your data type and the Python script -contains ``obj1('hello')``, the :c:member:`~PyTypeObject.tp_call` handler is invoked. - -This function takes three arguments: - -#. *self* is the instance of the data type which is the subject of the call. - If the call is ``obj1('hello')``, then *self* is ``obj1``. - -#. *args* is a tuple containing the arguments to the call. You can use - :c:func:`PyArg_ParseTuple` to extract the arguments. - -#. *kwds* is a dictionary of keyword arguments that were passed. If this is - non-``NULL`` and you support keyword arguments, use - :c:func:`PyArg_ParseTupleAndKeywords` to extract the arguments. If you - do not want to support keyword arguments and this is non-``NULL``, raise a - :exc:`TypeError` with a message saying that keyword arguments are not supported. - -Here is a toy ``tp_call`` implementation:: - - static PyObject * - newdatatype_call(newdatatypeobject *self, PyObject *args, PyObject *kwds) - { - PyObject *result; - const char *arg1; - const char *arg2; - const char *arg3; - - if (!PyArg_ParseTuple(args, "sss:call", &arg1, &arg2, &arg3)) { - return NULL; - } - result = PyUnicode_FromFormat( - "Returning -- value: [%d] arg1: [%s] arg2: [%s] arg3: [%s]\n", - obj->obj_UnderlyingDatatypePtr->size, - arg1, arg2, arg3); - return result; - } - -:: - - /* Iterators */ - getiterfunc tp_iter; - iternextfunc tp_iternext; - -These functions provide support for the iterator protocol. Both handlers -take exactly one parameter, the instance for which they are being called, -and return a new reference. In the case of an error, they should set an -exception and return ``NULL``. :c:member:`~PyTypeObject.tp_iter` corresponds -to the Python :meth:`~object.__iter__` method, while :c:member:`~PyTypeObject.tp_iternext` -corresponds to the Python :meth:`~iterator.__next__` method. - -Any :term:`iterable` object must implement the :c:member:`~PyTypeObject.tp_iter` -handler, which must return an :term:`iterator` object. Here the same guidelines -apply as for Python classes: - -* For collections (such as lists and tuples) which can support multiple - independent iterators, a new iterator should be created and returned by - each call to :c:member:`~PyTypeObject.tp_iter`. -* Objects which can only be iterated over once (usually due to side effects of - iteration, such as file objects) can implement :c:member:`~PyTypeObject.tp_iter` - by returning a new reference to themselves -- and should also therefore - implement the :c:member:`~PyTypeObject.tp_iternext` handler. - -Any :term:`iterator` object should implement both :c:member:`~PyTypeObject.tp_iter` -and :c:member:`~PyTypeObject.tp_iternext`. An iterator's -:c:member:`~PyTypeObject.tp_iter` handler should return a new reference -to the iterator. Its :c:member:`~PyTypeObject.tp_iternext` handler should -return a new reference to the next object in the iteration, if there is one. -If the iteration has reached the end, :c:member:`~PyTypeObject.tp_iternext` -may return ``NULL`` without setting an exception, or it may set -:exc:`StopIteration` *in addition* to returning ``NULL``; avoiding -the exception can yield slightly better performance. If an actual error -occurs, :c:member:`~PyTypeObject.tp_iternext` should always set an exception -and return ``NULL``. - - -.. _weakref-support: - -Weak Reference Support ----------------------- - -One of the goals of Python's weak reference implementation is to allow any type -to participate in the weak reference mechanism without incurring the overhead on -performance-critical objects (such as numbers). - -.. seealso:: - Documentation for the :mod:`weakref` module. - -For an object to be weakly referencable, the extension type must do two things: - -#. Include a :c:expr:`PyObject*` field in the C object structure dedicated to - the weak reference mechanism. The object's constructor should leave it - ``NULL`` (which is automatic when using the default - :c:member:`~PyTypeObject.tp_alloc`). - -#. Set the :c:member:`~PyTypeObject.tp_weaklistoffset` type member - to the offset of the aforementioned field in the C object structure, - so that the interpreter knows how to access and modify that field. - -Concretely, here is how a trivial object structure would be augmented -with the required field:: - - typedef struct { - PyObject_HEAD - PyObject *weakreflist; /* List of weak references */ - } TrivialObject; - -And the corresponding member in the statically declared type object:: - - static PyTypeObject TrivialType = { - PyVarObject_HEAD_INIT(NULL, 0) - /* ... other members omitted for brevity ... */ - .tp_weaklistoffset = offsetof(TrivialObject, weakreflist), - }; - -The only further addition is that ``tp_dealloc`` needs to clear any weak -references (by calling :c:func:`PyObject_ClearWeakRefs`) if the field is -non-``NULL``:: - - static void - Trivial_dealloc(TrivialObject *self) - { - /* Clear weakrefs first before calling any destructors */ - if (self->weakreflist != NULL) - PyObject_ClearWeakRefs((PyObject *) self); - /* ... remainder of destruction code omitted for brevity ... */ - Py_TYPE(self)->tp_free((PyObject *) self); - } - - -More Suggestions ----------------- - -In order to learn how to implement any specific method for your new data type, -get the :term:`CPython` source code. Go to the :file:`Objects` directory, -then search the C source files for ``tp_`` plus the function you want -(for example, ``tp_richcompare``). You will find examples of the function -you want to implement. - -When you need to verify that an object is a concrete instance of the type you -are implementing, use the :c:func:`PyObject_TypeCheck` function. A sample of -its use might be something like the following:: - - if (!PyObject_TypeCheck(some_object, &MyType)) { - PyErr_SetString(PyExc_TypeError, "arg #1 not a mything"); - return NULL; - } - -.. seealso:: - Download CPython source releases. - https://www.python.org/downloads/source/ - - The CPython project on GitHub, where the CPython source code is developed. - https://github.com/python/cpython diff --git a/Doc/extending/newtypes_tutorial.rst b/Doc/extending/newtypes_tutorial.rst deleted file mode 100644 index f4684ceb1a72a4..00000000000000 --- a/Doc/extending/newtypes_tutorial.rst +++ /dev/null @@ -1,908 +0,0 @@ -.. highlight:: c - -.. _defining-new-types: - -********************************** -Defining Extension Types: Tutorial -********************************** - -.. sectionauthor:: Michael Hudson <mwh@python.net> -.. sectionauthor:: Dave Kuhlman <dkuhlman@rexx.com> -.. sectionauthor:: Jim Fulton <jim@zope.com> - - -Python allows the writer of a C extension module to define new types that -can be manipulated from Python code, much like the built-in :class:`str` -and :class:`list` types. The code for all extension types follows a -pattern, but there are some details that you need to understand before you -can get started. This document is a gentle introduction to the topic. - - -.. _dnt-basics: - -The Basics -========== - -The :term:`CPython` runtime sees all Python objects as variables of type -:c:expr:`PyObject*`, which serves as a "base type" for all Python objects. -The :c:type:`PyObject` structure itself only contains the object's -:term:`reference count` and a pointer to the object's "type object". -This is where the action is; the type object determines which (C) functions -get called by the interpreter when, for instance, an attribute gets looked up -on an object, a method called, or it is multiplied by another object. These -C functions are called "type methods". - -So, if you want to define a new extension type, you need to create a new type -object. - -This sort of thing can only be explained by example, so here's a minimal, but -complete, module that defines a new type named :class:`!Custom` inside a C -extension module :mod:`!custom`: - -.. note:: - What we're showing here is the traditional way of defining *static* - extension types. It should be adequate for most uses. The C API also - allows defining heap-allocated extension types using the - :c:func:`PyType_FromSpec` function, which isn't covered in this tutorial. - -.. literalinclude:: ../includes/custom.c - -Now that's quite a bit to take in at once, but hopefully bits will seem familiar -from the previous chapter. This file defines three things: - -#. What a :class:`!Custom` **object** contains: this is the ``CustomObject`` - struct, which is allocated once for each :class:`!Custom` instance. -#. How the :class:`!Custom` **type** behaves: this is the ``CustomType`` struct, - which defines a set of flags and function pointers that the interpreter - inspects when specific operations are requested. -#. How to initialize the :mod:`!custom` module: this is the ``PyInit_custom`` - function and the associated ``custommodule`` struct. - -The first bit is:: - - typedef struct { - PyObject_HEAD - } CustomObject; - -This is what a Custom object will contain. ``PyObject_HEAD`` is mandatory -at the start of each object struct and defines a field called ``ob_base`` -of type :c:type:`PyObject`, containing a pointer to a type object and a -reference count (these can be accessed using the macros :c:macro:`Py_TYPE` -and :c:macro:`Py_REFCNT` respectively). The reason for the macro is to -abstract away the layout and to enable additional fields in :ref:`debug builds -<debug-build>`. - -.. note:: - There is no semicolon above after the :c:macro:`PyObject_HEAD` macro. - Be wary of adding one by accident: some compilers will complain. - -Of course, objects generally store additional data besides the standard -``PyObject_HEAD`` boilerplate; for example, here is the definition for -standard Python floats:: - - typedef struct { - PyObject_HEAD - double ob_fval; - } PyFloatObject; - -The second bit is the definition of the type object. :: - - static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom.Custom", - .tp_doc = PyDoc_STR("Custom objects"), - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = PyType_GenericNew, - }; - -.. note:: - We recommend using C99-style designated initializers as above, to - avoid listing all the :c:type:`PyTypeObject` fields that you don't care - about and also to avoid caring about the fields' declaration order. - -The actual definition of :c:type:`PyTypeObject` in :file:`object.h` has -many more :ref:`fields <type-structs>` than the definition above. The -remaining fields will be filled with zeros by the C compiler, and it's -common practice to not specify them explicitly unless you need them. - -We're going to pick it apart, one field at a time:: - - PyVarObject_HEAD_INIT(NULL, 0) - -This line is mandatory boilerplate to initialize the ``ob_base`` -field mentioned above. :: - - .tp_name = "custom.Custom", - -The name of our type. This will appear in the default textual representation of -our objects and in some error messages, for example: - -.. code-block:: pycon - - >>> "" + custom.Custom() - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: can only concatenate str (not "custom.Custom") to str - -Note that the name is a dotted name that includes both the module name and the -name of the type within the module. The module in this case is :mod:`!custom` and -the type is :class:`!Custom`, so we set the type name to :class:`!custom.Custom`. -Using the real dotted import path is important to make your type compatible -with the :mod:`pydoc` and :mod:`pickle` modules. :: - - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - -This is so that Python knows how much memory to allocate when creating -new :class:`!Custom` instances. :c:member:`~PyTypeObject.tp_itemsize` is -only used for variable-sized objects and should otherwise be zero. - -.. note:: - - If you want your type to be subclassable from Python, and your type has the same - :c:member:`~PyTypeObject.tp_basicsize` as its base type, you may have problems with multiple - inheritance. A Python subclass of your type will have to list your type first - in its :attr:`~class.__bases__`, or else it will not be able to call your type's - :meth:`~object.__new__` method without getting an error. You can avoid this problem by - ensuring that your type has a larger value for :c:member:`~PyTypeObject.tp_basicsize` than its - base type does. Most of the time, this will be true anyway, because either your - base type will be :class:`object`, or else you will be adding data members to - your base type, and therefore increasing its size. - -We set the class flags to :c:macro:`Py_TPFLAGS_DEFAULT`. :: - - .tp_flags = Py_TPFLAGS_DEFAULT, - -All types should include this constant in their flags. It enables all of the -members defined until at least Python 3.3. If you need further members, -you will need to OR the corresponding flags. - -We provide a doc string for the type in :c:member:`~PyTypeObject.tp_doc`. :: - - .tp_doc = PyDoc_STR("Custom objects"), - -To enable object creation, we have to provide a :c:member:`~PyTypeObject.tp_new` -handler. This is the equivalent of the Python method :meth:`~object.__new__`, but -has to be specified explicitly. In this case, we can just use the default -implementation provided by the API function :c:func:`PyType_GenericNew`. :: - - .tp_new = PyType_GenericNew, - -Everything else in the file should be familiar, except for some code in -:c:func:`!PyInit_custom`:: - - if (PyType_Ready(&CustomType) < 0) - return; - -This initializes the :class:`!Custom` type, filling in a number of members -to the appropriate default values, including :c:member:`~PyObject.ob_type` that we initially -set to ``NULL``. :: - - Py_INCREF(&CustomType); - if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(&CustomType); - Py_DECREF(m); - return NULL; - } - -This adds the type to the module dictionary. This allows us to create -:class:`!Custom` instances by calling the :class:`!Custom` class: - -.. code-block:: pycon - - >>> import custom - >>> mycustom = custom.Custom() - -That's it! All that remains is to build it; put the above code in a file called -:file:`custom.c` and: - -.. code-block:: python - - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[Extension("custom", ["custom.c"])]) - -in a file called :file:`setup.py`; then typing - -.. code-block:: shell-session - - $ python setup.py build - -at a shell should produce a file :file:`custom.so` in a subdirectory; move to -that directory and fire up Python --- you should be able to ``import custom`` and -play around with Custom objects. - -That wasn't so hard, was it? - -Of course, the current Custom type is pretty uninteresting. It has no data and -doesn't do anything. It can't even be subclassed. - -.. note:: - While this documentation showcases the standard :mod:`!distutils` module - for building C extensions, it is recommended in real-world use cases to - use the newer and better-maintained ``setuptools`` library. Documentation - on how to do this is out of scope for this document and can be found in - the `Python Packaging User's Guide <https://packaging.python.org/tutorials/distributing-packages/>`_. - - -Adding data and methods to the Basic example -============================================ - -Let's extend the basic example to add some data and methods. Let's also make -the type usable as a base class. We'll create a new module, :mod:`!custom2` that -adds these capabilities: - -.. literalinclude:: ../includes/custom2.c - - -This version of the module has a number of changes. - -We've added an extra include:: - - #include <structmember.h> - -This include provides declarations that we use to handle attributes, as -described a bit later. - -The :class:`!Custom` type now has three data attributes in its C struct, -*first*, *last*, and *number*. The *first* and *last* variables are Python -strings containing first and last names. The *number* attribute is a C integer. - -The object structure is updated accordingly:: - - typedef struct { - PyObject_HEAD - PyObject *first; /* first name */ - PyObject *last; /* last name */ - int number; - } CustomObject; - -Because we now have data to manage, we have to be more careful about object -allocation and deallocation. At a minimum, we need a deallocation method:: - - static void - Custom_dealloc(CustomObject *self) - { - Py_XDECREF(self->first); - Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); - } - -which is assigned to the :c:member:`~PyTypeObject.tp_dealloc` member:: - - .tp_dealloc = (destructor) Custom_dealloc, - -This method first clears the reference counts of the two Python attributes. -:c:func:`Py_XDECREF` correctly handles the case where its argument is -``NULL`` (which might happen here if ``tp_new`` failed midway). It then -calls the :c:member:`~PyTypeObject.tp_free` member of the object's type -(computed by ``Py_TYPE(self)``) to free the object's memory. Note that -the object's type might not be :class:`!CustomType`, because the object may -be an instance of a subclass. - -.. note:: - The explicit cast to ``destructor`` above is needed because we defined - ``Custom_dealloc`` to take a ``CustomObject *`` argument, but the ``tp_dealloc`` - function pointer expects to receive a ``PyObject *`` argument. Otherwise, - the compiler will emit a warning. This is object-oriented polymorphism, - in C! - -We want to make sure that the first and last names are initialized to empty -strings, so we provide a ``tp_new`` implementation:: - - static PyObject * - Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) - { - CustomObject *self; - self = (CustomObject *) type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - self->number = 0; - } - return (PyObject *) self; - } - -and install it in the :c:member:`~PyTypeObject.tp_new` member:: - - .tp_new = Custom_new, - -The ``tp_new`` handler is responsible for creating (as opposed to initializing) -objects of the type. It is exposed in Python as the :meth:`~object.__new__` method. -It is not required to define a ``tp_new`` member, and indeed many extension -types will simply reuse :c:func:`PyType_GenericNew` as done in the first -version of the :class:`!Custom` type above. In this case, we use the ``tp_new`` -handler to initialize the ``first`` and ``last`` attributes to non-``NULL`` -default values. - -``tp_new`` is passed the type being instantiated (not necessarily ``CustomType``, -if a subclass is instantiated) and any arguments passed when the type was -called, and is expected to return the instance created. ``tp_new`` handlers -always accept positional and keyword arguments, but they often ignore the -arguments, leaving the argument handling to initializer (a.k.a. ``tp_init`` -in C or ``__init__`` in Python) methods. - -.. note:: - ``tp_new`` shouldn't call ``tp_init`` explicitly, as the interpreter - will do it itself. - -The ``tp_new`` implementation calls the :c:member:`~PyTypeObject.tp_alloc` -slot to allocate memory:: - - self = (CustomObject *) type->tp_alloc(type, 0); - -Since memory allocation may fail, we must check the :c:member:`~PyTypeObject.tp_alloc` -result against ``NULL`` before proceeding. - -.. note:: - We didn't fill the :c:member:`~PyTypeObject.tp_alloc` slot ourselves. Rather - :c:func:`PyType_Ready` fills it for us by inheriting it from our base class, - which is :class:`object` by default. Most types use the default allocation - strategy. - -.. note:: - If you are creating a co-operative :c:member:`~PyTypeObject.tp_new` (one - that calls a base type's :c:member:`~PyTypeObject.tp_new` or :meth:`~object.__new__`), - you must *not* try to determine what method to call using method resolution - order at runtime. Always statically determine what type you are going to - call, and call its :c:member:`~PyTypeObject.tp_new` directly, or via - ``type->tp_base->tp_new``. If you do not do this, Python subclasses of your - type that also inherit from other Python-defined classes may not work correctly. - (Specifically, you may not be able to create instances of such subclasses - without getting a :exc:`TypeError`.) - -We also define an initialization function which accepts arguments to provide -initial values for our instance:: - - static int - Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) - { - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_XDECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_XDECREF(tmp); - } - return 0; - } - -by filling the :c:member:`~PyTypeObject.tp_init` slot. :: - - .tp_init = (initproc) Custom_init, - -The :c:member:`~PyTypeObject.tp_init` slot is exposed in Python as the -:meth:`~object.__init__` method. It is used to initialize an object after it's -created. Initializers always accept positional and keyword arguments, -and they should return either ``0`` on success or ``-1`` on error. - -Unlike the ``tp_new`` handler, there is no guarantee that ``tp_init`` -is called at all (for example, the :mod:`pickle` module by default -doesn't call :meth:`~object.__init__` on unpickled instances). It can also be -called multiple times. Anyone can call the :meth:`!__init__` method on -our objects. For this reason, we have to be extra careful when assigning -the new attribute values. We might be tempted, for example to assign the -``first`` member like this:: - - if (first) { - Py_XDECREF(self->first); - Py_INCREF(first); - self->first = first; - } - -But this would be risky. Our type doesn't restrict the type of the -``first`` member, so it could be any kind of object. It could have a -destructor that causes code to be executed that tries to access the -``first`` member; or that destructor could release the -:term:`Global interpreter Lock <GIL>` and let arbitrary code run in other -threads that accesses and modifies our object. - -To be paranoid and protect ourselves against this possibility, we almost -always reassign members before decrementing their reference counts. When -don't we have to do this? - -* when we absolutely know that the reference count is greater than 1; - -* when we know that deallocation of the object [#]_ will neither release - the :term:`GIL` nor cause any calls back into our type's code; - -* when decrementing a reference count in a :c:member:`~PyTypeObject.tp_dealloc` - handler on a type which doesn't support cyclic garbage collection [#]_. - -We want to expose our instance variables as attributes. There are a -number of ways to do that. The simplest way is to define member definitions:: - - static PyMemberDef Custom_members[] = { - {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0, - "first name"}, - {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0, - "last name"}, - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ - }; - -and put the definitions in the :c:member:`~PyTypeObject.tp_members` slot:: - - .tp_members = Custom_members, - -Each member definition has a member name, type, offset, access flags and -documentation string. See the :ref:`Generic-Attribute-Management` section -below for details. - -A disadvantage of this approach is that it doesn't provide a way to restrict the -types of objects that can be assigned to the Python attributes. We expect the -first and last names to be strings, but any Python objects can be assigned. -Further, the attributes can be deleted, setting the C pointers to ``NULL``. Even -though we can make sure the members are initialized to non-``NULL`` values, the -members can be set to ``NULL`` if the attributes are deleted. - -We define a single method, :meth:`!Custom.name()`, that outputs the objects name as the -concatenation of the first and last names. :: - - static PyObject * - Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) - { - if (self->first == NULL) { - PyErr_SetString(PyExc_AttributeError, "first"); - return NULL; - } - if (self->last == NULL) { - PyErr_SetString(PyExc_AttributeError, "last"); - return NULL; - } - return PyUnicode_FromFormat("%S %S", self->first, self->last); - } - -The method is implemented as a C function that takes a :class:`!Custom` (or -:class:`!Custom` subclass) instance as the first argument. Methods always take an -instance as the first argument. Methods often take positional and keyword -arguments as well, but in this case we don't take any and don't need to accept -a positional argument tuple or keyword argument dictionary. This method is -equivalent to the Python method: - -.. code-block:: python - - def name(self): - return "%s %s" % (self.first, self.last) - -Note that we have to check for the possibility that our :attr:`!first` and -:attr:`!last` members are ``NULL``. This is because they can be deleted, in which -case they are set to ``NULL``. It would be better to prevent deletion of these -attributes and to restrict the attribute values to be strings. We'll see how to -do that in the next section. - -Now that we've defined the method, we need to create an array of method -definitions:: - - static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ - }; - -(note that we used the :c:macro:`METH_NOARGS` flag to indicate that the method -is expecting no arguments other than *self*) - -and assign it to the :c:member:`~PyTypeObject.tp_methods` slot:: - - .tp_methods = Custom_methods, - -Finally, we'll make our type usable as a base class for subclassing. We've -written our methods carefully so far so that they don't make any assumptions -about the type of the object being created or used, so all we need to do is -to add the :c:macro:`Py_TPFLAGS_BASETYPE` to our class flag definition:: - - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - -We rename :c:func:`!PyInit_custom` to :c:func:`!PyInit_custom2`, update the -module name in the :c:type:`PyModuleDef` struct, and update the full class -name in the :c:type:`PyTypeObject` struct. - -Finally, we update our :file:`setup.py` file to build the new module: - -.. code-block:: python - - from distutils.core import setup, Extension - setup(name="custom", version="1.0", - ext_modules=[ - Extension("custom", ["custom.c"]), - Extension("custom2", ["custom2.c"]), - ]) - - -Providing finer control over data attributes -============================================ - -In this section, we'll provide finer control over how the :attr:`!first` and -:attr:`!last` attributes are set in the :class:`!Custom` example. In the previous -version of our module, the instance variables :attr:`!first` and :attr:`!last` -could be set to non-string values or even deleted. We want to make sure that -these attributes always contain strings. - -.. literalinclude:: ../includes/custom3.c - - -To provide greater control, over the :attr:`!first` and :attr:`!last` attributes, -we'll use custom getter and setter functions. Here are the functions for -getting and setting the :attr:`!first` attribute:: - - static PyObject * - Custom_getfirst(CustomObject *self, void *closure) - { - Py_INCREF(self->first); - return self->first; - } - - static int - Custom_setfirst(CustomObject *self, PyObject *value, void *closure) - { - PyObject *tmp; - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The first attribute value must be a string"); - return -1; - } - tmp = self->first; - Py_INCREF(value); - self->first = value; - Py_DECREF(tmp); - return 0; - } - -The getter function is passed a :class:`!Custom` object and a "closure", which is -a void pointer. In this case, the closure is ignored. (The closure supports an -advanced usage in which definition data is passed to the getter and setter. This -could, for example, be used to allow a single set of getter and setter functions -that decide the attribute to get or set based on data in the closure.) - -The setter function is passed the :class:`!Custom` object, the new value, and the -closure. The new value may be ``NULL``, in which case the attribute is being -deleted. In our setter, we raise an error if the attribute is deleted or if its -new value is not a string. - -We create an array of :c:type:`PyGetSetDef` structures:: - - static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, - "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, - "last name", NULL}, - {NULL} /* Sentinel */ - }; - -and register it in the :c:member:`~PyTypeObject.tp_getset` slot:: - - .tp_getset = Custom_getsetters, - -The last item in a :c:type:`PyGetSetDef` structure is the "closure" mentioned -above. In this case, we aren't using a closure, so we just pass ``NULL``. - -We also remove the member definitions for these attributes:: - - static PyMemberDef Custom_members[] = { - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ - }; - -We also need to update the :c:member:`~PyTypeObject.tp_init` handler to only -allow strings [#]_ to be passed:: - - static int - Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) - { - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_DECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_DECREF(tmp); - } - return 0; - } - -With these changes, we can assure that the ``first`` and ``last`` members are -never ``NULL`` so we can remove checks for ``NULL`` values in almost all cases. -This means that most of the :c:func:`Py_XDECREF` calls can be converted to -:c:func:`Py_DECREF` calls. The only place we can't change these calls is in -the ``tp_dealloc`` implementation, where there is the possibility that the -initialization of these members failed in ``tp_new``. - -We also rename the module initialization function and module name in the -initialization function, as we did before, and we add an extra definition to the -:file:`setup.py` file. - - -Supporting cyclic garbage collection -==================================== - -Python has a :term:`cyclic garbage collector (GC) <garbage collection>` that -can identify unneeded objects even when their reference counts are not zero. -This can happen when objects are involved in cycles. For example, consider: - -.. code-block:: pycon - - >>> l = [] - >>> l.append(l) - >>> del l - -In this example, we create a list that contains itself. When we delete it, it -still has a reference from itself. Its reference count doesn't drop to zero. -Fortunately, Python's cyclic garbage collector will eventually figure out that -the list is garbage and free it. - -In the second version of the :class:`!Custom` example, we allowed any kind of -object to be stored in the :attr:`!first` or :attr:`!last` attributes [#]_. -Besides, in the second and third versions, we allowed subclassing -:class:`!Custom`, and subclasses may add arbitrary attributes. For any of -those two reasons, :class:`!Custom` objects can participate in cycles: - -.. code-block:: pycon - - >>> import custom3 - >>> class Derived(custom3.Custom): pass - ... - >>> n = Derived() - >>> n.some_attribute = n - -To allow a :class:`!Custom` instance participating in a reference cycle to -be properly detected and collected by the cyclic GC, our :class:`!Custom` type -needs to fill two additional slots and to enable a flag that enables these slots: - -.. literalinclude:: ../includes/custom4.c - - -First, the traversal method lets the cyclic GC know about subobjects that could -participate in cycles:: - - static int - Custom_traverse(CustomObject *self, visitproc visit, void *arg) - { - int vret; - if (self->first) { - vret = visit(self->first, arg); - if (vret != 0) - return vret; - } - if (self->last) { - vret = visit(self->last, arg); - if (vret != 0) - return vret; - } - return 0; - } - -For each subobject that can participate in cycles, we need to call the -:c:func:`!visit` function, which is passed to the traversal method. The -:c:func:`!visit` function takes as arguments the subobject and the extra argument -*arg* passed to the traversal method. It returns an integer value that must be -returned if it is non-zero. - -Python provides a :c:func:`Py_VISIT` macro that automates calling visit -functions. With :c:func:`Py_VISIT`, we can minimize the amount of boilerplate -in ``Custom_traverse``:: - - static int - Custom_traverse(CustomObject *self, visitproc visit, void *arg) - { - Py_VISIT(self->first); - Py_VISIT(self->last); - return 0; - } - -.. note:: - The :c:member:`~PyTypeObject.tp_traverse` implementation must name its - arguments exactly *visit* and *arg* in order to use :c:func:`Py_VISIT`. - -Second, we need to provide a method for clearing any subobjects that can -participate in cycles:: - - static int - Custom_clear(CustomObject *self) - { - Py_CLEAR(self->first); - Py_CLEAR(self->last); - return 0; - } - -Notice the use of the :c:func:`Py_CLEAR` macro. It is the recommended and safe -way to clear data attributes of arbitrary types while decrementing -their reference counts. If you were to call :c:func:`Py_XDECREF` instead -on the attribute before setting it to ``NULL``, there is a possibility -that the attribute's destructor would call back into code that reads the -attribute again (*especially* if there is a reference cycle). - -.. note:: - You could emulate :c:func:`Py_CLEAR` by writing:: - - PyObject *tmp; - tmp = self->first; - self->first = NULL; - Py_XDECREF(tmp); - - Nevertheless, it is much easier and less error-prone to always - use :c:func:`Py_CLEAR` when deleting an attribute. Don't - try to micro-optimize at the expense of robustness! - -The deallocator ``Custom_dealloc`` may call arbitrary code when clearing -attributes. It means the circular GC can be triggered inside the function. -Since the GC assumes reference count is not zero, we need to untrack the object -from the GC by calling :c:func:`PyObject_GC_UnTrack` before clearing members. -Here is our reimplemented deallocator using :c:func:`PyObject_GC_UnTrack` -and ``Custom_clear``:: - - static void - Custom_dealloc(CustomObject *self) - { - PyObject_GC_UnTrack(self); - Custom_clear(self); - Py_TYPE(self)->tp_free((PyObject *) self); - } - -Finally, we add the :c:macro:`Py_TPFLAGS_HAVE_GC` flag to the class flags:: - - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - -That's pretty much it. If we had written custom :c:member:`~PyTypeObject.tp_alloc` or -:c:member:`~PyTypeObject.tp_free` handlers, we'd need to modify them for cyclic -garbage collection. Most extensions will use the versions automatically provided. - - -Subclassing other types -======================= - -It is possible to create new extension types that are derived from existing -types. It is easiest to inherit from the built in types, since an extension can -easily use the :c:type:`PyTypeObject` it needs. It can be difficult to share -these :c:type:`PyTypeObject` structures between extension modules. - -In this example we will create a :class:`!SubList` type that inherits from the -built-in :class:`list` type. The new type will be completely compatible with -regular lists, but will have an additional :meth:`!increment` method that -increases an internal counter: - -.. code-block:: pycon - - >>> import sublist - >>> s = sublist.SubList(range(3)) - >>> s.extend(s) - >>> print(len(s)) - 6 - >>> print(s.increment()) - 1 - >>> print(s.increment()) - 2 - -.. literalinclude:: ../includes/sublist.c - - -As you can see, the source code closely resembles the :class:`!Custom` examples in -previous sections. We will break down the main differences between them. :: - - typedef struct { - PyListObject list; - int state; - } SubListObject; - -The primary difference for derived type objects is that the base type's -object structure must be the first value. The base type will already include -the :c:func:`PyObject_HEAD` at the beginning of its structure. - -When a Python object is a :class:`!SubList` instance, its ``PyObject *`` pointer -can be safely cast to both ``PyListObject *`` and ``SubListObject *``:: - - static int - SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) - { - if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) - return -1; - self->state = 0; - return 0; - } - -We see above how to call through to the :meth:`~object.__init__` method of the base -type. - -This pattern is important when writing a type with custom -:c:member:`~PyTypeObject.tp_new` and :c:member:`~PyTypeObject.tp_dealloc` -members. The :c:member:`~PyTypeObject.tp_new` handler should not actually -create the memory for the object with its :c:member:`~PyTypeObject.tp_alloc`, -but let the base class handle it by calling its own :c:member:`~PyTypeObject.tp_new`. - -The :c:type:`PyTypeObject` struct supports a :c:member:`~PyTypeObject.tp_base` -specifying the type's concrete base class. Due to cross-platform compiler -issues, you can't fill that field directly with a reference to -:c:type:`PyList_Type`; it should be done later in the module initialization -function:: - - PyMODINIT_FUNC - PyInit_sublist(void) - { - PyObject* m; - SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; - - Py_INCREF(&SubListType); - if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(&SubListType); - Py_DECREF(m); - return NULL; - } - - return m; - } - -Before calling :c:func:`PyType_Ready`, the type structure must have the -:c:member:`~PyTypeObject.tp_base` slot filled in. When we are deriving an -existing type, it is not necessary to fill out the :c:member:`~PyTypeObject.tp_alloc` -slot with :c:func:`PyType_GenericNew` -- the allocation function from the base -type will be inherited. - -After that, calling :c:func:`PyType_Ready` and adding the type object to the -module is the same as with the basic :class:`!Custom` examples. - - -.. rubric:: Footnotes - -.. [#] This is true when we know that the object is a basic type, like a string or a - float. - -.. [#] We relied on this in the :c:member:`~PyTypeObject.tp_dealloc` handler - in this example, because our type doesn't support garbage collection. - -.. [#] We now know that the first and last members are strings, so perhaps we - could be less careful about decrementing their reference counts, however, - we accept instances of string subclasses. Even though deallocating normal - strings won't call back into our objects, we can't guarantee that deallocating - an instance of a string subclass won't call back into our objects. - -.. [#] Also, even with our attributes restricted to strings instances, the user - could pass arbitrary :class:`str` subclasses and therefore still create - reference cycles. diff --git a/Doc/extending/windows.rst b/Doc/extending/windows.rst deleted file mode 100644 index 6e17225d08f691..00000000000000 --- a/Doc/extending/windows.rst +++ /dev/null @@ -1,135 +0,0 @@ -.. highlight:: c - - -.. _building-on-windows: - -**************************************** -Building C and C++ Extensions on Windows -**************************************** - -This chapter briefly explains how to create a Windows extension module for -Python using Microsoft Visual C++, and follows with more detailed background -information on how it works. The explanatory material is useful for both the -Windows programmer learning to build Python extensions and the Unix programmer -interested in producing software which can be successfully built on both Unix -and Windows. - -Module authors are encouraged to use the distutils approach for building -extension modules, instead of the one described in this section. You will still -need the C compiler that was used to build Python; typically Microsoft Visual -C++. - -.. note:: - - This chapter mentions a number of filenames that include an encoded Python - version number. These filenames are represented with the version number shown - as ``XY``; in practice, ``'X'`` will be the major version number and ``'Y'`` - will be the minor version number of the Python release you're working with. For - example, if you are using Python 2.2.1, ``XY`` will actually be ``22``. - - -.. _win-cookbook: - -A Cookbook Approach -=================== - -There are two approaches to building extension modules on Windows, just as there -are on Unix: use the :mod:`distutils` package to control the build process, or -do things manually. The distutils approach works well for most extensions; -documentation on using :mod:`distutils` to build and package extension modules -is available in :ref:`distutils-index`. If you find you really need to do -things manually, it may be instructive to study the project file for the -:source:`winsound <PCbuild/winsound.vcxproj>` standard library module. - - -.. _dynamic-linking: - -Differences Between Unix and Windows -==================================== - -.. sectionauthor:: Chris Phoenix <cphoenix@best.com> - - -Unix and Windows use completely different paradigms for run-time loading of -code. Before you try to build a module that can be dynamically loaded, be aware -of how your system works. - -In Unix, a shared object (:file:`.so`) file contains code to be used by the -program, and also the names of functions and data that it expects to find in the -program. When the file is joined to the program, all references to those -functions and data in the file's code are changed to point to the actual -locations in the program where the functions and data are placed in memory. -This is basically a link operation. - -In Windows, a dynamic-link library (:file:`.dll`) file has no dangling -references. Instead, an access to functions or data goes through a lookup -table. So the DLL code does not have to be fixed up at runtime to refer to the -program's memory; instead, the code already uses the DLL's lookup table, and the -lookup table is modified at runtime to point to the functions and data. - -In Unix, there is only one type of library file (:file:`.a`) which contains code -from several object files (:file:`.o`). During the link step to create a shared -object file (:file:`.so`), the linker may find that it doesn't know where an -identifier is defined. The linker will look for it in the object files in the -libraries; if it finds it, it will include all the code from that object file. - -In Windows, there are two types of library, a static library and an import -library (both called :file:`.lib`). A static library is like a Unix :file:`.a` -file; it contains code to be included as necessary. An import library is -basically used only to reassure the linker that a certain identifier is legal, -and will be present in the program when the DLL is loaded. So the linker uses -the information from the import library to build the lookup table for using -identifiers that are not included in the DLL. When an application or a DLL is -linked, an import library may be generated, which will need to be used for all -future DLLs that depend on the symbols in the application or DLL. - -Suppose you are building two dynamic-load modules, B and C, which should share -another block of code A. On Unix, you would *not* pass :file:`A.a` to the -linker for :file:`B.so` and :file:`C.so`; that would cause it to be included -twice, so that B and C would each have their own copy. In Windows, building -:file:`A.dll` will also build :file:`A.lib`. You *do* pass :file:`A.lib` to the -linker for B and C. :file:`A.lib` does not contain code; it just contains -information which will be used at runtime to access A's code. - -In Windows, using an import library is sort of like using ``import spam``; it -gives you access to spam's names, but does not create a separate copy. On Unix, -linking with a library is more like ``from spam import *``; it does create a -separate copy. - - -.. _win-dlls: - -Using DLLs in Practice -====================== - -.. sectionauthor:: Chris Phoenix <cphoenix@best.com> - - -Windows Python is built in Microsoft Visual C++; using other compilers may or -may not work. The rest of this section is MSVC++ specific. - -When creating DLLs in Windows, you must pass :file:`pythonXY.lib` to the linker. -To build two DLLs, spam and ni (which uses C functions found in spam), you could -use these commands:: - - cl /LD /I/python/include spam.c ../libs/pythonXY.lib - cl /LD /I/python/include ni.c spam.lib ../libs/pythonXY.lib - -The first command created three files: :file:`spam.obj`, :file:`spam.dll` and -:file:`spam.lib`. :file:`Spam.dll` does not contain any Python functions (such -as :c:func:`PyArg_ParseTuple`), but it does know how to find the Python code -thanks to :file:`pythonXY.lib`. - -The second command created :file:`ni.dll` (and :file:`.obj` and :file:`.lib`), -which knows how to find the necessary functions from spam, and also from the -Python executable. - -Not every identifier is exported to the lookup table. If you want any other -modules (including Python) to be able to see your identifiers, you have to say -``_declspec(dllexport)``, as in ``void _declspec(dllexport) initspam(void)`` or -``PyObject _declspec(dllexport) *NiGetSpamData(void)``. - -Developer Studio will throw in a lot of import libraries that you do not really -need, adding about 100K to your executable. To get rid of them, use the Project -Settings dialog, Link tab, to specify *ignore default libraries*. Add the -correct :file:`msvcrt{xx}.lib` to the list of libraries. diff --git a/Doc/faq/design.rst b/Doc/faq/design.rst deleted file mode 100644 index 83c0152c85e84a..00000000000000 --- a/Doc/faq/design.rst +++ /dev/null @@ -1,772 +0,0 @@ -====================== -Design and History FAQ -====================== - -.. only:: html - - .. contents:: - - -Why does Python use indentation for grouping of statements? ------------------------------------------------------------ - -Guido van Rossum believes that using indentation for grouping is extremely -elegant and contributes a lot to the clarity of the average Python program. -Most people learn to love this feature after a while. - -Since there are no begin/end brackets there cannot be a disagreement between -grouping perceived by the parser and the human reader. Occasionally C -programmers will encounter a fragment of code like this:: - - if (x <= y) - x++; - y--; - z++; - -Only the ``x++`` statement is executed if the condition is true, but the -indentation leads many to believe otherwise. Even experienced C programmers will -sometimes stare at it a long time wondering as to why ``y`` is being decremented even -for ``x > y``. - -Because there are no begin/end brackets, Python is much less prone to -coding-style conflicts. In C there are many different ways to place the braces. -After becoming used to reading and writing code using a particular style, -it is normal to feel somewhat uneasy when reading (or being required to write) -in a different one. - - -Many coding styles place begin/end brackets on a line by themselves. This makes -programs considerably longer and wastes valuable screen space, making it harder -to get a good overview of a program. Ideally, a function should fit on one -screen (say, 20--30 lines). 20 lines of Python can do a lot more work than 20 -lines of C. This is not solely due to the lack of begin/end brackets -- the -lack of declarations and the high-level data types are also responsible -- but -the indentation-based syntax certainly helps. - - -Why am I getting strange results with simple arithmetic operations? -------------------------------------------------------------------- - -See the next question. - - -Why are floating-point calculations so inaccurate? --------------------------------------------------- - -Users are often surprised by results like this:: - - >>> 1.2 - 1.0 - 0.19999999999999996 - -and think it is a bug in Python. It's not. This has little to do with Python, -and much more to do with how the underlying platform handles floating-point -numbers. - -The :class:`float` type in CPython uses a C ``double`` for storage. A -:class:`float` object's value is stored in binary floating-point with a fixed -precision (typically 53 bits) and Python uses C operations, which in turn rely -on the hardware implementation in the processor, to perform floating-point -operations. This means that as far as floating-point operations are concerned, -Python behaves like many popular languages including C and Java. - -Many numbers that can be written easily in decimal notation cannot be expressed -exactly in binary floating-point. For example, after:: - - >>> x = 1.2 - -the value stored for ``x`` is a (very good) approximation to the decimal value -``1.2``, but is not exactly equal to it. On a typical machine, the actual -stored value is:: - - 1.0011001100110011001100110011001100110011001100110011 (binary) - -which is exactly:: - - 1.1999999999999999555910790149937383830547332763671875 (decimal) - -The typical precision of 53 bits provides Python floats with 15--16 -decimal digits of accuracy. - -For a fuller explanation, please see the :ref:`floating point arithmetic -<tut-fp-issues>` chapter in the Python tutorial. - - -Why are Python strings immutable? ---------------------------------- - -There are several advantages. - -One is performance: knowing that a string is immutable means we can allocate -space for it at creation time, and the storage requirements are fixed and -unchanging. This is also one of the reasons for the distinction between tuples -and lists. - -Another advantage is that strings in Python are considered as "elemental" as -numbers. No amount of activity will change the value 8 to anything else, and in -Python, no amount of activity will change the string "eight" to anything else. - - -.. _why-self: - -Why must 'self' be used explicitly in method definitions and calls? -------------------------------------------------------------------- - -The idea was borrowed from Modula-3. It turns out to be very useful, for a -variety of reasons. - -First, it's more obvious that you are using a method or instance attribute -instead of a local variable. Reading ``self.x`` or ``self.meth()`` makes it -absolutely clear that an instance variable or method is used even if you don't -know the class definition by heart. In C++, you can sort of tell by the lack of -a local variable declaration (assuming globals are rare or easily recognizable) --- but in Python, there are no local variable declarations, so you'd have to -look up the class definition to be sure. Some C++ and Java coding standards -call for instance attributes to have an ``m_`` prefix, so this explicitness is -still useful in those languages, too. - -Second, it means that no special syntax is necessary if you want to explicitly -reference or call the method from a particular class. In C++, if you want to -use a method from a base class which is overridden in a derived class, you have -to use the ``::`` operator -- in Python you can write -``baseclass.methodname(self, <argument list>)``. This is particularly useful -for :meth:`__init__` methods, and in general in cases where a derived class -method wants to extend the base class method of the same name and thus has to -call the base class method somehow. - -Finally, for instance variables it solves a syntactic problem with assignment: -since local variables in Python are (by definition!) those variables to which a -value is assigned in a function body (and that aren't explicitly declared -global), there has to be some way to tell the interpreter that an assignment was -meant to assign to an instance variable instead of to a local variable, and it -should preferably be syntactic (for efficiency reasons). C++ does this through -declarations, but Python doesn't have declarations and it would be a pity having -to introduce them just for this purpose. Using the explicit ``self.var`` solves -this nicely. Similarly, for using instance variables, having to write -``self.var`` means that references to unqualified names inside a method don't -have to search the instance's directories. To put it another way, local -variables and instance variables live in two different namespaces, and you need -to tell Python which namespace to use. - - -.. _why-can-t-i-use-an-assignment-in-an-expression: - -Why can't I use an assignment in an expression? ------------------------------------------------ - -Starting in Python 3.8, you can! - -Assignment expressions using the walrus operator ``:=`` assign a variable in an -expression:: - - while chunk := fp.read(200): - print(chunk) - -See :pep:`572` for more information. - - - -Why does Python use methods for some functionality (e.g. list.index()) but functions for other (e.g. len(list))? ----------------------------------------------------------------------------------------------------------------- - -As Guido said: - - (a) For some operations, prefix notation just reads better than - postfix -- prefix (and infix!) operations have a long tradition in - mathematics which likes notations where the visuals help the - mathematician thinking about a problem. Compare the easy with which we - rewrite a formula like x*(a+b) into x*a + x*b to the clumsiness of - doing the same thing using a raw OO notation. - - (b) When I read code that says len(x) I *know* that it is asking for - the length of something. This tells me two things: the result is an - integer, and the argument is some kind of container. To the contrary, - when I read x.len(), I have to already know that x is some kind of - container implementing an interface or inheriting from a class that - has a standard len(). Witness the confusion we occasionally have when - a class that is not implementing a mapping has a get() or keys() - method, or something that isn't a file has a write() method. - - -- https://mail.python.org/pipermail/python-3000/2006-November/004643.html - - -Why is join() a string method instead of a list or tuple method? ----------------------------------------------------------------- - -Strings became much more like other standard types starting in Python 1.6, when -methods were added which give the same functionality that has always been -available using the functions of the string module. Most of these new methods -have been widely accepted, but the one which appears to make some programmers -feel uncomfortable is:: - - ", ".join(['1', '2', '4', '8', '16']) - -which gives the result:: - - "1, 2, 4, 8, 16" - -There are two common arguments against this usage. - -The first runs along the lines of: "It looks really ugly using a method of a -string literal (string constant)", to which the answer is that it might, but a -string literal is just a fixed value. If the methods are to be allowed on names -bound to strings there is no logical reason to make them unavailable on -literals. - -The second objection is typically cast as: "I am really telling a sequence to -join its members together with a string constant". Sadly, you aren't. For some -reason there seems to be much less difficulty with having :meth:`~str.split` as -a string method, since in that case it is easy to see that :: - - "1, 2, 4, 8, 16".split(", ") - -is an instruction to a string literal to return the substrings delimited by the -given separator (or, by default, arbitrary runs of white space). - -:meth:`~str.join` is a string method because in using it you are telling the -separator string to iterate over a sequence of strings and insert itself between -adjacent elements. This method can be used with any argument which obeys the -rules for sequence objects, including any new classes you might define yourself. -Similar methods exist for bytes and bytearray objects. - - -How fast are exceptions? ------------------------- - -A try/except block is extremely efficient if no exceptions are raised. Actually -catching an exception is expensive. In versions of Python prior to 2.0 it was -common to use this idiom:: - - try: - value = mydict[key] - except KeyError: - mydict[key] = getvalue(key) - value = mydict[key] - -This only made sense when you expected the dict to have the key almost all the -time. If that wasn't the case, you coded it like this:: - - if key in mydict: - value = mydict[key] - else: - value = mydict[key] = getvalue(key) - -For this specific case, you could also use ``value = dict.setdefault(key, -getvalue(key))``, but only if the ``getvalue()`` call is cheap enough because it -is evaluated in all cases. - - -Why isn't there a switch or case statement in Python? ------------------------------------------------------ - -You can do this easily enough with a sequence of ``if... elif... elif... else``. -For literal values, or constants within a namespace, you can also use a -``match ... case`` statement. - -For cases where you need to choose from a very large number of possibilities, -you can create a dictionary mapping case values to functions to call. For -example:: - - functions = {'a': function_1, - 'b': function_2, - 'c': self.method_1} - - func = functions[value] - func() - -For calling methods on objects, you can simplify yet further by using the -:func:`getattr` built-in to retrieve methods with a particular name:: - - class MyVisitor: - def visit_a(self): - ... - - def dispatch(self, value): - method_name = 'visit_' + str(value) - method = getattr(self, method_name) - method() - -It's suggested that you use a prefix for the method names, such as ``visit_`` in -this example. Without such a prefix, if values are coming from an untrusted -source, an attacker would be able to call any method on your object. - - -Can't you emulate threads in the interpreter instead of relying on an OS-specific thread implementation? --------------------------------------------------------------------------------------------------------- - -Answer 1: Unfortunately, the interpreter pushes at least one C stack frame for -each Python stack frame. Also, extensions can call back into Python at almost -random moments. Therefore, a complete threads implementation requires thread -support for C. - -Answer 2: Fortunately, there is `Stackless Python <https://github.com/stackless-dev/stackless/wiki>`_, -which has a completely redesigned interpreter loop that avoids the C stack. - - -Why can't lambda expressions contain statements? ------------------------------------------------- - -Python lambda expressions cannot contain statements because Python's syntactic -framework can't handle statements nested inside expressions. However, in -Python, this is not a serious problem. Unlike lambda forms in other languages, -where they add functionality, Python lambdas are only a shorthand notation if -you're too lazy to define a function. - -Functions are already first class objects in Python, and can be declared in a -local scope. Therefore the only advantage of using a lambda instead of a -locally defined function is that you don't need to invent a name for the -function -- but that's just a local variable to which the function object (which -is exactly the same type of object that a lambda expression yields) is assigned! - - -Can Python be compiled to machine code, C or some other language? ------------------------------------------------------------------ - -`Cython <https://cython.org/>`_ compiles a modified version of Python with -optional annotations into C extensions. `Nuitka <https://www.nuitka.net/>`_ is -an up-and-coming compiler of Python into C++ code, aiming to support the full -Python language. - - -How does Python manage memory? ------------------------------- - -The details of Python memory management depend on the implementation. The -standard implementation of Python, :term:`CPython`, uses reference counting to -detect inaccessible objects, and another mechanism to collect reference cycles, -periodically executing a cycle detection algorithm which looks for inaccessible -cycles and deletes the objects involved. The :mod:`gc` module provides functions -to perform a garbage collection, obtain debugging statistics, and tune the -collector's parameters. - -Other implementations (such as `Jython <https://www.jython.org>`_ or -`PyPy <https://www.pypy.org>`_), however, can rely on a different mechanism -such as a full-blown garbage collector. This difference can cause some -subtle porting problems if your Python code depends on the behavior of the -reference counting implementation. - -In some Python implementations, the following code (which is fine in CPython) -will probably run out of file descriptors:: - - for file in very_long_list_of_files: - f = open(file) - c = f.read(1) - -Indeed, using CPython's reference counting and destructor scheme, each new -assignment to *f* closes the previous file. With a traditional GC, however, -those file objects will only get collected (and closed) at varying and possibly -long intervals. - -If you want to write code that will work with any Python implementation, -you should explicitly close the file or use the :keyword:`with` statement; -this will work regardless of memory management scheme:: - - for file in very_long_list_of_files: - with open(file) as f: - c = f.read(1) - - -Why doesn't CPython use a more traditional garbage collection scheme? ---------------------------------------------------------------------- - -For one thing, this is not a C standard feature and hence it's not portable. -(Yes, we know about the Boehm GC library. It has bits of assembler code for -*most* common platforms, not for all of them, and although it is mostly -transparent, it isn't completely transparent; patches are required to get -Python to work with it.) - -Traditional GC also becomes a problem when Python is embedded into other -applications. While in a standalone Python it's fine to replace the standard -malloc() and free() with versions provided by the GC library, an application -embedding Python may want to have its *own* substitute for malloc() and free(), -and may not want Python's. Right now, CPython works with anything that -implements malloc() and free() properly. - - -Why isn't all memory freed when CPython exits? ----------------------------------------------- - -Objects referenced from the global namespaces of Python modules are not always -deallocated when Python exits. This may happen if there are circular -references. There are also certain bits of memory that are allocated by the C -library that are impossible to free (e.g. a tool like Purify will complain about -these). Python is, however, aggressive about cleaning up memory on exit and -does try to destroy every single object. - -If you want to force Python to delete certain things on deallocation use the -:mod:`atexit` module to run a function that will force those deletions. - - -Why are there separate tuple and list data types? -------------------------------------------------- - -Lists and tuples, while similar in many respects, are generally used in -fundamentally different ways. Tuples can be thought of as being similar to -Pascal records or C structs; they're small collections of related data which may -be of different types which are operated on as a group. For example, a -Cartesian coordinate is appropriately represented as a tuple of two or three -numbers. - -Lists, on the other hand, are more like arrays in other languages. They tend to -hold a varying number of objects all of which have the same type and which are -operated on one-by-one. For example, ``os.listdir('.')`` returns a list of -strings representing the files in the current directory. Functions which -operate on this output would generally not break if you added another file or -two to the directory. - -Tuples are immutable, meaning that once a tuple has been created, you can't -replace any of its elements with a new value. Lists are mutable, meaning that -you can always change a list's elements. Only immutable elements can be used as -dictionary keys, and hence only tuples and not lists can be used as keys. - - -How are lists implemented in CPython? -------------------------------------- - -CPython's lists are really variable-length arrays, not Lisp-style linked lists. -The implementation uses a contiguous array of references to other objects, and -keeps a pointer to this array and the array's length in a list head structure. - -This makes indexing a list ``a[i]`` an operation whose cost is independent of -the size of the list or the value of the index. - -When items are appended or inserted, the array of references is resized. Some -cleverness is applied to improve the performance of appending items repeatedly; -when the array must be grown, some extra space is allocated so the next few -times don't require an actual resize. - - -How are dictionaries implemented in CPython? --------------------------------------------- - -CPython's dictionaries are implemented as resizable hash tables. Compared to -B-trees, this gives better performance for lookup (the most common operation by -far) under most circumstances, and the implementation is simpler. - -Dictionaries work by computing a hash code for each key stored in the dictionary -using the :func:`hash` built-in function. The hash code varies widely depending -on the key and a per-process seed; for example, "Python" could hash to --539294296 while "python", a string that differs by a single bit, could hash -to 1142331976. The hash code is then used to calculate a location in an -internal array where the value will be stored. Assuming that you're storing -keys that all have different hash values, this means that dictionaries take -constant time -- O(1), in Big-O notation -- to retrieve a key. - - -Why must dictionary keys be immutable? --------------------------------------- - -The hash table implementation of dictionaries uses a hash value calculated from -the key value to find the key. If the key were a mutable object, its value -could change, and thus its hash could also change. But since whoever changes -the key object can't tell that it was being used as a dictionary key, it can't -move the entry around in the dictionary. Then, when you try to look up the same -object in the dictionary it won't be found because its hash value is different. -If you tried to look up the old value it wouldn't be found either, because the -value of the object found in that hash bin would be different. - -If you want a dictionary indexed with a list, simply convert the list to a tuple -first; the function ``tuple(L)`` creates a tuple with the same entries as the -list ``L``. Tuples are immutable and can therefore be used as dictionary keys. - -Some unacceptable solutions that have been proposed: - -- Hash lists by their address (object ID). This doesn't work because if you - construct a new list with the same value it won't be found; e.g.:: - - mydict = {[1, 2]: '12'} - print(mydict[[1, 2]]) - - would raise a :exc:`KeyError` exception because the id of the ``[1, 2]`` used in the - second line differs from that in the first line. In other words, dictionary - keys should be compared using ``==``, not using :keyword:`is`. - -- Make a copy when using a list as a key. This doesn't work because the list, - being a mutable object, could contain a reference to itself, and then the - copying code would run into an infinite loop. - -- Allow lists as keys but tell the user not to modify them. This would allow a - class of hard-to-track bugs in programs when you forgot or modified a list by - accident. It also invalidates an important invariant of dictionaries: every - value in ``d.keys()`` is usable as a key of the dictionary. - -- Mark lists as read-only once they are used as a dictionary key. The problem - is that it's not just the top-level object that could change its value; you - could use a tuple containing a list as a key. Entering anything as a key into - a dictionary would require marking all objects reachable from there as - read-only -- and again, self-referential objects could cause an infinite loop. - -There is a trick to get around this if you need to, but use it at your own risk: -You can wrap a mutable structure inside a class instance which has both a -:meth:`__eq__` and a :meth:`__hash__` method. You must then make sure that the -hash value for all such wrapper objects that reside in a dictionary (or other -hash based structure), remain fixed while the object is in the dictionary (or -other structure). :: - - class ListWrapper: - def __init__(self, the_list): - self.the_list = the_list - - def __eq__(self, other): - return self.the_list == other.the_list - - def __hash__(self): - l = self.the_list - result = 98767 - len(l)*555 - for i, el in enumerate(l): - try: - result = result + (hash(el) % 9999999) * 1001 + i - except Exception: - result = (result % 7777777) + i * 333 - return result - -Note that the hash computation is complicated by the possibility that some -members of the list may be unhashable and also by the possibility of arithmetic -overflow. - -Furthermore it must always be the case that if ``o1 == o2`` (ie ``o1.__eq__(o2) -is True``) then ``hash(o1) == hash(o2)`` (ie, ``o1.__hash__() == o2.__hash__()``), -regardless of whether the object is in a dictionary or not. If you fail to meet -these restrictions dictionaries and other hash based structures will misbehave. - -In the case of ListWrapper, whenever the wrapper object is in a dictionary the -wrapped list must not change to avoid anomalies. Don't do this unless you are -prepared to think hard about the requirements and the consequences of not -meeting them correctly. Consider yourself warned. - - -Why doesn't list.sort() return the sorted list? ------------------------------------------------ - -In situations where performance matters, making a copy of the list just to sort -it would be wasteful. Therefore, :meth:`list.sort` sorts the list in place. In -order to remind you of that fact, it does not return the sorted list. This way, -you won't be fooled into accidentally overwriting a list when you need a sorted -copy but also need to keep the unsorted version around. - -If you want to return a new list, use the built-in :func:`sorted` function -instead. This function creates a new list from a provided iterable, sorts -it and returns it. For example, here's how to iterate over the keys of a -dictionary in sorted order:: - - for key in sorted(mydict): - ... # do whatever with mydict[key]... - - -How do you specify and enforce an interface spec in Python? ------------------------------------------------------------ - -An interface specification for a module as provided by languages such as C++ and -Java describes the prototypes for the methods and functions of the module. Many -feel that compile-time enforcement of interface specifications helps in the -construction of large programs. - -Python 2.6 adds an :mod:`abc` module that lets you define Abstract Base Classes -(ABCs). You can then use :func:`isinstance` and :func:`issubclass` to check -whether an instance or a class implements a particular ABC. The -:mod:`collections.abc` module defines a set of useful ABCs such as -:class:`~collections.abc.Iterable`, :class:`~collections.abc.Container`, and -:class:`~collections.abc.MutableMapping`. - -For Python, many of the advantages of interface specifications can be obtained -by an appropriate test discipline for components. - -A good test suite for a module can both provide a regression test and serve as a -module interface specification and a set of examples. Many Python modules can -be run as a script to provide a simple "self test." Even modules which use -complex external interfaces can often be tested in isolation using trivial -"stub" emulations of the external interface. The :mod:`doctest` and -:mod:`unittest` modules or third-party test frameworks can be used to construct -exhaustive test suites that exercise every line of code in a module. - -An appropriate testing discipline can help build large complex applications in -Python as well as having interface specifications would. In fact, it can be -better because an interface specification cannot test certain properties of a -program. For example, the :meth:`!list.append` method is expected to add new elements -to the end of some internal list; an interface specification cannot test that -your :meth:`!list.append` implementation will actually do this correctly, but it's -trivial to check this property in a test suite. - -Writing test suites is very helpful, and you might want to design your code to -make it easily tested. One increasingly popular technique, test-driven -development, calls for writing parts of the test suite first, before you write -any of the actual code. Of course Python allows you to be sloppy and not write -test cases at all. - - -Why is there no goto? ---------------------- - -In the 1970s people realized that unrestricted goto could lead -to messy "spaghetti" code that was hard to understand and revise. -In a high-level language, it is also unneeded as long as there -are ways to branch (in Python, with ``if`` statements and ``or``, -``and``, and ``if-else`` expressions) and loop (with ``while`` -and ``for`` statements, possibly containing ``continue`` and ``break``). - -One can also use exceptions to provide a "structured goto" -that works even across -function calls. Many feel that exceptions can conveniently emulate all -reasonable uses of the "go" or "goto" constructs of C, Fortran, and other -languages. For example:: - - class label(Exception): pass # declare a label - - try: - ... - if condition: raise label() # goto label - ... - except label: # where to goto - pass - ... - -This doesn't allow you to jump into the middle of a loop, but that's usually -considered an abuse of goto anyway. Use sparingly. - - -Why can't raw strings (r-strings) end with a backslash? -------------------------------------------------------- - -More precisely, they can't end with an odd number of backslashes: the unpaired -backslash at the end escapes the closing quote character, leaving an -unterminated string. - -Raw strings were designed to ease creating input for processors (chiefly regular -expression engines) that want to do their own backslash escape processing. Such -processors consider an unmatched trailing backslash to be an error anyway, so -raw strings disallow that. In return, they allow you to pass on the string -quote character by escaping it with a backslash. These rules work well when -r-strings are used for their intended purpose. - -If you're trying to build Windows pathnames, note that all Windows system calls -accept forward slashes too:: - - f = open("/mydir/file.txt") # works fine! - -If you're trying to build a pathname for a DOS command, try e.g. one of :: - - dir = r"\this\is\my\dos\dir" "\\" - dir = r"\this\is\my\dos\dir\ "[:-1] - dir = "\\this\\is\\my\\dos\\dir\\" - - -Why doesn't Python have a "with" statement for attribute assignments? ---------------------------------------------------------------------- - -Python has a 'with' statement that wraps the execution of a block, calling code -on the entrance and exit from the block. Some languages have a construct that -looks like this:: - - with obj: - a = 1 # equivalent to obj.a = 1 - total = total + 1 # obj.total = obj.total + 1 - -In Python, such a construct would be ambiguous. - -Other languages, such as Object Pascal, Delphi, and C++, use static types, so -it's possible to know, in an unambiguous way, what member is being assigned -to. This is the main point of static typing -- the compiler *always* knows the -scope of every variable at compile time. - -Python uses dynamic types. It is impossible to know in advance which attribute -will be referenced at runtime. Member attributes may be added or removed from -objects on the fly. This makes it impossible to know, from a simple reading, -what attribute is being referenced: a local one, a global one, or a member -attribute? - -For instance, take the following incomplete snippet:: - - def foo(a): - with a: - print(x) - -The snippet assumes that "a" must have a member attribute called "x". However, -there is nothing in Python that tells the interpreter this. What should happen -if "a" is, let us say, an integer? If there is a global variable named "x", -will it be used inside the with block? As you see, the dynamic nature of Python -makes such choices much harder. - -The primary benefit of "with" and similar language features (reduction of code -volume) can, however, easily be achieved in Python by assignment. Instead of:: - - function(args).mydict[index][index].a = 21 - function(args).mydict[index][index].b = 42 - function(args).mydict[index][index].c = 63 - -write this:: - - ref = function(args).mydict[index][index] - ref.a = 21 - ref.b = 42 - ref.c = 63 - -This also has the side-effect of increasing execution speed because name -bindings are resolved at run-time in Python, and the second version only needs -to perform the resolution once. - - -Why don't generators support the with statement? ------------------------------------------------- - -For technical reasons, a generator used directly as a context manager -would not work correctly. When, as is most common, a generator is used as -an iterator run to completion, no closing is needed. When it is, wrap -it as "contextlib.closing(generator)" in the 'with' statement. - - -Why are colons required for the if/while/def/class statements? --------------------------------------------------------------- - -The colon is required primarily to enhance readability (one of the results of -the experimental ABC language). Consider this:: - - if a == b - print(a) - -versus :: - - if a == b: - print(a) - -Notice how the second one is slightly easier to read. Notice further how a -colon sets off the example in this FAQ answer; it's a standard usage in English. - -Another minor reason is that the colon makes it easier for editors with syntax -highlighting; they can look for colons to decide when indentation needs to be -increased instead of having to do a more elaborate parsing of the program text. - - -Why does Python allow commas at the end of lists and tuples? ------------------------------------------------------------- - -Python lets you add a trailing comma at the end of lists, tuples, and -dictionaries:: - - [1, 2, 3,] - ('a', 'b', 'c',) - d = { - "A": [1, 5], - "B": [6, 7], # last trailing comma is optional but good style - } - - -There are several reasons to allow this. - -When you have a literal value for a list, tuple, or dictionary spread across -multiple lines, it's easier to add more elements because you don't have to -remember to add a comma to the previous line. The lines can also be reordered -without creating a syntax error. - -Accidentally omitting the comma can lead to errors that are hard to diagnose. -For example:: - - x = [ - "fee", - "fie" - "foo", - "fum" - ] - -This list looks like it has four elements, but it actually contains three: -"fee", "fiefoo" and "fum". Always adding the comma avoids this source of error. - -Allowing the trailing comma may also make programmatic code generation easier. diff --git a/Doc/faq/extending.rst b/Doc/faq/extending.rst deleted file mode 100644 index 2a8b976925d042..00000000000000 --- a/Doc/faq/extending.rst +++ /dev/null @@ -1,291 +0,0 @@ -======================= -Extending/Embedding FAQ -======================= - -.. only:: html - - .. contents:: - -.. highlight:: c - - -.. XXX need review for Python 3. - - -Can I create my own functions in C? ------------------------------------ - -Yes, you can create built-in modules containing functions, variables, exceptions -and even new types in C. This is explained in the document -:ref:`extending-index`. - -Most intermediate or advanced Python books will also cover this topic. - - -Can I create my own functions in C++? -------------------------------------- - -Yes, using the C compatibility features found in C++. Place ``extern "C" { -... }`` around the Python include files and put ``extern "C"`` before each -function that is going to be called by the Python interpreter. Global or static -C++ objects with constructors are probably not a good idea. - - -.. _c-wrapper-software: - -Writing C is hard; are there any alternatives? ----------------------------------------------- - -There are a number of alternatives to writing your own C extensions, depending -on what you're trying to do. - -.. XXX make sure these all work - -`Cython <https://cython.org>`_ and its relative `Pyrex -<https://www.csse.canterbury.ac.nz/greg.ewing/python/Pyrex/>`_ are compilers -that accept a slightly modified form of Python and generate the corresponding -C code. Cython and Pyrex make it possible to write an extension without having -to learn Python's C API. - -If you need to interface to some C or C++ library for which no Python extension -currently exists, you can try wrapping the library's data types and functions -with a tool such as `SWIG <https://www.swig.org>`_. `SIP -<https://riverbankcomputing.com/software/sip/intro>`__, `CXX -<https://cxx.sourceforge.net/>`_ `Boost -<https://www.boost.org/libs/python/doc/index.html>`_, or `Weave -<https://github.com/scipy/weave>`_ are also -alternatives for wrapping C++ libraries. - - -How can I execute arbitrary Python statements from C? ------------------------------------------------------ - -The highest-level function to do this is :c:func:`PyRun_SimpleString` which takes -a single string argument to be executed in the context of the module -``__main__`` and returns ``0`` for success and ``-1`` when an exception occurred -(including :exc:`SyntaxError`). If you want more control, use -:c:func:`PyRun_String`; see the source for :c:func:`PyRun_SimpleString` in -``Python/pythonrun.c``. - - -How can I evaluate an arbitrary Python expression from C? ---------------------------------------------------------- - -Call the function :c:func:`PyRun_String` from the previous question with the -start symbol :c:data:`Py_eval_input`; it parses an expression, evaluates it and -returns its value. - - -How do I extract C values from a Python object? ------------------------------------------------ - -That depends on the object's type. If it's a tuple, :c:func:`PyTuple_Size` -returns its length and :c:func:`PyTuple_GetItem` returns the item at a specified -index. Lists have similar functions, :c:func:`PyList_Size` and -:c:func:`PyList_GetItem`. - -For bytes, :c:func:`PyBytes_Size` returns its length and -:c:func:`PyBytes_AsStringAndSize` provides a pointer to its value and its -length. Note that Python bytes objects may contain null bytes so C's -:c:func:`!strlen` should not be used. - -To test the type of an object, first make sure it isn't ``NULL``, and then use -:c:func:`PyBytes_Check`, :c:func:`PyTuple_Check`, :c:func:`PyList_Check`, etc. - -There is also a high-level API to Python objects which is provided by the -so-called 'abstract' interface -- read ``Include/abstract.h`` for further -details. It allows interfacing with any kind of Python sequence using calls -like :c:func:`PySequence_Length`, :c:func:`PySequence_GetItem`, etc. as well -as many other useful protocols such as numbers (:c:func:`PyNumber_Index` et -al.) and mappings in the PyMapping APIs. - - -How do I use Py_BuildValue() to create a tuple of arbitrary length? -------------------------------------------------------------------- - -You can't. Use :c:func:`PyTuple_Pack` instead. - - -How do I call an object's method from C? ----------------------------------------- - -The :c:func:`PyObject_CallMethod` function can be used to call an arbitrary -method of an object. The parameters are the object, the name of the method to -call, a format string like that used with :c:func:`Py_BuildValue`, and the -argument values:: - - PyObject * - PyObject_CallMethod(PyObject *object, const char *method_name, - const char *arg_format, ...); - -This works for any object that has methods -- whether built-in or user-defined. -You are responsible for eventually :c:func:`Py_DECREF`\ 'ing the return value. - -To call, e.g., a file object's "seek" method with arguments 10, 0 (assuming the -file object pointer is "f"):: - - res = PyObject_CallMethod(f, "seek", "(ii)", 10, 0); - if (res == NULL) { - ... an exception occurred ... - } - else { - Py_DECREF(res); - } - -Note that since :c:func:`PyObject_CallObject` *always* wants a tuple for the -argument list, to call a function without arguments, pass "()" for the format, -and to call a function with one argument, surround the argument in parentheses, -e.g. "(i)". - - -How do I catch the output from PyErr_Print() (or anything that prints to stdout/stderr)? ----------------------------------------------------------------------------------------- - -In Python code, define an object that supports the ``write()`` method. Assign -this object to :data:`sys.stdout` and :data:`sys.stderr`. Call print_error, or -just allow the standard traceback mechanism to work. Then, the output will go -wherever your ``write()`` method sends it. - -The easiest way to do this is to use the :class:`io.StringIO` class: - -.. code-block:: pycon - - >>> import io, sys - >>> sys.stdout = io.StringIO() - >>> print('foo') - >>> print('hello world!') - >>> sys.stderr.write(sys.stdout.getvalue()) - foo - hello world! - -A custom object to do the same would look like this: - -.. code-block:: pycon - - >>> import io, sys - >>> class StdoutCatcher(io.TextIOBase): - ... def __init__(self): - ... self.data = [] - ... def write(self, stuff): - ... self.data.append(stuff) - ... - >>> import sys - >>> sys.stdout = StdoutCatcher() - >>> print('foo') - >>> print('hello world!') - >>> sys.stderr.write(''.join(sys.stdout.data)) - foo - hello world! - - -How do I access a module written in Python from C? --------------------------------------------------- - -You can get a pointer to the module object as follows:: - - module = PyImport_ImportModule("<modulename>"); - -If the module hasn't been imported yet (i.e. it is not yet present in -:data:`sys.modules`), this initializes the module; otherwise it simply returns -the value of ``sys.modules["<modulename>"]``. Note that it doesn't enter the -module into any namespace -- it only ensures it has been initialized and is -stored in :data:`sys.modules`. - -You can then access the module's attributes (i.e. any name defined in the -module) as follows:: - - attr = PyObject_GetAttrString(module, "<attrname>"); - -Calling :c:func:`PyObject_SetAttrString` to assign to variables in the module -also works. - - -How do I interface to C++ objects from Python? ----------------------------------------------- - -Depending on your requirements, there are many approaches. To do this manually, -begin by reading :ref:`the "Extending and Embedding" document -<extending-index>`. Realize that for the Python run-time system, there isn't a -whole lot of difference between C and C++ -- so the strategy of building a new -Python type around a C structure (pointer) type will also work for C++ objects. - -For C++ libraries, see :ref:`c-wrapper-software`. - - -I added a module using the Setup file and the make fails; why? --------------------------------------------------------------- - -Setup must end in a newline, if there is no newline there, the build process -fails. (Fixing this requires some ugly shell script hackery, and this bug is so -minor that it doesn't seem worth the effort.) - - -How do I debug an extension? ----------------------------- - -When using GDB with dynamically loaded extensions, you can't set a breakpoint in -your extension until your extension is loaded. - -In your ``.gdbinit`` file (or interactively), add the command: - -.. code-block:: none - - br _PyImport_LoadDynamicModule - -Then, when you run GDB: - -.. code-block:: shell-session - - $ gdb /local/bin/python - gdb) run myscript.py - gdb) continue # repeat until your extension is loaded - gdb) finish # so that your extension is loaded - gdb) br myfunction.c:50 - gdb) continue - -I want to compile a Python module on my Linux system, but some files are missing. Why? --------------------------------------------------------------------------------------- - -Most packaged versions of Python don't include the -:file:`/usr/lib/python2.{x}/config/` directory, which contains various files -required for compiling Python extensions. - -For Red Hat, install the python-devel RPM to get the necessary files. - -For Debian, run ``apt-get install python-dev``. - -How do I tell "incomplete input" from "invalid input"? ------------------------------------------------------- - -Sometimes you want to emulate the Python interactive interpreter's behavior, -where it gives you a continuation prompt when the input is incomplete (e.g. you -typed the start of an "if" statement or you didn't close your parentheses or -triple string quotes), but it gives you a syntax error message immediately when -the input is invalid. - -In Python you can use the :mod:`codeop` module, which approximates the parser's -behavior sufficiently. IDLE uses this, for example. - -The easiest way to do it in C is to call :c:func:`PyRun_InteractiveLoop` (perhaps -in a separate thread) and let the Python interpreter handle the input for -you. You can also set the :c:func:`PyOS_ReadlineFunctionPointer` to point at your -custom input function. See ``Modules/readline.c`` and ``Parser/myreadline.c`` -for more hints. - -How do I find undefined g++ symbols __builtin_new or __pure_virtual? --------------------------------------------------------------------- - -To dynamically load g++ extension modules, you must recompile Python, relink it -using g++ (change LINKCC in the Python Modules Makefile), and link your -extension module using g++ (e.g., ``g++ -shared -o mymodule.so mymodule.o``). - - -Can I create an object class with some methods implemented in C and others in Python (e.g. through inheritance)? ----------------------------------------------------------------------------------------------------------------- - -Yes, you can inherit from built-in classes such as :class:`int`, :class:`list`, -:class:`dict`, etc. - -The Boost Python Library (BPL, https://www.boost.org/libs/python/doc/index.html) -provides a way of doing this from C++ (i.e. you can inherit from an extension -class written in C++ using the BPL). diff --git a/Doc/faq/general.rst b/Doc/faq/general.rst deleted file mode 100644 index 8727332594bda6..00000000000000 --- a/Doc/faq/general.rst +++ /dev/null @@ -1,447 +0,0 @@ -:tocdepth: 2 - -================== -General Python FAQ -================== - -.. only:: html - - .. contents:: - - -General Information -=================== - -What is Python? ---------------- - -Python is an interpreted, interactive, object-oriented programming language. It -incorporates modules, exceptions, dynamic typing, very high level dynamic data -types, and classes. It supports multiple programming paradigms beyond -object-oriented programming, such as procedural and functional programming. -Python combines remarkable power with very clear syntax. It has interfaces to -many system calls and libraries, as well as to various window systems, and is -extensible in C or C++. It is also usable as an extension language for -applications that need a programmable interface. Finally, Python is portable: -it runs on many Unix variants including Linux and macOS, and on Windows. - -To find out more, start with :ref:`tutorial-index`. The `Beginner's Guide to -Python <https://wiki.python.org/moin/BeginnersGuide>`_ links to other -introductory tutorials and resources for learning Python. - - -What is the Python Software Foundation? ---------------------------------------- - -The Python Software Foundation is an independent non-profit organization that -holds the copyright on Python versions 2.1 and newer. The PSF's mission is to -advance open source technology related to the Python programming language and to -publicize the use of Python. The PSF's home page is at -https://www.python.org/psf/. - -Donations to the PSF are tax-exempt in the US. If you use Python and find it -helpful, please contribute via `the PSF donation page -<https://www.python.org/psf/donations/>`_. - - -Are there copyright restrictions on the use of Python? ------------------------------------------------------- - -You can do anything you want with the source, as long as you leave the -copyrights in and display those copyrights in any documentation about Python -that you produce. If you honor the copyright rules, it's OK to use Python for -commercial use, to sell copies of Python in source or binary form (modified or -unmodified), or to sell products that incorporate Python in some form. We would -still like to know about all commercial use of Python, of course. - -See `the license page <https://docs.python.org/3/license.html>`_ to find further -explanations and the full text of the PSF License. - -The Python logo is trademarked, and in certain cases permission is required to -use it. Consult `the Trademark Usage Policy -<https://www.python.org/psf/trademarks/>`__ for more information. - - -Why was Python created in the first place? ------------------------------------------- - -Here's a *very* brief summary of what started it all, written by Guido van -Rossum: - - I had extensive experience with implementing an interpreted language in the - ABC group at CWI, and from working with this group I had learned a lot about - language design. This is the origin of many Python features, including the - use of indentation for statement grouping and the inclusion of - very-high-level data types (although the details are all different in - Python). - - I had a number of gripes about the ABC language, but also liked many of its - features. It was impossible to extend the ABC language (or its - implementation) to remedy my complaints -- in fact its lack of extensibility - was one of its biggest problems. I had some experience with using Modula-2+ - and talked with the designers of Modula-3 and read the Modula-3 report. - Modula-3 is the origin of the syntax and semantics used for exceptions, and - some other Python features. - - I was working in the Amoeba distributed operating system group at CWI. We - needed a better way to do system administration than by writing either C - programs or Bourne shell scripts, since Amoeba had its own system call - interface which wasn't easily accessible from the Bourne shell. My - experience with error handling in Amoeba made me acutely aware of the - importance of exceptions as a programming language feature. - - It occurred to me that a scripting language with a syntax like ABC but with - access to the Amoeba system calls would fill the need. I realized that it - would be foolish to write an Amoeba-specific language, so I decided that I - needed a language that was generally extensible. - - During the 1989 Christmas holidays, I had a lot of time on my hand, so I - decided to give it a try. During the next year, while still mostly working - on it in my own time, Python was used in the Amoeba project with increasing - success, and the feedback from colleagues made me add many early - improvements. - - In February 1991, after just over a year of development, I decided to post to - USENET. The rest is in the ``Misc/HISTORY`` file. - - -What is Python good for? ------------------------- - -Python is a high-level general-purpose programming language that can be applied -to many different classes of problems. - -The language comes with a large standard library that covers areas such as -string processing (regular expressions, Unicode, calculating differences between -files), internet protocols (HTTP, FTP, SMTP, XML-RPC, POP, IMAP), -software engineering (unit testing, logging, profiling, parsing -Python code), and operating system interfaces (system calls, filesystems, TCP/IP -sockets). Look at the table of contents for :ref:`library-index` to get an idea -of what's available. A wide variety of third-party extensions are also -available. Consult `the Python Package Index <https://pypi.org>`_ to -find packages of interest to you. - - -How does the Python version numbering scheme work? --------------------------------------------------- - -Python versions are numbered "A.B.C" or "A.B": - -* *A* is the major version number -- it is only incremented for really major - changes in the language. -* *B* is the minor version number -- it is incremented for less earth-shattering - changes. -* *C* is the micro version number -- it is incremented for each bugfix release. - -See :pep:`6` for more information about bugfix releases. - -Not all releases are bugfix releases. In the run-up to a new feature release, a -series of development releases are made, denoted as alpha, beta, or release -candidate. Alphas are early releases in which interfaces aren't yet finalized; -it's not unexpected to see an interface change between two alpha releases. -Betas are more stable, preserving existing interfaces but possibly adding new -modules, and release candidates are frozen, making no changes except as needed -to fix critical bugs. - -Alpha, beta and release candidate versions have an additional suffix: - -* The suffix for an alpha version is "aN" for some small number *N*. -* The suffix for a beta version is "bN" for some small number *N*. -* The suffix for a release candidate version is "rcN" for some small number *N*. - -In other words, all versions labeled *2.0aN* precede the versions labeled -*2.0bN*, which precede versions labeled *2.0rcN*, and *those* precede 2.0. - -You may also find version numbers with a "+" suffix, e.g. "2.2+". These are -unreleased versions, built directly from the CPython development repository. In -practice, after a final minor release is made, the version is incremented to the -next minor version, which becomes the "a0" version, e.g. "2.4a0". - -See also the documentation for :data:`sys.version`, :data:`sys.hexversion`, and -:data:`sys.version_info`. - - -How do I obtain a copy of the Python source? --------------------------------------------- - -The latest Python source distribution is always available from python.org, at -https://www.python.org/downloads/. The latest development sources can be obtained -at https://github.com/python/cpython/. - -The source distribution is a gzipped tar file containing the complete C source, -Sphinx-formatted documentation, Python library modules, example programs, and -several useful pieces of freely distributable software. The source will compile -and run out of the box on most UNIX platforms. - -Consult the `Getting Started section of the Python Developer's Guide -<https://devguide.python.org/setup/>`__ for more -information on getting the source code and compiling it. - - -How do I get documentation on Python? -------------------------------------- - -.. XXX mention py3k - -The standard documentation for the current stable version of Python is available -at https://docs.python.org/3/. PDF, plain text, and downloadable HTML versions are -also available at https://docs.python.org/3/download.html. - -The documentation is written in reStructuredText and processed by `the Sphinx -documentation tool <https://www.sphinx-doc.org/>`__. The reStructuredText source for -the documentation is part of the Python source distribution. - - -I've never programmed before. Is there a Python tutorial? ---------------------------------------------------------- - -There are numerous tutorials and books available. The standard documentation -includes :ref:`tutorial-index`. - -Consult `the Beginner's Guide <https://wiki.python.org/moin/BeginnersGuide>`_ to -find information for beginning Python programmers, including lists of tutorials. - - -Is there a newsgroup or mailing list devoted to Python? -------------------------------------------------------- - -There is a newsgroup, :newsgroup:`comp.lang.python`, and a mailing list, -`python-list <https://mail.python.org/mailman/listinfo/python-list>`_. The -newsgroup and mailing list are gatewayed into each other -- if you can read news -it's unnecessary to subscribe to the mailing list. -:newsgroup:`comp.lang.python` is high-traffic, receiving hundreds of postings -every day, and Usenet readers are often more able to cope with this volume. - -Announcements of new software releases and events can be found in -comp.lang.python.announce, a low-traffic moderated list that receives about five -postings per day. It's available as `the python-announce mailing list -<https://mail.python.org/mailman3/lists/python-announce-list.python.org/>`_. - -More info about other mailing lists and newsgroups -can be found at https://www.python.org/community/lists/. - - -How do I get a beta test version of Python? -------------------------------------------- - -Alpha and beta releases are available from https://www.python.org/downloads/. All -releases are announced on the comp.lang.python and comp.lang.python.announce -newsgroups and on the Python home page at https://www.python.org/; an RSS feed of -news is available. - -You can also access the development version of Python through Git. See -`The Python Developer's Guide <https://devguide.python.org/>`_ for details. - - -How do I submit bug reports and patches for Python? ---------------------------------------------------- - -To report a bug or submit a patch, use the issue tracker at -https://github.com/python/cpython/issues. - -For more information on how Python is developed, consult `the Python Developer's -Guide <https://devguide.python.org/>`_. - - -Are there any published articles about Python that I can reference? -------------------------------------------------------------------- - -It's probably best to cite your favorite book about Python. - -The `very first article <https://ir.cwi.nl/pub/18204>`_ about Python was -written in 1991 and is now quite outdated. - - Guido van Rossum and Jelke de Boer, "Interactively Testing Remote Servers - Using the Python Programming Language", CWI Quarterly, Volume 4, Issue 4 - (December 1991), Amsterdam, pp 283--303. - - -Are there any books on Python? ------------------------------- - -Yes, there are many, and more are being published. See the python.org wiki at -https://wiki.python.org/moin/PythonBooks for a list. - -You can also search online bookstores for "Python" and filter out the Monty -Python references; or perhaps search for "Python" and "language". - - -Where in the world is www.python.org located? ---------------------------------------------- - -The Python project's infrastructure is located all over the world and is managed -by the Python Infrastructure Team. Details `here <https://infra.psf.io>`__. - - -Why is it called Python? ------------------------- - -When he began implementing Python, Guido van Rossum was also reading the -published scripts from `"Monty Python's Flying Circus" -<https://en.wikipedia.org/wiki/Monty_Python>`__, a BBC comedy series from the 1970s. Van Rossum -thought he needed a name that was short, unique, and slightly mysterious, so he -decided to call the language Python. - - -Do I have to like "Monty Python's Flying Circus"? -------------------------------------------------- - -No, but it helps. :) - - -Python in the real world -======================== - -How stable is Python? ---------------------- - -Very stable. New, stable releases have been coming out roughly every 6 to 18 -months since 1991, and this seems likely to continue. As of version 3.9, -Python will have a new feature release every 12 months (:pep:`602`). - -The developers issue bugfix releases of older versions, so the stability of -existing releases gradually improves. Bugfix releases, indicated by a third -component of the version number (e.g. 3.5.3, 3.6.2), are managed for stability; -only fixes for known problems are included in a bugfix release, and it's -guaranteed that interfaces will remain the same throughout a series of bugfix -releases. - -The latest stable releases can always be found on the `Python download page -<https://www.python.org/downloads/>`_. There are two production-ready versions -of Python: 2.x and 3.x. The recommended version is 3.x, which is supported by -most widely used libraries. Although 2.x is still widely used, `it is not -maintained anymore <https://peps.python.org/pep-0373/>`_. - -How many people are using Python? ---------------------------------- - -There are probably millions of users, though it's difficult to obtain an exact -count. - -Python is available for free download, so there are no sales figures, and it's -available from many different sites and packaged with many Linux distributions, -so download statistics don't tell the whole story either. - -The comp.lang.python newsgroup is very active, but not all Python users post to -the group or even read it. - - -Have any significant projects been done in Python? --------------------------------------------------- - -See https://www.python.org/about/success for a list of projects that use Python. -Consulting the proceedings for `past Python conferences -<https://www.python.org/community/workshops/>`_ will reveal contributions from many -different companies and organizations. - -High-profile Python projects include `the Mailman mailing list manager -<https://www.list.org>`_ and `the Zope application server -<https://www.zope.dev>`_. Several Linux distributions, most notably `Red Hat -<https://www.redhat.com>`_, have written part or all of their installer and -system administration software in Python. Companies that use Python internally -include Google, Yahoo, and Lucasfilm Ltd. - - -What new developments are expected for Python in the future? ------------------------------------------------------------- - -See https://peps.python.org/ for the Python Enhancement Proposals -(PEPs). PEPs are design documents describing a suggested new feature for Python, -providing a concise technical specification and a rationale. Look for a PEP -titled "Python X.Y Release Schedule", where X.Y is a version that hasn't been -publicly released yet. - -New development is discussed on `the python-dev mailing list -<https://mail.python.org/mailman3/lists/python-dev.python.org/>`_. - - -Is it reasonable to propose incompatible changes to Python? ------------------------------------------------------------ - -In general, no. There are already millions of lines of Python code around the -world, so any change in the language that invalidates more than a very small -fraction of existing programs has to be frowned upon. Even if you can provide a -conversion program, there's still the problem of updating all documentation; -many books have been written about Python, and we don't want to invalidate them -all at a single stroke. - -Providing a gradual upgrade path is necessary if a feature has to be changed. -:pep:`5` describes the procedure followed for introducing backward-incompatible -changes while minimizing disruption for users. - - -Is Python a good language for beginning programmers? ----------------------------------------------------- - -Yes. - -It is still common to start students with a procedural and statically typed -language such as Pascal, C, or a subset of C++ or Java. Students may be better -served by learning Python as their first language. Python has a very simple and -consistent syntax and a large standard library and, most importantly, using -Python in a beginning programming course lets students concentrate on important -programming skills such as problem decomposition and data type design. With -Python, students can be quickly introduced to basic concepts such as loops and -procedures. They can probably even work with user-defined objects in their very -first course. - -For a student who has never programmed before, using a statically typed language -seems unnatural. It presents additional complexity that the student must master -and slows the pace of the course. The students are trying to learn to think -like a computer, decompose problems, design consistent interfaces, and -encapsulate data. While learning to use a statically typed language is -important in the long term, it is not necessarily the best topic to address in -the students' first programming course. - -Many other aspects of Python make it a good first language. Like Java, Python -has a large standard library so that students can be assigned programming -projects very early in the course that *do* something. Assignments aren't -restricted to the standard four-function calculator and check balancing -programs. By using the standard library, students can gain the satisfaction of -working on realistic applications as they learn the fundamentals of programming. -Using the standard library also teaches students about code reuse. Third-party -modules such as PyGame are also helpful in extending the students' reach. - -Python's interactive interpreter enables students to test language features -while they're programming. They can keep a window with the interpreter running -while they enter their program's source in another window. If they can't -remember the methods for a list, they can do something like this:: - - >>> L = [] - >>> dir(L) # doctest: +NORMALIZE_WHITESPACE - ['__add__', '__class__', '__contains__', '__delattr__', '__delitem__', - '__dir__', '__doc__', '__eq__', '__format__', '__ge__', - '__getattribute__', '__getitem__', '__gt__', '__hash__', '__iadd__', - '__imul__', '__init__', '__iter__', '__le__', '__len__', '__lt__', - '__mul__', '__ne__', '__new__', '__reduce__', '__reduce_ex__', - '__repr__', '__reversed__', '__rmul__', '__setattr__', '__setitem__', - '__sizeof__', '__str__', '__subclasshook__', 'append', 'clear', - 'copy', 'count', 'extend', 'index', 'insert', 'pop', 'remove', - 'reverse', 'sort'] - >>> [d for d in dir(L) if '__' not in d] - ['append', 'clear', 'copy', 'count', 'extend', 'index', 'insert', 'pop', 'remove', 'reverse', 'sort'] - - >>> help(L.append) - Help on built-in function append: - <BLANKLINE> - append(...) - L.append(object) -> None -- append object to end - <BLANKLINE> - >>> L.append(1) - >>> L - [1] - -With the interpreter, documentation is never far from the student as they are -programming. - -There are also good IDEs for Python. IDLE is a cross-platform IDE for Python -that is written in Python using Tkinter. -Emacs users will be happy to know that there is a very good Python mode for -Emacs. All of these programming environments provide syntax highlighting, -auto-indenting, and access to the interactive interpreter while coding. Consult -`the Python wiki <https://wiki.python.org/moin/PythonEditors>`_ for a full list -of Python editing environments. - -If you want to discuss Python's use in education, you may be interested in -joining `the edu-sig mailing list -<https://www.python.org/community/sigs/current/edu-sig>`_. diff --git a/Doc/faq/gui.rst b/Doc/faq/gui.rst deleted file mode 100644 index 77c0a27a10c9a3..00000000000000 --- a/Doc/faq/gui.rst +++ /dev/null @@ -1,80 +0,0 @@ -:tocdepth: 2 - -========================== -Graphic User Interface FAQ -========================== - -.. only:: html - - .. contents:: - -.. XXX need review for Python 3. - - -General GUI Questions -===================== - -What GUI toolkits exist for Python? -=================================== - -Standard builds of Python include an object-oriented interface to the Tcl/Tk -widget set, called :ref:`tkinter <Tkinter>`. This is probably the easiest to -install (since it comes included with most -`binary distributions <https://www.python.org/downloads/>`_ of Python) and use. -For more info about Tk, including pointers to the source, see the -`Tcl/Tk home page <https://www.tcl.tk>`_. Tcl/Tk is fully portable to the -macOS, Windows, and Unix platforms. - -Depending on what platform(s) you are aiming at, there are also several -alternatives. A `list of cross-platform -<https://wiki.python.org/moin/GuiProgramming#Cross-Platform_Frameworks>`_ and -`platform-specific -<https://wiki.python.org/moin/GuiProgramming#Platform-specific_Frameworks>`_ GUI -frameworks can be found on the python wiki. - -Tkinter questions -================= - -How do I freeze Tkinter applications? -------------------------------------- - -Freeze is a tool to create stand-alone applications. When freezing Tkinter -applications, the applications will not be truly stand-alone, as the application -will still need the Tcl and Tk libraries. - -One solution is to ship the application with the Tcl and Tk libraries, and point -to them at run-time using the :envvar:`!TCL_LIBRARY` and :envvar:`!TK_LIBRARY` -environment variables. - -To get truly stand-alone applications, the Tcl scripts that form the library -have to be integrated into the application as well. One tool supporting that is -SAM (stand-alone modules), which is part of the Tix distribution -(https://tix.sourceforge.net/). - -Build Tix with SAM enabled, perform the appropriate call to -:c:func:`Tclsam_init`, etc. inside Python's -:file:`Modules/tkappinit.c`, and link with libtclsam and libtksam (you -might include the Tix libraries as well). - - -Can I have Tk events handled while waiting for I/O? ---------------------------------------------------- - -On platforms other than Windows, yes, and you don't even -need threads! But you'll have to restructure your I/O -code a bit. Tk has the equivalent of Xt's :c:func:`!XtAddInput` call, which allows you -to register a callback function which will be called from the Tk mainloop when -I/O is possible on a file descriptor. See :ref:`tkinter-file-handlers`. - - -I can't get key bindings to work in Tkinter: why? -------------------------------------------------- - -An often-heard complaint is that event handlers :ref:`bound <bindings-and-events>` -to events with the :meth:`!bind` method -don't get handled even when the appropriate key is pressed. - -The most common cause is that the widget to which the binding applies doesn't -have "keyboard focus". Check out the Tk documentation for the focus command. -Usually a widget is given the keyboard focus by clicking in it (but not for -labels; see the takefocus option). diff --git a/Doc/faq/index.rst b/Doc/faq/index.rst deleted file mode 100644 index 46ed3db2203d3a..00000000000000 --- a/Doc/faq/index.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. _faq-index: - -################################### - Python Frequently Asked Questions -################################### - -.. toctree:: - :maxdepth: 1 - - general.rst - programming.rst - design.rst - library.rst - extending.rst - windows.rst - gui.rst - installed.rst diff --git a/Doc/faq/installed.rst b/Doc/faq/installed.rst deleted file mode 100644 index 16c9a74daffb1d..00000000000000 --- a/Doc/faq/installed.rst +++ /dev/null @@ -1,53 +0,0 @@ -============================================= -"Why is Python Installed on my Computer?" FAQ -============================================= - -What is Python? ---------------- - -Python is a programming language. It's used for many different applications. -It's used in some high schools and colleges as an introductory programming -language because Python is easy to learn, but it's also used by professional -software developers at places such as Google, NASA, and Lucasfilm Ltd. - -If you wish to learn more about Python, start with the `Beginner's Guide to -Python <https://wiki.python.org/moin/BeginnersGuide>`_. - - -Why is Python installed on my machine? --------------------------------------- - -If you find Python installed on your system but don't remember installing it, -there are several possible ways it could have gotten there. - -* Perhaps another user on the computer wanted to learn programming and installed - it; you'll have to figure out who's been using the machine and might have - installed it. -* A third-party application installed on the machine might have been written in - Python and included a Python installation. There are many such applications, - from GUI programs to network servers and administrative scripts. -* Some Windows machines also have Python installed. At this writing we're aware - of computers from Hewlett-Packard and Compaq that include Python. Apparently - some of HP/Compaq's administrative tools are written in Python. -* Many Unix-compatible operating systems, such as macOS and some Linux - distributions, have Python installed by default; it's included in the base - installation. - - -Can I delete Python? --------------------- - -That depends on where Python came from. - -If someone installed it deliberately, you can remove it without hurting -anything. On Windows, use the Add/Remove Programs icon in the Control Panel. - -If Python was installed by a third-party application, you can also remove it, -but that application will no longer work. You should use that application's -uninstaller rather than removing Python directly. - -If Python came with your operating system, removing it is not recommended. If -you remove it, whatever tools were written in Python will no longer run, and -some of them might be important to you. Reinstalling the whole system would -then be required to fix things again. - diff --git a/Doc/faq/library.rst b/Doc/faq/library.rst deleted file mode 100644 index c69910718f0c92..00000000000000 --- a/Doc/faq/library.rst +++ /dev/null @@ -1,843 +0,0 @@ -:tocdepth: 2 - -========================= -Library and Extension FAQ -========================= - -.. only:: html - - .. contents:: - -General Library Questions -========================= - -How do I find a module or application to perform task X? --------------------------------------------------------- - -Check :ref:`the Library Reference <library-index>` to see if there's a relevant -standard library module. (Eventually you'll learn what's in the standard -library and will be able to skip this step.) - -For third-party packages, search the `Python Package Index -<https://pypi.org>`_ or try `Google <https://www.google.com>`_ or -another web search engine. Searching for "Python" plus a keyword or two for -your topic of interest will usually find something helpful. - - -Where is the math.py (socket.py, regex.py, etc.) source file? -------------------------------------------------------------- - -If you can't find a source file for a module it may be a built-in or -dynamically loaded module implemented in C, C++ or other compiled language. -In this case you may not have the source file or it may be something like -:file:`mathmodule.c`, somewhere in a C source directory (not on the Python Path). - -There are (at least) three kinds of modules in Python: - -1) modules written in Python (.py); -2) modules written in C and dynamically loaded (.dll, .pyd, .so, .sl, etc); -3) modules written in C and linked with the interpreter; to get a list of these, - type:: - - import sys - print(sys.builtin_module_names) - - -How do I make a Python script executable on Unix? -------------------------------------------------- - -You need to do two things: the script file's mode must be executable and the -first line must begin with ``#!`` followed by the path of the Python -interpreter. - -The first is done by executing ``chmod +x scriptfile`` or perhaps ``chmod 755 -scriptfile``. - -The second can be done in a number of ways. The most straightforward way is to -write :: - - #!/usr/local/bin/python - -as the very first line of your file, using the pathname for where the Python -interpreter is installed on your platform. - -If you would like the script to be independent of where the Python interpreter -lives, you can use the :program:`env` program. Almost all Unix variants support -the following, assuming the Python interpreter is in a directory on the user's -:envvar:`PATH`:: - - #!/usr/bin/env python - -*Don't* do this for CGI scripts. The :envvar:`PATH` variable for CGI scripts is -often very minimal, so you need to use the actual absolute pathname of the -interpreter. - -Occasionally, a user's environment is so full that the :program:`/usr/bin/env` -program fails; or there's no env program at all. In that case, you can try the -following hack (due to Alex Rezinsky): - -.. code-block:: sh - - #! /bin/sh - """:" - exec python $0 ${1+"$@"} - """ - -The minor disadvantage is that this defines the script's __doc__ string. -However, you can fix that by adding :: - - __doc__ = """...Whatever...""" - - - -Is there a curses/termcap package for Python? ---------------------------------------------- - -.. XXX curses *is* built by default, isn't it? - -For Unix variants: The standard Python source distribution comes with a curses -module in the :source:`Modules` subdirectory, though it's not compiled by default. -(Note that this is not available in the Windows distribution -- there is no -curses module for Windows.) - -The :mod:`curses` module supports basic curses features as well as many additional -functions from ncurses and SYSV curses such as colour, alternative character set -support, pads, and mouse support. This means the module isn't compatible with -operating systems that only have BSD curses, but there don't seem to be any -currently maintained OSes that fall into this category. - - -Is there an equivalent to C's onexit() in Python? -------------------------------------------------- - -The :mod:`atexit` module provides a register function that is similar to C's -:c:func:`!onexit`. - - -Why don't my signal handlers work? ----------------------------------- - -The most common problem is that the signal handler is declared with the wrong -argument list. It is called as :: - - handler(signum, frame) - -so it should be declared with two parameters:: - - def handler(signum, frame): - ... - - -Common tasks -============ - -How do I test a Python program or component? --------------------------------------------- - -Python comes with two testing frameworks. The :mod:`doctest` module finds -examples in the docstrings for a module and runs them, comparing the output with -the expected output given in the docstring. - -The :mod:`unittest` module is a fancier testing framework modelled on Java and -Smalltalk testing frameworks. - -To make testing easier, you should use good modular design in your program. -Your program should have almost all functionality -encapsulated in either functions or class methods -- and this sometimes has the -surprising and delightful effect of making the program run faster (because local -variable accesses are faster than global accesses). Furthermore the program -should avoid depending on mutating global variables, since this makes testing -much more difficult to do. - -The "global main logic" of your program may be as simple as :: - - if __name__ == "__main__": - main_logic() - -at the bottom of the main module of your program. - -Once your program is organized as a tractable collection of function and class -behaviours, you should write test functions that exercise the behaviours. A -test suite that automates a sequence of tests can be associated with each module. -This sounds like a lot of work, but since Python is so terse and flexible it's -surprisingly easy. You can make coding much more pleasant and fun by writing -your test functions in parallel with the "production code", since this makes it -easy to find bugs and even design flaws earlier. - -"Support modules" that are not intended to be the main module of a program may -include a self-test of the module. :: - - if __name__ == "__main__": - self_test() - -Even programs that interact with complex external interfaces may be tested when -the external interfaces are unavailable by using "fake" interfaces implemented -in Python. - - -How do I create documentation from doc strings? ------------------------------------------------ - -The :mod:`pydoc` module can create HTML from the doc strings in your Python -source code. An alternative for creating API documentation purely from -docstrings is `epydoc <https://epydoc.sourceforge.net/>`_. `Sphinx -<https://www.sphinx-doc.org>`_ can also include docstring content. - - -How do I get a single keypress at a time? ------------------------------------------ - -For Unix variants there are several solutions. It's straightforward to do this -using curses, but curses is a fairly large module to learn. - -.. XXX this doesn't work out of the box, some IO expert needs to check why - - Here's a solution without curses:: - - import termios, fcntl, sys, os - fd = sys.stdin.fileno() - - oldterm = termios.tcgetattr(fd) - newattr = termios.tcgetattr(fd) - newattr[3] = newattr[3] & ~termios.ICANON & ~termios.ECHO - termios.tcsetattr(fd, termios.TCSANOW, newattr) - - oldflags = fcntl.fcntl(fd, fcntl.F_GETFL) - fcntl.fcntl(fd, fcntl.F_SETFL, oldflags | os.O_NONBLOCK) - - try: - while True: - try: - c = sys.stdin.read(1) - print("Got character", repr(c)) - except OSError: - pass - finally: - termios.tcsetattr(fd, termios.TCSAFLUSH, oldterm) - fcntl.fcntl(fd, fcntl.F_SETFL, oldflags) - - You need the :mod:`termios` and the :mod:`fcntl` module for any of this to - work, and I've only tried it on Linux, though it should work elsewhere. In - this code, characters are read and printed one at a time. - - :func:`termios.tcsetattr` turns off stdin's echoing and disables canonical - mode. :func:`fcntl.fnctl` is used to obtain stdin's file descriptor flags - and modify them for non-blocking mode. Since reading stdin when it is empty - results in an :exc:`OSError`, this error is caught and ignored. - - .. versionchanged:: 3.3 - *sys.stdin.read* used to raise :exc:`IOError`. Starting from Python 3.3 - :exc:`IOError` is alias for :exc:`OSError`. - - -Threads -======= - -How do I program using threads? -------------------------------- - -Be sure to use the :mod:`threading` module and not the :mod:`_thread` module. -The :mod:`threading` module builds convenient abstractions on top of the -low-level primitives provided by the :mod:`_thread` module. - - -None of my threads seem to run: why? ------------------------------------- - -As soon as the main thread exits, all threads are killed. Your main thread is -running too quickly, giving the threads no time to do any work. - -A simple fix is to add a sleep to the end of the program that's long enough for -all the threads to finish:: - - import threading, time - - def thread_task(name, n): - for i in range(n): - print(name, i) - - for i in range(10): - T = threading.Thread(target=thread_task, args=(str(i), i)) - T.start() - - time.sleep(10) # <---------------------------! - -But now (on many platforms) the threads don't run in parallel, but appear to run -sequentially, one at a time! The reason is that the OS thread scheduler doesn't -start a new thread until the previous thread is blocked. - -A simple fix is to add a tiny sleep to the start of the run function:: - - def thread_task(name, n): - time.sleep(0.001) # <--------------------! - for i in range(n): - print(name, i) - - for i in range(10): - T = threading.Thread(target=thread_task, args=(str(i), i)) - T.start() - - time.sleep(10) - -Instead of trying to guess a good delay value for :func:`time.sleep`, -it's better to use some kind of semaphore mechanism. One idea is to use the -:mod:`queue` module to create a queue object, let each thread append a token to -the queue when it finishes, and let the main thread read as many tokens from the -queue as there are threads. - - -How do I parcel out work among a bunch of worker threads? ---------------------------------------------------------- - -The easiest way is to use the :mod:`concurrent.futures` module, -especially the :mod:`~concurrent.futures.ThreadPoolExecutor` class. - -Or, if you want fine control over the dispatching algorithm, you can write -your own logic manually. Use the :mod:`queue` module to create a queue -containing a list of jobs. The :class:`~queue.Queue` class maintains a -list of objects and has a ``.put(obj)`` method that adds items to the queue and -a ``.get()`` method to return them. The class will take care of the locking -necessary to ensure that each job is handed out exactly once. - -Here's a trivial example:: - - import threading, queue, time - - # The worker thread gets jobs off the queue. When the queue is empty, it - # assumes there will be no more work and exits. - # (Realistically workers will run until terminated.) - def worker(): - print('Running worker') - time.sleep(0.1) - while True: - try: - arg = q.get(block=False) - except queue.Empty: - print('Worker', threading.current_thread(), end=' ') - print('queue empty') - break - else: - print('Worker', threading.current_thread(), end=' ') - print('running with argument', arg) - time.sleep(0.5) - - # Create queue - q = queue.Queue() - - # Start a pool of 5 workers - for i in range(5): - t = threading.Thread(target=worker, name='worker %i' % (i+1)) - t.start() - - # Begin adding work to the queue - for i in range(50): - q.put(i) - - # Give threads time to run - print('Main thread sleeping') - time.sleep(5) - -When run, this will produce the following output: - -.. code-block:: none - - Running worker - Running worker - Running worker - Running worker - Running worker - Main thread sleeping - Worker <Thread(worker 1, started 130283832797456)> running with argument 0 - Worker <Thread(worker 2, started 130283824404752)> running with argument 1 - Worker <Thread(worker 3, started 130283816012048)> running with argument 2 - Worker <Thread(worker 4, started 130283807619344)> running with argument 3 - Worker <Thread(worker 5, started 130283799226640)> running with argument 4 - Worker <Thread(worker 1, started 130283832797456)> running with argument 5 - ... - -Consult the module's documentation for more details; the :class:`~queue.Queue` -class provides a featureful interface. - - -What kinds of global value mutation are thread-safe? ----------------------------------------------------- - -A :term:`global interpreter lock` (GIL) is used internally to ensure that only one -thread runs in the Python VM at a time. In general, Python offers to switch -among threads only between bytecode instructions; how frequently it switches can -be set via :func:`sys.setswitchinterval`. Each bytecode instruction and -therefore all the C implementation code reached from each instruction is -therefore atomic from the point of view of a Python program. - -In theory, this means an exact accounting requires an exact understanding of the -PVM bytecode implementation. In practice, it means that operations on shared -variables of built-in data types (ints, lists, dicts, etc) that "look atomic" -really are. - -For example, the following operations are all atomic (L, L1, L2 are lists, D, -D1, D2 are dicts, x, y are objects, i, j are ints):: - - L.append(x) - L1.extend(L2) - x = L[i] - x = L.pop() - L1[i:j] = L2 - L.sort() - x = y - x.field = y - D[x] = y - D1.update(D2) - D.keys() - -These aren't:: - - i = i+1 - L.append(L[-1]) - L[i] = L[j] - D[x] = D[x] + 1 - -Operations that replace other objects may invoke those other objects' -:meth:`~object.__del__` method when their reference count reaches zero, and that can -affect things. This is especially true for the mass updates to dictionaries and -lists. When in doubt, use a mutex! - - -Can't we get rid of the Global Interpreter Lock? ------------------------------------------------- - -.. XXX link to dbeazley's talk about GIL? - -The :term:`global interpreter lock` (GIL) is often seen as a hindrance to Python's -deployment on high-end multiprocessor server machines, because a multi-threaded -Python program effectively only uses one CPU, due to the insistence that -(almost) all Python code can only run while the GIL is held. - -Back in the days of Python 1.5, Greg Stein actually implemented a comprehensive -patch set (the "free threading" patches) that removed the GIL and replaced it -with fine-grained locking. Adam Olsen recently did a similar experiment -in his `python-safethread <https://code.google.com/archive/p/python-safethread>`_ -project. Unfortunately, both experiments exhibited a sharp drop in single-thread -performance (at least 30% slower), due to the amount of fine-grained locking -necessary to compensate for the removal of the GIL. - -This doesn't mean that you can't make good use of Python on multi-CPU machines! -You just have to be creative with dividing the work up between multiple -*processes* rather than multiple *threads*. The -:class:`~concurrent.futures.ProcessPoolExecutor` class in the new -:mod:`concurrent.futures` module provides an easy way of doing so; the -:mod:`multiprocessing` module provides a lower-level API in case you want -more control over dispatching of tasks. - -Judicious use of C extensions will also help; if you use a C extension to -perform a time-consuming task, the extension can release the GIL while the -thread of execution is in the C code and allow other threads to get some work -done. Some standard library modules such as :mod:`zlib` and :mod:`hashlib` -already do this. - -It has been suggested that the GIL should be a per-interpreter-state lock rather -than truly global; interpreters then wouldn't be able to share objects. -Unfortunately, this isn't likely to happen either. It would be a tremendous -amount of work, because many object implementations currently have global state. -For example, small integers and short strings are cached; these caches would -have to be moved to the interpreter state. Other object types have their own -free list; these free lists would have to be moved to the interpreter state. -And so on. - -And I doubt that it can even be done in finite time, because the same problem -exists for 3rd party extensions. It is likely that 3rd party extensions are -being written at a faster rate than you can convert them to store all their -global state in the interpreter state. - -And finally, once you have multiple interpreters not sharing any state, what -have you gained over running each interpreter in a separate process? - - -Input and Output -================ - -How do I delete a file? (And other file questions...) ------------------------------------------------------ - -Use ``os.remove(filename)`` or ``os.unlink(filename)``; for documentation, see -the :mod:`os` module. The two functions are identical; :func:`~os.unlink` is simply -the name of the Unix system call for this function. - -To remove a directory, use :func:`os.rmdir`; use :func:`os.mkdir` to create one. -``os.makedirs(path)`` will create any intermediate directories in ``path`` that -don't exist. ``os.removedirs(path)`` will remove intermediate directories as -long as they're empty; if you want to delete an entire directory tree and its -contents, use :func:`shutil.rmtree`. - -To rename a file, use ``os.rename(old_path, new_path)``. - -To truncate a file, open it using ``f = open(filename, "rb+")``, and use -``f.truncate(offset)``; offset defaults to the current seek position. There's -also ``os.ftruncate(fd, offset)`` for files opened with :func:`os.open`, where -*fd* is the file descriptor (a small integer). - -The :mod:`shutil` module also contains a number of functions to work on files -including :func:`~shutil.copyfile`, :func:`~shutil.copytree`, and -:func:`~shutil.rmtree`. - - -How do I copy a file? ---------------------- - -The :mod:`shutil` module contains a :func:`~shutil.copyfile` function. -Note that on Windows NTFS volumes, it does not copy -`alternate data streams -<https://en.wikipedia.org/wiki/NTFS#Alternate_data_stream_(ADS)>`_ -nor `resource forks <https://en.wikipedia.org/wiki/Resource_fork>`__ -on macOS HFS+ volumes, though both are now rarely used. -It also doesn't copy file permissions and metadata, though using -:func:`shutil.copy2` instead will preserve most (though not all) of it. - - -How do I read (or write) binary data? -------------------------------------- - -To read or write complex binary data formats, it's best to use the :mod:`struct` -module. It allows you to take a string containing binary data (usually numbers) -and convert it to Python objects; and vice versa. - -For example, the following code reads two 2-byte integers and one 4-byte integer -in big-endian format from a file:: - - import struct - - with open(filename, "rb") as f: - s = f.read(8) - x, y, z = struct.unpack(">hhl", s) - -The '>' in the format string forces big-endian data; the letter 'h' reads one -"short integer" (2 bytes), and 'l' reads one "long integer" (4 bytes) from the -string. - -For data that is more regular (e.g. a homogeneous list of ints or floats), -you can also use the :mod:`array` module. - -.. note:: - - To read and write binary data, it is mandatory to open the file in - binary mode (here, passing ``"rb"`` to :func:`open`). If you use - ``"r"`` instead (the default), the file will be open in text mode - and ``f.read()`` will return :class:`str` objects rather than - :class:`bytes` objects. - - -I can't seem to use os.read() on a pipe created with os.popen(); why? ---------------------------------------------------------------------- - -:func:`os.read` is a low-level function which takes a file descriptor, a small -integer representing the opened file. :func:`os.popen` creates a high-level -file object, the same type returned by the built-in :func:`open` function. -Thus, to read *n* bytes from a pipe *p* created with :func:`os.popen`, you need to -use ``p.read(n)``. - - -.. XXX update to use subprocess. See the :ref:`subprocess-replacements` section. - - How do I run a subprocess with pipes connected to both input and output? - ------------------------------------------------------------------------ - - Use the :mod:`popen2` module. For example:: - - import popen2 - fromchild, tochild = popen2.popen2("command") - tochild.write("input\n") - tochild.flush() - output = fromchild.readline() - - Warning: in general it is unwise to do this because you can easily cause a - deadlock where your process is blocked waiting for output from the child - while the child is blocked waiting for input from you. This can be caused - by the parent expecting the child to output more text than it does or - by data being stuck in stdio buffers due to lack of flushing. - The Python parent can of course explicitly flush the data it sends to the - child before it reads any output, but if the child is a naive C program it - may have been written to never explicitly flush its output, even if it is - interactive, since flushing is normally automatic. - - Note that a deadlock is also possible if you use :func:`popen3` to read - stdout and stderr. If one of the two is too large for the internal buffer - (increasing the buffer size does not help) and you ``read()`` the other one - first, there is a deadlock, too. - - Note on a bug in popen2: unless your program calls ``wait()`` or - ``waitpid()``, finished child processes are never removed, and eventually - calls to popen2 will fail because of a limit on the number of child - processes. Calling :func:`os.waitpid` with the :const:`os.WNOHANG` option can - prevent this; a good place to insert such a call would be before calling - ``popen2`` again. - - In many cases, all you really need is to run some data through a command and - get the result back. Unless the amount of data is very large, the easiest - way to do this is to write it to a temporary file and run the command with - that temporary file as input. The standard module :mod:`tempfile` exports a - :func:`~tempfile.mktemp` function to generate unique temporary file names. :: - - import tempfile - import os - - class Popen3: - """ - This is a deadlock-safe version of popen that returns - an object with errorlevel, out (a string) and err (a string). - (capturestderr may not work under windows.) - Example: print(Popen3('grep spam','\n\nhere spam\n\n').out) - """ - def __init__(self,command,input=None,capturestderr=None): - outfile=tempfile.mktemp() - command="( %s ) > %s" % (command,outfile) - if input: - infile=tempfile.mktemp() - open(infile,"w").write(input) - command=command+" <"+infile - if capturestderr: - errfile=tempfile.mktemp() - command=command+" 2>"+errfile - self.errorlevel=os.system(command) >> 8 - self.out=open(outfile,"r").read() - os.remove(outfile) - if input: - os.remove(infile) - if capturestderr: - self.err=open(errfile,"r").read() - os.remove(errfile) - - Note that many interactive programs (e.g. vi) don't work well with pipes - substituted for standard input and output. You will have to use pseudo ttys - ("ptys") instead of pipes. Or you can use a Python interface to Don Libes' - "expect" library. A Python extension that interfaces to expect is called - "expy" and available from https://expectpy.sourceforge.net. A pure Python - solution that works like expect is `pexpect - <https://pypi.org/project/pexpect/>`_. - - -How do I access the serial (RS232) port? ----------------------------------------- - -For Win32, OSX, Linux, BSD, Jython, IronPython: - - https://pypi.org/project/pyserial/ - -For Unix, see a Usenet post by Mitch Chapman: - - https://groups.google.com/groups?selm=34A04430.CF9@ohioee.com - - -Why doesn't closing sys.stdout (stdin, stderr) really close it? ---------------------------------------------------------------- - -Python :term:`file objects <file object>` are a high-level layer of -abstraction on low-level C file descriptors. - -For most file objects you create in Python via the built-in :func:`open` -function, ``f.close()`` marks the Python file object as being closed from -Python's point of view, and also arranges to close the underlying C file -descriptor. This also happens automatically in ``f``'s destructor, when -``f`` becomes garbage. - -But stdin, stdout and stderr are treated specially by Python, because of the -special status also given to them by C. Running ``sys.stdout.close()`` marks -the Python-level file object as being closed, but does *not* close the -associated C file descriptor. - -To close the underlying C file descriptor for one of these three, you should -first be sure that's what you really want to do (e.g., you may confuse -extension modules trying to do I/O). If it is, use :func:`os.close`:: - - os.close(stdin.fileno()) - os.close(stdout.fileno()) - os.close(stderr.fileno()) - -Or you can use the numeric constants 0, 1 and 2, respectively. - - -Network/Internet Programming -============================ - -What WWW tools are there for Python? ------------------------------------- - -See the chapters titled :ref:`internet` and :ref:`netdata` in the Library -Reference Manual. Python has many modules that will help you build server-side -and client-side web systems. - -.. XXX check if wiki page is still up to date - -A summary of available frameworks is maintained by Paul Boddie at -https://wiki.python.org/moin/WebProgramming\ . - -Cameron Laird maintains a useful set of pages about Python web technologies at -https://web.archive.org/web/20210224183619/http://phaseit.net/claird/comp.lang.python/web_python. - - -How can I mimic CGI form submission (METHOD=POST)? --------------------------------------------------- - -I would like to retrieve web pages that are the result of POSTing a form. Is -there existing code that would let me do this easily? - -Yes. Here's a simple example that uses :mod:`urllib.request`:: - - #!/usr/local/bin/python - - import urllib.request - - # build the query string - qs = "First=Josephine&MI=Q&Last=Public" - - # connect and send the server a path - req = urllib.request.urlopen('http://www.some-server.out-there' - '/cgi-bin/some-cgi-script', data=qs) - with req: - msg, hdrs = req.read(), req.info() - -Note that in general for percent-encoded POST operations, query strings must be -quoted using :func:`urllib.parse.urlencode`. For example, to send -``name=Guy Steele, Jr.``:: - - >>> import urllib.parse - >>> urllib.parse.urlencode({'name': 'Guy Steele, Jr.'}) - 'name=Guy+Steele%2C+Jr.' - -.. seealso:: :ref:`urllib-howto` for extensive examples. - - -What module should I use to help with generating HTML? ------------------------------------------------------- - -.. XXX add modern template languages - -You can find a collection of useful links on the `Web Programming wiki page -<https://wiki.python.org/moin/WebProgramming>`_. - - -How do I send mail from a Python script? ----------------------------------------- - -Use the standard library module :mod:`smtplib`. - -Here's a very simple interactive mail sender that uses it. This method will -work on any host that supports an SMTP listener. :: - - import sys, smtplib - - fromaddr = input("From: ") - toaddrs = input("To: ").split(',') - print("Enter message, end with ^D:") - msg = '' - while True: - line = sys.stdin.readline() - if not line: - break - msg += line - - # The actual mail send - server = smtplib.SMTP('localhost') - server.sendmail(fromaddr, toaddrs, msg) - server.quit() - -A Unix-only alternative uses sendmail. The location of the sendmail program -varies between systems; sometimes it is ``/usr/lib/sendmail``, sometimes -``/usr/sbin/sendmail``. The sendmail manual page will help you out. Here's -some sample code:: - - import os - - SENDMAIL = "/usr/sbin/sendmail" # sendmail location - p = os.popen("%s -t -i" % SENDMAIL, "w") - p.write("To: receiver@example.com\n") - p.write("Subject: test\n") - p.write("\n") # blank line separating headers from body - p.write("Some text\n") - p.write("some more text\n") - sts = p.close() - if sts != 0: - print("Sendmail exit status", sts) - - -How do I avoid blocking in the connect() method of a socket? ------------------------------------------------------------- - -The :mod:`select` module is commonly used to help with asynchronous I/O on -sockets. - -To prevent the TCP connect from blocking, you can set the socket to non-blocking -mode. Then when you do the :meth:`~socket.socket.connect`, -you will either connect immediately -(unlikely) or get an exception that contains the error number as ``.errno``. -``errno.EINPROGRESS`` indicates that the connection is in progress, but hasn't -finished yet. Different OSes will return different values, so you're going to -have to check what's returned on your system. - -You can use the :meth:`~socket.socket.connect_ex` method -to avoid creating an exception. -It will just return the errno value. -To poll, you can call :meth:`~socket.socket.connect_ex` again later --- ``0`` or ``errno.EISCONN`` indicate that you're connected -- or you can pass this -socket to :meth:`select.select` to check if it's writable. - -.. note:: - The :mod:`asyncio` module provides a general purpose single-threaded and - concurrent asynchronous library, which can be used for writing non-blocking - network code. - The third-party `Twisted <https://twisted.org/>`_ library is - a popular and feature-rich alternative. - - -Databases -========= - -Are there any interfaces to database packages in Python? --------------------------------------------------------- - -Yes. - -Interfaces to disk-based hashes such as :mod:`DBM <dbm.ndbm>` and :mod:`GDBM -<dbm.gnu>` are also included with standard Python. There is also the -:mod:`sqlite3` module, which provides a lightweight disk-based relational -database. - -Support for most relational databases is available. See the -`DatabaseProgramming wiki page -<https://wiki.python.org/moin/DatabaseProgramming>`_ for details. - - -How do you implement persistent objects in Python? --------------------------------------------------- - -The :mod:`pickle` library module solves this in a very general way (though you -still can't store things like open files, sockets or windows), and the -:mod:`shelve` library module uses pickle and (g)dbm to create persistent -mappings containing arbitrary Python objects. - - -Mathematics and Numerics -======================== - -How do I generate random numbers in Python? -------------------------------------------- - -The standard module :mod:`random` implements a random number generator. Usage -is simple:: - - import random - random.random() - -This returns a random floating point number in the range [0, 1). - -There are also many other specialized generators in this module, such as: - -* ``randrange(a, b)`` chooses an integer in the range [a, b). -* ``uniform(a, b)`` chooses a floating point number in the range [a, b). -* ``normalvariate(mean, sdev)`` samples the normal (Gaussian) distribution. - -Some higher-level functions operate on sequences directly, such as: - -* ``choice(S)`` chooses a random element from a given sequence. -* ``shuffle(L)`` shuffles a list in-place, i.e. permutes it randomly. - -There's also a ``Random`` class you can instantiate to create independent -multiple random number generators. diff --git a/Doc/faq/programming.rst b/Doc/faq/programming.rst deleted file mode 100644 index f43f69b8a1ea91..00000000000000 --- a/Doc/faq/programming.rst +++ /dev/null @@ -1,2216 +0,0 @@ -:tocdepth: 2 - -=============== -Programming FAQ -=============== - -.. only:: html - - .. contents:: - -General Questions -================= - -Is there a source code level debugger with breakpoints, single-stepping, etc.? ------------------------------------------------------------------------------- - -Yes. - -Several debuggers for Python are described below, and the built-in function -:func:`breakpoint` allows you to drop into any of them. - -The pdb module is a simple but adequate console-mode debugger for Python. It is -part of the standard Python library, and is :mod:`documented in the Library -Reference Manual <pdb>`. You can also write your own debugger by using the code -for pdb as an example. - -The IDLE interactive development environment, which is part of the standard -Python distribution (normally available as -`Tools/scripts/idle3 <https://github.com/python/cpython/blob/main/Tools/scripts/idle3>`_), -includes a graphical debugger. - -PythonWin is a Python IDE that includes a GUI debugger based on pdb. The -PythonWin debugger colors breakpoints and has quite a few cool features such as -debugging non-PythonWin programs. PythonWin is available as part of -`pywin32 <https://github.com/mhammond/pywin32>`_ project and -as a part of the -`ActivePython <https://www.activestate.com/products/python/>`_ distribution. - -`Eric <https://eric-ide.python-projects.org/>`_ is an IDE built on PyQt -and the Scintilla editing component. - -`trepan3k <https://github.com/rocky/python3-trepan/>`_ is a gdb-like debugger. - -`Visual Studio Code <https://code.visualstudio.com/>`_ is an IDE with debugging -tools that integrates with version-control software. - -There are a number of commercial Python IDEs that include graphical debuggers. -They include: - -* `Wing IDE <https://wingware.com/>`_ -* `Komodo IDE <https://www.activestate.com/products/komodo-ide/>`_ -* `PyCharm <https://www.jetbrains.com/pycharm/>`_ - - -Are there tools to help find bugs or perform static analysis? -------------------------------------------------------------- - -Yes. - -`Pylint <https://pylint.pycqa.org/en/latest/index.html>`_ and -`Pyflakes <https://github.com/PyCQA/pyflakes>`_ do basic checking that will -help you catch bugs sooner. - -Static type checkers such as `Mypy <https://mypy-lang.org/>`_, -`Pyre <https://pyre-check.org/>`_, and -`Pytype <https://github.com/google/pytype>`_ can check type hints in Python -source code. - - -.. _faq-create-standalone-binary: - -How can I create a stand-alone binary from a Python script? ------------------------------------------------------------ - -You don't need the ability to compile Python to C code if all you want is a -stand-alone program that users can download and run without having to install -the Python distribution first. There are a number of tools that determine the -set of modules required by a program and bind these modules together with a -Python binary to produce a single executable. - -One is to use the freeze tool, which is included in the Python source tree as -`Tools/freeze <https://github.com/python/cpython/tree/main/Tools/freeze>`_. -It converts Python byte code to C arrays; with a C compiler you can -embed all your modules into a new program, which is then linked with the -standard Python modules. - -It works by scanning your source recursively for import statements (in both -forms) and looking for the modules in the standard Python path as well as in the -source directory (for built-in modules). It then turns the bytecode for modules -written in Python into C code (array initializers that can be turned into code -objects using the marshal module) and creates a custom-made config file that -only contains those built-in modules which are actually used in the program. It -then compiles the generated C code and links it with the rest of the Python -interpreter to form a self-contained binary which acts exactly like your script. - -The following packages can help with the creation of console and GUI -executables: - -* `Nuitka <https://nuitka.net/>`_ (Cross-platform) -* `PyInstaller <https://pyinstaller.org/>`_ (Cross-platform) -* `PyOxidizer <https://pyoxidizer.readthedocs.io/en/stable/>`_ (Cross-platform) -* `cx_Freeze <https://marcelotduarte.github.io/cx_Freeze/>`_ (Cross-platform) -* `py2app <https://github.com/ronaldoussoren/py2app>`_ (macOS only) -* `py2exe <https://www.py2exe.org/>`_ (Windows only) - -Are there coding standards or a style guide for Python programs? ----------------------------------------------------------------- - -Yes. The coding style required for standard library modules is documented as -:pep:`8`. - - -Core Language -============= - -.. _faq-unboundlocalerror: - -Why am I getting an UnboundLocalError when the variable has a value? --------------------------------------------------------------------- - -It can be a surprise to get the :exc:`UnboundLocalError` in previously working -code when it is modified by adding an assignment statement somewhere in -the body of a function. - -This code: - - >>> x = 10 - >>> def bar(): - ... print(x) - ... - >>> bar() - 10 - -works, but this code: - - >>> x = 10 - >>> def foo(): - ... print(x) - ... x += 1 - -results in an :exc:`!UnboundLocalError`: - - >>> foo() - Traceback (most recent call last): - ... - UnboundLocalError: local variable 'x' referenced before assignment - -This is because when you make an assignment to a variable in a scope, that -variable becomes local to that scope and shadows any similarly named variable -in the outer scope. Since the last statement in foo assigns a new value to -``x``, the compiler recognizes it as a local variable. Consequently when the -earlier ``print(x)`` attempts to print the uninitialized local variable and -an error results. - -In the example above you can access the outer scope variable by declaring it -global: - - >>> x = 10 - >>> def foobar(): - ... global x - ... print(x) - ... x += 1 - ... - >>> foobar() - 10 - -This explicit declaration is required in order to remind you that (unlike the -superficially analogous situation with class and instance variables) you are -actually modifying the value of the variable in the outer scope: - - >>> print(x) - 11 - -You can do a similar thing in a nested scope using the :keyword:`nonlocal` -keyword: - - >>> def foo(): - ... x = 10 - ... def bar(): - ... nonlocal x - ... print(x) - ... x += 1 - ... bar() - ... print(x) - ... - >>> foo() - 10 - 11 - - -What are the rules for local and global variables in Python? ------------------------------------------------------------- - -In Python, variables that are only referenced inside a function are implicitly -global. If a variable is assigned a value anywhere within the function's body, -it's assumed to be a local unless explicitly declared as global. - -Though a bit surprising at first, a moment's consideration explains this. On -one hand, requiring :keyword:`global` for assigned variables provides a bar -against unintended side-effects. On the other hand, if ``global`` was required -for all global references, you'd be using ``global`` all the time. You'd have -to declare as global every reference to a built-in function or to a component of -an imported module. This clutter would defeat the usefulness of the ``global`` -declaration for identifying side-effects. - - -Why do lambdas defined in a loop with different values all return the same result? ----------------------------------------------------------------------------------- - -Assume you use a for loop to define a few different lambdas (or even plain -functions), e.g.:: - - >>> squares = [] - >>> for x in range(5): - ... squares.append(lambda: x**2) - -This gives you a list that contains 5 lambdas that calculate ``x**2``. You -might expect that, when called, they would return, respectively, ``0``, ``1``, -``4``, ``9``, and ``16``. However, when you actually try you will see that -they all return ``16``:: - - >>> squares[2]() - 16 - >>> squares[4]() - 16 - -This happens because ``x`` is not local to the lambdas, but is defined in -the outer scope, and it is accessed when the lambda is called --- not when it -is defined. At the end of the loop, the value of ``x`` is ``4``, so all the -functions now return ``4**2``, i.e. ``16``. You can also verify this by -changing the value of ``x`` and see how the results of the lambdas change:: - - >>> x = 8 - >>> squares[2]() - 64 - -In order to avoid this, you need to save the values in variables local to the -lambdas, so that they don't rely on the value of the global ``x``:: - - >>> squares = [] - >>> for x in range(5): - ... squares.append(lambda n=x: n**2) - -Here, ``n=x`` creates a new variable ``n`` local to the lambda and computed -when the lambda is defined so that it has the same value that ``x`` had at -that point in the loop. This means that the value of ``n`` will be ``0`` -in the first lambda, ``1`` in the second, ``2`` in the third, and so on. -Therefore each lambda will now return the correct result:: - - >>> squares[2]() - 4 - >>> squares[4]() - 16 - -Note that this behaviour is not peculiar to lambdas, but applies to regular -functions too. - - -How do I share global variables across modules? ------------------------------------------------- - -The canonical way to share information across modules within a single program is -to create a special module (often called config or cfg). Just import the config -module in all modules of your application; the module then becomes available as -a global name. Because there is only one instance of each module, any changes -made to the module object get reflected everywhere. For example: - -config.py:: - - x = 0 # Default value of the 'x' configuration setting - -mod.py:: - - import config - config.x = 1 - -main.py:: - - import config - import mod - print(config.x) - -Note that using a module is also the basis for implementing the singleton design -pattern, for the same reason. - - -What are the "best practices" for using import in a module? ------------------------------------------------------------ - -In general, don't use ``from modulename import *``. Doing so clutters the -importer's namespace, and makes it much harder for linters to detect undefined -names. - -Import modules at the top of a file. Doing so makes it clear what other modules -your code requires and avoids questions of whether the module name is in scope. -Using one import per line makes it easy to add and delete module imports, but -using multiple imports per line uses less screen space. - -It's good practice if you import modules in the following order: - -1. standard library modules -- e.g. :mod:`sys`, :mod:`os`, :mod:`argparse`, :mod:`re` -2. third-party library modules (anything installed in Python's site-packages - directory) -- e.g. :mod:`!dateutil`, :mod:`!requests`, :mod:`!PIL.Image` -3. locally developed modules - -It is sometimes necessary to move imports to a function or class to avoid -problems with circular imports. Gordon McMillan says: - - Circular imports are fine where both modules use the "import <module>" form - of import. They fail when the 2nd module wants to grab a name out of the - first ("from module import name") and the import is at the top level. That's - because names in the 1st are not yet available, because the first module is - busy importing the 2nd. - -In this case, if the second module is only used in one function, then the import -can easily be moved into that function. By the time the import is called, the -first module will have finished initializing, and the second module can do its -import. - -It may also be necessary to move imports out of the top level of code if some of -the modules are platform-specific. In that case, it may not even be possible to -import all of the modules at the top of the file. In this case, importing the -correct modules in the corresponding platform-specific code is a good option. - -Only move imports into a local scope, such as inside a function definition, if -it's necessary to solve a problem such as avoiding a circular import or are -trying to reduce the initialization time of a module. This technique is -especially helpful if many of the imports are unnecessary depending on how the -program executes. You may also want to move imports into a function if the -modules are only ever used in that function. Note that loading a module the -first time may be expensive because of the one time initialization of the -module, but loading a module multiple times is virtually free, costing only a -couple of dictionary lookups. Even if the module name has gone out of scope, -the module is probably available in :data:`sys.modules`. - - -Why are default values shared between objects? ----------------------------------------------- - -This type of bug commonly bites neophyte programmers. Consider this function:: - - def foo(mydict={}): # Danger: shared reference to one dict for all calls - ... compute something ... - mydict[key] = value - return mydict - -The first time you call this function, ``mydict`` contains a single item. The -second time, ``mydict`` contains two items because when ``foo()`` begins -executing, ``mydict`` starts out with an item already in it. - -It is often expected that a function call creates new objects for default -values. This is not what happens. Default values are created exactly once, when -the function is defined. If that object is changed, like the dictionary in this -example, subsequent calls to the function will refer to this changed object. - -By definition, immutable objects such as numbers, strings, tuples, and ``None``, -are safe from change. Changes to mutable objects such as dictionaries, lists, -and class instances can lead to confusion. - -Because of this feature, it is good programming practice to not use mutable -objects as default values. Instead, use ``None`` as the default value and -inside the function, check if the parameter is ``None`` and create a new -list/dictionary/whatever if it is. For example, don't write:: - - def foo(mydict={}): - ... - -but:: - - def foo(mydict=None): - if mydict is None: - mydict = {} # create a new dict for local namespace - -This feature can be useful. When you have a function that's time-consuming to -compute, a common technique is to cache the parameters and the resulting value -of each call to the function, and return the cached value if the same value is -requested again. This is called "memoizing", and can be implemented like this:: - - # Callers can only provide two parameters and optionally pass _cache by keyword - def expensive(arg1, arg2, *, _cache={}): - if (arg1, arg2) in _cache: - return _cache[(arg1, arg2)] - - # Calculate the value - result = ... expensive computation ... - _cache[(arg1, arg2)] = result # Store result in the cache - return result - -You could use a global variable containing a dictionary instead of the default -value; it's a matter of taste. - - -How can I pass optional or keyword parameters from one function to another? ---------------------------------------------------------------------------- - -Collect the arguments using the ``*`` and ``**`` specifiers in the function's -parameter list; this gives you the positional arguments as a tuple and the -keyword arguments as a dictionary. You can then pass these arguments when -calling another function by using ``*`` and ``**``:: - - def f(x, *args, **kwargs): - ... - kwargs['width'] = '14.3c' - ... - g(x, *args, **kwargs) - - -.. index:: - single: argument; difference from parameter - single: parameter; difference from argument - -.. _faq-argument-vs-parameter: - -What is the difference between arguments and parameters? --------------------------------------------------------- - -:term:`Parameters <parameter>` are defined by the names that appear in a -function definition, whereas :term:`arguments <argument>` are the values -actually passed to a function when calling it. Parameters define what -:term:`kind of arguments <parameter>` a function can accept. For -example, given the function definition:: - - def func(foo, bar=None, **kwargs): - pass - -*foo*, *bar* and *kwargs* are parameters of ``func``. However, when calling -``func``, for example:: - - func(42, bar=314, extra=somevar) - -the values ``42``, ``314``, and ``somevar`` are arguments. - - -Why did changing list 'y' also change list 'x'? ------------------------------------------------- - -If you wrote code like:: - - >>> x = [] - >>> y = x - >>> y.append(10) - >>> y - [10] - >>> x - [10] - -you might be wondering why appending an element to ``y`` changed ``x`` too. - -There are two factors that produce this result: - -1) Variables are simply names that refer to objects. Doing ``y = x`` doesn't - create a copy of the list -- it creates a new variable ``y`` that refers to - the same object ``x`` refers to. This means that there is only one object - (the list), and both ``x`` and ``y`` refer to it. -2) Lists are :term:`mutable`, which means that you can change their content. - -After the call to :meth:`!append`, the content of the mutable object has -changed from ``[]`` to ``[10]``. Since both the variables refer to the same -object, using either name accesses the modified value ``[10]``. - -If we instead assign an immutable object to ``x``:: - - >>> x = 5 # ints are immutable - >>> y = x - >>> x = x + 1 # 5 can't be mutated, we are creating a new object here - >>> x - 6 - >>> y - 5 - -we can see that in this case ``x`` and ``y`` are not equal anymore. This is -because integers are :term:`immutable`, and when we do ``x = x + 1`` we are not -mutating the int ``5`` by incrementing its value; instead, we are creating a -new object (the int ``6``) and assigning it to ``x`` (that is, changing which -object ``x`` refers to). After this assignment we have two objects (the ints -``6`` and ``5``) and two variables that refer to them (``x`` now refers to -``6`` but ``y`` still refers to ``5``). - -Some operations (for example ``y.append(10)`` and ``y.sort()``) mutate the -object, whereas superficially similar operations (for example ``y = y + [10]`` -and :func:`sorted(y) <sorted>`) create a new object. In general in Python (and in all cases -in the standard library) a method that mutates an object will return ``None`` -to help avoid getting the two types of operations confused. So if you -mistakenly write ``y.sort()`` thinking it will give you a sorted copy of ``y``, -you'll instead end up with ``None``, which will likely cause your program to -generate an easily diagnosed error. - -However, there is one class of operations where the same operation sometimes -has different behaviors with different types: the augmented assignment -operators. For example, ``+=`` mutates lists but not tuples or ints (``a_list -+= [1, 2, 3]`` is equivalent to ``a_list.extend([1, 2, 3])`` and mutates -``a_list``, whereas ``some_tuple += (1, 2, 3)`` and ``some_int += 1`` create -new objects). - -In other words: - -* If we have a mutable object (:class:`list`, :class:`dict`, :class:`set`, - etc.), we can use some specific operations to mutate it and all the variables - that refer to it will see the change. -* If we have an immutable object (:class:`str`, :class:`int`, :class:`tuple`, - etc.), all the variables that refer to it will always see the same value, - but operations that transform that value into a new value always return a new - object. - -If you want to know if two variables refer to the same object or not, you can -use the :keyword:`is` operator, or the built-in function :func:`id`. - - -How do I write a function with output parameters (call by reference)? ---------------------------------------------------------------------- - -Remember that arguments are passed by assignment in Python. Since assignment -just creates references to objects, there's no alias between an argument name in -the caller and callee, and so no call-by-reference per se. You can achieve the -desired effect in a number of ways. - -1) By returning a tuple of the results:: - - >>> def func1(a, b): - ... a = 'new-value' # a and b are local names - ... b = b + 1 # assigned to new objects - ... return a, b # return new values - ... - >>> x, y = 'old-value', 99 - >>> func1(x, y) - ('new-value', 100) - - This is almost always the clearest solution. - -2) By using global variables. This isn't thread-safe, and is not recommended. - -3) By passing a mutable (changeable in-place) object:: - - >>> def func2(a): - ... a[0] = 'new-value' # 'a' references a mutable list - ... a[1] = a[1] + 1 # changes a shared object - ... - >>> args = ['old-value', 99] - >>> func2(args) - >>> args - ['new-value', 100] - -4) By passing in a dictionary that gets mutated:: - - >>> def func3(args): - ... args['a'] = 'new-value' # args is a mutable dictionary - ... args['b'] = args['b'] + 1 # change it in-place - ... - >>> args = {'a': 'old-value', 'b': 99} - >>> func3(args) - >>> args - {'a': 'new-value', 'b': 100} - -5) Or bundle up values in a class instance:: - - >>> class Namespace: - ... def __init__(self, /, **args): - ... for key, value in args.items(): - ... setattr(self, key, value) - ... - >>> def func4(args): - ... args.a = 'new-value' # args is a mutable Namespace - ... args.b = args.b + 1 # change object in-place - ... - >>> args = Namespace(a='old-value', b=99) - >>> func4(args) - >>> vars(args) - {'a': 'new-value', 'b': 100} - - - There's almost never a good reason to get this complicated. - -Your best choice is to return a tuple containing the multiple results. - - -How do you make a higher order function in Python? --------------------------------------------------- - -You have two choices: you can use nested scopes or you can use callable objects. -For example, suppose you wanted to define ``linear(a,b)`` which returns a -function ``f(x)`` that computes the value ``a*x+b``. Using nested scopes:: - - def linear(a, b): - def result(x): - return a * x + b - return result - -Or using a callable object:: - - class linear: - - def __init__(self, a, b): - self.a, self.b = a, b - - def __call__(self, x): - return self.a * x + self.b - -In both cases, :: - - taxes = linear(0.3, 2) - -gives a callable object where ``taxes(10e6) == 0.3 * 10e6 + 2``. - -The callable object approach has the disadvantage that it is a bit slower and -results in slightly longer code. However, note that a collection of callables -can share their signature via inheritance:: - - class exponential(linear): - # __init__ inherited - def __call__(self, x): - return self.a * (x ** self.b) - -Object can encapsulate state for several methods:: - - class counter: - - value = 0 - - def set(self, x): - self.value = x - - def up(self): - self.value = self.value + 1 - - def down(self): - self.value = self.value - 1 - - count = counter() - inc, dec, reset = count.up, count.down, count.set - -Here ``inc()``, ``dec()`` and ``reset()`` act like functions which share the -same counting variable. - - -How do I copy an object in Python? ----------------------------------- - -In general, try :func:`copy.copy` or :func:`copy.deepcopy` for the general case. -Not all objects can be copied, but most can. - -Some objects can be copied more easily. Dictionaries have a :meth:`~dict.copy` -method:: - - newdict = olddict.copy() - -Sequences can be copied by slicing:: - - new_l = l[:] - - -How can I find the methods or attributes of an object? ------------------------------------------------------- - -For an instance ``x`` of a user-defined class, :func:`dir(x) <dir>` returns an alphabetized -list of the names containing the instance attributes and methods and attributes -defined by its class. - - -How can my code discover the name of an object? ------------------------------------------------ - -Generally speaking, it can't, because objects don't really have names. -Essentially, assignment always binds a name to a value; the same is true of -``def`` and ``class`` statements, but in that case the value is a -callable. Consider the following code:: - - >>> class A: - ... pass - ... - >>> B = A - >>> a = B() - >>> b = a - >>> print(b) - <__main__.A object at 0x16D07CC> - >>> print(a) - <__main__.A object at 0x16D07CC> - -Arguably the class has a name: even though it is bound to two names and invoked -through the name ``B`` the created instance is still reported as an instance of -class ``A``. However, it is impossible to say whether the instance's name is ``a`` or -``b``, since both names are bound to the same value. - -Generally speaking it should not be necessary for your code to "know the names" -of particular values. Unless you are deliberately writing introspective -programs, this is usually an indication that a change of approach might be -beneficial. - -In comp.lang.python, Fredrik Lundh once gave an excellent analogy in answer to -this question: - - The same way as you get the name of that cat you found on your porch: the cat - (object) itself cannot tell you its name, and it doesn't really care -- so - the only way to find out what it's called is to ask all your neighbours - (namespaces) if it's their cat (object)... - - ....and don't be surprised if you'll find that it's known by many names, or - no name at all! - - -What's up with the comma operator's precedence? ------------------------------------------------ - -Comma is not an operator in Python. Consider this session:: - - >>> "a" in "b", "a" - (False, 'a') - -Since the comma is not an operator, but a separator between expressions the -above is evaluated as if you had entered:: - - ("a" in "b"), "a" - -not:: - - "a" in ("b", "a") - -The same is true of the various assignment operators (``=``, ``+=`` etc). They -are not truly operators but syntactic delimiters in assignment statements. - - -Is there an equivalent of C's "?:" ternary operator? ----------------------------------------------------- - -Yes, there is. The syntax is as follows:: - - [on_true] if [expression] else [on_false] - - x, y = 50, 25 - small = x if x < y else y - -Before this syntax was introduced in Python 2.5, a common idiom was to use -logical operators:: - - [expression] and [on_true] or [on_false] - -However, this idiom is unsafe, as it can give wrong results when *on_true* -has a false boolean value. Therefore, it is always better to use -the ``... if ... else ...`` form. - - -Is it possible to write obfuscated one-liners in Python? --------------------------------------------------------- - -Yes. Usually this is done by nesting :keyword:`lambda` within -:keyword:`!lambda`. See the following three examples, slightly adapted from Ulf Bartelt:: - - from functools import reduce - - # Primes < 1000 - print(list(filter(None,map(lambda y:y*reduce(lambda x,y:x*y!=0, - map(lambda x,y=y:y%x,range(2,int(pow(y,0.5)+1))),1),range(2,1000))))) - - # First 10 Fibonacci numbers - print(list(map(lambda x,f=lambda x,f:(f(x-1,f)+f(x-2,f)) if x>1 else 1: - f(x,f), range(10)))) - - # Mandelbrot set - print((lambda Ru,Ro,Iu,Io,IM,Sx,Sy:reduce(lambda x,y:x+'\n'+y,map(lambda y, - Iu=Iu,Io=Io,Ru=Ru,Ro=Ro,Sy=Sy,L=lambda yc,Iu=Iu,Io=Io,Ru=Ru,Ro=Ro,i=IM, - Sx=Sx,Sy=Sy:reduce(lambda x,y:x+y,map(lambda x,xc=Ru,yc=yc,Ru=Ru,Ro=Ro, - i=i,Sx=Sx,F=lambda xc,yc,x,y,k,f=lambda xc,yc,x,y,k,f:(k<=0)or (x*x+y*y - >=4.0) or 1+f(xc,yc,x*x-y*y+xc,2.0*x*y+yc,k-1,f):f(xc,yc,x,y,k,f):chr( - 64+F(Ru+x*(Ro-Ru)/Sx,yc,0,0,i)),range(Sx))):L(Iu+y*(Io-Iu)/Sy),range(Sy - ))))(-2.1, 0.7, -1.2, 1.2, 30, 80, 24)) - # \___ ___/ \___ ___/ | | |__ lines on screen - # V V | |______ columns on screen - # | | |__________ maximum of "iterations" - # | |_________________ range on y axis - # |____________________________ range on x axis - -Don't try this at home, kids! - - -.. _faq-positional-only-arguments: - -What does the slash(/) in the parameter list of a function mean? ----------------------------------------------------------------- - -A slash in the argument list of a function denotes that the parameters prior to -it are positional-only. Positional-only parameters are the ones without an -externally usable name. Upon calling a function that accepts positional-only -parameters, arguments are mapped to parameters based solely on their position. -For example, :func:`divmod` is a function that accepts positional-only -parameters. Its documentation looks like this:: - - >>> help(divmod) - Help on built-in function divmod in module builtins: - - divmod(x, y, /) - Return the tuple (x//y, x%y). Invariant: div*y + mod == x. - -The slash at the end of the parameter list means that both parameters are -positional-only. Thus, calling :func:`divmod` with keyword arguments would lead -to an error:: - - >>> divmod(x=3, y=4) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - TypeError: divmod() takes no keyword arguments - - -Numbers and strings -=================== - -How do I specify hexadecimal and octal integers? ------------------------------------------------- - -To specify an octal digit, precede the octal value with a zero, and then a lower -or uppercase "o". For example, to set the variable "a" to the octal value "10" -(8 in decimal), type:: - - >>> a = 0o10 - >>> a - 8 - -Hexadecimal is just as easy. Simply precede the hexadecimal number with a zero, -and then a lower or uppercase "x". Hexadecimal digits can be specified in lower -or uppercase. For example, in the Python interpreter:: - - >>> a = 0xa5 - >>> a - 165 - >>> b = 0XB2 - >>> b - 178 - - -Why does -22 // 10 return -3? ------------------------------ - -It's primarily driven by the desire that ``i % j`` have the same sign as ``j``. -If you want that, and also want:: - - i == (i // j) * j + (i % j) - -then integer division has to return the floor. C also requires that identity to -hold, and then compilers that truncate ``i // j`` need to make ``i % j`` have -the same sign as ``i``. - -There are few real use cases for ``i % j`` when ``j`` is negative. When ``j`` -is positive, there are many, and in virtually all of them it's more useful for -``i % j`` to be ``>= 0``. If the clock says 10 now, what did it say 200 hours -ago? ``-190 % 12 == 2`` is useful; ``-190 % 12 == -10`` is a bug waiting to -bite. - - -How do I get int literal attribute instead of SyntaxError? ----------------------------------------------------------- - -Trying to lookup an ``int`` literal attribute in the normal manner gives -a :exc:`SyntaxError` because the period is seen as a decimal point:: - - >>> 1.__class__ - File "<stdin>", line 1 - 1.__class__ - ^ - SyntaxError: invalid decimal literal - -The solution is to separate the literal from the period -with either a space or parentheses. - - >>> 1 .__class__ - <class 'int'> - >>> (1).__class__ - <class 'int'> - - -How do I convert a string to a number? --------------------------------------- - -For integers, use the built-in :func:`int` type constructor, e.g. ``int('144') -== 144``. Similarly, :func:`float` converts to floating-point, -e.g. ``float('144') == 144.0``. - -By default, these interpret the number as decimal, so that ``int('0144') == -144`` holds true, and ``int('0x144')`` raises :exc:`ValueError`. ``int(string, -base)`` takes the base to convert from as a second optional argument, so ``int( -'0x144', 16) == 324``. If the base is specified as 0, the number is interpreted -using Python's rules: a leading '0o' indicates octal, and '0x' indicates a hex -number. - -Do not use the built-in function :func:`eval` if all you need is to convert -strings to numbers. :func:`eval` will be significantly slower and it presents a -security risk: someone could pass you a Python expression that might have -unwanted side effects. For example, someone could pass -``__import__('os').system("rm -rf $HOME")`` which would erase your home -directory. - -:func:`eval` also has the effect of interpreting numbers as Python expressions, -so that e.g. ``eval('09')`` gives a syntax error because Python does not allow -leading '0' in a decimal number (except '0'). - - -How do I convert a number to a string? --------------------------------------- - -To convert, e.g., the number ``144`` to the string ``'144'``, use the built-in type -constructor :func:`str`. If you want a hexadecimal or octal representation, use -the built-in functions :func:`hex` or :func:`oct`. For fancy formatting, see -the :ref:`f-strings` and :ref:`formatstrings` sections, -e.g. ``"{:04d}".format(144)`` yields -``'0144'`` and ``"{:.3f}".format(1.0/3.0)`` yields ``'0.333'``. - - -How do I modify a string in place? ----------------------------------- - -You can't, because strings are immutable. In most situations, you should -simply construct a new string from the various parts you want to assemble -it from. However, if you need an object with the ability to modify in-place -unicode data, try using an :class:`io.StringIO` object or the :mod:`array` -module:: - - >>> import io - >>> s = "Hello, world" - >>> sio = io.StringIO(s) - >>> sio.getvalue() - 'Hello, world' - >>> sio.seek(7) - 7 - >>> sio.write("there!") - 6 - >>> sio.getvalue() - 'Hello, there!' - - >>> import array - >>> a = array.array('u', s) - >>> print(a) - array('u', 'Hello, world') - >>> a[0] = 'y' - >>> print(a) - array('u', 'yello, world') - >>> a.tounicode() - 'yello, world' - - -How do I use strings to call functions/methods? ------------------------------------------------ - -There are various techniques. - -* The best is to use a dictionary that maps strings to functions. The primary - advantage of this technique is that the strings do not need to match the names - of the functions. This is also the primary technique used to emulate a case - construct:: - - def a(): - pass - - def b(): - pass - - dispatch = {'go': a, 'stop': b} # Note lack of parens for funcs - - dispatch[get_input()]() # Note trailing parens to call function - -* Use the built-in function :func:`getattr`:: - - import foo - getattr(foo, 'bar')() - - Note that :func:`getattr` works on any object, including classes, class - instances, modules, and so on. - - This is used in several places in the standard library, like this:: - - class Foo: - def do_foo(self): - ... - - def do_bar(self): - ... - - f = getattr(foo_instance, 'do_' + opname) - f() - - -* Use :func:`locals` to resolve the function name:: - - def myFunc(): - print("hello") - - fname = "myFunc" - - f = locals()[fname] - f() - - -Is there an equivalent to Perl's chomp() for removing trailing newlines from strings? -------------------------------------------------------------------------------------- - -You can use ``S.rstrip("\r\n")`` to remove all occurrences of any line -terminator from the end of the string ``S`` without removing other trailing -whitespace. If the string ``S`` represents more than one line, with several -empty lines at the end, the line terminators for all the blank lines will -be removed:: - - >>> lines = ("line 1 \r\n" - ... "\r\n" - ... "\r\n") - >>> lines.rstrip("\n\r") - 'line 1 ' - -Since this is typically only desired when reading text one line at a time, using -``S.rstrip()`` this way works well. - - -Is there a scanf() or sscanf() equivalent? ------------------------------------------- - -Not as such. - -For simple input parsing, the easiest approach is usually to split the line into -whitespace-delimited words using the :meth:`~str.split` method of string objects -and then convert decimal strings to numeric values using :func:`int` or -:func:`float`. :meth:`!split()` supports an optional "sep" parameter which is useful -if the line uses something other than whitespace as a separator. - -For more complicated input parsing, regular expressions are more powerful -than C's ``sscanf`` and better suited for the task. - - -What does 'UnicodeDecodeError' or 'UnicodeEncodeError' error mean? -------------------------------------------------------------------- - -See the :ref:`unicode-howto`. - - -.. _faq-programming-raw-string-backslash: - -Can I end a raw string with an odd number of backslashes? ---------------------------------------------------------- - -A raw string ending with an odd number of backslashes will escape the string's quote:: - - >>> r'C:\this\will\not\work\' - File "<stdin>", line 1 - r'C:\this\will\not\work\' - ^ - SyntaxError: unterminated string literal (detected at line 1) - -There are several workarounds for this. One is to use regular strings and double -the backslashes:: - - >>> 'C:\\this\\will\\work\\' - 'C:\\this\\will\\work\\' - -Another is to concatenate a regular string containing an escaped backslash to the -raw string:: - - >>> r'C:\this\will\work' '\\' - 'C:\\this\\will\\work\\' - -It is also possible to use :func:`os.path.join` to append a backslash on Windows:: - - >>> os.path.join(r'C:\this\will\work', '') - 'C:\\this\\will\\work\\' - -Note that while a backslash will "escape" a quote for the purposes of -determining where the raw string ends, no escaping occurs when interpreting the -value of the raw string. That is, the backslash remains present in the value of -the raw string:: - - >>> r'backslash\'preserved' - "backslash\\'preserved" - -Also see the specification in the :ref:`language reference <strings>`. - -Performance -=========== - -My program is too slow. How do I speed it up? ---------------------------------------------- - -That's a tough one, in general. First, here are a list of things to -remember before diving further: - -* Performance characteristics vary across Python implementations. This FAQ - focuses on :term:`CPython`. -* Behaviour can vary across operating systems, especially when talking about - I/O or multi-threading. -* You should always find the hot spots in your program *before* attempting to - optimize any code (see the :mod:`profile` module). -* Writing benchmark scripts will allow you to iterate quickly when searching - for improvements (see the :mod:`timeit` module). -* It is highly recommended to have good code coverage (through unit testing - or any other technique) before potentially introducing regressions hidden - in sophisticated optimizations. - -That being said, there are many tricks to speed up Python code. Here are -some general principles which go a long way towards reaching acceptable -performance levels: - -* Making your algorithms faster (or changing to faster ones) can yield - much larger benefits than trying to sprinkle micro-optimization tricks - all over your code. - -* Use the right data structures. Study documentation for the :ref:`bltin-types` - and the :mod:`collections` module. - -* When the standard library provides a primitive for doing something, it is - likely (although not guaranteed) to be faster than any alternative you - may come up with. This is doubly true for primitives written in C, such - as builtins and some extension types. For example, be sure to use - either the :meth:`list.sort` built-in method or the related :func:`sorted` - function to do sorting (and see the :ref:`sortinghowto` for examples - of moderately advanced usage). - -* Abstractions tend to create indirections and force the interpreter to work - more. If the levels of indirection outweigh the amount of useful work - done, your program will be slower. You should avoid excessive abstraction, - especially under the form of tiny functions or methods (which are also often - detrimental to readability). - -If you have reached the limit of what pure Python can allow, there are tools -to take you further away. For example, `Cython <https://cython.org>`_ can -compile a slightly modified version of Python code into a C extension, and -can be used on many different platforms. Cython can take advantage of -compilation (and optional type annotations) to make your code significantly -faster than when interpreted. If you are confident in your C programming -skills, you can also :ref:`write a C extension module <extending-index>` -yourself. - -.. seealso:: - The wiki page devoted to `performance tips - <https://wiki.python.org/moin/PythonSpeed/PerformanceTips>`_. - -.. _efficient_string_concatenation: - -What is the most efficient way to concatenate many strings together? --------------------------------------------------------------------- - -:class:`str` and :class:`bytes` objects are immutable, therefore concatenating -many strings together is inefficient as each concatenation creates a new -object. In the general case, the total runtime cost is quadratic in the -total string length. - -To accumulate many :class:`str` objects, the recommended idiom is to place -them into a list and call :meth:`str.join` at the end:: - - chunks = [] - for s in my_strings: - chunks.append(s) - result = ''.join(chunks) - -(another reasonably efficient idiom is to use :class:`io.StringIO`) - -To accumulate many :class:`bytes` objects, the recommended idiom is to extend -a :class:`bytearray` object using in-place concatenation (the ``+=`` operator):: - - result = bytearray() - for b in my_bytes_objects: - result += b - - -Sequences (Tuples/Lists) -======================== - -How do I convert between tuples and lists? ------------------------------------------- - -The type constructor ``tuple(seq)`` converts any sequence (actually, any -iterable) into a tuple with the same items in the same order. - -For example, ``tuple([1, 2, 3])`` yields ``(1, 2, 3)`` and ``tuple('abc')`` -yields ``('a', 'b', 'c')``. If the argument is a tuple, it does not make a copy -but returns the same object, so it is cheap to call :func:`tuple` when you -aren't sure that an object is already a tuple. - -The type constructor ``list(seq)`` converts any sequence or iterable into a list -with the same items in the same order. For example, ``list((1, 2, 3))`` yields -``[1, 2, 3]`` and ``list('abc')`` yields ``['a', 'b', 'c']``. If the argument -is a list, it makes a copy just like ``seq[:]`` would. - - -What's a negative index? ------------------------- - -Python sequences are indexed with positive numbers and negative numbers. For -positive numbers 0 is the first index 1 is the second index and so forth. For -negative indices -1 is the last index and -2 is the penultimate (next to last) -index and so forth. Think of ``seq[-n]`` as the same as ``seq[len(seq)-n]``. - -Using negative indices can be very convenient. For example ``S[:-1]`` is all of -the string except for its last character, which is useful for removing the -trailing newline from a string. - - -How do I iterate over a sequence in reverse order? --------------------------------------------------- - -Use the :func:`reversed` built-in function:: - - for x in reversed(sequence): - ... # do something with x ... - -This won't touch your original sequence, but build a new copy with reversed -order to iterate over. - - -How do you remove duplicates from a list? ------------------------------------------ - -See the Python Cookbook for a long discussion of many ways to do this: - - https://code.activestate.com/recipes/52560/ - -If you don't mind reordering the list, sort it and then scan from the end of the -list, deleting duplicates as you go:: - - if mylist: - mylist.sort() - last = mylist[-1] - for i in range(len(mylist)-2, -1, -1): - if last == mylist[i]: - del mylist[i] - else: - last = mylist[i] - -If all elements of the list may be used as set keys (i.e. they are all -:term:`hashable`) this is often faster :: - - mylist = list(set(mylist)) - -This converts the list into a set, thereby removing duplicates, and then back -into a list. - - -How do you remove multiple items from a list --------------------------------------------- - -As with removing duplicates, explicitly iterating in reverse with a -delete condition is one possibility. However, it is easier and faster -to use slice replacement with an implicit or explicit forward iteration. -Here are three variations.:: - - mylist[:] = filter(keep_function, mylist) - mylist[:] = (x for x in mylist if keep_condition) - mylist[:] = [x for x in mylist if keep_condition] - -The list comprehension may be fastest. - - -How do you make an array in Python? ------------------------------------ - -Use a list:: - - ["this", 1, "is", "an", "array"] - -Lists are equivalent to C or Pascal arrays in their time complexity; the primary -difference is that a Python list can contain objects of many different types. - -The ``array`` module also provides methods for creating arrays of fixed types -with compact representations, but they are slower to index than lists. Also -note that `NumPy <https://numpy.org/>`_ -and other third party packages define array-like structures with -various characteristics as well. - -To get Lisp-style linked lists, you can emulate *cons cells* using tuples:: - - lisp_list = ("like", ("this", ("example", None) ) ) - -If mutability is desired, you could use lists instead of tuples. Here the -analogue of a Lisp *car* is ``lisp_list[0]`` and the analogue of *cdr* is -``lisp_list[1]``. Only do this if you're sure you really need to, because it's -usually a lot slower than using Python lists. - - -.. _faq-multidimensional-list: - -How do I create a multidimensional list? ----------------------------------------- - -You probably tried to make a multidimensional array like this:: - - >>> A = [[None] * 2] * 3 - -This looks correct if you print it: - -.. testsetup:: - - A = [[None] * 2] * 3 - -.. doctest:: - - >>> A - [[None, None], [None, None], [None, None]] - -But when you assign a value, it shows up in multiple places: - -.. testsetup:: - - A = [[None] * 2] * 3 - -.. doctest:: - - >>> A[0][0] = 5 - >>> A - [[5, None], [5, None], [5, None]] - -The reason is that replicating a list with ``*`` doesn't create copies, it only -creates references to the existing objects. The ``*3`` creates a list -containing 3 references to the same list of length two. Changes to one row will -show in all rows, which is almost certainly not what you want. - -The suggested approach is to create a list of the desired length first and then -fill in each element with a newly created list:: - - A = [None] * 3 - for i in range(3): - A[i] = [None] * 2 - -This generates a list containing 3 different lists of length two. You can also -use a list comprehension:: - - w, h = 2, 3 - A = [[None] * w for i in range(h)] - -Or, you can use an extension that provides a matrix datatype; `NumPy -<https://numpy.org/>`_ is the best known. - - -How do I apply a method or function to a sequence of objects? -------------------------------------------------------------- - -To call a method or function and accumulate the return values is a list, -a :term:`list comprehension` is an elegant solution:: - - result = [obj.method() for obj in mylist] - - result = [function(obj) for obj in mylist] - -To just run the method or function without saving the return values, -a plain :keyword:`for` loop will suffice:: - - for obj in mylist: - obj.method() - - for obj in mylist: - function(obj) - -.. _faq-augmented-assignment-tuple-error: - -Why does a_tuple[i] += ['item'] raise an exception when the addition works? ---------------------------------------------------------------------------- - -This is because of a combination of the fact that augmented assignment -operators are *assignment* operators, and the difference between mutable and -immutable objects in Python. - -This discussion applies in general when augmented assignment operators are -applied to elements of a tuple that point to mutable objects, but we'll use -a ``list`` and ``+=`` as our exemplar. - -If you wrote:: - - >>> a_tuple = (1, 2) - >>> a_tuple[0] += 1 - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -The reason for the exception should be immediately clear: ``1`` is added to the -object ``a_tuple[0]`` points to (``1``), producing the result object, ``2``, -but when we attempt to assign the result of the computation, ``2``, to element -``0`` of the tuple, we get an error because we can't change what an element of -a tuple points to. - -Under the covers, what this augmented assignment statement is doing is -approximately this:: - - >>> result = a_tuple[0] + 1 - >>> a_tuple[0] = result - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -It is the assignment part of the operation that produces the error, since a -tuple is immutable. - -When you write something like:: - - >>> a_tuple = (['foo'], 'bar') - >>> a_tuple[0] += ['item'] - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -The exception is a bit more surprising, and even more surprising is the fact -that even though there was an error, the append worked:: - - >>> a_tuple[0] - ['foo', 'item'] - -To see why this happens, you need to know that (a) if an object implements an -:meth:`~object.__iadd__` magic method, it gets called when the ``+=`` augmented -assignment -is executed, and its return value is what gets used in the assignment statement; -and (b) for lists, :meth:`!__iadd__` is equivalent to calling :meth:`!extend` on the list -and returning the list. That's why we say that for lists, ``+=`` is a -"shorthand" for :meth:`!list.extend`:: - - >>> a_list = [] - >>> a_list += [1] - >>> a_list - [1] - -This is equivalent to:: - - >>> result = a_list.__iadd__([1]) - >>> a_list = result - -The object pointed to by a_list has been mutated, and the pointer to the -mutated object is assigned back to ``a_list``. The end result of the -assignment is a no-op, since it is a pointer to the same object that ``a_list`` -was previously pointing to, but the assignment still happens. - -Thus, in our tuple example what is happening is equivalent to:: - - >>> result = a_tuple[0].__iadd__(['item']) - >>> a_tuple[0] = result - Traceback (most recent call last): - ... - TypeError: 'tuple' object does not support item assignment - -The :meth:`!__iadd__` succeeds, and thus the list is extended, but even though -``result`` points to the same object that ``a_tuple[0]`` already points to, -that final assignment still results in an error, because tuples are immutable. - - -I want to do a complicated sort: can you do a Schwartzian Transform in Python? ------------------------------------------------------------------------------- - -The technique, attributed to Randal Schwartz of the Perl community, sorts the -elements of a list by a metric which maps each element to its "sort value". In -Python, use the ``key`` argument for the :meth:`list.sort` method:: - - Isorted = L[:] - Isorted.sort(key=lambda s: int(s[10:15])) - - -How can I sort one list by values from another list? ----------------------------------------------------- - -Merge them into an iterator of tuples, sort the resulting list, and then pick -out the element you want. :: - - >>> list1 = ["what", "I'm", "sorting", "by"] - >>> list2 = ["something", "else", "to", "sort"] - >>> pairs = zip(list1, list2) - >>> pairs = sorted(pairs) - >>> pairs - [("I'm", 'else'), ('by', 'sort'), ('sorting', 'to'), ('what', 'something')] - >>> result = [x[1] for x in pairs] - >>> result - ['else', 'sort', 'to', 'something'] - - -Objects -======= - -What is a class? ----------------- - -A class is the particular object type created by executing a class statement. -Class objects are used as templates to create instance objects, which embody -both the data (attributes) and code (methods) specific to a datatype. - -A class can be based on one or more other classes, called its base class(es). It -then inherits the attributes and methods of its base classes. This allows an -object model to be successively refined by inheritance. You might have a -generic ``Mailbox`` class that provides basic accessor methods for a mailbox, -and subclasses such as ``MboxMailbox``, ``MaildirMailbox``, ``OutlookMailbox`` -that handle various specific mailbox formats. - - -What is a method? ------------------ - -A method is a function on some object ``x`` that you normally call as -``x.name(arguments...)``. Methods are defined as functions inside the class -definition:: - - class C: - def meth(self, arg): - return arg * 2 + self.attribute - - -What is self? -------------- - -Self is merely a conventional name for the first argument of a method. A method -defined as ``meth(self, a, b, c)`` should be called as ``x.meth(a, b, c)`` for -some instance ``x`` of the class in which the definition occurs; the called -method will think it is called as ``meth(x, a, b, c)``. - -See also :ref:`why-self`. - - -How do I check if an object is an instance of a given class or of a subclass of it? ------------------------------------------------------------------------------------ - -Use the built-in function :func:`isinstance(obj, cls) <isinstance>`. You can -check if an object -is an instance of any of a number of classes by providing a tuple instead of a -single class, e.g. ``isinstance(obj, (class1, class2, ...))``, and can also -check whether an object is one of Python's built-in types, e.g. -``isinstance(obj, str)`` or ``isinstance(obj, (int, float, complex))``. - -Note that :func:`isinstance` also checks for virtual inheritance from an -:term:`abstract base class`. So, the test will return ``True`` for a -registered class even if hasn't directly or indirectly inherited from it. To -test for "true inheritance", scan the :term:`MRO` of the class: - -.. testcode:: - - from collections.abc import Mapping - - class P: - pass - - class C(P): - pass - - Mapping.register(P) - -.. doctest:: - - >>> c = C() - >>> isinstance(c, C) # direct - True - >>> isinstance(c, P) # indirect - True - >>> isinstance(c, Mapping) # virtual - True - - # Actual inheritance chain - >>> type(c).__mro__ - (<class 'C'>, <class 'P'>, <class 'object'>) - - # Test for "true inheritance" - >>> Mapping in type(c).__mro__ - False - -Note that most programs do not use :func:`isinstance` on user-defined classes -very often. If you are developing the classes yourself, a more proper -object-oriented style is to define methods on the classes that encapsulate a -particular behaviour, instead of checking the object's class and doing a -different thing based on what class it is. For example, if you have a function -that does something:: - - def search(obj): - if isinstance(obj, Mailbox): - ... # code to search a mailbox - elif isinstance(obj, Document): - ... # code to search a document - elif ... - -A better approach is to define a ``search()`` method on all the classes and just -call it:: - - class Mailbox: - def search(self): - ... # code to search a mailbox - - class Document: - def search(self): - ... # code to search a document - - obj.search() - - -What is delegation? -------------------- - -Delegation is an object oriented technique (also called a design pattern). -Let's say you have an object ``x`` and want to change the behaviour of just one -of its methods. You can create a new class that provides a new implementation -of the method you're interested in changing and delegates all other methods to -the corresponding method of ``x``. - -Python programmers can easily implement delegation. For example, the following -class implements a class that behaves like a file but converts all written data -to uppercase:: - - class UpperOut: - - def __init__(self, outfile): - self._outfile = outfile - - def write(self, s): - self._outfile.write(s.upper()) - - def __getattr__(self, name): - return getattr(self._outfile, name) - -Here the ``UpperOut`` class redefines the ``write()`` method to convert the -argument string to uppercase before calling the underlying -``self._outfile.write()`` method. All other methods are delegated to the -underlying ``self._outfile`` object. The delegation is accomplished via the -:meth:`~object.__getattr__` method; consult :ref:`the language reference <attribute-access>` -for more information about controlling attribute access. - -Note that for more general cases delegation can get trickier. When attributes -must be set as well as retrieved, the class must define a :meth:`~object.__setattr__` -method too, and it must do so carefully. The basic implementation of -:meth:`!__setattr__` is roughly equivalent to the following:: - - class X: - ... - def __setattr__(self, name, value): - self.__dict__[name] = value - ... - -Most :meth:`!__setattr__` implementations must modify -:meth:`self.__dict__ <object.__dict__>` to store -local state for self without causing an infinite recursion. - - -How do I call a method defined in a base class from a derived class that extends it? ------------------------------------------------------------------------------------- - -Use the built-in :func:`super` function:: - - class Derived(Base): - def meth(self): - super().meth() # calls Base.meth - -In the example, :func:`super` will automatically determine the instance from -which it was called (the ``self`` value), look up the :term:`method resolution -order` (MRO) with ``type(self).__mro__``, and return the next in line after -``Derived`` in the MRO: ``Base``. - - -How can I organize my code to make it easier to change the base class? ----------------------------------------------------------------------- - -You could assign the base class to an alias and derive from the alias. Then all -you have to change is the value assigned to the alias. Incidentally, this trick -is also handy if you want to decide dynamically (e.g. depending on availability -of resources) which base class to use. Example:: - - class Base: - ... - - BaseAlias = Base - - class Derived(BaseAlias): - ... - - -How do I create static class data and static class methods? ------------------------------------------------------------ - -Both static data and static methods (in the sense of C++ or Java) are supported -in Python. - -For static data, simply define a class attribute. To assign a new value to the -attribute, you have to explicitly use the class name in the assignment:: - - class C: - count = 0 # number of times C.__init__ called - - def __init__(self): - C.count = C.count + 1 - - def getcount(self): - return C.count # or return self.count - -``c.count`` also refers to ``C.count`` for any ``c`` such that ``isinstance(c, -C)`` holds, unless overridden by ``c`` itself or by some class on the base-class -search path from ``c.__class__`` back to ``C``. - -Caution: within a method of C, an assignment like ``self.count = 42`` creates a -new and unrelated instance named "count" in ``self``'s own dict. Rebinding of a -class-static data name must always specify the class whether inside a method or -not:: - - C.count = 314 - -Static methods are possible:: - - class C: - @staticmethod - def static(arg1, arg2, arg3): - # No 'self' parameter! - ... - -However, a far more straightforward way to get the effect of a static method is -via a simple module-level function:: - - def getcount(): - return C.count - -If your code is structured so as to define one class (or tightly related class -hierarchy) per module, this supplies the desired encapsulation. - - -How can I overload constructors (or methods) in Python? -------------------------------------------------------- - -This answer actually applies to all methods, but the question usually comes up -first in the context of constructors. - -In C++ you'd write - -.. code-block:: c - - class C { - C() { cout << "No arguments\n"; } - C(int i) { cout << "Argument is " << i << "\n"; } - } - -In Python you have to write a single constructor that catches all cases using -default arguments. For example:: - - class C: - def __init__(self, i=None): - if i is None: - print("No arguments") - else: - print("Argument is", i) - -This is not entirely equivalent, but close enough in practice. - -You could also try a variable-length argument list, e.g. :: - - def __init__(self, *args): - ... - -The same approach works for all method definitions. - - -I try to use __spam and I get an error about _SomeClassName__spam. ------------------------------------------------------------------- - -Variable names with double leading underscores are "mangled" to provide a simple -but effective way to define class private variables. Any identifier of the form -``__spam`` (at least two leading underscores, at most one trailing underscore) -is textually replaced with ``_classname__spam``, where ``classname`` is the -current class name with any leading underscores stripped. - -This doesn't guarantee privacy: an outside user can still deliberately access -the "_classname__spam" attribute, and private values are visible in the object's -``__dict__``. Many Python programmers never bother to use private variable -names at all. - - -My class defines __del__ but it is not called when I delete the object. ------------------------------------------------------------------------ - -There are several possible reasons for this. - -The :keyword:`del` statement does not necessarily call :meth:`~object.__del__` -- it simply -decrements the object's reference count, and if this reaches zero -:meth:`!__del__` is called. - -If your data structures contain circular links (e.g. a tree where each child has -a parent reference and each parent has a list of children) the reference counts -will never go back to zero. Once in a while Python runs an algorithm to detect -such cycles, but the garbage collector might run some time after the last -reference to your data structure vanishes, so your :meth:`!__del__` method may be -called at an inconvenient and random time. This is inconvenient if you're trying -to reproduce a problem. Worse, the order in which object's :meth:`!__del__` -methods are executed is arbitrary. You can run :func:`gc.collect` to force a -collection, but there *are* pathological cases where objects will never be -collected. - -Despite the cycle collector, it's still a good idea to define an explicit -``close()`` method on objects to be called whenever you're done with them. The -``close()`` method can then remove attributes that refer to subobjects. Don't -call :meth:`!__del__` directly -- :meth:`!__del__` should call ``close()`` and -``close()`` should make sure that it can be called more than once for the same -object. - -Another way to avoid cyclical references is to use the :mod:`weakref` module, -which allows you to point to objects without incrementing their reference count. -Tree data structures, for instance, should use weak references for their parent -and sibling references (if they need them!). - -.. XXX relevant for Python 3? - - If the object has ever been a local variable in a function that caught an - expression in an except clause, chances are that a reference to the object - still exists in that function's stack frame as contained in the stack trace. - Normally, calling :func:`sys.exc_clear` will take care of this by clearing - the last recorded exception. - -Finally, if your :meth:`!__del__` method raises an exception, a warning message -is printed to :data:`sys.stderr`. - - -How do I get a list of all instances of a given class? ------------------------------------------------------- - -Python does not keep track of all instances of a class (or of a built-in type). -You can program the class's constructor to keep track of all instances by -keeping a list of weak references to each instance. - - -Why does the result of ``id()`` appear to be not unique? --------------------------------------------------------- - -The :func:`id` builtin returns an integer that is guaranteed to be unique during -the lifetime of the object. Since in CPython, this is the object's memory -address, it happens frequently that after an object is deleted from memory, the -next freshly created object is allocated at the same position in memory. This -is illustrated by this example: - ->>> id(1000) # doctest: +SKIP -13901272 ->>> id(2000) # doctest: +SKIP -13901272 - -The two ids belong to different integer objects that are created before, and -deleted immediately after execution of the ``id()`` call. To be sure that -objects whose id you want to examine are still alive, create another reference -to the object: - ->>> a = 1000; b = 2000 ->>> id(a) # doctest: +SKIP -13901272 ->>> id(b) # doctest: +SKIP -13891296 - - -When can I rely on identity tests with the *is* operator? ---------------------------------------------------------- - -The ``is`` operator tests for object identity. The test ``a is b`` is -equivalent to ``id(a) == id(b)``. - -The most important property of an identity test is that an object is always -identical to itself, ``a is a`` always returns ``True``. Identity tests are -usually faster than equality tests. And unlike equality tests, identity tests -are guaranteed to return a boolean ``True`` or ``False``. - -However, identity tests can *only* be substituted for equality tests when -object identity is assured. Generally, there are three circumstances where -identity is guaranteed: - -1) Assignments create new names but do not change object identity. After the -assignment ``new = old``, it is guaranteed that ``new is old``. - -2) Putting an object in a container that stores object references does not -change object identity. After the list assignment ``s[0] = x``, it is -guaranteed that ``s[0] is x``. - -3) If an object is a singleton, it means that only one instance of that object -can exist. After the assignments ``a = None`` and ``b = None``, it is -guaranteed that ``a is b`` because ``None`` is a singleton. - -In most other circumstances, identity tests are inadvisable and equality tests -are preferred. In particular, identity tests should not be used to check -constants such as :class:`int` and :class:`str` which aren't guaranteed to be -singletons:: - - >>> a = 1000 - >>> b = 500 - >>> c = b + 500 - >>> a is c - False - - >>> a = 'Python' - >>> b = 'Py' - >>> c = b + 'thon' - >>> a is c - False - -Likewise, new instances of mutable containers are never identical:: - - >>> a = [] - >>> b = [] - >>> a is b - False - -In the standard library code, you will see several common patterns for -correctly using identity tests: - -1) As recommended by :pep:`8`, an identity test is the preferred way to check -for ``None``. This reads like plain English in code and avoids confusion with -other objects that may have boolean values that evaluate to false. - -2) Detecting optional arguments can be tricky when ``None`` is a valid input -value. In those situations, you can create a singleton sentinel object -guaranteed to be distinct from other objects. For example, here is how -to implement a method that behaves like :meth:`dict.pop`:: - - _sentinel = object() - - def pop(self, key, default=_sentinel): - if key in self: - value = self[key] - del self[key] - return value - if default is _sentinel: - raise KeyError(key) - return default - -3) Container implementations sometimes need to augment equality tests with -identity tests. This prevents the code from being confused by objects such as -``float('NaN')`` that are not equal to themselves. - -For example, here is the implementation of -:meth:`!collections.abc.Sequence.__contains__`:: - - def __contains__(self, value): - for v in self: - if v is value or v == value: - return True - return False - - -How can a subclass control what data is stored in an immutable instance? ------------------------------------------------------------------------- - -When subclassing an immutable type, override the :meth:`~object.__new__` method -instead of the :meth:`~object.__init__` method. The latter only runs *after* an -instance is created, which is too late to alter data in an immutable -instance. - -All of these immutable classes have a different signature than their -parent class: - -.. testcode:: - - from datetime import date - - class FirstOfMonthDate(date): - "Always choose the first day of the month" - def __new__(cls, year, month, day): - return super().__new__(cls, year, month, 1) - - class NamedInt(int): - "Allow text names for some numbers" - xlat = {'zero': 0, 'one': 1, 'ten': 10} - def __new__(cls, value): - value = cls.xlat.get(value, value) - return super().__new__(cls, value) - - class TitleStr(str): - "Convert str to name suitable for a URL path" - def __new__(cls, s): - s = s.lower().replace(' ', '-') - s = ''.join([c for c in s if c.isalnum() or c == '-']) - return super().__new__(cls, s) - -The classes can be used like this: - -.. doctest:: - - >>> FirstOfMonthDate(2012, 2, 14) - FirstOfMonthDate(2012, 2, 1) - >>> NamedInt('ten') - 10 - >>> NamedInt(20) - 20 - >>> TitleStr('Blog: Why Python Rocks') - 'blog-why-python-rocks' - - -.. _faq-cache-method-calls: - -How do I cache method calls? ----------------------------- - -The two principal tools for caching methods are -:func:`functools.cached_property` and :func:`functools.lru_cache`. The -former stores results at the instance level and the latter at the class -level. - -The *cached_property* approach only works with methods that do not take -any arguments. It does not create a reference to the instance. The -cached method result will be kept only as long as the instance is alive. - -The advantage is that when an instance is no longer used, the cached -method result will be released right away. The disadvantage is that if -instances accumulate, so too will the accumulated method results. They -can grow without bound. - -The *lru_cache* approach works with methods that have :term:`hashable` -arguments. It creates a reference to the instance unless special -efforts are made to pass in weak references. - -The advantage of the least recently used algorithm is that the cache is -bounded by the specified *maxsize*. The disadvantage is that instances -are kept alive until they age out of the cache or until the cache is -cleared. - -This example shows the various techniques:: - - class Weather: - "Lookup weather information on a government website" - - def __init__(self, station_id): - self._station_id = station_id - # The _station_id is private and immutable - - def current_temperature(self): - "Latest hourly observation" - # Do not cache this because old results - # can be out of date. - - @cached_property - def location(self): - "Return the longitude/latitude coordinates of the station" - # Result only depends on the station_id - - @lru_cache(maxsize=20) - def historic_rainfall(self, date, units='mm'): - "Rainfall on a given date" - # Depends on the station_id, date, and units. - -The above example assumes that the *station_id* never changes. If the -relevant instance attributes are mutable, the *cached_property* approach -can't be made to work because it cannot detect changes to the -attributes. - -To make the *lru_cache* approach work when the *station_id* is mutable, -the class needs to define the :meth:`~object.__eq__` and :meth:`~object.__hash__` -methods so that the cache can detect relevant attribute updates:: - - class Weather: - "Example with a mutable station identifier" - - def __init__(self, station_id): - self.station_id = station_id - - def change_station(self, station_id): - self.station_id = station_id - - def __eq__(self, other): - return self.station_id == other.station_id - - def __hash__(self): - return hash(self.station_id) - - @lru_cache(maxsize=20) - def historic_rainfall(self, date, units='cm'): - 'Rainfall on a given date' - # Depends on the station_id, date, and units. - - -Modules -======= - -How do I create a .pyc file? ----------------------------- - -When a module is imported for the first time (or when the source file has -changed since the current compiled file was created) a ``.pyc`` file containing -the compiled code should be created in a ``__pycache__`` subdirectory of the -directory containing the ``.py`` file. The ``.pyc`` file will have a -filename that starts with the same name as the ``.py`` file, and ends with -``.pyc``, with a middle component that depends on the particular ``python`` -binary that created it. (See :pep:`3147` for details.) - -One reason that a ``.pyc`` file may not be created is a permissions problem -with the directory containing the source file, meaning that the ``__pycache__`` -subdirectory cannot be created. This can happen, for example, if you develop as -one user but run as another, such as if you are testing with a web server. - -Unless the :envvar:`PYTHONDONTWRITEBYTECODE` environment variable is set, -creation of a .pyc file is automatic if you're importing a module and Python -has the ability (permissions, free space, etc...) to create a ``__pycache__`` -subdirectory and write the compiled module to that subdirectory. - -Running Python on a top level script is not considered an import and no -``.pyc`` will be created. For example, if you have a top-level module -``foo.py`` that imports another module ``xyz.py``, when you run ``foo`` (by -typing ``python foo.py`` as a shell command), a ``.pyc`` will be created for -``xyz`` because ``xyz`` is imported, but no ``.pyc`` file will be created for -``foo`` since ``foo.py`` isn't being imported. - -If you need to create a ``.pyc`` file for ``foo`` -- that is, to create a -``.pyc`` file for a module that is not imported -- you can, using the -:mod:`py_compile` and :mod:`compileall` modules. - -The :mod:`py_compile` module can manually compile any module. One way is to use -the ``compile()`` function in that module interactively:: - - >>> import py_compile - >>> py_compile.compile('foo.py') # doctest: +SKIP - -This will write the ``.pyc`` to a ``__pycache__`` subdirectory in the same -location as ``foo.py`` (or you can override that with the optional parameter -``cfile``). - -You can also automatically compile all files in a directory or directories using -the :mod:`compileall` module. You can do it from the shell prompt by running -``compileall.py`` and providing the path of a directory containing Python files -to compile:: - - python -m compileall . - - -How do I find the current module name? --------------------------------------- - -A module can find out its own module name by looking at the predefined global -variable ``__name__``. If this has the value ``'__main__'``, the program is -running as a script. Many modules that are usually used by importing them also -provide a command-line interface or a self-test, and only execute this code -after checking ``__name__``:: - - def main(): - print('Running test...') - ... - - if __name__ == '__main__': - main() - - -How can I have modules that mutually import each other? -------------------------------------------------------- - -Suppose you have the following modules: - -:file:`foo.py`:: - - from bar import bar_var - foo_var = 1 - -:file:`bar.py`:: - - from foo import foo_var - bar_var = 2 - -The problem is that the interpreter will perform the following steps: - -* main imports ``foo`` -* Empty globals for ``foo`` are created -* ``foo`` is compiled and starts executing -* ``foo`` imports ``bar`` -* Empty globals for ``bar`` are created -* ``bar`` is compiled and starts executing -* ``bar`` imports ``foo`` (which is a no-op since there already is a module named ``foo``) -* The import mechanism tries to read ``foo_var`` from ``foo`` globals, to set ``bar.foo_var = foo.foo_var`` - -The last step fails, because Python isn't done with interpreting ``foo`` yet and -the global symbol dictionary for ``foo`` is still empty. - -The same thing happens when you use ``import foo``, and then try to access -``foo.foo_var`` in global code. - -There are (at least) three possible workarounds for this problem. - -Guido van Rossum recommends avoiding all uses of ``from <module> import ...``, -and placing all code inside functions. Initializations of global variables and -class variables should use constants or built-in functions only. This means -everything from an imported module is referenced as ``<module>.<name>``. - -Jim Roskind suggests performing steps in the following order in each module: - -* exports (globals, functions, and classes that don't need imported base - classes) -* ``import`` statements -* active code (including globals that are initialized from imported values). - -Van Rossum doesn't like this approach much because the imports appear in a -strange place, but it does work. - -Matthias Urlichs recommends restructuring your code so that the recursive import -is not necessary in the first place. - -These solutions are not mutually exclusive. - - -__import__('x.y.z') returns <module 'x'>; how do I get z? ---------------------------------------------------------- - -Consider using the convenience function :func:`~importlib.import_module` from -:mod:`importlib` instead:: - - z = importlib.import_module('x.y.z') - - -When I edit an imported module and reimport it, the changes don't show up. Why does this happen? -------------------------------------------------------------------------------------------------- - -For reasons of efficiency as well as consistency, Python only reads the module -file on the first time a module is imported. If it didn't, in a program -consisting of many modules where each one imports the same basic module, the -basic module would be parsed and re-parsed many times. To force re-reading of a -changed module, do this:: - - import importlib - import modname - importlib.reload(modname) - -Warning: this technique is not 100% fool-proof. In particular, modules -containing statements like :: - - from modname import some_objects - -will continue to work with the old version of the imported objects. If the -module contains class definitions, existing class instances will *not* be -updated to use the new class definition. This can result in the following -paradoxical behaviour:: - - >>> import importlib - >>> import cls - >>> c = cls.C() # Create an instance of C - >>> importlib.reload(cls) - <module 'cls' from 'cls.py'> - >>> isinstance(c, cls.C) # isinstance is false?!? - False - -The nature of the problem is made clear if you print out the "identity" of the -class objects:: - - >>> hex(id(c.__class__)) - '0x7352a0' - >>> hex(id(cls.C)) - '0x4198d0' diff --git a/Doc/faq/python-video-icon.png b/Doc/faq/python-video-icon.png deleted file mode 100644 index 265da50c7b38fc..00000000000000 Binary files a/Doc/faq/python-video-icon.png and /dev/null differ diff --git a/Doc/faq/windows.rst b/Doc/faq/windows.rst deleted file mode 100644 index c0c92fdbbc84d1..00000000000000 --- a/Doc/faq/windows.rst +++ /dev/null @@ -1,286 +0,0 @@ -:tocdepth: 2 - -.. highlight:: none - -.. _windows-faq: - -===================== -Python on Windows FAQ -===================== - -.. only:: html - - .. contents:: - -.. XXX need review for Python 3. - XXX need review for Windows Vista/Seven? - -.. _faq-run-program-under-windows: - - -How do I run a Python program under Windows? --------------------------------------------- - -This is not necessarily a straightforward question. If you are already familiar -with running programs from the Windows command line then everything will seem -obvious; otherwise, you might need a little more guidance. - -Unless you use some sort of integrated development environment, you will end up -*typing* Windows commands into what is referred to as a -"Command prompt window". Usually you can create such a window from your -search bar by searching for ``cmd``. You should be able to recognize -when you have started such a window because you will see a Windows "command -prompt", which usually looks like this: - -.. code-block:: doscon - - C:\> - -The letter may be different, and there might be other things after it, so you -might just as easily see something like: - -.. code-block:: doscon - - D:\YourName\Projects\Python> - -depending on how your computer has been set up and what else you have recently -done with it. Once you have started such a window, you are well on the way to -running Python programs. - -You need to realize that your Python scripts have to be processed by another -program called the Python *interpreter*. The interpreter reads your script, -compiles it into bytecodes, and then executes the bytecodes to run your -program. So, how do you arrange for the interpreter to handle your Python? - -First, you need to make sure that your command window recognises the word -"py" as an instruction to start the interpreter. If you have opened a -command window, you should try entering the command ``py`` and hitting -return: - -.. code-block:: doscon - - C:\Users\YourName> py - -You should then see something like: - -.. code-block:: pycon - - Python 3.6.4 (v3.6.4:d48eceb, Dec 19 2017, 06:04:45) [MSC v.1900 32 bit (Intel)] on win32 - Type "help", "copyright", "credits" or "license" for more information. - >>> - -You have started the interpreter in "interactive mode". That means you can enter -Python statements or expressions interactively and have them executed or -evaluated while you wait. This is one of Python's strongest features. Check it -by entering a few expressions of your choice and seeing the results: - -.. code-block:: pycon - - >>> print("Hello") - Hello - >>> "Hello" * 3 - 'HelloHelloHello' - -Many people use the interactive mode as a convenient yet highly programmable -calculator. When you want to end your interactive Python session, -call the :func:`exit` function or hold the :kbd:`Ctrl` key down -while you enter a :kbd:`Z`, then hit the ":kbd:`Enter`" key to get -back to your Windows command prompt. - -You may also find that you have a Start-menu entry such as :menuselection:`Start ---> Programs --> Python 3.x --> Python (command line)` that results in you -seeing the ``>>>`` prompt in a new window. If so, the window will disappear -after you call the :func:`exit` function or enter the :kbd:`Ctrl-Z` -character; Windows is running a single "python" -command in the window, and closes it when you terminate the interpreter. - -Now that we know the ``py`` command is recognized, you can give your -Python script to it. You'll have to give either an absolute or a -relative path to the Python script. Let's say your Python script is -located in your desktop and is named ``hello.py``, and your command -prompt is nicely opened in your home directory so you're seeing something -similar to:: - - C:\Users\YourName> - -So now you'll ask the ``py`` command to give your script to Python by -typing ``py`` followed by your script path:: - - - C:\Users\YourName> py Desktop\hello.py - hello - -How do I make Python scripts executable? ----------------------------------------- - -On Windows, the standard Python installer already associates the .py -extension with a file type (Python.File) and gives that file type an open -command that runs the interpreter (``D:\Program Files\Python\python.exe "%1" -%*``). This is enough to make scripts executable from the command prompt as -'foo.py'. If you'd rather be able to execute the script by simple typing 'foo' -with no extension you need to add .py to the PATHEXT environment variable. - -Why does Python sometimes take so long to start? ------------------------------------------------- - -Usually Python starts very quickly on Windows, but occasionally there are bug -reports that Python suddenly begins to take a long time to start up. This is -made even more puzzling because Python will work fine on other Windows systems -which appear to be configured identically. - -The problem may be caused by a misconfiguration of virus checking software on -the problem machine. Some virus scanners have been known to introduce startup -overhead of two orders of magnitude when the scanner is configured to monitor -all reads from the filesystem. Try checking the configuration of virus scanning -software on your systems to ensure that they are indeed configured identically. -McAfee, when configured to scan all file system read activity, is a particular -offender. - - -How do I make an executable from a Python script? -------------------------------------------------- - -See :ref:`faq-create-standalone-binary` for a list of tools that can be used to -make executables. - - -Is a ``*.pyd`` file the same as a DLL? --------------------------------------- - -Yes, .pyd files are dll's, but there are a few differences. If you have a DLL -named ``foo.pyd``, then it must have a function ``PyInit_foo()``. You can then -write Python "import foo", and Python will search for foo.pyd (as well as -foo.py, foo.pyc) and if it finds it, will attempt to call ``PyInit_foo()`` to -initialize it. You do not link your .exe with foo.lib, as that would cause -Windows to require the DLL to be present. - -Note that the search path for foo.pyd is PYTHONPATH, not the same as the path -that Windows uses to search for foo.dll. Also, foo.pyd need not be present to -run your program, whereas if you linked your program with a dll, the dll is -required. Of course, foo.pyd is required if you want to say ``import foo``. In -a DLL, linkage is declared in the source code with ``__declspec(dllexport)``. -In a .pyd, linkage is defined in a list of available functions. - - -How can I embed Python into a Windows application? --------------------------------------------------- - -Embedding the Python interpreter in a Windows app can be summarized as follows: - -1. Do **not** build Python into your .exe file directly. On Windows, Python must - be a DLL to handle importing modules that are themselves DLL's. (This is the - first key undocumented fact.) Instead, link to :file:`python{NN}.dll`; it is - typically installed in ``C:\Windows\System``. *NN* is the Python version, a - number such as "33" for Python 3.3. - - You can link to Python in two different ways. Load-time linking means - linking against :file:`python{NN}.lib`, while run-time linking means linking - against :file:`python{NN}.dll`. (General note: :file:`python{NN}.lib` is the - so-called "import lib" corresponding to :file:`python{NN}.dll`. It merely - defines symbols for the linker.) - - Run-time linking greatly simplifies link options; everything happens at run - time. Your code must load :file:`python{NN}.dll` using the Windows - ``LoadLibraryEx()`` routine. The code must also use access routines and data - in :file:`python{NN}.dll` (that is, Python's C API's) using pointers obtained - by the Windows ``GetProcAddress()`` routine. Macros can make using these - pointers transparent to any C code that calls routines in Python's C API. - - .. XXX what about static linking? - -2. If you use SWIG, it is easy to create a Python "extension module" that will - make the app's data and methods available to Python. SWIG will handle just - about all the grungy details for you. The result is C code that you link - *into* your .exe file (!) You do **not** have to create a DLL file, and this - also simplifies linking. - -3. SWIG will create an init function (a C function) whose name depends on the - name of the extension module. For example, if the name of the module is leo, - the init function will be called initleo(). If you use SWIG shadow classes, - as you should, the init function will be called initleoc(). This initializes - a mostly hidden helper class used by the shadow class. - - The reason you can link the C code in step 2 into your .exe file is that - calling the initialization function is equivalent to importing the module - into Python! (This is the second key undocumented fact.) - -4. In short, you can use the following code to initialize the Python interpreter - with your extension module. - - .. code-block:: c - - #include <Python.h> - ... - Py_Initialize(); // Initialize Python. - initmyAppc(); // Initialize (import) the helper class. - PyRun_SimpleString("import myApp"); // Import the shadow class. - -5. There are two problems with Python's C API which will become apparent if you - use a compiler other than MSVC, the compiler used to build pythonNN.dll. - - Problem 1: The so-called "Very High Level" functions that take ``FILE *`` - arguments will not work in a multi-compiler environment because each - compiler's notion of a ``struct FILE`` will be different. From an implementation - standpoint these are very low level functions. - - Problem 2: SWIG generates the following code when generating wrappers to void - functions: - - .. code-block:: c - - Py_INCREF(Py_None); - _resultobj = Py_None; - return _resultobj; - - Alas, Py_None is a macro that expands to a reference to a complex data - structure called _Py_NoneStruct inside pythonNN.dll. Again, this code will - fail in a mult-compiler environment. Replace such code by: - - .. code-block:: c - - return Py_BuildValue(""); - - It may be possible to use SWIG's ``%typemap`` command to make the change - automatically, though I have not been able to get this to work (I'm a - complete SWIG newbie). - -6. Using a Python shell script to put up a Python interpreter window from inside - your Windows app is not a good idea; the resulting window will be independent - of your app's windowing system. Rather, you (or the wxPythonWindow class) - should create a "native" interpreter window. It is easy to connect that - window to the Python interpreter. You can redirect Python's i/o to _any_ - object that supports read and write, so all you need is a Python object - (defined in your extension module) that contains read() and write() methods. - -How do I keep editors from inserting tabs into my Python source? ----------------------------------------------------------------- - -The FAQ does not recommend using tabs, and the Python style guide, :pep:`8`, -recommends 4 spaces for distributed Python code; this is also the Emacs -python-mode default. - -Under any editor, mixing tabs and spaces is a bad idea. MSVC is no different in -this respect, and is easily configured to use spaces: Take :menuselection:`Tools ---> Options --> Tabs`, and for file type "Default" set "Tab size" and "Indent -size" to 4, and select the "Insert spaces" radio button. - -Python raises :exc:`IndentationError` or :exc:`TabError` if mixed tabs -and spaces are causing problems in leading whitespace. -You may also run the :mod:`tabnanny` module to check a directory tree -in batch mode. - - -How do I check for a keypress without blocking? ------------------------------------------------ - -Use the :mod:`msvcrt` module. This is a standard Windows-specific extension module. -It defines a function ``kbhit()`` which checks whether a keyboard hit is -present, and ``getch()`` which gets one character without echoing it. - -How do I solve the missing api-ms-win-crt-runtime-l1-1-0.dll error? -------------------------------------------------------------------- - -This can occur on Python 3.5 and later when using Windows 8.1 or earlier without all updates having been installed. -First ensure your operating system is supported and is up to date, and if that does not resolve the issue, -visit the `Microsoft support page <https://support.microsoft.com/en-us/help/3118401/>`_ -for guidance on manually installing the C Runtime update. diff --git a/Doc/glossary.rst b/Doc/glossary.rst deleted file mode 100644 index f318a37b654e9d..00000000000000 --- a/Doc/glossary.rst +++ /dev/null @@ -1,1261 +0,0 @@ -.. _glossary: - -******** -Glossary -******** - -.. if you add new entries, keep the alphabetical sorting! - -.. glossary:: - - ``>>>`` - The default Python prompt of the interactive shell. Often seen for code - examples which can be executed interactively in the interpreter. - - ``...`` - Can refer to: - - * The default Python prompt of the interactive shell when entering the - code for an indented code block, when within a pair of matching left and - right delimiters (parentheses, square brackets, curly braces or triple - quotes), or after specifying a decorator. - - * The :const:`Ellipsis` built-in constant. - - 2to3 - A tool that tries to convert Python 2.x code to Python 3.x code by - handling most of the incompatibilities which can be detected by parsing the - source and traversing the parse tree. - - 2to3 is available in the standard library as :mod:`lib2to3`; a standalone - entry point is provided as :file:`Tools/scripts/2to3`. See - :ref:`2to3-reference`. - - abstract base class - Abstract base classes complement :term:`duck-typing` by - providing a way to define interfaces when other techniques like - :func:`hasattr` would be clumsy or subtly wrong (for example with - :ref:`magic methods <special-lookup>`). ABCs introduce virtual - subclasses, which are classes that don't inherit from a class but are - still recognized by :func:`isinstance` and :func:`issubclass`; see the - :mod:`abc` module documentation. Python comes with many built-in ABCs for - data structures (in the :mod:`collections.abc` module), numbers (in the - :mod:`numbers` module), streams (in the :mod:`io` module), import finders - and loaders (in the :mod:`importlib.abc` module). You can create your own - ABCs with the :mod:`abc` module. - - annotation - A label associated with a variable, a class - attribute or a function parameter or return value, - used by convention as a :term:`type hint`. - - Annotations of local variables cannot be accessed at runtime, but - annotations of global variables, class attributes, and functions - are stored in the :attr:`__annotations__` - special attribute of modules, classes, and functions, - respectively. - - See :term:`variable annotation`, :term:`function annotation`, :pep:`484` - and :pep:`526`, which describe this functionality. - Also see :ref:`annotations-howto` - for best practices on working with annotations. - - argument - A value passed to a :term:`function` (or :term:`method`) when calling the - function. There are two kinds of argument: - - * :dfn:`keyword argument`: an argument preceded by an identifier (e.g. - ``name=``) in a function call or passed as a value in a dictionary - preceded by ``**``. For example, ``3`` and ``5`` are both keyword - arguments in the following calls to :func:`complex`:: - - complex(real=3, imag=5) - complex(**{'real': 3, 'imag': 5}) - - * :dfn:`positional argument`: an argument that is not a keyword argument. - Positional arguments can appear at the beginning of an argument list - and/or be passed as elements of an :term:`iterable` preceded by ``*``. - For example, ``3`` and ``5`` are both positional arguments in the - following calls:: - - complex(3, 5) - complex(*(3, 5)) - - Arguments are assigned to the named local variables in a function body. - See the :ref:`calls` section for the rules governing this assignment. - Syntactically, any expression can be used to represent an argument; the - evaluated value is assigned to the local variable. - - See also the :term:`parameter` glossary entry, the FAQ question on - :ref:`the difference between arguments and parameters - <faq-argument-vs-parameter>`, and :pep:`362`. - - asynchronous context manager - An object which controls the environment seen in an - :keyword:`async with` statement by defining :meth:`~object.__aenter__` and - :meth:`~object.__aexit__` methods. Introduced by :pep:`492`. - - asynchronous generator - A function which returns an :term:`asynchronous generator iterator`. It - looks like a coroutine function defined with :keyword:`async def` except - that it contains :keyword:`yield` expressions for producing a series of - values usable in an :keyword:`async for` loop. - - Usually refers to an asynchronous generator function, but may refer to an - *asynchronous generator iterator* in some contexts. In cases where the - intended meaning isn't clear, using the full terms avoids ambiguity. - - An asynchronous generator function may contain :keyword:`await` - expressions as well as :keyword:`async for`, and :keyword:`async with` - statements. - - asynchronous generator iterator - An object created by a :term:`asynchronous generator` function. - - This is an :term:`asynchronous iterator` which when called using the - :meth:`~object.__anext__` method returns an awaitable object which will execute - the body of the asynchronous generator function until the next - :keyword:`yield` expression. - - Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending - try-statements). When the *asynchronous generator iterator* effectively - resumes with another awaitable returned by :meth:`~object.__anext__`, it - picks up where it left off. See :pep:`492` and :pep:`525`. - - asynchronous iterable - An object, that can be used in an :keyword:`async for` statement. - Must return an :term:`asynchronous iterator` from its - :meth:`~object.__aiter__` method. Introduced by :pep:`492`. - - asynchronous iterator - An object that implements the :meth:`~object.__aiter__` and :meth:`~object.__anext__` - methods. :meth:`~object.__anext__` must return an :term:`awaitable` object. - :keyword:`async for` resolves the awaitables returned by an asynchronous - iterator's :meth:`~object.__anext__` method until it raises a - :exc:`StopAsyncIteration` exception. Introduced by :pep:`492`. - - attribute - A value associated with an object which is usually referenced by name - using dotted expressions. - For example, if an object *o* has an attribute - *a* it would be referenced as *o.a*. - - It is possible to give an object an attribute whose name is not an - identifier as defined by :ref:`identifiers`, for example using - :func:`setattr`, if the object allows it. - Such an attribute will not be accessible using a dotted expression, - and would instead need to be retrieved with :func:`getattr`. - - awaitable - An object that can be used in an :keyword:`await` expression. Can be - a :term:`coroutine` or an object with an :meth:`~object.__await__` method. - See also :pep:`492`. - - BDFL - Benevolent Dictator For Life, a.k.a. `Guido van Rossum - <https://gvanrossum.github.io/>`_, Python's creator. - - binary file - A :term:`file object` able to read and write - :term:`bytes-like objects <bytes-like object>`. - Examples of binary files are files opened in binary mode (``'rb'``, - ``'wb'`` or ``'rb+'``), :data:`sys.stdin.buffer`, - :data:`sys.stdout.buffer`, and instances of :class:`io.BytesIO` and - :class:`gzip.GzipFile`. - - See also :term:`text file` for a file object able to read and write - :class:`str` objects. - - borrowed reference - In Python's C API, a borrowed reference is a reference to an object, - where the code using the object does not own the reference. - It becomes a dangling - pointer if the object is destroyed. For example, a garbage collection can - remove the last :term:`strong reference` to the object and so destroy it. - - Calling :c:func:`Py_INCREF` on the :term:`borrowed reference` is - recommended to convert it to a :term:`strong reference` in-place, except - when the object cannot be destroyed before the last usage of the borrowed - reference. The :c:func:`Py_NewRef` function can be used to create a new - :term:`strong reference`. - - bytes-like object - An object that supports the :ref:`bufferobjects` and can - export a C-:term:`contiguous` buffer. This includes all :class:`bytes`, - :class:`bytearray`, and :class:`array.array` objects, as well as many - common :class:`memoryview` objects. Bytes-like objects can - be used for various operations that work with binary data; these include - compression, saving to a binary file, and sending over a socket. - - Some operations need the binary data to be mutable. The documentation - often refers to these as "read-write bytes-like objects". Example - mutable buffer objects include :class:`bytearray` and a - :class:`memoryview` of a :class:`bytearray`. - Other operations require the binary data to be stored in - immutable objects ("read-only bytes-like objects"); examples - of these include :class:`bytes` and a :class:`memoryview` - of a :class:`bytes` object. - - bytecode - Python source code is compiled into bytecode, the internal representation - of a Python program in the CPython interpreter. The bytecode is also - cached in ``.pyc`` files so that executing the same file is - faster the second time (recompilation from source to bytecode can be - avoided). This "intermediate language" is said to run on a - :term:`virtual machine` that executes the machine code corresponding to - each bytecode. Do note that bytecodes are not expected to work between - different Python virtual machines, nor to be stable between Python - releases. - - A list of bytecode instructions can be found in the documentation for - :ref:`the dis module <bytecodes>`. - - callable - A callable is an object that can be called, possibly with a set - of arguments (see :term:`argument`), with the following syntax:: - - callable(argument1, argument2, argumentN) - - A :term:`function`, and by extension a :term:`method`, is a callable. - An instance of a class that implements the :meth:`~object.__call__` - method is also a callable. - - callback - A subroutine function which is passed as an argument to be executed at - some point in the future. - - class - A template for creating user-defined objects. Class definitions - normally contain method definitions which operate on instances of the - class. - - class variable - A variable defined in a class and intended to be modified only at - class level (i.e., not in an instance of the class). - - complex number - An extension of the familiar real number system in which all numbers are - expressed as a sum of a real part and an imaginary part. Imaginary - numbers are real multiples of the imaginary unit (the square root of - ``-1``), often written ``i`` in mathematics or ``j`` in - engineering. Python has built-in support for complex numbers, which are - written with this latter notation; the imaginary part is written with a - ``j`` suffix, e.g., ``3+1j``. To get access to complex equivalents of the - :mod:`math` module, use :mod:`cmath`. Use of complex numbers is a fairly - advanced mathematical feature. If you're not aware of a need for them, - it's almost certain you can safely ignore them. - - context manager - An object which controls the environment seen in a :keyword:`with` - statement by defining :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. - See :pep:`343`. - - context variable - A variable which can have different values depending on its context. - This is similar to Thread-Local Storage in which each execution - thread may have a different value for a variable. However, with context - variables, there may be several contexts in one execution thread and the - main usage for context variables is to keep track of variables in - concurrent asynchronous tasks. - See :mod:`contextvars`. - - contiguous - .. index:: C-contiguous, Fortran contiguous - - A buffer is considered contiguous exactly if it is either - *C-contiguous* or *Fortran contiguous*. Zero-dimensional buffers are - C and Fortran contiguous. In one-dimensional arrays, the items - must be laid out in memory next to each other, in order of - increasing indexes starting from zero. In multidimensional - C-contiguous arrays, the last index varies the fastest when - visiting items in order of memory address. However, in - Fortran contiguous arrays, the first index varies the fastest. - - coroutine - Coroutines are a more generalized form of subroutines. Subroutines are - entered at one point and exited at another point. Coroutines can be - entered, exited, and resumed at many different points. They can be - implemented with the :keyword:`async def` statement. See also - :pep:`492`. - - coroutine function - A function which returns a :term:`coroutine` object. A coroutine - function may be defined with the :keyword:`async def` statement, - and may contain :keyword:`await`, :keyword:`async for`, and - :keyword:`async with` keywords. These were introduced - by :pep:`492`. - - CPython - The canonical implementation of the Python programming language, as - distributed on `python.org <https://www.python.org>`_. The term "CPython" - is used when necessary to distinguish this implementation from others - such as Jython or IronPython. - - decorator - A function returning another function, usually applied as a function - transformation using the ``@wrapper`` syntax. Common examples for - decorators are :func:`classmethod` and :func:`staticmethod`. - - The decorator syntax is merely syntactic sugar, the following two - function definitions are semantically equivalent:: - - def f(arg): - ... - f = staticmethod(f) - - @staticmethod - def f(arg): - ... - - The same concept exists for classes, but is less commonly used there. See - the documentation for :ref:`function definitions <function>` and - :ref:`class definitions <class>` for more about decorators. - - descriptor - Any object which defines the methods :meth:`__get__`, :meth:`__set__`, or - :meth:`__delete__`. When a class attribute is a descriptor, its special - binding behavior is triggered upon attribute lookup. Normally, using - *a.b* to get, set or delete an attribute looks up the object named *b* in - the class dictionary for *a*, but if *b* is a descriptor, the respective - descriptor method gets called. Understanding descriptors is a key to a - deep understanding of Python because they are the basis for many features - including functions, methods, properties, class methods, static methods, - and reference to super classes. - - For more information about descriptors' methods, see :ref:`descriptors` - or the :ref:`Descriptor How To Guide <descriptorhowto>`. - - dictionary - An associative array, where arbitrary keys are mapped to values. The - keys can be any object with :meth:`__hash__` and :meth:`__eq__` methods. - Called a hash in Perl. - - dictionary comprehension - A compact way to process all or part of the elements in an iterable and - return a dictionary with the results. ``results = {n: n ** 2 for n in - range(10)}`` generates a dictionary containing key ``n`` mapped to - value ``n ** 2``. See :ref:`comprehensions`. - - dictionary view - The objects returned from :meth:`dict.keys`, :meth:`dict.values`, and - :meth:`dict.items` are called dictionary views. They provide a dynamic - view on the dictionary’s entries, which means that when the dictionary - changes, the view reflects these changes. To force the - dictionary view to become a full list use ``list(dictview)``. See - :ref:`dict-views`. - - docstring - A string literal which appears as the first expression in a class, - function or module. While ignored when the suite is executed, it is - recognized by the compiler and put into the :attr:`__doc__` attribute - of the enclosing class, function or module. Since it is available via - introspection, it is the canonical place for documentation of the - object. - - duck-typing - A programming style which does not look at an object's type to determine - if it has the right interface; instead, the method or attribute is simply - called or used ("If it looks like a duck and quacks like a duck, it - must be a duck.") By emphasizing interfaces rather than specific types, - well-designed code improves its flexibility by allowing polymorphic - substitution. Duck-typing avoids tests using :func:`type` or - :func:`isinstance`. (Note, however, that duck-typing can be complemented - with :term:`abstract base classes <abstract base class>`.) Instead, it - typically employs :func:`hasattr` tests or :term:`EAFP` programming. - - EAFP - Easier to ask for forgiveness than permission. This common Python coding - style assumes the existence of valid keys or attributes and catches - exceptions if the assumption proves false. This clean and fast style is - characterized by the presence of many :keyword:`try` and :keyword:`except` - statements. The technique contrasts with the :term:`LBYL` style - common to many other languages such as C. - - expression - A piece of syntax which can be evaluated to some value. In other words, - an expression is an accumulation of expression elements like literals, - names, attribute access, operators or function calls which all return a - value. In contrast to many other languages, not all language constructs - are expressions. There are also :term:`statement`\s which cannot be used - as expressions, such as :keyword:`while`. Assignments are also statements, - not expressions. - - extension module - A module written in C or C++, using Python's C API to interact with the - core and with user code. - - f-string - String literals prefixed with ``'f'`` or ``'F'`` are commonly called - "f-strings" which is short for - :ref:`formatted string literals <f-strings>`. See also :pep:`498`. - - file object - An object exposing a file-oriented API (with methods such as - :meth:`read()` or :meth:`write()`) to an underlying resource. Depending - on the way it was created, a file object can mediate access to a real - on-disk file or to another type of storage or communication device - (for example standard input/output, in-memory buffers, sockets, pipes, - etc.). File objects are also called :dfn:`file-like objects` or - :dfn:`streams`. - - There are actually three categories of file objects: raw - :term:`binary files <binary file>`, buffered - :term:`binary files <binary file>` and :term:`text files <text file>`. - Their interfaces are defined in the :mod:`io` module. The canonical - way to create a file object is by using the :func:`open` function. - - file-like object - A synonym for :term:`file object`. - - filesystem encoding and error handler - Encoding and error handler used by Python to decode bytes from the - operating system and encode Unicode to the operating system. - - The filesystem encoding must guarantee to successfully decode all bytes - below 128. If the file system encoding fails to provide this guarantee, - API functions can raise :exc:`UnicodeError`. - - The :func:`sys.getfilesystemencoding` and - :func:`sys.getfilesystemencodeerrors` functions can be used to get the - filesystem encoding and error handler. - - The :term:`filesystem encoding and error handler` are configured at - Python startup by the :c:func:`PyConfig_Read` function: see - :c:member:`~PyConfig.filesystem_encoding` and - :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. - - See also the :term:`locale encoding`. - - finder - An object that tries to find the :term:`loader` for a module that is - being imported. - - Since Python 3.3, there are two types of finder: :term:`meta path finders - <meta path finder>` for use with :data:`sys.meta_path`, and :term:`path - entry finders <path entry finder>` for use with :data:`sys.path_hooks`. - - See :pep:`302`, :pep:`420` and :pep:`451` for much more detail. - - floor division - Mathematical division that rounds down to nearest integer. The floor - division operator is ``//``. For example, the expression ``11 // 4`` - evaluates to ``2`` in contrast to the ``2.75`` returned by float true - division. Note that ``(-11) // 4`` is ``-3`` because that is ``-2.75`` - rounded *downward*. See :pep:`238`. - - function - A series of statements which returns some value to a caller. It can also - be passed zero or more :term:`arguments <argument>` which may be used in - the execution of the body. See also :term:`parameter`, :term:`method`, - and the :ref:`function` section. - - function annotation - An :term:`annotation` of a function parameter or return value. - - Function annotations are usually used for - :term:`type hints <type hint>`: for example, this function is expected to take two - :class:`int` arguments and is also expected to have an :class:`int` - return value:: - - def sum_two_numbers(a: int, b: int) -> int: - return a + b - - Function annotation syntax is explained in section :ref:`function`. - - See :term:`variable annotation` and :pep:`484`, - which describe this functionality. - Also see :ref:`annotations-howto` - for best practices on working with annotations. - - __future__ - A :ref:`future statement <future>`, ``from __future__ import <feature>``, - directs the compiler to compile the current module using syntax or - semantics that will become standard in a future release of Python. - The :mod:`__future__` module documents the possible values of - *feature*. By importing this module and evaluating its variables, - you can see when a new feature was first added to the language and - when it will (or did) become the default:: - - >>> import __future__ - >>> __future__.division - _Feature((2, 2, 0, 'alpha', 2), (3, 0, 0, 'alpha', 0), 8192) - - garbage collection - The process of freeing memory when it is not used anymore. Python - performs garbage collection via reference counting and a cyclic garbage - collector that is able to detect and break reference cycles. The - garbage collector can be controlled using the :mod:`gc` module. - - .. index:: single: generator - - generator - A function which returns a :term:`generator iterator`. It looks like a - normal function except that it contains :keyword:`yield` expressions - for producing a series of values usable in a for-loop or that can be - retrieved one at a time with the :func:`next` function. - - Usually refers to a generator function, but may refer to a - *generator iterator* in some contexts. In cases where the intended - meaning isn't clear, using the full terms avoids ambiguity. - - generator iterator - An object created by a :term:`generator` function. - - Each :keyword:`yield` temporarily suspends processing, remembering the - location execution state (including local variables and pending - try-statements). When the *generator iterator* resumes, it picks up where - it left off (in contrast to functions which start fresh on every - invocation). - - .. index:: single: generator expression - - generator expression - An expression that returns an iterator. It looks like a normal expression - followed by a :keyword:`!for` clause defining a loop variable, range, - and an optional :keyword:`!if` clause. The combined expression - generates values for an enclosing function:: - - >>> sum(i*i for i in range(10)) # sum of squares 0, 1, 4, ... 81 - 285 - - generic function - A function composed of multiple functions implementing the same operation - for different types. Which implementation should be used during a call is - determined by the dispatch algorithm. - - See also the :term:`single dispatch` glossary entry, the - :func:`functools.singledispatch` decorator, and :pep:`443`. - - generic type - A :term:`type` that can be parameterized; typically a - :ref:`container class<sequence-types>` such as :class:`list` or - :class:`dict`. Used for :term:`type hints <type hint>` and - :term:`annotations <annotation>`. - - For more details, see :ref:`generic alias types<types-genericalias>`, - :pep:`483`, :pep:`484`, :pep:`585`, and the :mod:`typing` module. - - GIL - See :term:`global interpreter lock`. - - global interpreter lock - The mechanism used by the :term:`CPython` interpreter to assure that - only one thread executes Python :term:`bytecode` at a time. - This simplifies the CPython implementation by making the object model - (including critical built-in types such as :class:`dict`) implicitly - safe against concurrent access. Locking the entire interpreter - makes it easier for the interpreter to be multi-threaded, at the - expense of much of the parallelism afforded by multi-processor - machines. - - However, some extension modules, either standard or third-party, - are designed so as to release the GIL when doing computationally intensive - tasks such as compression or hashing. Also, the GIL is always released - when doing I/O. - - Past efforts to create a "free-threaded" interpreter (one which locks - shared data at a much finer granularity) have not been successful - because performance suffered in the common single-processor case. It - is believed that overcoming this performance issue would make the - implementation much more complicated and therefore costlier to maintain. - - - hash-based pyc - A bytecode cache file that uses the hash rather than the last-modified - time of the corresponding source file to determine its validity. See - :ref:`pyc-invalidation`. - - hashable - An object is *hashable* if it has a hash value which never changes during - its lifetime (it needs a :meth:`__hash__` method), and can be compared to - other objects (it needs an :meth:`__eq__` method). Hashable objects which - compare equal must have the same hash value. - - Hashability makes an object usable as a dictionary key and a set member, - because these data structures use the hash value internally. - - Most of Python's immutable built-in objects are hashable; mutable - containers (such as lists or dictionaries) are not; immutable - containers (such as tuples and frozensets) are only hashable if - their elements are hashable. Objects which are - instances of user-defined classes are hashable by default. They all - compare unequal (except with themselves), and their hash value is derived - from their :func:`id`. - - IDLE - An Integrated Development and Learning Environment for Python. - :ref:`idle` is a basic editor and interpreter environment - which ships with the standard distribution of Python. - - immutable - An object with a fixed value. Immutable objects include numbers, strings and - tuples. Such an object cannot be altered. A new object has to - be created if a different value has to be stored. They play an important - role in places where a constant hash value is needed, for example as a key - in a dictionary. - - import path - A list of locations (or :term:`path entries <path entry>`) that are - searched by the :term:`path based finder` for modules to import. During - import, this list of locations usually comes from :data:`sys.path`, but - for subpackages it may also come from the parent package's ``__path__`` - attribute. - - importing - The process by which Python code in one module is made available to - Python code in another module. - - importer - An object that both finds and loads a module; both a - :term:`finder` and :term:`loader` object. - - interactive - Python has an interactive interpreter which means you can enter - statements and expressions at the interpreter prompt, immediately - execute them and see their results. Just launch ``python`` with no - arguments (possibly by selecting it from your computer's main - menu). It is a very powerful way to test out new ideas or inspect - modules and packages (remember ``help(x)``). - - interpreted - Python is an interpreted language, as opposed to a compiled one, - though the distinction can be blurry because of the presence of the - bytecode compiler. This means that source files can be run directly - without explicitly creating an executable which is then run. - Interpreted languages typically have a shorter development/debug cycle - than compiled ones, though their programs generally also run more - slowly. See also :term:`interactive`. - - interpreter shutdown - When asked to shut down, the Python interpreter enters a special phase - where it gradually releases all allocated resources, such as modules - and various critical internal structures. It also makes several calls - to the :term:`garbage collector <garbage collection>`. This can trigger - the execution of code in user-defined destructors or weakref callbacks. - Code executed during the shutdown phase can encounter various - exceptions as the resources it relies on may not function anymore - (common examples are library modules or the warnings machinery). - - The main reason for interpreter shutdown is that the ``__main__`` module - or the script being run has finished executing. - - iterable - An object capable of returning its members one at a time. Examples of - iterables include all sequence types (such as :class:`list`, :class:`str`, - and :class:`tuple`) and some non-sequence types like :class:`dict`, - :term:`file objects <file object>`, and objects of any classes you define - with an :meth:`__iter__` method or with a :meth:`~object.__getitem__` method - that implements :term:`sequence` semantics. - - Iterables can be - used in a :keyword:`for` loop and in many other places where a sequence is - needed (:func:`zip`, :func:`map`, ...). When an iterable object is passed - as an argument to the built-in function :func:`iter`, it returns an - iterator for the object. This iterator is good for one pass over the set - of values. When using iterables, it is usually not necessary to call - :func:`iter` or deal with iterator objects yourself. The ``for`` - statement does that automatically for you, creating a temporary unnamed - variable to hold the iterator for the duration of the loop. See also - :term:`iterator`, :term:`sequence`, and :term:`generator`. - - iterator - An object representing a stream of data. Repeated calls to the iterator's - :meth:`~iterator.__next__` method (or passing it to the built-in function - :func:`next`) return successive items in the stream. When no more data - are available a :exc:`StopIteration` exception is raised instead. At this - point, the iterator object is exhausted and any further calls to its - :meth:`__next__` method just raise :exc:`StopIteration` again. Iterators - are required to have an :meth:`__iter__` method that returns the iterator - object itself so every iterator is also iterable and may be used in most - places where other iterables are accepted. One notable exception is code - which attempts multiple iteration passes. A container object (such as a - :class:`list`) produces a fresh new iterator each time you pass it to the - :func:`iter` function or use it in a :keyword:`for` loop. Attempting this - with an iterator will just return the same exhausted iterator object used - in the previous iteration pass, making it appear like an empty container. - - More information can be found in :ref:`typeiter`. - - .. impl-detail:: - - CPython does not consistently apply the requirement that an iterator - define :meth:`__iter__`. - - key function - A key function or collation function is a callable that returns a value - used for sorting or ordering. For example, :func:`locale.strxfrm` is - used to produce a sort key that is aware of locale specific sort - conventions. - - A number of tools in Python accept key functions to control how elements - are ordered or grouped. They include :func:`min`, :func:`max`, - :func:`sorted`, :meth:`list.sort`, :func:`heapq.merge`, - :func:`heapq.nsmallest`, :func:`heapq.nlargest`, and - :func:`itertools.groupby`. - - There are several ways to create a key function. For example. the - :meth:`str.lower` method can serve as a key function for case insensitive - sorts. Alternatively, a key function can be built from a - :keyword:`lambda` expression such as ``lambda r: (r[0], r[2])``. Also, - :func:`operator.attrgetter`, :func:`operator.itemgetter`, and - :func:`operator.methodcaller` are three key function constructors. See the :ref:`Sorting HOW TO - <sortinghowto>` for examples of how to create and use key functions. - - keyword argument - See :term:`argument`. - - lambda - An anonymous inline function consisting of a single :term:`expression` - which is evaluated when the function is called. The syntax to create - a lambda function is ``lambda [parameters]: expression`` - - LBYL - Look before you leap. This coding style explicitly tests for - pre-conditions before making calls or lookups. This style contrasts with - the :term:`EAFP` approach and is characterized by the presence of many - :keyword:`if` statements. - - In a multi-threaded environment, the LBYL approach can risk introducing a - race condition between "the looking" and "the leaping". For example, the - code, ``if key in mapping: return mapping[key]`` can fail if another - thread removes *key* from *mapping* after the test, but before the lookup. - This issue can be solved with locks or by using the EAFP approach. - - locale encoding - On Unix, it is the encoding of the LC_CTYPE locale. It can be set with - :func:`locale.setlocale(locale.LC_CTYPE, new_locale) <locale.setlocale>`. - - On Windows, it is the ANSI code page (ex: ``"cp1252"``). - - On Android and VxWorks, Python uses ``"utf-8"`` as the locale encoding. - - ``locale.getencoding()`` can be used to get the locale encoding. - - See also the :term:`filesystem encoding and error handler`. - - list - A built-in Python :term:`sequence`. Despite its name it is more akin - to an array in other languages than to a linked list since access to - elements is O(1). - - list comprehension - A compact way to process all or part of the elements in a sequence and - return a list with the results. ``result = ['{:#04x}'.format(x) for x in - range(256) if x % 2 == 0]`` generates a list of strings containing - even hex numbers (0x..) in the range from 0 to 255. The :keyword:`if` - clause is optional. If omitted, all elements in ``range(256)`` are - processed. - - loader - An object that loads a module. It must define a method named - :meth:`load_module`. A loader is typically returned by a - :term:`finder`. See :pep:`302` for details and - :class:`importlib.abc.Loader` for an :term:`abstract base class`. - - magic method - .. index:: pair: magic; method - - An informal synonym for :term:`special method`. - - mapping - A container object that supports arbitrary key lookups and implements the - methods specified in the :class:`collections.abc.Mapping` or - :class:`collections.abc.MutableMapping` - :ref:`abstract base classes <collections-abstract-base-classes>`. Examples - include :class:`dict`, :class:`collections.defaultdict`, - :class:`collections.OrderedDict` and :class:`collections.Counter`. - - meta path finder - A :term:`finder` returned by a search of :data:`sys.meta_path`. Meta path - finders are related to, but different from :term:`path entry finders - <path entry finder>`. - - See :class:`importlib.abc.MetaPathFinder` for the methods that meta path - finders implement. - - metaclass - The class of a class. Class definitions create a class name, a class - dictionary, and a list of base classes. The metaclass is responsible for - taking those three arguments and creating the class. Most object oriented - programming languages provide a default implementation. What makes Python - special is that it is possible to create custom metaclasses. Most users - never need this tool, but when the need arises, metaclasses can provide - powerful, elegant solutions. They have been used for logging attribute - access, adding thread-safety, tracking object creation, implementing - singletons, and many other tasks. - - More information can be found in :ref:`metaclasses`. - - method - A function which is defined inside a class body. If called as an attribute - of an instance of that class, the method will get the instance object as - its first :term:`argument` (which is usually called ``self``). - See :term:`function` and :term:`nested scope`. - - method resolution order - Method Resolution Order is the order in which base classes are searched - for a member during lookup. See `The Python 2.3 Method Resolution Order - <https://www.python.org/download/releases/2.3/mro/>`_ for details of the - algorithm used by the Python interpreter since the 2.3 release. - - module - An object that serves as an organizational unit of Python code. Modules - have a namespace containing arbitrary Python objects. Modules are loaded - into Python by the process of :term:`importing`. - - See also :term:`package`. - - module spec - A namespace containing the import-related information used to load a - module. An instance of :class:`importlib.machinery.ModuleSpec`. - - MRO - See :term:`method resolution order`. - - mutable - Mutable objects can change their value but keep their :func:`id`. See - also :term:`immutable`. - - named tuple - The term "named tuple" applies to any type or class that inherits from - tuple and whose indexable elements are also accessible using named - attributes. The type or class may have other features as well. - - Several built-in types are named tuples, including the values returned - by :func:`time.localtime` and :func:`os.stat`. Another example is - :data:`sys.float_info`:: - - >>> sys.float_info[1] # indexed access - 1024 - >>> sys.float_info.max_exp # named field access - 1024 - >>> isinstance(sys.float_info, tuple) # kind of tuple - True - - Some named tuples are built-in types (such as the above examples). - Alternatively, a named tuple can be created from a regular class - definition that inherits from :class:`tuple` and that defines named - fields. Such a class can be written by hand or it can be created with - the factory function :func:`collections.namedtuple`. The latter - technique also adds some extra methods that may not be found in - hand-written or built-in named tuples. - - namespace - The place where a variable is stored. Namespaces are implemented as - dictionaries. There are the local, global and built-in namespaces as well - as nested namespaces in objects (in methods). Namespaces support - modularity by preventing naming conflicts. For instance, the functions - :func:`builtins.open <.open>` and :func:`os.open` are distinguished by - their namespaces. Namespaces also aid readability and maintainability by - making it clear which module implements a function. For instance, writing - :func:`random.seed` or :func:`itertools.islice` makes it clear that those - functions are implemented by the :mod:`random` and :mod:`itertools` - modules, respectively. - - namespace package - A :pep:`420` :term:`package` which serves only as a container for - subpackages. Namespace packages may have no physical representation, - and specifically are not like a :term:`regular package` because they - have no ``__init__.py`` file. - - See also :term:`module`. - - nested scope - The ability to refer to a variable in an enclosing definition. For - instance, a function defined inside another function can refer to - variables in the outer function. Note that nested scopes by default work - only for reference and not for assignment. Local variables both read and - write in the innermost scope. Likewise, global variables read and write - to the global namespace. The :keyword:`nonlocal` allows writing to outer - scopes. - - new-style class - Old name for the flavor of classes now used for all class objects. In - earlier Python versions, only new-style classes could use Python's newer, - versatile features like :attr:`~object.__slots__`, descriptors, - properties, :meth:`__getattribute__`, class methods, and static methods. - - object - Any data with state (attributes or value) and defined behavior - (methods). Also the ultimate base class of any :term:`new-style - class`. - - package - A Python :term:`module` which can contain submodules or recursively, - subpackages. Technically, a package is a Python module with a - ``__path__`` attribute. - - See also :term:`regular package` and :term:`namespace package`. - - parameter - A named entity in a :term:`function` (or method) definition that - specifies an :term:`argument` (or in some cases, arguments) that the - function can accept. There are five kinds of parameter: - - * :dfn:`positional-or-keyword`: specifies an argument that can be passed - either :term:`positionally <argument>` or as a :term:`keyword argument - <argument>`. This is the default kind of parameter, for example *foo* - and *bar* in the following:: - - def func(foo, bar=None): ... - - .. _positional-only_parameter: - - * :dfn:`positional-only`: specifies an argument that can be supplied only - by position. Positional-only parameters can be defined by including a - ``/`` character in the parameter list of the function definition after - them, for example *posonly1* and *posonly2* in the following:: - - def func(posonly1, posonly2, /, positional_or_keyword): ... - - .. _keyword-only_parameter: - - * :dfn:`keyword-only`: specifies an argument that can be supplied only - by keyword. Keyword-only parameters can be defined by including a - single var-positional parameter or bare ``*`` in the parameter list - of the function definition before them, for example *kw_only1* and - *kw_only2* in the following:: - - def func(arg, *, kw_only1, kw_only2): ... - - * :dfn:`var-positional`: specifies that an arbitrary sequence of - positional arguments can be provided (in addition to any positional - arguments already accepted by other parameters). Such a parameter can - be defined by prepending the parameter name with ``*``, for example - *args* in the following:: - - def func(*args, **kwargs): ... - - * :dfn:`var-keyword`: specifies that arbitrarily many keyword arguments - can be provided (in addition to any keyword arguments already accepted - by other parameters). Such a parameter can be defined by prepending - the parameter name with ``**``, for example *kwargs* in the example - above. - - Parameters can specify both optional and required arguments, as well as - default values for some optional arguments. - - See also the :term:`argument` glossary entry, the FAQ question on - :ref:`the difference between arguments and parameters - <faq-argument-vs-parameter>`, the :class:`inspect.Parameter` class, the - :ref:`function` section, and :pep:`362`. - - path entry - A single location on the :term:`import path` which the :term:`path - based finder` consults to find modules for importing. - - path entry finder - A :term:`finder` returned by a callable on :data:`sys.path_hooks` - (i.e. a :term:`path entry hook`) which knows how to locate modules given - a :term:`path entry`. - - See :class:`importlib.abc.PathEntryFinder` for the methods that path entry - finders implement. - - path entry hook - A callable on the :data:`sys.path_hook` list which returns a :term:`path - entry finder` if it knows how to find modules on a specific :term:`path - entry`. - - path based finder - One of the default :term:`meta path finders <meta path finder>` which - searches an :term:`import path` for modules. - - path-like object - An object representing a file system path. A path-like object is either - a :class:`str` or :class:`bytes` object representing a path, or an object - implementing the :class:`os.PathLike` protocol. An object that supports - the :class:`os.PathLike` protocol can be converted to a :class:`str` or - :class:`bytes` file system path by calling the :func:`os.fspath` function; - :func:`os.fsdecode` and :func:`os.fsencode` can be used to guarantee a - :class:`str` or :class:`bytes` result instead, respectively. Introduced - by :pep:`519`. - - PEP - Python Enhancement Proposal. A PEP is a design document - providing information to the Python community, or describing a new - feature for Python or its processes or environment. PEPs should - provide a concise technical specification and a rationale for proposed - features. - - PEPs are intended to be the primary mechanisms for proposing major new - features, for collecting community input on an issue, and for documenting - the design decisions that have gone into Python. The PEP author is - responsible for building consensus within the community and documenting - dissenting opinions. - - See :pep:`1`. - - portion - A set of files in a single directory (possibly stored in a zip file) - that contribute to a namespace package, as defined in :pep:`420`. - - positional argument - See :term:`argument`. - - provisional API - A provisional API is one which has been deliberately excluded from - the standard library's backwards compatibility guarantees. While major - changes to such interfaces are not expected, as long as they are marked - provisional, backwards incompatible changes (up to and including removal - of the interface) may occur if deemed necessary by core developers. Such - changes will not be made gratuitously -- they will occur only if serious - fundamental flaws are uncovered that were missed prior to the inclusion - of the API. - - Even for provisional APIs, backwards incompatible changes are seen as - a "solution of last resort" - every attempt will still be made to find - a backwards compatible resolution to any identified problems. - - This process allows the standard library to continue to evolve over - time, without locking in problematic design errors for extended periods - of time. See :pep:`411` for more details. - - provisional package - See :term:`provisional API`. - - Python 3000 - Nickname for the Python 3.x release line (coined long ago when the - release of version 3 was something in the distant future.) This is also - abbreviated "Py3k". - - Pythonic - An idea or piece of code which closely follows the most common idioms - of the Python language, rather than implementing code using concepts - common to other languages. For example, a common idiom in Python is - to loop over all elements of an iterable using a :keyword:`for` - statement. Many other languages don't have this type of construct, so - people unfamiliar with Python sometimes use a numerical counter instead:: - - for i in range(len(food)): - print(food[i]) - - As opposed to the cleaner, Pythonic method:: - - for piece in food: - print(piece) - - qualified name - A dotted name showing the "path" from a module's global scope to a - class, function or method defined in that module, as defined in - :pep:`3155`. For top-level functions and classes, the qualified name - is the same as the object's name:: - - >>> class C: - ... class D: - ... def meth(self): - ... pass - ... - >>> C.__qualname__ - 'C' - >>> C.D.__qualname__ - 'C.D' - >>> C.D.meth.__qualname__ - 'C.D.meth' - - When used to refer to modules, the *fully qualified name* means the - entire dotted path to the module, including any parent packages, - e.g. ``email.mime.text``:: - - >>> import email.mime.text - >>> email.mime.text.__name__ - 'email.mime.text' - - reference count - The number of references to an object. When the reference count of an - object drops to zero, it is deallocated. Reference counting is - generally not visible to Python code, but it is a key element of the - :term:`CPython` implementation. Programmers can call the - :func:`sys.getrefcount` function to return the - reference count for a particular object. - - regular package - A traditional :term:`package`, such as a directory containing an - ``__init__.py`` file. - - See also :term:`namespace package`. - - __slots__ - A declaration inside a class that saves memory by pre-declaring space for - instance attributes and eliminating instance dictionaries. Though - popular, the technique is somewhat tricky to get right and is best - reserved for rare cases where there are large numbers of instances in a - memory-critical application. - - sequence - An :term:`iterable` which supports efficient element access using integer - indices via the :meth:`~object.__getitem__` special method and defines a - :meth:`__len__` method that returns the length of the sequence. - Some built-in sequence types are :class:`list`, :class:`str`, - :class:`tuple`, and :class:`bytes`. Note that :class:`dict` also - supports :meth:`~object.__getitem__` and :meth:`__len__`, but is considered a - mapping rather than a sequence because the lookups use arbitrary - :term:`immutable` keys rather than integers. - - The :class:`collections.abc.Sequence` abstract base class - defines a much richer interface that goes beyond just - :meth:`~object.__getitem__` and :meth:`__len__`, adding :meth:`count`, - :meth:`index`, :meth:`__contains__`, and - :meth:`__reversed__`. Types that implement this expanded - interface can be registered explicitly using - :func:`~abc.ABCMeta.register`. - - set comprehension - A compact way to process all or part of the elements in an iterable and - return a set with the results. ``results = {c for c in 'abracadabra' if - c not in 'abc'}`` generates the set of strings ``{'r', 'd'}``. See - :ref:`comprehensions`. - - single dispatch - A form of :term:`generic function` dispatch where the implementation is - chosen based on the type of a single argument. - - slice - An object usually containing a portion of a :term:`sequence`. A slice is - created using the subscript notation, ``[]`` with colons between numbers - when several are given, such as in ``variable_name[1:3:5]``. The bracket - (subscript) notation uses :class:`slice` objects internally. - - special method - .. index:: pair: special; method - - A method that is called implicitly by Python to execute a certain - operation on a type, such as addition. Such methods have names starting - and ending with double underscores. Special methods are documented in - :ref:`specialnames`. - - statement - A statement is part of a suite (a "block" of code). A statement is either - an :term:`expression` or one of several constructs with a keyword, such - as :keyword:`if`, :keyword:`while` or :keyword:`for`. - - strong reference - In Python's C API, a strong reference is a reference to an object - which is owned by the code holding the reference. The strong - reference is taken by calling :c:func:`Py_INCREF` when the - reference is created and released with :c:func:`Py_DECREF` - when the reference is deleted. - - The :c:func:`Py_NewRef` function can be used to create a strong reference - to an object. Usually, the :c:func:`Py_DECREF` function must be called on - the strong reference before exiting the scope of the strong reference, to - avoid leaking one reference. - - See also :term:`borrowed reference`. - - text encoding - A string in Python is a sequence of Unicode code points (in range - ``U+0000``--``U+10FFFF``). To store or transfer a string, it needs to be - serialized as a sequence of bytes. - - Serializing a string into a sequence of bytes is known as "encoding", and - recreating the string from the sequence of bytes is known as "decoding". - - There are a variety of different text serialization - :ref:`codecs <standard-encodings>`, which are collectively referred to as - "text encodings". - - text file - A :term:`file object` able to read and write :class:`str` objects. - Often, a text file actually accesses a byte-oriented datastream - and handles the :term:`text encoding` automatically. - Examples of text files are files opened in text mode (``'r'`` or ``'w'``), - :data:`sys.stdin`, :data:`sys.stdout`, and instances of - :class:`io.StringIO`. - - See also :term:`binary file` for a file object able to read and write - :term:`bytes-like objects <bytes-like object>`. - - triple-quoted string - A string which is bound by three instances of either a quotation mark - (") or an apostrophe ('). While they don't provide any functionality - not available with single-quoted strings, they are useful for a number - of reasons. They allow you to include unescaped single and double - quotes within a string and they can span multiple lines without the - use of the continuation character, making them especially useful when - writing docstrings. - - type - The type of a Python object determines what kind of object it is; every - object has a type. An object's type is accessible as its - :attr:`~instance.__class__` attribute or can be retrieved with - ``type(obj)``. - - type alias - A synonym for a type, created by assigning the type to an identifier. - - Type aliases are useful for simplifying :term:`type hints <type hint>`. - For example:: - - def remove_gray_shades( - colors: list[tuple[int, int, int]]) -> list[tuple[int, int, int]]: - pass - - could be made more readable like this:: - - Color = tuple[int, int, int] - - def remove_gray_shades(colors: list[Color]) -> list[Color]: - pass - - See :mod:`typing` and :pep:`484`, which describe this functionality. - - type hint - An :term:`annotation` that specifies the expected type for a variable, a class - attribute, or a function parameter or return value. - - Type hints are optional and are not enforced by Python but - they are useful to static type analysis tools, and aid IDEs with code - completion and refactoring. - - Type hints of global variables, class attributes, and functions, - but not local variables, can be accessed using - :func:`typing.get_type_hints`. - - See :mod:`typing` and :pep:`484`, which describe this functionality. - - universal newlines - A manner of interpreting text streams in which all of the following are - recognized as ending a line: the Unix end-of-line convention ``'\n'``, - the Windows convention ``'\r\n'``, and the old Macintosh convention - ``'\r'``. See :pep:`278` and :pep:`3116`, as well as - :func:`bytes.splitlines` for an additional use. - - variable annotation - An :term:`annotation` of a variable or a class attribute. - - When annotating a variable or a class attribute, assignment is optional:: - - class C: - field: 'annotation' - - Variable annotations are usually used for - :term:`type hints <type hint>`: for example this variable is expected to take - :class:`int` values:: - - count: int = 0 - - Variable annotation syntax is explained in section :ref:`annassign`. - - See :term:`function annotation`, :pep:`484` - and :pep:`526`, which describe this functionality. - Also see :ref:`annotations-howto` - for best practices on working with annotations. - - virtual environment - A cooperatively isolated runtime environment that allows Python users - and applications to install and upgrade Python distribution packages - without interfering with the behaviour of other Python applications - running on the same system. - - See also :mod:`venv`. - - virtual machine - A computer defined entirely in software. Python's virtual machine - executes the :term:`bytecode` emitted by the bytecode compiler. - - Zen of Python - Listing of Python design principles and philosophies that are helpful in - understanding and using the language. The listing can be found by typing - "``import this``" at the interactive prompt. diff --git a/Doc/howto/argparse.rst b/Doc/howto/argparse.rst deleted file mode 100644 index 06a301a120c1c7..00000000000000 --- a/Doc/howto/argparse.rst +++ /dev/null @@ -1,821 +0,0 @@ -.. _argparse-tutorial: - -***************** -Argparse Tutorial -***************** - -:author: Tshepang Lekhonkhobe - -.. currentmodule:: argparse - -This tutorial is intended to be a gentle introduction to :mod:`argparse`, the -recommended command-line parsing module in the Python standard library. - -.. note:: - - There are two other modules that fulfill the same task, namely - :mod:`getopt` (an equivalent for ``getopt()`` from the C - language) and the deprecated :mod:`optparse`. - Note also that :mod:`argparse` is based on :mod:`optparse`, - and therefore very similar in terms of usage. - - -Concepts -======== - -Let's show the sort of functionality that we are going to explore in this -introductory tutorial by making use of the :command:`ls` command: - -.. code-block:: shell-session - - $ ls - cpython devguide prog.py pypy rm-unused-function.patch - $ ls pypy - ctypes_configure demo dotviewer include lib_pypy lib-python ... - $ ls -l - total 20 - drwxr-xr-x 19 wena wena 4096 Feb 18 18:51 cpython - drwxr-xr-x 4 wena wena 4096 Feb 8 12:04 devguide - -rwxr-xr-x 1 wena wena 535 Feb 19 00:05 prog.py - drwxr-xr-x 14 wena wena 4096 Feb 7 00:59 pypy - -rw-r--r-- 1 wena wena 741 Feb 18 01:01 rm-unused-function.patch - $ ls --help - Usage: ls [OPTION]... [FILE]... - List information about the FILEs (the current directory by default). - Sort entries alphabetically if none of -cftuvSUX nor --sort is specified. - ... - -A few concepts we can learn from the four commands: - -* The :command:`ls` command is useful when run without any options at all. It defaults - to displaying the contents of the current directory. - -* If we want beyond what it provides by default, we tell it a bit more. In - this case, we want it to display a different directory, ``pypy``. - What we did is specify what is known as a positional argument. It's named so - because the program should know what to do with the value, solely based on - where it appears on the command line. This concept is more relevant - to a command like :command:`cp`, whose most basic usage is ``cp SRC DEST``. - The first position is *what you want copied,* and the second - position is *where you want it copied to*. - -* Now, say we want to change behaviour of the program. In our example, - we display more info for each file instead of just showing the file names. - The ``-l`` in that case is known as an optional argument. - -* That's a snippet of the help text. It's very useful in that you can - come across a program you have never used before, and can figure out - how it works simply by reading its help text. - - -The basics -========== - -Let us start with a very simple example which does (almost) nothing:: - - import argparse - parser = argparse.ArgumentParser() - parser.parse_args() - -Following is a result of running the code: - -.. code-block:: shell-session - - $ python3 prog.py - $ python3 prog.py --help - usage: prog.py [-h] - - options: - -h, --help show this help message and exit - $ python3 prog.py --verbose - usage: prog.py [-h] - prog.py: error: unrecognized arguments: --verbose - $ python3 prog.py foo - usage: prog.py [-h] - prog.py: error: unrecognized arguments: foo - -Here is what is happening: - -* Running the script without any options results in nothing displayed to - stdout. Not so useful. - -* The second one starts to display the usefulness of the :mod:`argparse` - module. We have done almost nothing, but already we get a nice help message. - -* The ``--help`` option, which can also be shortened to ``-h``, is the only - option we get for free (i.e. no need to specify it). Specifying anything - else results in an error. But even then, we do get a useful usage message, - also for free. - - -Introducing Positional arguments -================================ - -An example:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("echo") - args = parser.parse_args() - print(args.echo) - -And running the code: - -.. code-block:: shell-session - - $ python3 prog.py - usage: prog.py [-h] echo - prog.py: error: the following arguments are required: echo - $ python3 prog.py --help - usage: prog.py [-h] echo - - positional arguments: - echo - - options: - -h, --help show this help message and exit - $ python3 prog.py foo - foo - -Here is what's happening: - -* We've added the :meth:`~ArgumentParser.add_argument` method, which is what we use to specify - which command-line options the program is willing to accept. In this case, - I've named it ``echo`` so that it's in line with its function. - -* Calling our program now requires us to specify an option. - -* The :meth:`~ArgumentParser.parse_args` method actually returns some data from the - options specified, in this case, ``echo``. - -* The variable is some form of 'magic' that :mod:`argparse` performs for free - (i.e. no need to specify which variable that value is stored in). - You will also notice that its name matches the string argument given - to the method, ``echo``. - -Note however that, although the help display looks nice and all, it currently -is not as helpful as it can be. For example we see that we got ``echo`` as a -positional argument, but we don't know what it does, other than by guessing or -by reading the source code. So, let's make it a bit more useful:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("echo", help="echo the string you use here") - args = parser.parse_args() - print(args.echo) - -And we get: - -.. code-block:: shell-session - - $ python3 prog.py -h - usage: prog.py [-h] echo - - positional arguments: - echo echo the string you use here - - options: - -h, --help show this help message and exit - -Now, how about doing something even more useful:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", help="display a square of a given number") - args = parser.parse_args() - print(args.square**2) - -Following is a result of running the code: - -.. code-block:: shell-session - - $ python3 prog.py 4 - Traceback (most recent call last): - File "prog.py", line 5, in <module> - print(args.square**2) - TypeError: unsupported operand type(s) for ** or pow(): 'str' and 'int' - -That didn't go so well. That's because :mod:`argparse` treats the options we -give it as strings, unless we tell it otherwise. So, let's tell -:mod:`argparse` to treat that input as an integer:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", help="display a square of a given number", - type=int) - args = parser.parse_args() - print(args.square**2) - -Following is a result of running the code: - -.. code-block:: shell-session - - $ python3 prog.py 4 - 16 - $ python3 prog.py four - usage: prog.py [-h] square - prog.py: error: argument square: invalid int value: 'four' - -That went well. The program now even helpfully quits on bad illegal input -before proceeding. - - -Introducing Optional arguments -============================== - -So far we have been playing with positional arguments. Let us -have a look on how to add optional ones:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--verbosity", help="increase output verbosity") - args = parser.parse_args() - if args.verbosity: - print("verbosity turned on") - -And the output: - -.. code-block:: shell-session - - $ python3 prog.py --verbosity 1 - verbosity turned on - $ python3 prog.py - $ python3 prog.py --help - usage: prog.py [-h] [--verbosity VERBOSITY] - - options: - -h, --help show this help message and exit - --verbosity VERBOSITY - increase output verbosity - $ python3 prog.py --verbosity - usage: prog.py [-h] [--verbosity VERBOSITY] - prog.py: error: argument --verbosity: expected one argument - -Here is what is happening: - -* The program is written so as to display something when ``--verbosity`` is - specified and display nothing when not. - -* To show that the option is actually optional, there is no error when running - the program without it. Note that by default, if an optional argument isn't - used, the relevant variable, in this case ``args.verbosity``, is - given ``None`` as a value, which is the reason it fails the truth - test of the :keyword:`if` statement. - -* The help message is a bit different. - -* When using the ``--verbosity`` option, one must also specify some value, - any value. - -The above example accepts arbitrary integer values for ``--verbosity``, but for -our simple program, only two values are actually useful, ``True`` or ``False``. -Let's modify the code accordingly:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("--verbose", help="increase output verbosity", - action="store_true") - args = parser.parse_args() - if args.verbose: - print("verbosity turned on") - -And the output: - -.. code-block:: shell-session - - $ python3 prog.py --verbose - verbosity turned on - $ python3 prog.py --verbose 1 - usage: prog.py [-h] [--verbose] - prog.py: error: unrecognized arguments: 1 - $ python3 prog.py --help - usage: prog.py [-h] [--verbose] - - options: - -h, --help show this help message and exit - --verbose increase output verbosity - -Here is what is happening: - -* The option is now more of a flag than something that requires a value. - We even changed the name of the option to match that idea. - Note that we now specify a new keyword, ``action``, and give it the value - ``"store_true"``. This means that, if the option is specified, - assign the value ``True`` to ``args.verbose``. - Not specifying it implies ``False``. - -* It complains when you specify a value, in true spirit of what flags - actually are. - -* Notice the different help text. - - -Short options -------------- - -If you are familiar with command line usage, -you will notice that I haven't yet touched on the topic of short -versions of the options. It's quite simple:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("-v", "--verbose", help="increase output verbosity", - action="store_true") - args = parser.parse_args() - if args.verbose: - print("verbosity turned on") - -And here goes: - -.. code-block:: shell-session - - $ python3 prog.py -v - verbosity turned on - $ python3 prog.py --help - usage: prog.py [-h] [-v] - - options: - -h, --help show this help message and exit - -v, --verbose increase output verbosity - -Note that the new ability is also reflected in the help text. - - -Combining Positional and Optional arguments -=========================================== - -Our program keeps growing in complexity:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbose", action="store_true", - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbose: - print(f"the square of {args.square} equals {answer}") - else: - print(answer) - -And now the output: - -.. code-block:: shell-session - - $ python3 prog.py - usage: prog.py [-h] [-v] square - prog.py: error: the following arguments are required: square - $ python3 prog.py 4 - 16 - $ python3 prog.py 4 --verbose - the square of 4 equals 16 - $ python3 prog.py --verbose 4 - the square of 4 equals 16 - -* We've brought back a positional argument, hence the complaint. - -* Note that the order does not matter. - -How about we give this program of ours back the ability to have -multiple verbosity values, and actually get to use them:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", type=int, - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity == 2: - print(f"the square of {args.square} equals {answer}") - elif args.verbosity == 1: - print(f"{args.square}^2 == {answer}") - else: - print(answer) - -And the output: - -.. code-block:: shell-session - - $ python3 prog.py 4 - 16 - $ python3 prog.py 4 -v - usage: prog.py [-h] [-v VERBOSITY] square - prog.py: error: argument -v/--verbosity: expected one argument - $ python3 prog.py 4 -v 1 - 4^2 == 16 - $ python3 prog.py 4 -v 2 - the square of 4 equals 16 - $ python3 prog.py 4 -v 3 - 16 - -These all look good except the last one, which exposes a bug in our program. -Let's fix it by restricting the values the ``--verbosity`` option can accept:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", type=int, choices=[0, 1, 2], - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity == 2: - print(f"the square of {args.square} equals {answer}") - elif args.verbosity == 1: - print(f"{args.square}^2 == {answer}") - else: - print(answer) - -And the output: - -.. code-block:: shell-session - - $ python3 prog.py 4 -v 3 - usage: prog.py [-h] [-v {0,1,2}] square - prog.py: error: argument -v/--verbosity: invalid choice: 3 (choose from 0, 1, 2) - $ python3 prog.py 4 -h - usage: prog.py [-h] [-v {0,1,2}] square - - positional arguments: - square display a square of a given number - - options: - -h, --help show this help message and exit - -v {0,1,2}, --verbosity {0,1,2} - increase output verbosity - -Note that the change also reflects both in the error message as well as the -help string. - -Now, let's use a different approach of playing with verbosity, which is pretty -common. It also matches the way the CPython executable handles its own -verbosity argument (check the output of ``python --help``):: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display the square of a given number") - parser.add_argument("-v", "--verbosity", action="count", - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity == 2: - print(f"the square of {args.square} equals {answer}") - elif args.verbosity == 1: - print(f"{args.square}^2 == {answer}") - else: - print(answer) - -We have introduced another action, "count", -to count the number of occurrences of specific options. - - -.. code-block:: shell-session - - $ python3 prog.py 4 - 16 - $ python3 prog.py 4 -v - 4^2 == 16 - $ python3 prog.py 4 -vv - the square of 4 equals 16 - $ python3 prog.py 4 --verbosity --verbosity - the square of 4 equals 16 - $ python3 prog.py 4 -v 1 - usage: prog.py [-h] [-v] square - prog.py: error: unrecognized arguments: 1 - $ python3 prog.py 4 -h - usage: prog.py [-h] [-v] square - - positional arguments: - square display a square of a given number - - options: - -h, --help show this help message and exit - -v, --verbosity increase output verbosity - $ python3 prog.py 4 -vvv - 16 - -* Yes, it's now more of a flag (similar to ``action="store_true"``) in the - previous version of our script. That should explain the complaint. - -* It also behaves similar to "store_true" action. - -* Now here's a demonstration of what the "count" action gives. You've probably - seen this sort of usage before. - -* And if you don't specify the ``-v`` flag, that flag is considered to have - ``None`` value. - -* As should be expected, specifying the long form of the flag, we should get - the same output. - -* Sadly, our help output isn't very informative on the new ability our script - has acquired, but that can always be fixed by improving the documentation for - our script (e.g. via the ``help`` keyword argument). - -* That last output exposes a bug in our program. - - -Let's fix:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", action="count", - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - - # bugfix: replace == with >= - if args.verbosity >= 2: - print(f"the square of {args.square} equals {answer}") - elif args.verbosity >= 1: - print(f"{args.square}^2 == {answer}") - else: - print(answer) - -And this is what it gives: - -.. code-block:: shell-session - - $ python3 prog.py 4 -vvv - the square of 4 equals 16 - $ python3 prog.py 4 -vvvv - the square of 4 equals 16 - $ python3 prog.py 4 - Traceback (most recent call last): - File "prog.py", line 11, in <module> - if args.verbosity >= 2: - TypeError: '>=' not supported between instances of 'NoneType' and 'int' - - -* First output went well, and fixes the bug we had before. - That is, we want any value >= 2 to be as verbose as possible. - -* Third output not so good. - -Let's fix that bug:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("square", type=int, - help="display a square of a given number") - parser.add_argument("-v", "--verbosity", action="count", default=0, - help="increase output verbosity") - args = parser.parse_args() - answer = args.square**2 - if args.verbosity >= 2: - print(f"the square of {args.square} equals {answer}") - elif args.verbosity >= 1: - print(f"{args.square}^2 == {answer}") - else: - print(answer) - -We've just introduced yet another keyword, ``default``. -We've set it to ``0`` in order to make it comparable to the other int values. -Remember that by default, -if an optional argument isn't specified, -it gets the ``None`` value, and that cannot be compared to an int value -(hence the :exc:`TypeError` exception). - -And: - -.. code-block:: shell-session - - $ python3 prog.py 4 - 16 - -You can go quite far just with what we've learned so far, -and we have only scratched the surface. -The :mod:`argparse` module is very powerful, -and we'll explore a bit more of it before we end this tutorial. - - -Getting a little more advanced -============================== - -What if we wanted to expand our tiny program to perform other powers, -not just squares:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - parser.add_argument("-v", "--verbosity", action="count", default=0) - args = parser.parse_args() - answer = args.x**args.y - if args.verbosity >= 2: - print(f"{args.x} to the power {args.y} equals {answer}") - elif args.verbosity >= 1: - print(f"{args.x}^{args.y} == {answer}") - else: - print(answer) - -Output: - -.. code-block:: shell-session - - $ python3 prog.py - usage: prog.py [-h] [-v] x y - prog.py: error: the following arguments are required: x, y - $ python3 prog.py -h - usage: prog.py [-h] [-v] x y - - positional arguments: - x the base - y the exponent - - options: - -h, --help show this help message and exit - -v, --verbosity - $ python3 prog.py 4 2 -v - 4^2 == 16 - - -Notice that so far we've been using verbosity level to *change* the text -that gets displayed. The following example instead uses verbosity level -to display *more* text instead:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - parser.add_argument("-v", "--verbosity", action="count", default=0) - args = parser.parse_args() - answer = args.x**args.y - if args.verbosity >= 2: - print(f"Running '{__file__}'") - if args.verbosity >= 1: - print(f"{args.x}^{args.y} == ", end="") - print(answer) - -Output: - -.. code-block:: shell-session - - $ python3 prog.py 4 2 - 16 - $ python3 prog.py 4 2 -v - 4^2 == 16 - $ python3 prog.py 4 2 -vv - Running 'prog.py' - 4^2 == 16 - - -Conflicting options -------------------- - -So far, we have been working with two methods of an -:class:`argparse.ArgumentParser` instance. Let's introduce a third one, -:meth:`~ArgumentParser.add_mutually_exclusive_group`. It allows for us to specify options that -conflict with each other. Let's also change the rest of the program so that -the new functionality makes more sense: -we'll introduce the ``--quiet`` option, -which will be the opposite of the ``--verbose`` one:: - - import argparse - - parser = argparse.ArgumentParser() - group = parser.add_mutually_exclusive_group() - group.add_argument("-v", "--verbose", action="store_true") - group.add_argument("-q", "--quiet", action="store_true") - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - args = parser.parse_args() - answer = args.x**args.y - - if args.quiet: - print(answer) - elif args.verbose: - print(f"{args.x} to the power {args.y} equals {answer}") - else: - print(f"{args.x}^{args.y} == {answer}") - -Our program is now simpler, and we've lost some functionality for the sake of -demonstration. Anyways, here's the output: - -.. code-block:: shell-session - - $ python3 prog.py 4 2 - 4^2 == 16 - $ python3 prog.py 4 2 -q - 16 - $ python3 prog.py 4 2 -v - 4 to the power 2 equals 16 - $ python3 prog.py 4 2 -vq - usage: prog.py [-h] [-v | -q] x y - prog.py: error: argument -q/--quiet: not allowed with argument -v/--verbose - $ python3 prog.py 4 2 -v --quiet - usage: prog.py [-h] [-v | -q] x y - prog.py: error: argument -q/--quiet: not allowed with argument -v/--verbose - -That should be easy to follow. I've added that last output so you can see the -sort of flexibility you get, i.e. mixing long form options with short form -ones. - -Before we conclude, you probably want to tell your users the main purpose of -your program, just in case they don't know:: - - import argparse - - parser = argparse.ArgumentParser(description="calculate X to the power of Y") - group = parser.add_mutually_exclusive_group() - group.add_argument("-v", "--verbose", action="store_true") - group.add_argument("-q", "--quiet", action="store_true") - parser.add_argument("x", type=int, help="the base") - parser.add_argument("y", type=int, help="the exponent") - args = parser.parse_args() - answer = args.x**args.y - - if args.quiet: - print(answer) - elif args.verbose: - print(f"{args.x} to the power {args.y} equals {answer}") - else: - print(f"{args.x}^{args.y} == {answer}") - -Note that slight difference in the usage text. Note the ``[-v | -q]``, -which tells us that we can either use ``-v`` or ``-q``, -but not both at the same time: - -.. code-block:: shell-session - - $ python3 prog.py --help - usage: prog.py [-h] [-v | -q] x y - - calculate X to the power of Y - - positional arguments: - x the base - y the exponent - - options: - -h, --help show this help message and exit - -v, --verbose - -q, --quiet - - -How to translate the argparse output -==================================== - -The output of the :mod:`argparse` module such as its help text and error -messages are all made translatable using the :mod:`gettext` module. This -allows applications to easily localize messages produced by -:mod:`argparse`. See also :ref:`i18n-howto`. - -For instance, in this :mod:`argparse` output: - -.. code-block:: shell-session - - $ python prog.py --help - usage: prog.py [-h] [-v | -q] x y - - calculate X to the power of Y - - positional arguments: - x the base - y the exponent - - options: - -h, --help show this help message and exit - -v, --verbose - -q, --quiet - -The strings ``usage:``, ``positional arguments:``, ``options:`` and -``show this help message and exit`` are all translatable. - -In order to translate these strings, they must first be extracted -into a ``.po`` file. For example, using `Babel <https://babel.pocoo.org/>`__, -run this command: - -.. code-block:: shell-session - - $ pybabel extract -o messages.po /usr/lib/python3.12/argparse.py - -This command will extract all translatable strings from the :mod:`argparse` -module and output them into a file named ``messages.po``. This command assumes -that your Python installation is in ``/usr/lib``. - -You can find out the location of the :mod:`argparse` module on your system -using this script:: - - import argparse - print(argparse.__file__) - -Once the messages in the ``.po`` file are translated and the translations are -installed using :mod:`gettext`, :mod:`argparse` will be able to display the -translated messages. - -To translate your own strings in the :mod:`argparse` output, use :mod:`gettext`. - -Conclusion -========== - -The :mod:`argparse` module offers a lot more than shown here. -Its docs are quite detailed and thorough, and full of examples. -Having gone through this tutorial, you should easily digest them -without feeling overwhelmed. diff --git a/Doc/howto/clinic.rst b/Doc/howto/clinic.rst deleted file mode 100644 index 060977246149cf..00000000000000 --- a/Doc/howto/clinic.rst +++ /dev/null @@ -1,14 +0,0 @@ -:orphan: - -.. This page is retained solely for existing links to /howto/clinic.html. - Direct readers to the devguide. - -********************** -Argument Clinic How-To -********************** - - -.. note:: - - The Argument Clinic How-TO has been moved to the `Python Developer's Guide - <https://devguide.python.org/development-tools/clinic/>`__. diff --git a/Doc/howto/cporting.rst b/Doc/howto/cporting.rst deleted file mode 100644 index 7773620b40b973..00000000000000 --- a/Doc/howto/cporting.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. highlight:: c - -.. _cporting-howto: - -************************************* -Porting Extension Modules to Python 3 -************************************* - -We recommend the following resources for porting extension modules to Python 3: - -* The `Migrating C extensions`_ chapter from - *Supporting Python 3: An in-depth guide*, a book on moving from Python 2 - to Python 3 in general, guides the reader through porting an extension - module. -* The `Porting guide`_ from the *py3c* project provides opinionated - suggestions with supporting code. -* The `Cython`_ and `CFFI`_ libraries offer abstractions over - Python's C API. - Extensions generally need to be re-written to use one of them, - but the library then handles differences between various Python - versions and implementations. - -.. _Migrating C extensions: http://python3porting.com/cextensions.html -.. _Porting guide: https://py3c.readthedocs.io/en/latest/guide.html -.. _Cython: https://cython.org/ -.. _CFFI: https://cffi.readthedocs.io/en/latest/ diff --git a/Doc/howto/curses.rst b/Doc/howto/curses.rst deleted file mode 100644 index 4828e2fa29bd24..00000000000000 --- a/Doc/howto/curses.rst +++ /dev/null @@ -1,549 +0,0 @@ -.. _curses-howto: - -********************************** - Curses Programming with Python -********************************** - -.. currentmodule:: curses - -:Author: A.M. Kuchling, Eric S. Raymond -:Release: 2.04 - - -.. topic:: Abstract - - This document describes how to use the :mod:`curses` extension - module to control text-mode displays. - - -What is curses? -=============== - -The curses library supplies a terminal-independent screen-painting and -keyboard-handling facility for text-based terminals; such terminals -include VT100s, the Linux console, and the simulated terminal provided -by various programs. Display terminals support various control codes -to perform common operations such as moving the cursor, scrolling the -screen, and erasing areas. Different terminals use widely differing -codes, and often have their own minor quirks. - -In a world of graphical displays, one might ask "why bother"? It's -true that character-cell display terminals are an obsolete technology, -but there are niches in which being able to do fancy things with them -are still valuable. One niche is on small-footprint or embedded -Unixes that don't run an X server. Another is tools such as OS -installers and kernel configurators that may have to run before any -graphical support is available. - -The curses library provides fairly basic functionality, providing the -programmer with an abstraction of a display containing multiple -non-overlapping windows of text. The contents of a window can be -changed in various ways---adding text, erasing it, changing its -appearance---and the curses library will figure out what control codes -need to be sent to the terminal to produce the right output. curses -doesn't provide many user-interface concepts such as buttons, checkboxes, -or dialogs; if you need such features, consider a user interface library such as -`Urwid <https://pypi.org/project/urwid/>`_. - -The curses library was originally written for BSD Unix; the later System V -versions of Unix from AT&T added many enhancements and new functions. BSD curses -is no longer maintained, having been replaced by ncurses, which is an -open-source implementation of the AT&T interface. If you're using an -open-source Unix such as Linux or FreeBSD, your system almost certainly uses -ncurses. Since most current commercial Unix versions are based on System V -code, all the functions described here will probably be available. The older -versions of curses carried by some proprietary Unixes may not support -everything, though. - -The Windows version of Python doesn't include the :mod:`curses` -module. A ported version called `UniCurses -<https://pypi.org/project/UniCurses>`_ is available. - - -The Python curses module ------------------------- - -The Python module is a fairly simple wrapper over the C functions provided by -curses; if you're already familiar with curses programming in C, it's really -easy to transfer that knowledge to Python. The biggest difference is that the -Python interface makes things simpler by merging different C functions such as -:c:func:`!addstr`, :c:func:`!mvaddstr`, and :c:func:`!mvwaddstr` into a single -:meth:`~curses.window.addstr` method. You'll see this covered in more -detail later. - -This HOWTO is an introduction to writing text-mode programs with curses -and Python. It doesn't attempt to be a complete guide to the curses API; for -that, see the Python library guide's section on ncurses, and the C manual pages -for ncurses. It will, however, give you the basic ideas. - - -Starting and ending a curses application -======================================== - -Before doing anything, curses must be initialized. This is done by -calling the :func:`~curses.initscr` function, which will determine the -terminal type, send any required setup codes to the terminal, and -create various internal data structures. If successful, -:func:`!initscr` returns a window object representing the entire -screen; this is usually called ``stdscr`` after the name of the -corresponding C variable. :: - - import curses - stdscr = curses.initscr() - -Usually curses applications turn off automatic echoing of keys to the -screen, in order to be able to read keys and only display them under -certain circumstances. This requires calling the -:func:`~curses.noecho` function. :: - - curses.noecho() - -Applications will also commonly need to react to keys instantly, -without requiring the Enter key to be pressed; this is called cbreak -mode, as opposed to the usual buffered input mode. :: - - curses.cbreak() - -Terminals usually return special keys, such as the cursor keys or navigation -keys such as Page Up and Home, as a multibyte escape sequence. While you could -write your application to expect such sequences and process them accordingly, -curses can do it for you, returning a special value such as -:const:`curses.KEY_LEFT`. To get curses to do the job, you'll have to enable -keypad mode. :: - - stdscr.keypad(True) - -Terminating a curses application is much easier than starting one. You'll need -to call:: - - curses.nocbreak() - stdscr.keypad(False) - curses.echo() - -to reverse the curses-friendly terminal settings. Then call the -:func:`~curses.endwin` function to restore the terminal to its original -operating mode. :: - - curses.endwin() - -A common problem when debugging a curses application is to get your terminal -messed up when the application dies without restoring the terminal to its -previous state. In Python this commonly happens when your code is buggy and -raises an uncaught exception. Keys are no longer echoed to the screen when -you type them, for example, which makes using the shell difficult. - -In Python you can avoid these complications and make debugging much easier by -importing the :func:`curses.wrapper` function and using it like this:: - - from curses import wrapper - - def main(stdscr): - # Clear screen - stdscr.clear() - - # This raises ZeroDivisionError when i == 10. - for i in range(0, 11): - v = i-10 - stdscr.addstr(i, 0, '10 divided by {} is {}'.format(v, 10/v)) - - stdscr.refresh() - stdscr.getkey() - - wrapper(main) - -The :func:`~curses.wrapper` function takes a callable object and does the -initializations described above, also initializing colors if color -support is present. :func:`!wrapper` then runs your provided callable. -Once the callable returns, :func:`!wrapper` will restore the original -state of the terminal. The callable is called inside a -:keyword:`try`...\ :keyword:`except` that catches exceptions, restores -the state of the terminal, and then re-raises the exception. Therefore -your terminal won't be left in a funny state on exception and you'll be -able to read the exception's message and traceback. - - -Windows and Pads -================ - -Windows are the basic abstraction in curses. A window object represents a -rectangular area of the screen, and supports methods to display text, -erase it, allow the user to input strings, and so forth. - -The ``stdscr`` object returned by the :func:`~curses.initscr` function is a -window object that covers the entire screen. Many programs may need -only this single window, but you might wish to divide the screen into -smaller windows, in order to redraw or clear them separately. The -:func:`~curses.newwin` function creates a new window of a given size, -returning the new window object. :: - - begin_x = 20; begin_y = 7 - height = 5; width = 40 - win = curses.newwin(height, width, begin_y, begin_x) - -Note that the coordinate system used in curses is unusual. -Coordinates are always passed in the order *y,x*, and the top-left -corner of a window is coordinate (0,0). This breaks the normal -convention for handling coordinates where the *x* coordinate comes -first. This is an unfortunate difference from most other computer -applications, but it's been part of curses since it was first written, -and it's too late to change things now. - -Your application can determine the size of the screen by using the -:data:`curses.LINES` and :data:`curses.COLS` variables to obtain the *y* and -*x* sizes. Legal coordinates will then extend from ``(0,0)`` to -``(curses.LINES - 1, curses.COLS - 1)``. - -When you call a method to display or erase text, the effect doesn't -immediately show up on the display. Instead you must call the -:meth:`~curses.window.refresh` method of window objects to update the -screen. - -This is because curses was originally written with slow 300-baud -terminal connections in mind; with these terminals, minimizing the -time required to redraw the screen was very important. Instead curses -accumulates changes to the screen and displays them in the most -efficient manner when you call :meth:`!refresh`. For example, if your -program displays some text in a window and then clears the window, -there's no need to send the original text because they're never -visible. - -In practice, explicitly telling curses to redraw a window doesn't -really complicate programming with curses much. Most programs go into a flurry -of activity, and then pause waiting for a keypress or some other action on the -part of the user. All you have to do is to be sure that the screen has been -redrawn before pausing to wait for user input, by first calling -:meth:`!stdscr.refresh` or the :meth:`!refresh` method of some other relevant -window. - -A pad is a special case of a window; it can be larger than the actual display -screen, and only a portion of the pad displayed at a time. Creating a pad -requires the pad's height and width, while refreshing a pad requires giving the -coordinates of the on-screen area where a subsection of the pad will be -displayed. :: - - pad = curses.newpad(100, 100) - # These loops fill the pad with letters; addch() is - # explained in the next section - for y in range(0, 99): - for x in range(0, 99): - pad.addch(y,x, ord('a') + (x*x+y*y) % 26) - - # Displays a section of the pad in the middle of the screen. - # (0,0) : coordinate of upper-left corner of pad area to display. - # (5,5) : coordinate of upper-left corner of window area to be filled - # with pad content. - # (20, 75) : coordinate of lower-right corner of window area to be - # : filled with pad content. - pad.refresh( 0,0, 5,5, 20,75) - -The :meth:`!refresh` call displays a section of the pad in the rectangle -extending from coordinate (5,5) to coordinate (20,75) on the screen; the upper -left corner of the displayed section is coordinate (0,0) on the pad. Beyond -that difference, pads are exactly like ordinary windows and support the same -methods. - -If you have multiple windows and pads on screen there is a more -efficient way to update the screen and prevent annoying screen flicker -as each part of the screen gets updated. :meth:`!refresh` actually -does two things: - -1) Calls the :meth:`~curses.window.noutrefresh` method of each window - to update an underlying data structure representing the desired - state of the screen. -2) Calls the function :func:`~curses.doupdate` function to change the - physical screen to match the desired state recorded in the data structure. - -Instead you can call :meth:`!noutrefresh` on a number of windows to -update the data structure, and then call :func:`!doupdate` to update -the screen. - - -Displaying Text -=============== - -From a C programmer's point of view, curses may sometimes look like a -twisty maze of functions, all subtly different. For example, -:c:func:`!addstr` displays a string at the current cursor location in -the ``stdscr`` window, while :c:func:`!mvaddstr` moves to a given y,x -coordinate first before displaying the string. :c:func:`!waddstr` is just -like :c:func:`!addstr`, but allows specifying a window to use instead of -using ``stdscr`` by default. :c:func:`!mvwaddstr` allows specifying both -a window and a coordinate. - -Fortunately the Python interface hides all these details. ``stdscr`` -is a window object like any other, and methods such as -:meth:`~curses.window.addstr` accept multiple argument forms. Usually there -are four different forms. - -+---------------------------------+-----------------------------------------------+ -| Form | Description | -+=================================+===============================================+ -| *str* or *ch* | Display the string *str* or character *ch* at | -| | the current position | -+---------------------------------+-----------------------------------------------+ -| *str* or *ch*, *attr* | Display the string *str* or character *ch*, | -| | using attribute *attr* at the current | -| | position | -+---------------------------------+-----------------------------------------------+ -| *y*, *x*, *str* or *ch* | Move to position *y,x* within the window, and | -| | display *str* or *ch* | -+---------------------------------+-----------------------------------------------+ -| *y*, *x*, *str* or *ch*, *attr* | Move to position *y,x* within the window, and | -| | display *str* or *ch*, using attribute *attr* | -+---------------------------------+-----------------------------------------------+ - -Attributes allow displaying text in highlighted forms such as boldface, -underline, reverse code, or in color. They'll be explained in more detail in -the next subsection. - - -The :meth:`~curses.window.addstr` method takes a Python string or -bytestring as the value to be displayed. The contents of bytestrings -are sent to the terminal as-is. Strings are encoded to bytes using -the value of the window's :attr:`~window.encoding` attribute; this defaults to -the default system encoding as returned by :func:`locale.getencoding`. - -The :meth:`~curses.window.addch` methods take a character, which can be -either a string of length 1, a bytestring of length 1, or an integer. - -Constants are provided for extension characters; these constants are -integers greater than 255. For example, :const:`ACS_PLMINUS` is a +/- -symbol, and :const:`ACS_ULCORNER` is the upper left corner of a box -(handy for drawing borders). You can also use the appropriate Unicode -character. - -Windows remember where the cursor was left after the last operation, so if you -leave out the *y,x* coordinates, the string or character will be displayed -wherever the last operation left off. You can also move the cursor with the -``move(y,x)`` method. Because some terminals always display a flashing cursor, -you may want to ensure that the cursor is positioned in some location where it -won't be distracting; it can be confusing to have the cursor blinking at some -apparently random location. - -If your application doesn't need a blinking cursor at all, you can -call ``curs_set(False)`` to make it invisible. For compatibility -with older curses versions, there's a ``leaveok(bool)`` function -that's a synonym for :func:`~curses.curs_set`. When *bool* is true, the -curses library will attempt to suppress the flashing cursor, and you -won't need to worry about leaving it in odd locations. - - -Attributes and Color --------------------- - -Characters can be displayed in different ways. Status lines in a text-based -application are commonly shown in reverse video, or a text viewer may need to -highlight certain words. curses supports this by allowing you to specify an -attribute for each cell on the screen. - -An attribute is an integer, each bit representing a different -attribute. You can try to display text with multiple attribute bits -set, but curses doesn't guarantee that all the possible combinations -are available, or that they're all visually distinct. That depends on -the ability of the terminal being used, so it's safest to stick to the -most commonly available attributes, listed here. - -+----------------------+--------------------------------------+ -| Attribute | Description | -+======================+======================================+ -| :const:`A_BLINK` | Blinking text | -+----------------------+--------------------------------------+ -| :const:`A_BOLD` | Extra bright or bold text | -+----------------------+--------------------------------------+ -| :const:`A_DIM` | Half bright text | -+----------------------+--------------------------------------+ -| :const:`A_REVERSE` | Reverse-video text | -+----------------------+--------------------------------------+ -| :const:`A_STANDOUT` | The best highlighting mode available | -+----------------------+--------------------------------------+ -| :const:`A_UNDERLINE` | Underlined text | -+----------------------+--------------------------------------+ - -So, to display a reverse-video status line on the top line of the screen, you -could code:: - - stdscr.addstr(0, 0, "Current mode: Typing mode", - curses.A_REVERSE) - stdscr.refresh() - -The curses library also supports color on those terminals that provide it. The -most common such terminal is probably the Linux console, followed by color -xterms. - -To use color, you must call the :func:`~curses.start_color` function soon -after calling :func:`~curses.initscr`, to initialize the default color set -(the :func:`curses.wrapper` function does this automatically). Once that's -done, the :func:`~curses.has_colors` function returns TRUE if the terminal -in use can -actually display color. (Note: curses uses the American spelling 'color', -instead of the Canadian/British spelling 'colour'. If you're used to the -British spelling, you'll have to resign yourself to misspelling it for the sake -of these functions.) - -The curses library maintains a finite number of color pairs, containing a -foreground (or text) color and a background color. You can get the attribute -value corresponding to a color pair with the :func:`~curses.color_pair` -function; this can be bitwise-OR'ed with other attributes such as -:const:`A_REVERSE`, but again, such combinations are not guaranteed to work -on all terminals. - -An example, which displays a line of text using color pair 1:: - - stdscr.addstr("Pretty text", curses.color_pair(1)) - stdscr.refresh() - -As I said before, a color pair consists of a foreground and background color. -The ``init_pair(n, f, b)`` function changes the definition of color pair *n*, to -foreground color f and background color b. Color pair 0 is hard-wired to white -on black, and cannot be changed. - -Colors are numbered, and :func:`start_color` initializes 8 basic -colors when it activates color mode. They are: 0:black, 1:red, -2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The :mod:`curses` -module defines named constants for each of these colors: -:const:`curses.COLOR_BLACK`, :const:`curses.COLOR_RED`, and so forth. - -Let's put all this together. To change color 1 to red text on a white -background, you would call:: - - curses.init_pair(1, curses.COLOR_RED, curses.COLOR_WHITE) - -When you change a color pair, any text already displayed using that color pair -will change to the new colors. You can also display new text in this color -with:: - - stdscr.addstr(0,0, "RED ALERT!", curses.color_pair(1)) - -Very fancy terminals can change the definitions of the actual colors to a given -RGB value. This lets you change color 1, which is usually red, to purple or -blue or any other color you like. Unfortunately, the Linux console doesn't -support this, so I'm unable to try it out, and can't provide any examples. You -can check if your terminal can do this by calling -:func:`~curses.can_change_color`, which returns ``True`` if the capability is -there. If you're lucky enough to have such a talented terminal, consult your -system's man pages for more information. - - -User Input -========== - -The C curses library offers only very simple input mechanisms. Python's -:mod:`curses` module adds a basic text-input widget. (Other libraries -such as `Urwid <https://pypi.org/project/urwid/>`_ have more extensive -collections of widgets.) - -There are two methods for getting input from a window: - -* :meth:`~curses.window.getch` refreshes the screen and then waits for - the user to hit a key, displaying the key if :func:`~curses.echo` has been - called earlier. You can optionally specify a coordinate to which - the cursor should be moved before pausing. - -* :meth:`~curses.window.getkey` does the same thing but converts the - integer to a string. Individual characters are returned as - 1-character strings, and special keys such as function keys return - longer strings containing a key name such as ``KEY_UP`` or ``^G``. - -It's possible to not wait for the user using the -:meth:`~curses.window.nodelay` window method. After ``nodelay(True)``, -:meth:`!getch` and :meth:`!getkey` for the window become -non-blocking. To signal that no input is ready, :meth:`!getch` returns -``curses.ERR`` (a value of -1) and :meth:`!getkey` raises an exception. -There's also a :func:`~curses.halfdelay` function, which can be used to (in -effect) set a timer on each :meth:`!getch`; if no input becomes -available within a specified delay (measured in tenths of a second), -curses raises an exception. - -The :meth:`!getch` method returns an integer; if it's between 0 and 255, it -represents the ASCII code of the key pressed. Values greater than 255 are -special keys such as Page Up, Home, or the cursor keys. You can compare the -value returned to constants such as :const:`curses.KEY_PPAGE`, -:const:`curses.KEY_HOME`, or :const:`curses.KEY_LEFT`. The main loop of -your program may look something like this:: - - while True: - c = stdscr.getch() - if c == ord('p'): - PrintDocument() - elif c == ord('q'): - break # Exit the while loop - elif c == curses.KEY_HOME: - x = y = 0 - -The :mod:`curses.ascii` module supplies ASCII class membership functions that -take either integer or 1-character string arguments; these may be useful in -writing more readable tests for such loops. It also supplies -conversion functions that take either integer or 1-character-string arguments -and return the same type. For example, :func:`curses.ascii.ctrl` returns the -control character corresponding to its argument. - -There's also a method to retrieve an entire string, -:meth:`~curses.window.getstr`. It isn't used very often, because its -functionality is quite limited; the only editing keys available are -the backspace key and the Enter key, which terminates the string. It -can optionally be limited to a fixed number of characters. :: - - curses.echo() # Enable echoing of characters - - # Get a 15-character string, with the cursor on the top line - s = stdscr.getstr(0,0, 15) - -The :mod:`curses.textpad` module supplies a text box that supports an -Emacs-like set of keybindings. Various methods of the -:class:`~curses.textpad.Textbox` class support editing with input -validation and gathering the edit results either with or without -trailing spaces. Here's an example:: - - import curses - from curses.textpad import Textbox, rectangle - - def main(stdscr): - stdscr.addstr(0, 0, "Enter IM message: (hit Ctrl-G to send)") - - editwin = curses.newwin(5,30, 2,1) - rectangle(stdscr, 1,0, 1+5+1, 1+30+1) - stdscr.refresh() - - box = Textbox(editwin) - - # Let the user edit until Ctrl-G is struck. - box.edit() - - # Get resulting contents - message = box.gather() - -See the library documentation on :mod:`curses.textpad` for more details. - - -For More Information -==================== - -This HOWTO doesn't cover some advanced topics, such as reading the -contents of the screen or capturing mouse events from an xterm -instance, but the Python library page for the :mod:`curses` module is now -reasonably complete. You should browse it next. - -If you're in doubt about the detailed behavior of the curses -functions, consult the manual pages for your curses implementation, -whether it's ncurses or a proprietary Unix vendor's. The manual pages -will document any quirks, and provide complete lists of all the -functions, attributes, and :ref:`ACS_\* <curses-acs-codes>` characters available to -you. - -Because the curses API is so large, some functions aren't supported in -the Python interface. Often this isn't because they're difficult to -implement, but because no one has needed them yet. Also, Python -doesn't yet support the menu library associated with ncurses. -Patches adding support for these would be welcome; see -`the Python Developer's Guide <https://devguide.python.org/>`_ to -learn more about submitting patches to Python. - -* `Writing Programs with NCURSES <https://invisible-island.net/ncurses/ncurses-intro.html>`_: - a lengthy tutorial for C programmers. -* `The ncurses man page <https://linux.die.net/man/3/ncurses>`_ -* `The ncurses FAQ <https://invisible-island.net/ncurses/ncurses.faq.html>`_ -* `"Use curses... don't swear" <https://www.youtube.com/watch?v=eN1eZtjLEnU>`_: - video of a PyCon 2013 talk on controlling terminals using curses or Urwid. -* `"Console Applications with Urwid" <https://pyvideo.org/video/1568/console-applications-with-urwid>`_: - video of a PyCon CA 2012 talk demonstrating some applications written using - Urwid. diff --git a/Doc/howto/descriptor.rst b/Doc/howto/descriptor.rst deleted file mode 100644 index 0a7263614670a1..00000000000000 --- a/Doc/howto/descriptor.rst +++ /dev/null @@ -1,1770 +0,0 @@ -.. _descriptorhowto: - -====================== -Descriptor HowTo Guide -====================== - -:Author: Raymond Hettinger -:Contact: <python at rcn dot com> - -.. Contents:: - - -:term:`Descriptors <descriptor>` let objects customize attribute lookup, -storage, and deletion. - -This guide has four major sections: - -1) The "primer" gives a basic overview, moving gently from simple examples, - adding one feature at a time. Start here if you're new to descriptors. - -2) The second section shows a complete, practical descriptor example. If you - already know the basics, start there. - -3) The third section provides a more technical tutorial that goes into the - detailed mechanics of how descriptors work. Most people don't need this - level of detail. - -4) The last section has pure Python equivalents for built-in descriptors that - are written in C. Read this if you're curious about how functions turn - into bound methods or about the implementation of common tools like - :func:`classmethod`, :func:`staticmethod`, :func:`property`, and - :term:`__slots__`. - - -Primer -^^^^^^ - -In this primer, we start with the most basic possible example and then we'll -add new capabilities one by one. - - -Simple example: A descriptor that returns a constant ----------------------------------------------------- - -The :class:`Ten` class is a descriptor whose :meth:`__get__` method always -returns the constant ``10``: - -.. testcode:: - - class Ten: - def __get__(self, obj, objtype=None): - return 10 - -To use the descriptor, it must be stored as a class variable in another class: - -.. testcode:: - - class A: - x = 5 # Regular class attribute - y = Ten() # Descriptor instance - -An interactive session shows the difference between normal attribute lookup -and descriptor lookup: - -.. doctest:: - - >>> a = A() # Make an instance of class A - >>> a.x # Normal attribute lookup - 5 - >>> a.y # Descriptor lookup - 10 - -In the ``a.x`` attribute lookup, the dot operator finds ``'x': 5`` -in the class dictionary. In the ``a.y`` lookup, the dot operator -finds a descriptor instance, recognized by its ``__get__`` method. -Calling that method returns ``10``. - -Note that the value ``10`` is not stored in either the class dictionary or the -instance dictionary. Instead, the value ``10`` is computed on demand. - -This example shows how a simple descriptor works, but it isn't very useful. -For retrieving constants, normal attribute lookup would be better. - -In the next section, we'll create something more useful, a dynamic lookup. - - -Dynamic lookups ---------------- - -Interesting descriptors typically run computations instead of returning -constants: - -.. testcode:: - - import os - - class DirectorySize: - - def __get__(self, obj, objtype=None): - return len(os.listdir(obj.dirname)) - - class Directory: - - size = DirectorySize() # Descriptor instance - - def __init__(self, dirname): - self.dirname = dirname # Regular instance attribute - -An interactive session shows that the lookup is dynamic — it computes -different, updated answers each time:: - - >>> s = Directory('songs') - >>> g = Directory('games') - >>> s.size # The songs directory has twenty files - 20 - >>> g.size # The games directory has three files - 3 - >>> os.remove('games/chess') # Delete a game - >>> g.size # File count is automatically updated - 2 - -Besides showing how descriptors can run computations, this example also -reveals the purpose of the parameters to :meth:`__get__`. The *self* -parameter is *size*, an instance of *DirectorySize*. The *obj* parameter is -either *g* or *s*, an instance of *Directory*. It is the *obj* parameter that -lets the :meth:`__get__` method learn the target directory. The *objtype* -parameter is the class *Directory*. - - -Managed attributes ------------------- - -A popular use for descriptors is managing access to instance data. The -descriptor is assigned to a public attribute in the class dictionary while the -actual data is stored as a private attribute in the instance dictionary. The -descriptor's :meth:`__get__` and :meth:`__set__` methods are triggered when -the public attribute is accessed. - -In the following example, *age* is the public attribute and *_age* is the -private attribute. When the public attribute is accessed, the descriptor logs -the lookup or update: - -.. testcode:: - - import logging - - logging.basicConfig(level=logging.INFO) - - class LoggedAgeAccess: - - def __get__(self, obj, objtype=None): - value = obj._age - logging.info('Accessing %r giving %r', 'age', value) - return value - - def __set__(self, obj, value): - logging.info('Updating %r to %r', 'age', value) - obj._age = value - - class Person: - - age = LoggedAgeAccess() # Descriptor instance - - def __init__(self, name, age): - self.name = name # Regular instance attribute - self.age = age # Calls __set__() - - def birthday(self): - self.age += 1 # Calls both __get__() and __set__() - - -An interactive session shows that all access to the managed attribute *age* is -logged, but that the regular attribute *name* is not logged: - -.. testcode:: - :hide: - - import logging, sys - logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) - -.. doctest:: - - >>> mary = Person('Mary M', 30) # The initial age update is logged - INFO:root:Updating 'age' to 30 - >>> dave = Person('David D', 40) - INFO:root:Updating 'age' to 40 - - >>> vars(mary) # The actual data is in a private attribute - {'name': 'Mary M', '_age': 30} - >>> vars(dave) - {'name': 'David D', '_age': 40} - - >>> mary.age # Access the data and log the lookup - INFO:root:Accessing 'age' giving 30 - 30 - >>> mary.birthday() # Updates are logged as well - INFO:root:Accessing 'age' giving 30 - INFO:root:Updating 'age' to 31 - - >>> dave.name # Regular attribute lookup isn't logged - 'David D' - >>> dave.age # Only the managed attribute is logged - INFO:root:Accessing 'age' giving 40 - 40 - -One major issue with this example is that the private name *_age* is hardwired in -the *LoggedAgeAccess* class. That means that each instance can only have one -logged attribute and that its name is unchangeable. In the next example, -we'll fix that problem. - - -Customized names ----------------- - -When a class uses descriptors, it can inform each descriptor about which -variable name was used. - -In this example, the :class:`Person` class has two descriptor instances, -*name* and *age*. When the :class:`Person` class is defined, it makes a -callback to :meth:`__set_name__` in *LoggedAccess* so that the field names can -be recorded, giving each descriptor its own *public_name* and *private_name*: - -.. testcode:: - - import logging - - logging.basicConfig(level=logging.INFO) - - class LoggedAccess: - - def __set_name__(self, owner, name): - self.public_name = name - self.private_name = '_' + name - - def __get__(self, obj, objtype=None): - value = getattr(obj, self.private_name) - logging.info('Accessing %r giving %r', self.public_name, value) - return value - - def __set__(self, obj, value): - logging.info('Updating %r to %r', self.public_name, value) - setattr(obj, self.private_name, value) - - class Person: - - name = LoggedAccess() # First descriptor instance - age = LoggedAccess() # Second descriptor instance - - def __init__(self, name, age): - self.name = name # Calls the first descriptor - self.age = age # Calls the second descriptor - - def birthday(self): - self.age += 1 - -An interactive session shows that the :class:`Person` class has called -:meth:`__set_name__` so that the field names would be recorded. Here -we call :func:`vars` to look up the descriptor without triggering it: - -.. doctest:: - - >>> vars(vars(Person)['name']) - {'public_name': 'name', 'private_name': '_name'} - >>> vars(vars(Person)['age']) - {'public_name': 'age', 'private_name': '_age'} - -The new class now logs access to both *name* and *age*: - -.. testcode:: - :hide: - - import logging, sys - logging.basicConfig(level=logging.INFO, stream=sys.stdout, force=True) - -.. doctest:: - - >>> pete = Person('Peter P', 10) - INFO:root:Updating 'name' to 'Peter P' - INFO:root:Updating 'age' to 10 - >>> kate = Person('Catherine C', 20) - INFO:root:Updating 'name' to 'Catherine C' - INFO:root:Updating 'age' to 20 - -The two *Person* instances contain only the private names: - -.. doctest:: - - >>> vars(pete) - {'_name': 'Peter P', '_age': 10} - >>> vars(kate) - {'_name': 'Catherine C', '_age': 20} - - -Closing thoughts ----------------- - -A :term:`descriptor` is what we call any object that defines :meth:`__get__`, -:meth:`__set__`, or :meth:`__delete__`. - -Optionally, descriptors can have a :meth:`__set_name__` method. This is only -used in cases where a descriptor needs to know either the class where it was -created or the name of class variable it was assigned to. (This method, if -present, is called even if the class is not a descriptor.) - -Descriptors get invoked by the dot operator during attribute lookup. If a -descriptor is accessed indirectly with ``vars(some_class)[descriptor_name]``, -the descriptor instance is returned without invoking it. - -Descriptors only work when used as class variables. When put in instances, -they have no effect. - -The main motivation for descriptors is to provide a hook allowing objects -stored in class variables to control what happens during attribute lookup. - -Traditionally, the calling class controls what happens during lookup. -Descriptors invert that relationship and allow the data being looked-up to -have a say in the matter. - -Descriptors are used throughout the language. It is how functions turn into -bound methods. Common tools like :func:`classmethod`, :func:`staticmethod`, -:func:`property`, and :func:`functools.cached_property` are all implemented as -descriptors. - - -Complete Practical Example -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In this example, we create a practical and powerful tool for locating -notoriously hard to find data corruption bugs. - - -Validator class ---------------- - -A validator is a descriptor for managed attribute access. Prior to storing -any data, it verifies that the new value meets various type and range -restrictions. If those restrictions aren't met, it raises an exception to -prevent data corruption at its source. - -This :class:`Validator` class is both an :term:`abstract base class` and a -managed attribute descriptor: - -.. testcode:: - - from abc import ABC, abstractmethod - - class Validator(ABC): - - def __set_name__(self, owner, name): - self.private_name = '_' + name - - def __get__(self, obj, objtype=None): - return getattr(obj, self.private_name) - - def __set__(self, obj, value): - self.validate(value) - setattr(obj, self.private_name, value) - - @abstractmethod - def validate(self, value): - pass - -Custom validators need to inherit from :class:`Validator` and must supply a -:meth:`validate` method to test various restrictions as needed. - - -Custom validators ------------------ - -Here are three practical data validation utilities: - -1) :class:`OneOf` verifies that a value is one of a restricted set of options. - -2) :class:`Number` verifies that a value is either an :class:`int` or - :class:`float`. Optionally, it verifies that a value is between a given - minimum or maximum. - -3) :class:`String` verifies that a value is a :class:`str`. Optionally, it - validates a given minimum or maximum length. It can validate a - user-defined `predicate - <https://en.wikipedia.org/wiki/Predicate_(mathematical_logic)>`_ as well. - -.. testcode:: - - class OneOf(Validator): - - def __init__(self, *options): - self.options = set(options) - - def validate(self, value): - if value not in self.options: - raise ValueError(f'Expected {value!r} to be one of {self.options!r}') - - class Number(Validator): - - def __init__(self, minvalue=None, maxvalue=None): - self.minvalue = minvalue - self.maxvalue = maxvalue - - def validate(self, value): - if not isinstance(value, (int, float)): - raise TypeError(f'Expected {value!r} to be an int or float') - if self.minvalue is not None and value < self.minvalue: - raise ValueError( - f'Expected {value!r} to be at least {self.minvalue!r}' - ) - if self.maxvalue is not None and value > self.maxvalue: - raise ValueError( - f'Expected {value!r} to be no more than {self.maxvalue!r}' - ) - - class String(Validator): - - def __init__(self, minsize=None, maxsize=None, predicate=None): - self.minsize = minsize - self.maxsize = maxsize - self.predicate = predicate - - def validate(self, value): - if not isinstance(value, str): - raise TypeError(f'Expected {value!r} to be an str') - if self.minsize is not None and len(value) < self.minsize: - raise ValueError( - f'Expected {value!r} to be no smaller than {self.minsize!r}' - ) - if self.maxsize is not None and len(value) > self.maxsize: - raise ValueError( - f'Expected {value!r} to be no bigger than {self.maxsize!r}' - ) - if self.predicate is not None and not self.predicate(value): - raise ValueError( - f'Expected {self.predicate} to be true for {value!r}' - ) - - -Practical application ---------------------- - -Here's how the data validators can be used in a real class: - -.. testcode:: - - class Component: - - name = String(minsize=3, maxsize=10, predicate=str.isupper) - kind = OneOf('wood', 'metal', 'plastic') - quantity = Number(minvalue=0) - - def __init__(self, name, kind, quantity): - self.name = name - self.kind = kind - self.quantity = quantity - -The descriptors prevent invalid instances from being created: - -.. doctest:: - - >>> Component('Widget', 'metal', 5) # Blocked: 'Widget' is not all uppercase - Traceback (most recent call last): - ... - ValueError: Expected <method 'isupper' of 'str' objects> to be true for 'Widget' - - >>> Component('WIDGET', 'metle', 5) # Blocked: 'metle' is misspelled - Traceback (most recent call last): - ... - ValueError: Expected 'metle' to be one of {'metal', 'plastic', 'wood'} - - >>> Component('WIDGET', 'metal', -5) # Blocked: -5 is negative - Traceback (most recent call last): - ... - ValueError: Expected -5 to be at least 0 - >>> Component('WIDGET', 'metal', 'V') # Blocked: 'V' isn't a number - Traceback (most recent call last): - ... - TypeError: Expected 'V' to be an int or float - - >>> c = Component('WIDGET', 'metal', 5) # Allowed: The inputs are valid - - -Technical Tutorial -^^^^^^^^^^^^^^^^^^ - -What follows is a more technical tutorial for the mechanics and details of how -descriptors work. - - -Abstract --------- - -Defines descriptors, summarizes the protocol, and shows how descriptors are -called. Provides an example showing how object relational mappings work. - -Learning about descriptors not only provides access to a larger toolset, it -creates a deeper understanding of how Python works. - - -Definition and introduction ---------------------------- - -In general, a descriptor is an attribute value that has one of the methods in -the descriptor protocol. Those methods are :meth:`__get__`, :meth:`__set__`, -and :meth:`__delete__`. If any of those methods are defined for an -attribute, it is said to be a :term:`descriptor`. - -The default behavior for attribute access is to get, set, or delete the -attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain -starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and -continuing through the method resolution order of ``type(a)``. If the -looked-up value is an object defining one of the descriptor methods, then Python -may override the default behavior and invoke the descriptor method instead. -Where this occurs in the precedence chain depends on which descriptor methods -were defined. - -Descriptors are a powerful, general purpose protocol. They are the mechanism -behind properties, methods, static methods, class methods, and -:func:`super()`. They are used throughout Python itself. Descriptors -simplify the underlying C code and offer a flexible set of new tools for -everyday Python programs. - - -Descriptor protocol -------------------- - -``descr.__get__(self, obj, type=None) -> value`` - -``descr.__set__(self, obj, value) -> None`` - -``descr.__delete__(self, obj) -> None`` - -That is all there is to it. Define any of these methods and an object is -considered a descriptor and can override default behavior upon being looked up -as an attribute. - -If an object defines :meth:`__set__` or :meth:`__delete__`, it is considered -a data descriptor. Descriptors that only define :meth:`__get__` are called -non-data descriptors (they are often used for methods but other uses are -possible). - -Data and non-data descriptors differ in how overrides are calculated with -respect to entries in an instance's dictionary. If an instance's dictionary -has an entry with the same name as a data descriptor, the data descriptor -takes precedence. If an instance's dictionary has an entry with the same -name as a non-data descriptor, the dictionary entry takes precedence. - -To make a read-only data descriptor, define both :meth:`__get__` and -:meth:`__set__` with the :meth:`__set__` raising an :exc:`AttributeError` when -called. Defining the :meth:`__set__` method with an exception raising -placeholder is enough to make it a data descriptor. - - -Overview of descriptor invocation ---------------------------------- - -A descriptor can be called directly with ``desc.__get__(obj)`` or -``desc.__get__(None, cls)``. - -But it is more common for a descriptor to be invoked automatically from -attribute access. - -The expression ``obj.x`` looks up the attribute ``x`` in the chain of -namespaces for ``obj``. If the search finds a descriptor outside of the -instance ``__dict__``, its :meth:`__get__` method is invoked according to the -precedence rules listed below. - -The details of invocation depend on whether ``obj`` is an object, class, or -instance of super. - - -Invocation from an instance ---------------------------- - -Instance lookup scans through a chain of namespaces giving data descriptors -the highest priority, followed by instance variables, then non-data -descriptors, then class variables, and lastly :meth:`__getattr__` if it is -provided. - -If a descriptor is found for ``a.x``, then it is invoked with: -``desc.__get__(a, type(a))``. - -The logic for a dotted lookup is in :meth:`object.__getattribute__`. Here is -a pure Python equivalent: - -.. testcode:: - - def find_name_in_mro(cls, name, default): - "Emulate _PyType_Lookup() in Objects/typeobject.c" - for base in cls.__mro__: - if name in vars(base): - return vars(base)[name] - return default - - def object_getattribute(obj, name): - "Emulate PyObject_GenericGetAttr() in Objects/object.c" - null = object() - objtype = type(obj) - cls_var = find_name_in_mro(objtype, name, null) - descr_get = getattr(type(cls_var), '__get__', null) - if descr_get is not null: - if (hasattr(type(cls_var), '__set__') - or hasattr(type(cls_var), '__delete__')): - return descr_get(cls_var, obj, objtype) # data descriptor - if hasattr(obj, '__dict__') and name in vars(obj): - return vars(obj)[name] # instance variable - if descr_get is not null: - return descr_get(cls_var, obj, objtype) # non-data descriptor - if cls_var is not null: - return cls_var # class variable - raise AttributeError(name) - - -.. testcode:: - :hide: - - # Test the fidelity of object_getattribute() by comparing it with the - # normal object.__getattribute__(). The former will be accessed by - # square brackets and the latter by the dot operator. - - class Object: - - def __getitem__(obj, name): - try: - return object_getattribute(obj, name) - except AttributeError: - if not hasattr(type(obj), '__getattr__'): - raise - return type(obj).__getattr__(obj, name) # __getattr__ - - class DualOperator(Object): - - x = 10 - - def __init__(self, z): - self.z = z - - @property - def p2(self): - return 2 * self.x - - @property - def p3(self): - return 3 * self.x - - def m5(self, y): - return 5 * y - - def m7(self, y): - return 7 * y - - def __getattr__(self, name): - return ('getattr_hook', self, name) - - class DualOperatorWithSlots: - - __getitem__ = Object.__getitem__ - - __slots__ = ['z'] - - x = 15 - - def __init__(self, z): - self.z = z - - @property - def p2(self): - return 2 * self.x - - def m5(self, y): - return 5 * y - - def __getattr__(self, name): - return ('getattr_hook', self, name) - - class D1: - def __get__(self, obj, objtype=None): - return type(self), obj, objtype - - class U1: - x = D1() - - class U2(U1): - pass - -.. doctest:: - :hide: - - >>> a = DualOperator(11) - >>> vars(a).update(p3 = '_p3', m7 = '_m7') - >>> a.x == a['x'] == 10 - True - >>> a.z == a['z'] == 11 - True - >>> a.p2 == a['p2'] == 20 - True - >>> a.p3 == a['p3'] == 30 - True - >>> a.m5(100) == a.m5(100) == 500 - True - >>> a.m7 == a['m7'] == '_m7' - True - >>> a.g == a['g'] == ('getattr_hook', a, 'g') - True - - >>> b = DualOperatorWithSlots(22) - >>> b.x == b['x'] == 15 - True - >>> b.z == b['z'] == 22 - True - >>> b.p2 == b['p2'] == 30 - True - >>> b.m5(200) == b['m5'](200) == 1000 - True - >>> b.g == b['g'] == ('getattr_hook', b, 'g') - True - - >>> u2 = U2() - >>> object_getattribute(u2, 'x') == u2.x == (D1, u2, U2) - True - -Note, there is no :meth:`__getattr__` hook in the :meth:`__getattribute__` -code. That is why calling :meth:`__getattribute__` directly or with -``super().__getattribute__`` will bypass :meth:`__getattr__` entirely. - -Instead, it is the dot operator and the :func:`getattr` function that are -responsible for invoking :meth:`__getattr__` whenever :meth:`__getattribute__` -raises an :exc:`AttributeError`. Their logic is encapsulated in a helper -function: - -.. testcode:: - - def getattr_hook(obj, name): - "Emulate slot_tp_getattr_hook() in Objects/typeobject.c" - try: - return obj.__getattribute__(name) - except AttributeError: - if not hasattr(type(obj), '__getattr__'): - raise - return type(obj).__getattr__(obj, name) # __getattr__ - -.. doctest:: - :hide: - - - >>> class ClassWithGetAttr: - ... x = 123 - ... def __getattr__(self, attr): - ... return attr.upper() - ... - >>> cw = ClassWithGetAttr() - >>> cw.y = 456 - >>> getattr_hook(cw, 'x') - 123 - >>> getattr_hook(cw, 'y') - 456 - >>> getattr_hook(cw, 'z') - 'Z' - - >>> class ClassWithoutGetAttr: - ... x = 123 - ... - >>> cwo = ClassWithoutGetAttr() - >>> cwo.y = 456 - >>> getattr_hook(cwo, 'x') - 123 - >>> getattr_hook(cwo, 'y') - 456 - >>> getattr_hook(cwo, 'z') - Traceback (most recent call last): - ... - AttributeError: 'ClassWithoutGetAttr' object has no attribute 'z' - - -Invocation from a class ------------------------ - -The logic for a dotted lookup such as ``A.x`` is in -:meth:`type.__getattribute__`. The steps are similar to those for -:meth:`object.__getattribute__` but the instance dictionary lookup is replaced -by a search through the class's :term:`method resolution order`. - -If a descriptor is found, it is invoked with ``desc.__get__(None, A)``. - -The full C implementation can be found in :c:func:`!type_getattro` and -:c:func:`!_PyType_Lookup` in :source:`Objects/typeobject.c`. - - -Invocation from super ---------------------- - -The logic for super's dotted lookup is in the :meth:`__getattribute__` method for -object returned by :class:`super()`. - -A dotted lookup such as ``super(A, obj).m`` searches ``obj.__class__.__mro__`` -for the base class ``B`` immediately following ``A`` and then returns -``B.__dict__['m'].__get__(obj, A)``. If not a descriptor, ``m`` is returned -unchanged. - -The full C implementation can be found in :c:func:`!super_getattro` in -:source:`Objects/typeobject.c`. A pure Python equivalent can be found in -`Guido's Tutorial -<https://www.python.org/download/releases/2.2.3/descrintro/#cooperation>`_. - - -Summary of invocation logic ---------------------------- - -The mechanism for descriptors is embedded in the :meth:`__getattribute__()` -methods for :class:`object`, :class:`type`, and :func:`super`. - -The important points to remember are: - -* Descriptors are invoked by the :meth:`__getattribute__` method. - -* Classes inherit this machinery from :class:`object`, :class:`type`, or - :func:`super`. - -* Overriding :meth:`__getattribute__` prevents automatic descriptor calls - because all the descriptor logic is in that method. - -* :meth:`object.__getattribute__` and :meth:`type.__getattribute__` make - different calls to :meth:`__get__`. The first includes the instance and may - include the class. The second puts in ``None`` for the instance and always - includes the class. - -* Data descriptors always override instance dictionaries. - -* Non-data descriptors may be overridden by instance dictionaries. - - -Automatic name notification ---------------------------- - -Sometimes it is desirable for a descriptor to know what class variable name it -was assigned to. When a new class is created, the :class:`type` metaclass -scans the dictionary of the new class. If any of the entries are descriptors -and if they define :meth:`__set_name__`, that method is called with two -arguments. The *owner* is the class where the descriptor is used, and the -*name* is the class variable the descriptor was assigned to. - -The implementation details are in :c:func:`!type_new` and -:c:func:`!set_names` in :source:`Objects/typeobject.c`. - -Since the update logic is in :meth:`type.__new__`, notifications only take -place at the time of class creation. If descriptors are added to the class -afterwards, :meth:`__set_name__` will need to be called manually. - - -ORM example ------------ - -The following code is a simplified skeleton showing how data descriptors could -be used to implement an `object relational mapping -<https://en.wikipedia.org/wiki/Object%E2%80%93relational_mapping>`_. - -The essential idea is that the data is stored in an external database. The -Python instances only hold keys to the database's tables. Descriptors take -care of lookups or updates: - -.. testcode:: - - class Field: - - def __set_name__(self, owner, name): - self.fetch = f'SELECT {name} FROM {owner.table} WHERE {owner.key}=?;' - self.store = f'UPDATE {owner.table} SET {name}=? WHERE {owner.key}=?;' - - def __get__(self, obj, objtype=None): - return conn.execute(self.fetch, [obj.key]).fetchone()[0] - - def __set__(self, obj, value): - conn.execute(self.store, [value, obj.key]) - conn.commit() - -We can use the :class:`Field` class to define `models -<https://en.wikipedia.org/wiki/Database_model>`_ that describe the schema for -each table in a database: - -.. testcode:: - - class Movie: - table = 'Movies' # Table name - key = 'title' # Primary key - director = Field() - year = Field() - - def __init__(self, key): - self.key = key - - class Song: - table = 'Music' - key = 'title' - artist = Field() - year = Field() - genre = Field() - - def __init__(self, key): - self.key = key - -To use the models, first connect to the database:: - - >>> import sqlite3 - >>> conn = sqlite3.connect('entertainment.db') - -An interactive session shows how data is retrieved from the database and how -it can be updated: - -.. testsetup:: - - song_data = [ - ('Country Roads', 'John Denver', 1972), - ('Me and Bobby McGee', 'Janice Joplin', 1971), - ('Coal Miners Daughter', 'Loretta Lynn', 1970), - ] - - movie_data = [ - ('Star Wars', 'George Lucas', 1977), - ('Jaws', 'Steven Spielberg', 1975), - ('Aliens', 'James Cameron', 1986), - ] - - import sqlite3 - - conn = sqlite3.connect(':memory:') - conn.execute('CREATE TABLE Music (title text, artist text, year integer);') - conn.execute('CREATE INDEX MusicNdx ON Music (title);') - conn.executemany('INSERT INTO Music VALUES (?, ?, ?);', song_data) - conn.execute('CREATE TABLE Movies (title text, director text, year integer);') - conn.execute('CREATE INDEX MovieNdx ON Music (title);') - conn.executemany('INSERT INTO Movies VALUES (?, ?, ?);', movie_data) - conn.commit() - -.. doctest:: - - >>> Movie('Star Wars').director - 'George Lucas' - >>> jaws = Movie('Jaws') - >>> f'Released in {jaws.year} by {jaws.director}' - 'Released in 1975 by Steven Spielberg' - - >>> Song('Country Roads').artist - 'John Denver' - - >>> Movie('Star Wars').director = 'J.J. Abrams' - >>> Movie('Star Wars').director - 'J.J. Abrams' - -.. testcleanup:: - - conn.close() - - -Pure Python Equivalents -^^^^^^^^^^^^^^^^^^^^^^^ - -The descriptor protocol is simple and offers exciting possibilities. Several -use cases are so common that they have been prepackaged into built-in tools. -Properties, bound methods, static methods, class methods, and \_\_slots\_\_ are -all based on the descriptor protocol. - - -Properties ----------- - -Calling :func:`property` is a succinct way of building a data descriptor that -triggers a function call upon access to an attribute. Its signature is:: - - property(fget=None, fset=None, fdel=None, doc=None) -> property - -The documentation shows a typical use to define a managed attribute ``x``: - -.. testcode:: - - class C: - def getx(self): return self.__x - def setx(self, value): self.__x = value - def delx(self): del self.__x - x = property(getx, setx, delx, "I'm the 'x' property.") - -.. doctest:: - :hide: - - >>> C.x.__doc__ - "I'm the 'x' property." - >>> c.x = 2.71828 - >>> c.x - 2.71828 - >>> del c.x - >>> c.x - Traceback (most recent call last): - ... - AttributeError: 'C' object has no attribute '_C__x' - -To see how :func:`property` is implemented in terms of the descriptor protocol, -here is a pure Python equivalent: - -.. testcode:: - - class Property: - "Emulate PyProperty_Type() in Objects/descrobject.c" - - def __init__(self, fget=None, fset=None, fdel=None, doc=None): - self.fget = fget - self.fset = fset - self.fdel = fdel - if doc is None and fget is not None: - doc = fget.__doc__ - self.__doc__ = doc - self._name = '' - - def __set_name__(self, owner, name): - self._name = name - - def __get__(self, obj, objtype=None): - if obj is None: - return self - if self.fget is None: - raise AttributeError(f"property '{self._name}' has no getter") - return self.fget(obj) - - def __set__(self, obj, value): - if self.fset is None: - raise AttributeError(f"property '{self._name}' has no setter") - self.fset(obj, value) - - def __delete__(self, obj): - if self.fdel is None: - raise AttributeError(f"property '{self._name}' has no deleter") - self.fdel(obj) - - def getter(self, fget): - prop = type(self)(fget, self.fset, self.fdel, self.__doc__) - prop._name = self._name - return prop - - def setter(self, fset): - prop = type(self)(self.fget, fset, self.fdel, self.__doc__) - prop._name = self._name - return prop - - def deleter(self, fdel): - prop = type(self)(self.fget, self.fset, fdel, self.__doc__) - prop._name = self._name - return prop - -.. testcode:: - :hide: - - # Verify the Property() emulation - - class CC: - def getx(self): - return self.__x - def setx(self, value): - self.__x = value - def delx(self): - del self.__x - x = Property(getx, setx, delx, "I'm the 'x' property.") - - # Now do it again but use the decorator style - - class CCC: - @Property - def x(self): - return self.__x - @x.setter - def x(self, value): - self.__x = value - @x.deleter - def x(self): - del self.__x - - -.. doctest:: - :hide: - - >>> cc = CC() - >>> hasattr(cc, 'x') - False - >>> cc.x = 33 - >>> cc.x - 33 - >>> del cc.x - >>> hasattr(cc, 'x') - False - - >>> ccc = CCC() - >>> hasattr(ccc, 'x') - False - >>> ccc.x = 333 - >>> ccc.x == 333 - True - >>> del ccc.x - >>> hasattr(ccc, 'x') - False - -The :func:`property` builtin helps whenever a user interface has granted -attribute access and then subsequent changes require the intervention of a -method. - -For instance, a spreadsheet class may grant access to a cell value through -``Cell('b10').value``. Subsequent improvements to the program require the cell -to be recalculated on every access; however, the programmer does not want to -affect existing client code accessing the attribute directly. The solution is -to wrap access to the value attribute in a property data descriptor: - -.. testcode:: - - class Cell: - ... - - @property - def value(self): - "Recalculate the cell before returning value" - self.recalc() - return self._value - -Either the built-in :func:`property` or our :func:`Property` equivalent would -work in this example. - - -Functions and methods ---------------------- - -Python's object oriented features are built upon a function based environment. -Using non-data descriptors, the two are merged seamlessly. - -Functions stored in class dictionaries get turned into methods when invoked. -Methods only differ from regular functions in that the object instance is -prepended to the other arguments. By convention, the instance is called -*self* but could be called *this* or any other variable name. - -Methods can be created manually with :class:`types.MethodType` which is -roughly equivalent to: - -.. testcode:: - - class MethodType: - "Emulate PyMethod_Type in Objects/classobject.c" - - def __init__(self, func, obj): - self.__func__ = func - self.__self__ = obj - - def __call__(self, *args, **kwargs): - func = self.__func__ - obj = self.__self__ - return func(obj, *args, **kwargs) - -To support automatic creation of methods, functions include the -:meth:`__get__` method for binding methods during attribute access. This -means that functions are non-data descriptors that return bound methods -during dotted lookup from an instance. Here's how it works: - -.. testcode:: - - class Function: - ... - - def __get__(self, obj, objtype=None): - "Simulate func_descr_get() in Objects/funcobject.c" - if obj is None: - return self - return MethodType(self, obj) - -Running the following class in the interpreter shows how the function -descriptor works in practice: - -.. testcode:: - - class D: - def f(self, x): - return x - -The function has a :term:`qualified name` attribute to support introspection: - -.. doctest:: - - >>> D.f.__qualname__ - 'D.f' - -Accessing the function through the class dictionary does not invoke -:meth:`__get__`. Instead, it just returns the underlying function object:: - - >>> D.__dict__['f'] - <function D.f at 0x00C45070> - -Dotted access from a class calls :meth:`__get__` which just returns the -underlying function unchanged:: - - >>> D.f - <function D.f at 0x00C45070> - -The interesting behavior occurs during dotted access from an instance. The -dotted lookup calls :meth:`__get__` which returns a bound method object:: - - >>> d = D() - >>> d.f - <bound method D.f of <__main__.D object at 0x00B18C90>> - -Internally, the bound method stores the underlying function and the bound -instance:: - - >>> d.f.__func__ - <function D.f at 0x00C45070> - - >>> d.f.__self__ - <__main__.D object at 0x1012e1f98> - -If you have ever wondered where *self* comes from in regular methods or where -*cls* comes from in class methods, this is it! - - -Kinds of methods ----------------- - -Non-data descriptors provide a simple mechanism for variations on the usual -patterns of binding functions into methods. - -To recap, functions have a :meth:`__get__` method so that they can be converted -to a method when accessed as attributes. The non-data descriptor transforms an -``obj.f(*args)`` call into ``f(obj, *args)``. Calling ``cls.f(*args)`` -becomes ``f(*args)``. - -This chart summarizes the binding and its two most useful variants: - - +-----------------+----------------------+------------------+ - | Transformation | Called from an | Called from a | - | | object | class | - +=================+======================+==================+ - | function | f(obj, \*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | staticmethod | f(\*args) | f(\*args) | - +-----------------+----------------------+------------------+ - | classmethod | f(type(obj), \*args) | f(cls, \*args) | - +-----------------+----------------------+------------------+ - - -Static methods --------------- - -Static methods return the underlying function without changes. Calling either -``c.f`` or ``C.f`` is the equivalent of a direct lookup into -``object.__getattribute__(c, "f")`` or ``object.__getattribute__(C, "f")``. As a -result, the function becomes identically accessible from either an object or a -class. - -Good candidates for static methods are methods that do not reference the -``self`` variable. - -For instance, a statistics package may include a container class for -experimental data. The class provides normal methods for computing the average, -mean, median, and other descriptive statistics that depend on the data. However, -there may be useful functions which are conceptually related but do not depend -on the data. For instance, ``erf(x)`` is handy conversion routine that comes up -in statistical work but does not directly depend on a particular dataset. -It can be called either from an object or the class: ``s.erf(1.5) --> .9332`` or -``Sample.erf(1.5) --> .9332``. - -Since static methods return the underlying function with no changes, the -example calls are unexciting: - -.. testcode:: - - class E: - @staticmethod - def f(x): - return x * 10 - -.. doctest:: - - >>> E.f(3) - 30 - >>> E().f(3) - 30 - -Using the non-data descriptor protocol, a pure Python version of -:func:`staticmethod` would look like this: - -.. testcode:: - - import functools - - class StaticMethod: - "Emulate PyStaticMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - functools.update_wrapper(self, f) - - def __get__(self, obj, objtype=None): - return self.f - - def __call__(self, *args, **kwds): - return self.f(*args, **kwds) - -The :func:`functools.update_wrapper` call adds a ``__wrapped__`` attribute -that refers to the underlying function. Also it carries forward -the attributes necessary to make the wrapper look like the wrapped -function: ``__name__``, ``__qualname__``, ``__doc__``, and ``__annotations__``. - -.. testcode:: - :hide: - - class E_sim: - @StaticMethod - def f(x: int) -> str: - "Simple function example" - return "!" * x - - wrapped_ord = StaticMethod(ord) - -.. doctest:: - :hide: - - >>> E_sim.f(3) - '!!!' - >>> E_sim().f(3) - '!!!' - - >>> sm = vars(E_sim)['f'] - >>> type(sm).__name__ - 'StaticMethod' - >>> f = E_sim.f - >>> type(f).__name__ - 'function' - >>> sm.__name__ - 'f' - >>> f.__name__ - 'f' - >>> sm.__qualname__ - 'E_sim.f' - >>> f.__qualname__ - 'E_sim.f' - >>> sm.__doc__ - 'Simple function example' - >>> f.__doc__ - 'Simple function example' - >>> sm.__annotations__ - {'x': <class 'int'>, 'return': <class 'str'>} - >>> f.__annotations__ - {'x': <class 'int'>, 'return': <class 'str'>} - >>> sm.__module__ == f.__module__ - True - >>> sm(3) - '!!!' - >>> f(3) - '!!!' - - >>> wrapped_ord('A') - 65 - >>> wrapped_ord.__module__ == ord.__module__ - True - >>> wrapped_ord.__wrapped__ == ord - True - >>> wrapped_ord.__name__ == ord.__name__ - True - >>> wrapped_ord.__qualname__ == ord.__qualname__ - True - >>> wrapped_ord.__doc__ == ord.__doc__ - True - - -Class methods -------------- - -Unlike static methods, class methods prepend the class reference to the -argument list before calling the function. This format is the same -for whether the caller is an object or a class: - -.. testcode:: - - class F: - @classmethod - def f(cls, x): - return cls.__name__, x - -.. doctest:: - - >>> F.f(3) - ('F', 3) - >>> F().f(3) - ('F', 3) - -This behavior is useful whenever the method only needs to have a class -reference and does not rely on data stored in a specific instance. One use for -class methods is to create alternate class constructors. For example, the -classmethod :func:`dict.fromkeys` creates a new dictionary from a list of -keys. The pure Python equivalent is: - -.. testcode:: - - class Dict(dict): - @classmethod - def fromkeys(cls, iterable, value=None): - "Emulate dict_fromkeys() in Objects/dictobject.c" - d = cls() - for key in iterable: - d[key] = value - return d - -Now a new dictionary of unique keys can be constructed like this: - -.. doctest:: - - >>> d = Dict.fromkeys('abracadabra') - >>> type(d) is Dict - True - >>> d - {'a': None, 'b': None, 'r': None, 'c': None, 'd': None} - -Using the non-data descriptor protocol, a pure Python version of -:func:`classmethod` would look like this: - -.. testcode:: - - import functools - - class ClassMethod: - "Emulate PyClassMethod_Type() in Objects/funcobject.c" - - def __init__(self, f): - self.f = f - functools.update_wrapper(self, f) - - def __get__(self, obj, cls=None): - if cls is None: - cls = type(obj) - if hasattr(type(self.f), '__get__'): - # This code path was added in Python 3.9 - # and was deprecated in Python 3.11. - return self.f.__get__(cls, cls) - return MethodType(self.f, cls) - -.. testcode:: - :hide: - - # Verify the emulation works - class T: - @ClassMethod - def cm(cls, x: int, y: str) -> tuple[str, int, str]: - "Class method that returns a tuple" - return (cls.__name__, x, y) - - @ClassMethod - @property - def __doc__(cls): - return f'A doc for {cls.__name__!r}' - - -.. doctest:: - :hide: - - >>> T.cm(11, 22) - ('T', 11, 22) - - # Also call it from an instance - >>> t = T() - >>> t.cm(11, 22) - ('T', 11, 22) - - # Check the alternate path for chained descriptors - >>> T.__doc__ - "A doc for 'T'" - - # Verify that T uses our emulation - >>> type(vars(T)['cm']).__name__ - 'ClassMethod' - - # Verify that update_wrapper() correctly copied attributes - >>> T.cm.__name__ - 'cm' - >>> T.cm.__qualname__ - 'T.cm' - >>> T.cm.__doc__ - 'Class method that returns a tuple' - >>> T.cm.__annotations__ - {'x': <class 'int'>, 'y': <class 'str'>, 'return': tuple[str, int, str]} - - # Verify that __wrapped__ was added and works correctly - >>> f = vars(T)['cm'].__wrapped__ - >>> type(f).__name__ - 'function' - >>> f.__name__ - 'cm' - >>> f(T, 11, 22) - ('T', 11, 22) - - -The code path for ``hasattr(type(self.f), '__get__')`` was added in -Python 3.9 and makes it possible for :func:`classmethod` to support -chained decorators. For example, a classmethod and property could be -chained together. In Python 3.11, this functionality was deprecated. - -.. testcode:: - - class G: - @classmethod - @property - def __doc__(cls): - return f'A doc for {cls.__name__!r}' - -.. doctest:: - - >>> G.__doc__ - "A doc for 'G'" - -The :func:`functools.update_wrapper` call in ``ClassMethod`` adds a -``__wrapped__`` attribute that refers to the underlying function. Also -it carries forward the attributes necessary to make the wrapper look -like the wrapped function: ``__name__``, ``__qualname__``, ``__doc__``, -and ``__annotations__``. - - -Member objects and __slots__ ----------------------------- - -When a class defines ``__slots__``, it replaces instance dictionaries with a -fixed-length array of slot values. From a user point of view that has -several effects: - -1. Provides immediate detection of bugs due to misspelled attribute -assignments. Only attribute names specified in ``__slots__`` are allowed: - -.. testcode:: - - class Vehicle: - __slots__ = ('id_number', 'make', 'model') - -.. doctest:: - - >>> auto = Vehicle() - >>> auto.id_nubmer = 'VYE483814LQEX' - Traceback (most recent call last): - ... - AttributeError: 'Vehicle' object has no attribute 'id_nubmer' - -2. Helps create immutable objects where descriptors manage access to private -attributes stored in ``__slots__``: - -.. testcode:: - - class Immutable: - - __slots__ = ('_dept', '_name') # Replace the instance dictionary - - def __init__(self, dept, name): - self._dept = dept # Store to private attribute - self._name = name # Store to private attribute - - @property # Read-only descriptor - def dept(self): - return self._dept - - @property - def name(self): # Read-only descriptor - return self._name - -.. doctest:: - - >>> mark = Immutable('Botany', 'Mark Watney') - >>> mark.dept - 'Botany' - >>> mark.dept = 'Space Pirate' - Traceback (most recent call last): - ... - AttributeError: property 'dept' of 'Immutable' object has no setter - >>> mark.location = 'Mars' - Traceback (most recent call last): - ... - AttributeError: 'Immutable' object has no attribute 'location' - -3. Saves memory. On a 64-bit Linux build, an instance with two attributes -takes 48 bytes with ``__slots__`` and 152 bytes without. This `flyweight -design pattern <https://en.wikipedia.org/wiki/Flyweight_pattern>`_ likely only -matters when a large number of instances are going to be created. - -4. Improves speed. Reading instance variables is 35% faster with -``__slots__`` (as measured with Python 3.10 on an Apple M1 processor). - -5. Blocks tools like :func:`functools.cached_property` which require an -instance dictionary to function correctly: - -.. testcode:: - - from functools import cached_property - - class CP: - __slots__ = () # Eliminates the instance dict - - @cached_property # Requires an instance dict - def pi(self): - return 4 * sum((-1.0)**n / (2.0*n + 1.0) - for n in reversed(range(100_000))) - -.. doctest:: - - >>> CP().pi - Traceback (most recent call last): - ... - TypeError: No '__dict__' attribute on 'CP' instance to cache 'pi' property. - -It is not possible to create an exact drop-in pure Python version of -``__slots__`` because it requires direct access to C structures and control -over object memory allocation. However, we can build a mostly faithful -simulation where the actual C structure for slots is emulated by a private -``_slotvalues`` list. Reads and writes to that private structure are managed -by member descriptors: - -.. testcode:: - - null = object() - - class Member: - - def __init__(self, name, clsname, offset): - 'Emulate PyMemberDef in Include/structmember.h' - # Also see descr_new() in Objects/descrobject.c - self.name = name - self.clsname = clsname - self.offset = offset - - def __get__(self, obj, objtype=None): - 'Emulate member_get() in Objects/descrobject.c' - # Also see PyMember_GetOne() in Python/structmember.c - if obj is None: - return self - value = obj._slotvalues[self.offset] - if value is null: - raise AttributeError(self.name) - return value - - def __set__(self, obj, value): - 'Emulate member_set() in Objects/descrobject.c' - obj._slotvalues[self.offset] = value - - def __delete__(self, obj): - 'Emulate member_delete() in Objects/descrobject.c' - value = obj._slotvalues[self.offset] - if value is null: - raise AttributeError(self.name) - obj._slotvalues[self.offset] = null - - def __repr__(self): - 'Emulate member_repr() in Objects/descrobject.c' - return f'<Member {self.name!r} of {self.clsname!r}>' - -The :meth:`type.__new__` method takes care of adding member objects to class -variables: - -.. testcode:: - - class Type(type): - 'Simulate how the type metaclass adds member objects for slots' - - def __new__(mcls, clsname, bases, mapping, **kwargs): - 'Emulate type_new() in Objects/typeobject.c' - # type_new() calls PyTypeReady() which calls add_methods() - slot_names = mapping.get('slot_names', []) - for offset, name in enumerate(slot_names): - mapping[name] = Member(name, clsname, offset) - return type.__new__(mcls, clsname, bases, mapping, **kwargs) - -The :meth:`object.__new__` method takes care of creating instances that have -slots instead of an instance dictionary. Here is a rough simulation in pure -Python: - -.. testcode:: - - class Object: - 'Simulate how object.__new__() allocates memory for __slots__' - - def __new__(cls, *args, **kwargs): - 'Emulate object_new() in Objects/typeobject.c' - inst = super().__new__(cls) - if hasattr(cls, 'slot_names'): - empty_slots = [null] * len(cls.slot_names) - object.__setattr__(inst, '_slotvalues', empty_slots) - return inst - - def __setattr__(self, name, value): - 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' - cls = type(self) - if hasattr(cls, 'slot_names') and name not in cls.slot_names: - raise AttributeError( - f'{cls.__name__!r} object has no attribute {name!r}' - ) - super().__setattr__(name, value) - - def __delattr__(self, name): - 'Emulate _PyObject_GenericSetAttrWithDict() Objects/object.c' - cls = type(self) - if hasattr(cls, 'slot_names') and name not in cls.slot_names: - raise AttributeError( - f'{cls.__name__!r} object has no attribute {name!r}' - ) - super().__delattr__(name) - -To use the simulation in a real class, just inherit from :class:`Object` and -set the :term:`metaclass` to :class:`Type`: - -.. testcode:: - - class H(Object, metaclass=Type): - 'Instance variables stored in slots' - - slot_names = ['x', 'y'] - - def __init__(self, x, y): - self.x = x - self.y = y - -At this point, the metaclass has loaded member objects for *x* and *y*:: - - >>> from pprint import pp - >>> pp(dict(vars(H))) - {'__module__': '__main__', - '__doc__': 'Instance variables stored in slots', - 'slot_names': ['x', 'y'], - '__init__': <function H.__init__ at 0x7fb5d302f9d0>, - 'x': <Member 'x' of 'H'>, - 'y': <Member 'y' of 'H'>} - -.. doctest:: - :hide: - - # We test this separately because the preceding section is not - # doctestable due to the hex memory address for the __init__ function - >>> isinstance(vars(H)['x'], Member) - True - >>> isinstance(vars(H)['y'], Member) - True - -When instances are created, they have a ``slot_values`` list where the -attributes are stored: - -.. doctest:: - - >>> h = H(10, 20) - >>> vars(h) - {'_slotvalues': [10, 20]} - >>> h.x = 55 - >>> vars(h) - {'_slotvalues': [55, 20]} - -Misspelled or unassigned attributes will raise an exception: - -.. doctest:: - - >>> h.xz - Traceback (most recent call last): - ... - AttributeError: 'H' object has no attribute 'xz' - -.. doctest:: - :hide: - - # Examples for deleted attributes are not shown because this section - # is already a bit lengthy. We still test that code here. - >>> del h.x - >>> hasattr(h, 'x') - False - - # Also test the code for uninitialized slots - >>> class HU(Object, metaclass=Type): - ... slot_names = ['x', 'y'] - ... - >>> hu = HU() - >>> hasattr(hu, 'x') - False - >>> hasattr(hu, 'y') - False diff --git a/Doc/howto/functional.rst b/Doc/howto/functional.rst deleted file mode 100644 index efc25dac695da0..00000000000000 --- a/Doc/howto/functional.rst +++ /dev/null @@ -1,1268 +0,0 @@ -******************************** - Functional Programming HOWTO -******************************** - -:Author: A. M. Kuchling -:Release: 0.32 - -In this document, we'll take a tour of Python's features suitable for -implementing programs in a functional style. After an introduction to the -concepts of functional programming, we'll look at language features such as -:term:`iterator`\s and :term:`generator`\s and relevant library modules such as -:mod:`itertools` and :mod:`functools`. - - -Introduction -============ - -This section explains the basic concept of functional programming; if -you're just interested in learning about Python language features, -skip to the next section on :ref:`functional-howto-iterators`. - -Programming languages support decomposing problems in several different ways: - -* Most programming languages are **procedural**: programs are lists of - instructions that tell the computer what to do with the program's input. C, - Pascal, and even Unix shells are procedural languages. - -* In **declarative** languages, you write a specification that describes the - problem to be solved, and the language implementation figures out how to - perform the computation efficiently. SQL is the declarative language you're - most likely to be familiar with; a SQL query describes the data set you want - to retrieve, and the SQL engine decides whether to scan tables or use indexes, - which subclauses should be performed first, etc. - -* **Object-oriented** programs manipulate collections of objects. Objects have - internal state and support methods that query or modify this internal state in - some way. Smalltalk and Java are object-oriented languages. C++ and Python - are languages that support object-oriented programming, but don't force the - use of object-oriented features. - -* **Functional** programming decomposes a problem into a set of functions. - Ideally, functions only take inputs and produce outputs, and don't have any - internal state that affects the output produced for a given input. Well-known - functional languages include the ML family (Standard ML, OCaml, and other - variants) and Haskell. - -The designers of some computer languages choose to emphasize one -particular approach to programming. This often makes it difficult to -write programs that use a different approach. Other languages are -multi-paradigm languages that support several different approaches. -Lisp, C++, and Python are multi-paradigm; you can write programs or -libraries that are largely procedural, object-oriented, or functional -in all of these languages. In a large program, different sections -might be written using different approaches; the GUI might be -object-oriented while the processing logic is procedural or -functional, for example. - -In a functional program, input flows through a set of functions. Each function -operates on its input and produces some output. Functional style discourages -functions with side effects that modify internal state or make other changes -that aren't visible in the function's return value. Functions that have no side -effects at all are called **purely functional**. Avoiding side effects means -not using data structures that get updated as a program runs; every function's -output must only depend on its input. - -Some languages are very strict about purity and don't even have assignment -statements such as ``a=3`` or ``c = a + b``, but it's difficult to avoid all -side effects, such as printing to the screen or writing to a disk file. Another -example is a call to the :func:`print` or :func:`time.sleep` function, neither -of which returns a useful value. Both are called only for their side effects -of sending some text to the screen or pausing execution for a second. - -Python programs written in functional style usually won't go to the extreme of -avoiding all I/O or all assignments; instead, they'll provide a -functional-appearing interface but will use non-functional features internally. -For example, the implementation of a function will still use assignments to -local variables, but won't modify global variables or have other side effects. - -Functional programming can be considered the opposite of object-oriented -programming. Objects are little capsules containing some internal state along -with a collection of method calls that let you modify this state, and programs -consist of making the right set of state changes. Functional programming wants -to avoid state changes as much as possible and works with data flowing between -functions. In Python you might combine the two approaches by writing functions -that take and return instances representing objects in your application (e-mail -messages, transactions, etc.). - -Functional design may seem like an odd constraint to work under. Why should you -avoid objects and side effects? There are theoretical and practical advantages -to the functional style: - -* Formal provability. -* Modularity. -* Composability. -* Ease of debugging and testing. - - -Formal provability ------------------- - -A theoretical benefit is that it's easier to construct a mathematical proof that -a functional program is correct. - -For a long time researchers have been interested in finding ways to -mathematically prove programs correct. This is different from testing a program -on numerous inputs and concluding that its output is usually correct, or reading -a program's source code and concluding that the code looks right; the goal is -instead a rigorous proof that a program produces the right result for all -possible inputs. - -The technique used to prove programs correct is to write down **invariants**, -properties of the input data and of the program's variables that are always -true. For each line of code, you then show that if invariants X and Y are true -**before** the line is executed, the slightly different invariants X' and Y' are -true **after** the line is executed. This continues until you reach the end of -the program, at which point the invariants should match the desired conditions -on the program's output. - -Functional programming's avoidance of assignments arose because assignments are -difficult to handle with this technique; assignments can break invariants that -were true before the assignment without producing any new invariants that can be -propagated onward. - -Unfortunately, proving programs correct is largely impractical and not relevant -to Python software. Even trivial programs require proofs that are several pages -long; the proof of correctness for a moderately complicated program would be -enormous, and few or none of the programs you use daily (the Python interpreter, -your XML parser, your web browser) could be proven correct. Even if you wrote -down or generated a proof, there would then be the question of verifying the -proof; maybe there's an error in it, and you wrongly believe you've proved the -program correct. - - -Modularity ----------- - -A more practical benefit of functional programming is that it forces you to -break apart your problem into small pieces. Programs are more modular as a -result. It's easier to specify and write a small function that does one thing -than a large function that performs a complicated transformation. Small -functions are also easier to read and to check for errors. - - -Ease of debugging and testing ------------------------------ - -Testing and debugging a functional-style program is easier. - -Debugging is simplified because functions are generally small and clearly -specified. When a program doesn't work, each function is an interface point -where you can check that the data are correct. You can look at the intermediate -inputs and outputs to quickly isolate the function that's responsible for a bug. - -Testing is easier because each function is a potential subject for a unit test. -Functions don't depend on system state that needs to be replicated before -running a test; instead you only have to synthesize the right input and then -check that the output matches expectations. - - -Composability -------------- - -As you work on a functional-style program, you'll write a number of functions -with varying inputs and outputs. Some of these functions will be unavoidably -specialized to a particular application, but others will be useful in a wide -variety of programs. For example, a function that takes a directory path and -returns all the XML files in the directory, or a function that takes a filename -and returns its contents, can be applied to many different situations. - -Over time you'll form a personal library of utilities. Often you'll assemble -new programs by arranging existing functions in a new configuration and writing -a few functions specialized for the current task. - - -.. _functional-howto-iterators: - -Iterators -========= - -I'll start by looking at a Python language feature that's an important -foundation for writing functional-style programs: iterators. - -An iterator is an object representing a stream of data; this object returns the -data one element at a time. A Python iterator must support a method called -:meth:`~iterator.__next__` that takes no arguments and always returns the next -element of the stream. If there are no more elements in the stream, -:meth:`~iterator.__next__` must raise the :exc:`StopIteration` exception. -Iterators don't have to be finite, though; it's perfectly reasonable to write -an iterator that produces an infinite stream of data. - -The built-in :func:`iter` function takes an arbitrary object and tries to return -an iterator that will return the object's contents or elements, raising -:exc:`TypeError` if the object doesn't support iteration. Several of Python's -built-in data types support iteration, the most common being lists and -dictionaries. An object is called :term:`iterable` if you can get an iterator -for it. - -You can experiment with the iteration interface manually: - - >>> L = [1, 2, 3] - >>> it = iter(L) - >>> it #doctest: +ELLIPSIS - <...iterator object at ...> - >>> it.__next__() # same as next(it) - 1 - >>> next(it) - 2 - >>> next(it) - 3 - >>> next(it) - Traceback (most recent call last): - File "<stdin>", line 1, in <module> - StopIteration - >>> - -Python expects iterable objects in several different contexts, the most -important being the :keyword:`for` statement. In the statement ``for X in Y``, -Y must be an iterator or some object for which :func:`iter` can create an -iterator. These two statements are equivalent:: - - - for i in iter(obj): - print(i) - - for i in obj: - print(i) - -Iterators can be materialized as lists or tuples by using the :func:`list` or -:func:`tuple` constructor functions: - - >>> L = [1, 2, 3] - >>> iterator = iter(L) - >>> t = tuple(iterator) - >>> t - (1, 2, 3) - -Sequence unpacking also supports iterators: if you know an iterator will return -N elements, you can unpack them into an N-tuple: - - >>> L = [1, 2, 3] - >>> iterator = iter(L) - >>> a, b, c = iterator - >>> a, b, c - (1, 2, 3) - -Built-in functions such as :func:`max` and :func:`min` can take a single -iterator argument and will return the largest or smallest element. The ``"in"`` -and ``"not in"`` operators also support iterators: ``X in iterator`` is true if -X is found in the stream returned by the iterator. You'll run into obvious -problems if the iterator is infinite; :func:`max`, :func:`min` -will never return, and if the element X never appears in the stream, the -``"in"`` and ``"not in"`` operators won't return either. - -Note that you can only go forward in an iterator; there's no way to get the -previous element, reset the iterator, or make a copy of it. Iterator objects -can optionally provide these additional capabilities, but the iterator protocol -only specifies the :meth:`~iterator.__next__` method. Functions may therefore -consume all of the iterator's output, and if you need to do something different -with the same stream, you'll have to create a new iterator. - - - -Data Types That Support Iterators ---------------------------------- - -We've already seen how lists and tuples support iterators. In fact, any Python -sequence type, such as strings, will automatically support creation of an -iterator. - -Calling :func:`iter` on a dictionary returns an iterator that will loop over the -dictionary's keys:: - - >>> m = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'Jun': 6, - ... 'Jul': 7, 'Aug': 8, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12} - >>> for key in m: - ... print(key, m[key]) - Jan 1 - Feb 2 - Mar 3 - Apr 4 - May 5 - Jun 6 - Jul 7 - Aug 8 - Sep 9 - Oct 10 - Nov 11 - Dec 12 - -Note that starting with Python 3.7, dictionary iteration order is guaranteed -to be the same as the insertion order. In earlier versions, the behaviour was -unspecified and could vary between implementations. - -Applying :func:`iter` to a dictionary always loops over the keys, but -dictionaries have methods that return other iterators. If you want to iterate -over values or key/value pairs, you can explicitly call the -:meth:`~dict.values` or :meth:`~dict.items` methods to get an appropriate -iterator. - -The :func:`dict` constructor can accept an iterator that returns a finite stream -of ``(key, value)`` tuples: - - >>> L = [('Italy', 'Rome'), ('France', 'Paris'), ('US', 'Washington DC')] - >>> dict(iter(L)) - {'Italy': 'Rome', 'France': 'Paris', 'US': 'Washington DC'} - -Files also support iteration by calling the :meth:`~io.TextIOBase.readline` -method until there are no more lines in the file. This means you can read each -line of a file like this:: - - for line in file: - # do something for each line - ... - -Sets can take their contents from an iterable and let you iterate over the set's -elements:: - - >>> S = {2, 3, 5, 7, 11, 13} - >>> for i in S: - ... print(i) - 2 - 3 - 5 - 7 - 11 - 13 - - - -Generator expressions and list comprehensions -============================================= - -Two common operations on an iterator's output are 1) performing some operation -for every element, 2) selecting a subset of elements that meet some condition. -For example, given a list of strings, you might want to strip off trailing -whitespace from each line or extract all the strings containing a given -substring. - -List comprehensions and generator expressions (short form: "listcomps" and -"genexps") are a concise notation for such operations, borrowed from the -functional programming language Haskell (https://www.haskell.org/). You can strip -all the whitespace from a stream of strings with the following code:: - - >>> line_list = [' line 1\n', 'line 2 \n', ' \n', ''] - - >>> # Generator expression -- returns iterator - >>> stripped_iter = (line.strip() for line in line_list) - - >>> # List comprehension -- returns list - >>> stripped_list = [line.strip() for line in line_list] - -You can select only certain elements by adding an ``"if"`` condition:: - - >>> stripped_list = [line.strip() for line in line_list - ... if line != ""] - -With a list comprehension, you get back a Python list; ``stripped_list`` is a -list containing the resulting lines, not an iterator. Generator expressions -return an iterator that computes the values as necessary, not needing to -materialize all the values at once. This means that list comprehensions aren't -useful if you're working with iterators that return an infinite stream or a very -large amount of data. Generator expressions are preferable in these situations. - -Generator expressions are surrounded by parentheses ("()") and list -comprehensions are surrounded by square brackets ("[]"). Generator expressions -have the form:: - - ( expression for expr in sequence1 - if condition1 - for expr2 in sequence2 - if condition2 - for expr3 in sequence3 - ... - if condition3 - for exprN in sequenceN - if conditionN ) - -Again, for a list comprehension only the outside brackets are different (square -brackets instead of parentheses). - -The elements of the generated output will be the successive values of -``expression``. The ``if`` clauses are all optional; if present, ``expression`` -is only evaluated and added to the result when ``condition`` is true. - -Generator expressions always have to be written inside parentheses, but the -parentheses signalling a function call also count. If you want to create an -iterator that will be immediately passed to a function you can write:: - - obj_total = sum(obj.count for obj in list_all_objects()) - -The ``for...in`` clauses contain the sequences to be iterated over. The -sequences do not have to be the same length, because they are iterated over from -left to right, **not** in parallel. For each element in ``sequence1``, -``sequence2`` is looped over from the beginning. ``sequence3`` is then looped -over for each resulting pair of elements from ``sequence1`` and ``sequence2``. - -To put it another way, a list comprehension or generator expression is -equivalent to the following Python code:: - - for expr1 in sequence1: - if not (condition1): - continue # Skip this element - for expr2 in sequence2: - if not (condition2): - continue # Skip this element - ... - for exprN in sequenceN: - if not (conditionN): - continue # Skip this element - - # Output the value of - # the expression. - -This means that when there are multiple ``for...in`` clauses but no ``if`` -clauses, the length of the resulting output will be equal to the product of the -lengths of all the sequences. If you have two lists of length 3, the output -list is 9 elements long: - - >>> seq1 = 'abc' - >>> seq2 = (1, 2, 3) - >>> [(x, y) for x in seq1 for y in seq2] #doctest: +NORMALIZE_WHITESPACE - [('a', 1), ('a', 2), ('a', 3), - ('b', 1), ('b', 2), ('b', 3), - ('c', 1), ('c', 2), ('c', 3)] - -To avoid introducing an ambiguity into Python's grammar, if ``expression`` is -creating a tuple, it must be surrounded with parentheses. The first list -comprehension below is a syntax error, while the second one is correct:: - - # Syntax error - [x, y for x in seq1 for y in seq2] - # Correct - [(x, y) for x in seq1 for y in seq2] - - -Generators -========== - -Generators are a special class of functions that simplify the task of writing -iterators. Regular functions compute a value and return it, but generators -return an iterator that returns a stream of values. - -You're doubtless familiar with how regular function calls work in Python or C. -When you call a function, it gets a private namespace where its local variables -are created. When the function reaches a ``return`` statement, the local -variables are destroyed and the value is returned to the caller. A later call -to the same function creates a new private namespace and a fresh set of local -variables. But, what if the local variables weren't thrown away on exiting a -function? What if you could later resume the function where it left off? This -is what generators provide; they can be thought of as resumable functions. - -Here's the simplest example of a generator function: - - >>> def generate_ints(N): - ... for i in range(N): - ... yield i - -Any function containing a :keyword:`yield` keyword is a generator function; -this is detected by Python's :term:`bytecode` compiler which compiles the -function specially as a result. - -When you call a generator function, it doesn't return a single value; instead it -returns a generator object that supports the iterator protocol. On executing -the ``yield`` expression, the generator outputs the value of ``i``, similar to a -``return`` statement. The big difference between ``yield`` and a ``return`` -statement is that on reaching a ``yield`` the generator's state of execution is -suspended and local variables are preserved. On the next call to the -generator's :meth:`~generator.__next__` method, the function will resume -executing. - -Here's a sample usage of the ``generate_ints()`` generator: - - >>> gen = generate_ints(3) - >>> gen #doctest: +ELLIPSIS - <generator object generate_ints at ...> - >>> next(gen) - 0 - >>> next(gen) - 1 - >>> next(gen) - 2 - >>> next(gen) - Traceback (most recent call last): - File "stdin", line 1, in <module> - File "stdin", line 2, in generate_ints - StopIteration - -You could equally write ``for i in generate_ints(5)``, or ``a, b, c = -generate_ints(3)``. - -Inside a generator function, ``return value`` causes ``StopIteration(value)`` -to be raised from the :meth:`~generator.__next__` method. Once this happens, or -the bottom of the function is reached, the procession of values ends and the -generator cannot yield any further values. - -You could achieve the effect of generators manually by writing your own class -and storing all the local variables of the generator as instance variables. For -example, returning a list of integers could be done by setting ``self.count`` to -0, and having the :meth:`~iterator.__next__` method increment ``self.count`` and -return it. -However, for a moderately complicated generator, writing a corresponding class -can be much messier. - -The test suite included with Python's library, -:source:`Lib/test/test_generators.py`, contains -a number of more interesting examples. Here's one generator that implements an -in-order traversal of a tree using generators recursively. :: - - # A recursive generator that generates Tree leaves in in-order. - def inorder(t): - if t: - for x in inorder(t.left): - yield x - - yield t.label - - for x in inorder(t.right): - yield x - -Two other examples in ``test_generators.py`` produce solutions for the N-Queens -problem (placing N queens on an NxN chess board so that no queen threatens -another) and the Knight's Tour (finding a route that takes a knight to every -square of an NxN chessboard without visiting any square twice). - - - -Passing values into a generator -------------------------------- - -In Python 2.4 and earlier, generators only produced output. Once a generator's -code was invoked to create an iterator, there was no way to pass any new -information into the function when its execution is resumed. You could hack -together this ability by making the generator look at a global variable or by -passing in some mutable object that callers then modify, but these approaches -are messy. - -In Python 2.5 there's a simple way to pass values into a generator. -:keyword:`yield` became an expression, returning a value that can be assigned to -a variable or otherwise operated on:: - - val = (yield i) - -I recommend that you **always** put parentheses around a ``yield`` expression -when you're doing something with the returned value, as in the above example. -The parentheses aren't always necessary, but it's easier to always add them -instead of having to remember when they're needed. - -(:pep:`342` explains the exact rules, which are that a ``yield``-expression must -always be parenthesized except when it occurs at the top-level expression on the -right-hand side of an assignment. This means you can write ``val = yield i`` -but have to use parentheses when there's an operation, as in ``val = (yield i) -+ 12``.) - -Values are sent into a generator by calling its :meth:`send(value) -<generator.send>` method. This method resumes the generator's code and the -``yield`` expression returns the specified value. If the regular -:meth:`~generator.__next__` method is called, the ``yield`` returns ``None``. - -Here's a simple counter that increments by 1 and allows changing the value of -the internal counter. - -.. testcode:: - - def counter(maximum): - i = 0 - while i < maximum: - val = (yield i) - # If value provided, change counter - if val is not None: - i = val - else: - i += 1 - -And here's an example of changing the counter: - - >>> it = counter(10) #doctest: +SKIP - >>> next(it) #doctest: +SKIP - 0 - >>> next(it) #doctest: +SKIP - 1 - >>> it.send(8) #doctest: +SKIP - 8 - >>> next(it) #doctest: +SKIP - 9 - >>> next(it) #doctest: +SKIP - Traceback (most recent call last): - File "t.py", line 15, in <module> - it.next() - StopIteration - -Because ``yield`` will often be returning ``None``, you should always check for -this case. Don't just use its value in expressions unless you're sure that the -:meth:`~generator.send` method will be the only method used to resume your -generator function. - -In addition to :meth:`~generator.send`, there are two other methods on -generators: - -* :meth:`throw(value) <generator.throw>` is used to - raise an exception inside the generator; the exception is raised by the - ``yield`` expression where the generator's execution is paused. - -* :meth:`~generator.close` raises a :exc:`GeneratorExit` exception inside the - generator to terminate the iteration. On receiving this exception, the - generator's code must either raise :exc:`GeneratorExit` or - :exc:`StopIteration`; catching the exception and doing anything else is - illegal and will trigger a :exc:`RuntimeError`. :meth:`~generator.close` - will also be called by Python's garbage collector when the generator is - garbage-collected. - - If you need to run cleanup code when a :exc:`GeneratorExit` occurs, I suggest - using a ``try: ... finally:`` suite instead of catching :exc:`GeneratorExit`. - -The cumulative effect of these changes is to turn generators from one-way -producers of information into both producers and consumers. - -Generators also become **coroutines**, a more generalized form of subroutines. -Subroutines are entered at one point and exited at another point (the top of the -function, and a ``return`` statement), but coroutines can be entered, exited, -and resumed at many different points (the ``yield`` statements). - - -Built-in functions -================== - -Let's look in more detail at built-in functions often used with iterators. - -Two of Python's built-in functions, :func:`map` and :func:`filter` duplicate the -features of generator expressions: - -:func:`map(f, iterA, iterB, ...) <map>` returns an iterator over the sequence - ``f(iterA[0], iterB[0]), f(iterA[1], iterB[1]), f(iterA[2], iterB[2]), ...``. - - >>> def upper(s): - ... return s.upper() - - >>> list(map(upper, ['sentence', 'fragment'])) - ['SENTENCE', 'FRAGMENT'] - >>> [upper(s) for s in ['sentence', 'fragment']] - ['SENTENCE', 'FRAGMENT'] - -You can of course achieve the same effect with a list comprehension. - -:func:`filter(predicate, iter) <filter>` returns an iterator over all the -sequence elements that meet a certain condition, and is similarly duplicated by -list comprehensions. A **predicate** is a function that returns the truth -value of some condition; for use with :func:`filter`, the predicate must take a -single value. - - >>> def is_even(x): - ... return (x % 2) == 0 - - >>> list(filter(is_even, range(10))) - [0, 2, 4, 6, 8] - - -This can also be written as a list comprehension: - - >>> list(x for x in range(10) if is_even(x)) - [0, 2, 4, 6, 8] - - -:func:`enumerate(iter, start=0) <enumerate>` counts off the elements in the -iterable returning 2-tuples containing the count (from *start*) and -each element. :: - - >>> for item in enumerate(['subject', 'verb', 'object']): - ... print(item) - (0, 'subject') - (1, 'verb') - (2, 'object') - -:func:`enumerate` is often used when looping through a list and recording the -indexes at which certain conditions are met:: - - f = open('data.txt', 'r') - for i, line in enumerate(f): - if line.strip() == '': - print('Blank line at line #%i' % i) - -:func:`sorted(iterable, key=None, reverse=False) <sorted>` collects all the -elements of the iterable into a list, sorts the list, and returns the sorted -result. The *key* and *reverse* arguments are passed through to the -constructed list's :meth:`~list.sort` method. :: - - >>> import random - >>> # Generate 8 random numbers between [0, 10000) - >>> rand_list = random.sample(range(10000), 8) - >>> rand_list #doctest: +SKIP - [769, 7953, 9828, 6431, 8442, 9878, 6213, 2207] - >>> sorted(rand_list) #doctest: +SKIP - [769, 2207, 6213, 6431, 7953, 8442, 9828, 9878] - >>> sorted(rand_list, reverse=True) #doctest: +SKIP - [9878, 9828, 8442, 7953, 6431, 6213, 2207, 769] - -(For a more detailed discussion of sorting, see the :ref:`sortinghowto`.) - - -The :func:`any(iter) <any>` and :func:`all(iter) <all>` built-ins look at the -truth values of an iterable's contents. :func:`any` returns ``True`` if any element -in the iterable is a true value, and :func:`all` returns ``True`` if all of the -elements are true values: - - >>> any([0, 1, 0]) - True - >>> any([0, 0, 0]) - False - >>> any([1, 1, 1]) - True - >>> all([0, 1, 0]) - False - >>> all([0, 0, 0]) - False - >>> all([1, 1, 1]) - True - - -:func:`zip(iterA, iterB, ...) <zip>` takes one element from each iterable and -returns them in a tuple:: - - zip(['a', 'b', 'c'], (1, 2, 3)) => - ('a', 1), ('b', 2), ('c', 3) - -It doesn't construct an in-memory list and exhaust all the input iterators -before returning; instead tuples are constructed and returned only if they're -requested. (The technical term for this behaviour is `lazy evaluation -<https://en.wikipedia.org/wiki/Lazy_evaluation>`__.) - -This iterator is intended to be used with iterables that are all of the same -length. If the iterables are of different lengths, the resulting stream will be -the same length as the shortest iterable. :: - - zip(['a', 'b'], (1, 2, 3)) => - ('a', 1), ('b', 2) - -You should avoid doing this, though, because an element may be taken from the -longer iterators and discarded. This means you can't go on to use the iterators -further because you risk skipping a discarded element. - - -The itertools module -==================== - -The :mod:`itertools` module contains a number of commonly used iterators as well -as functions for combining several iterators. This section will introduce the -module's contents by showing small examples. - -The module's functions fall into a few broad classes: - -* Functions that create a new iterator based on an existing iterator. -* Functions for treating an iterator's elements as function arguments. -* Functions for selecting portions of an iterator's output. -* A function for grouping an iterator's output. - -Creating new iterators ----------------------- - -:func:`itertools.count(start, step) <itertools.count>` returns an infinite -stream of evenly spaced values. You can optionally supply the starting number, -which defaults to 0, and the interval between numbers, which defaults to 1:: - - itertools.count() => - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... - itertools.count(10) => - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... - itertools.count(10, 5) => - 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, ... - -:func:`itertools.cycle(iter) <itertools.cycle>` saves a copy of the contents of -a provided iterable and returns a new iterator that returns its elements from -first to last. The new iterator will repeat these elements infinitely. :: - - itertools.cycle([1, 2, 3, 4, 5]) => - 1, 2, 3, 4, 5, 1, 2, 3, 4, 5, ... - -:func:`itertools.repeat(elem, [n]) <itertools.repeat>` returns the provided -element *n* times, or returns the element endlessly if *n* is not provided. :: - - itertools.repeat('abc') => - abc, abc, abc, abc, abc, abc, abc, abc, abc, abc, ... - itertools.repeat('abc', 5) => - abc, abc, abc, abc, abc - -:func:`itertools.chain(iterA, iterB, ...) <itertools.chain>` takes an arbitrary -number of iterables as input, and returns all the elements of the first -iterator, then all the elements of the second, and so on, until all of the -iterables have been exhausted. :: - - itertools.chain(['a', 'b', 'c'], (1, 2, 3)) => - a, b, c, 1, 2, 3 - -:func:`itertools.islice(iter, [start], stop, [step]) <itertools.islice>` returns -a stream that's a slice of the iterator. With a single *stop* argument, it -will return the first *stop* elements. If you supply a starting index, you'll -get *stop-start* elements, and if you supply a value for *step*, elements -will be skipped accordingly. Unlike Python's string and list slicing, you can't -use negative values for *start*, *stop*, or *step*. :: - - itertools.islice(range(10), 8) => - 0, 1, 2, 3, 4, 5, 6, 7 - itertools.islice(range(10), 2, 8) => - 2, 3, 4, 5, 6, 7 - itertools.islice(range(10), 2, 8, 2) => - 2, 4, 6 - -:func:`itertools.tee(iter, [n]) <itertools.tee>` replicates an iterator; it -returns *n* independent iterators that will all return the contents of the -source iterator. -If you don't supply a value for *n*, the default is 2. Replicating iterators -requires saving some of the contents of the source iterator, so this can consume -significant memory if the iterator is large and one of the new iterators is -consumed more than the others. :: - - itertools.tee( itertools.count() ) => - iterA, iterB - - where iterA -> - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... - - and iterB -> - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, ... - - -Calling functions on elements ------------------------------ - -The :mod:`operator` module contains a set of functions corresponding to Python's -operators. Some examples are :func:`operator.add(a, b) <operator.add>` (adds -two values), :func:`operator.ne(a, b) <operator.ne>` (same as ``a != b``), and -:func:`operator.attrgetter('id') <operator.attrgetter>` -(returns a callable that fetches the ``.id`` attribute). - -:func:`itertools.starmap(func, iter) <itertools.starmap>` assumes that the -iterable will return a stream of tuples, and calls *func* using these tuples as -the arguments:: - - itertools.starmap(os.path.join, - [('/bin', 'python'), ('/usr', 'bin', 'java'), - ('/usr', 'bin', 'perl'), ('/usr', 'bin', 'ruby')]) - => - /bin/python, /usr/bin/java, /usr/bin/perl, /usr/bin/ruby - - -Selecting elements ------------------- - -Another group of functions chooses a subset of an iterator's elements based on a -predicate. - -:func:`itertools.filterfalse(predicate, iter) <itertools.filterfalse>` is the -opposite of :func:`filter`, returning all elements for which the predicate -returns false:: - - itertools.filterfalse(is_even, itertools.count()) => - 1, 3, 5, 7, 9, 11, 13, 15, ... - -:func:`itertools.takewhile(predicate, iter) <itertools.takewhile>` returns -elements for as long as the predicate returns true. Once the predicate returns -false, the iterator will signal the end of its results. :: - - def less_than_10(x): - return x < 10 - - itertools.takewhile(less_than_10, itertools.count()) => - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 - - itertools.takewhile(is_even, itertools.count()) => - 0 - -:func:`itertools.dropwhile(predicate, iter) <itertools.dropwhile>` discards -elements while the predicate returns true, and then returns the rest of the -iterable's results. :: - - itertools.dropwhile(less_than_10, itertools.count()) => - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, ... - - itertools.dropwhile(is_even, itertools.count()) => - 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, ... - -:func:`itertools.compress(data, selectors) <itertools.compress>` takes two -iterators and returns only those elements of *data* for which the corresponding -element of *selectors* is true, stopping whenever either one is exhausted:: - - itertools.compress([1, 2, 3, 4, 5], [True, True, False, False, True]) => - 1, 2, 5 - - -Combinatoric functions ----------------------- - -The :func:`itertools.combinations(iterable, r) <itertools.combinations>` -returns an iterator giving all possible *r*-tuple combinations of the -elements contained in *iterable*. :: - - itertools.combinations([1, 2, 3, 4, 5], 2) => - (1, 2), (1, 3), (1, 4), (1, 5), - (2, 3), (2, 4), (2, 5), - (3, 4), (3, 5), - (4, 5) - - itertools.combinations([1, 2, 3, 4, 5], 3) => - (1, 2, 3), (1, 2, 4), (1, 2, 5), (1, 3, 4), (1, 3, 5), (1, 4, 5), - (2, 3, 4), (2, 3, 5), (2, 4, 5), - (3, 4, 5) - -The elements within each tuple remain in the same order as -*iterable* returned them. For example, the number 1 is always before -2, 3, 4, or 5 in the examples above. A similar function, -:func:`itertools.permutations(iterable, r=None) <itertools.permutations>`, -removes this constraint on the order, returning all possible -arrangements of length *r*:: - - itertools.permutations([1, 2, 3, 4, 5], 2) => - (1, 2), (1, 3), (1, 4), (1, 5), - (2, 1), (2, 3), (2, 4), (2, 5), - (3, 1), (3, 2), (3, 4), (3, 5), - (4, 1), (4, 2), (4, 3), (4, 5), - (5, 1), (5, 2), (5, 3), (5, 4) - - itertools.permutations([1, 2, 3, 4, 5]) => - (1, 2, 3, 4, 5), (1, 2, 3, 5, 4), (1, 2, 4, 3, 5), - ... - (5, 4, 3, 2, 1) - -If you don't supply a value for *r* the length of the iterable is used, -meaning that all the elements are permuted. - -Note that these functions produce all of the possible combinations by -position and don't require that the contents of *iterable* are unique:: - - itertools.permutations('aba', 3) => - ('a', 'b', 'a'), ('a', 'a', 'b'), ('b', 'a', 'a'), - ('b', 'a', 'a'), ('a', 'a', 'b'), ('a', 'b', 'a') - -The identical tuple ``('a', 'a', 'b')`` occurs twice, but the two 'a' -strings came from different positions. - -The :func:`itertools.combinations_with_replacement(iterable, r) <itertools.combinations_with_replacement>` -function relaxes a different constraint: elements can be repeated -within a single tuple. Conceptually an element is selected for the -first position of each tuple and then is replaced before the second -element is selected. :: - - itertools.combinations_with_replacement([1, 2, 3, 4, 5], 2) => - (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), - (2, 2), (2, 3), (2, 4), (2, 5), - (3, 3), (3, 4), (3, 5), - (4, 4), (4, 5), - (5, 5) - - -Grouping elements ------------------ - -The last function I'll discuss, :func:`itertools.groupby(iter, key_func=None) -<itertools.groupby>`, is the most complicated. ``key_func(elem)`` is a function -that can compute a key value for each element returned by the iterable. If you -don't supply a key function, the key is simply each element itself. - -:func:`~itertools.groupby` collects all the consecutive elements from the -underlying iterable that have the same key value, and returns a stream of -2-tuples containing a key value and an iterator for the elements with that key. - -:: - - city_list = [('Decatur', 'AL'), ('Huntsville', 'AL'), ('Selma', 'AL'), - ('Anchorage', 'AK'), ('Nome', 'AK'), - ('Flagstaff', 'AZ'), ('Phoenix', 'AZ'), ('Tucson', 'AZ'), - ... - ] - - def get_state(city_state): - return city_state[1] - - itertools.groupby(city_list, get_state) => - ('AL', iterator-1), - ('AK', iterator-2), - ('AZ', iterator-3), ... - - where - iterator-1 => - ('Decatur', 'AL'), ('Huntsville', 'AL'), ('Selma', 'AL') - iterator-2 => - ('Anchorage', 'AK'), ('Nome', 'AK') - iterator-3 => - ('Flagstaff', 'AZ'), ('Phoenix', 'AZ'), ('Tucson', 'AZ') - -:func:`~itertools.groupby` assumes that the underlying iterable's contents will -already be sorted based on the key. Note that the returned iterators also use -the underlying iterable, so you have to consume the results of iterator-1 before -requesting iterator-2 and its corresponding key. - - -The functools module -==================== - -The :mod:`functools` module contains some higher-order functions. -A **higher-order function** takes one or more functions as input and returns a -new function. The most useful tool in this module is the -:func:`functools.partial` function. - -For programs written in a functional style, you'll sometimes want to construct -variants of existing functions that have some of the parameters filled in. -Consider a Python function ``f(a, b, c)``; you may wish to create a new function -``g(b, c)`` that's equivalent to ``f(1, b, c)``; you're filling in a value for -one of ``f()``'s parameters. This is called "partial function application". - -The constructor for :func:`~functools.partial` takes the arguments -``(function, arg1, arg2, ..., kwarg1=value1, kwarg2=value2)``. The resulting -object is callable, so you can just call it to invoke ``function`` with the -filled-in arguments. - -Here's a small but realistic example:: - - import functools - - def log(message, subsystem): - """Write the contents of 'message' to the specified subsystem.""" - print('%s: %s' % (subsystem, message)) - ... - - server_log = functools.partial(log, subsystem='server') - server_log('Unable to open socket') - -:func:`functools.reduce(func, iter, [initial_value]) <functools.reduce>` -cumulatively performs an operation on all the iterable's elements and, -therefore, can't be applied to infinite iterables. *func* must be a function -that takes two elements and returns a single value. :func:`functools.reduce` -takes the first two elements A and B returned by the iterator and calculates -``func(A, B)``. It then requests the third element, C, calculates -``func(func(A, B), C)``, combines this result with the fourth element returned, -and continues until the iterable is exhausted. If the iterable returns no -values at all, a :exc:`TypeError` exception is raised. If the initial value is -supplied, it's used as a starting point and ``func(initial_value, A)`` is the -first calculation. :: - - >>> import operator, functools - >>> functools.reduce(operator.concat, ['A', 'BB', 'C']) - 'ABBC' - >>> functools.reduce(operator.concat, []) - Traceback (most recent call last): - ... - TypeError: reduce() of empty sequence with no initial value - >>> functools.reduce(operator.mul, [1, 2, 3], 1) - 6 - >>> functools.reduce(operator.mul, [], 1) - 1 - -If you use :func:`operator.add` with :func:`functools.reduce`, you'll add up all the -elements of the iterable. This case is so common that there's a special -built-in called :func:`sum` to compute it: - - >>> import functools, operator - >>> functools.reduce(operator.add, [1, 2, 3, 4], 0) - 10 - >>> sum([1, 2, 3, 4]) - 10 - >>> sum([]) - 0 - -For many uses of :func:`functools.reduce`, though, it can be clearer to just -write the obvious :keyword:`for` loop:: - - import functools - # Instead of: - product = functools.reduce(operator.mul, [1, 2, 3], 1) - - # You can write: - product = 1 - for i in [1, 2, 3]: - product *= i - -A related function is :func:`itertools.accumulate(iterable, func=operator.add) -<itertools.accumulate>`. It performs the same calculation, but instead of -returning only the final result, :func:`~itertools.accumulate` returns an iterator -that also yields each partial result:: - - itertools.accumulate([1, 2, 3, 4, 5]) => - 1, 3, 6, 10, 15 - - itertools.accumulate([1, 2, 3, 4, 5], operator.mul) => - 1, 2, 6, 24, 120 - - -The operator module -------------------- - -The :mod:`operator` module was mentioned earlier. It contains a set of -functions corresponding to Python's operators. These functions are often useful -in functional-style code because they save you from writing trivial functions -that perform a single operation. - -Some of the functions in this module are: - -* Math operations: ``add()``, ``sub()``, ``mul()``, ``floordiv()``, ``abs()``, ... -* Logical operations: ``not_()``, ``truth()``. -* Bitwise operations: ``and_()``, ``or_()``, ``invert()``. -* Comparisons: ``eq()``, ``ne()``, ``lt()``, ``le()``, ``gt()``, and ``ge()``. -* Object identity: ``is_()``, ``is_not()``. - -Consult the operator module's documentation for a complete list. - - -Small functions and the lambda expression -========================================= - -When writing functional-style programs, you'll often need little functions that -act as predicates or that combine elements in some way. - -If there's a Python built-in or a module function that's suitable, you don't -need to define a new function at all:: - - stripped_lines = [line.strip() for line in lines] - existing_files = filter(os.path.exists, file_list) - -If the function you need doesn't exist, you need to write it. One way to write -small functions is to use the :keyword:`lambda` expression. ``lambda`` takes a -number of parameters and an expression combining these parameters, and creates -an anonymous function that returns the value of the expression:: - - adder = lambda x, y: x+y - - print_assign = lambda name, value: name + '=' + str(value) - -An alternative is to just use the ``def`` statement and define a function in the -usual way:: - - def adder(x, y): - return x + y - - def print_assign(name, value): - return name + '=' + str(value) - -Which alternative is preferable? That's a style question; my usual course is to -avoid using ``lambda``. - -One reason for my preference is that ``lambda`` is quite limited in the -functions it can define. The result has to be computable as a single -expression, which means you can't have multiway ``if... elif... else`` -comparisons or ``try... except`` statements. If you try to do too much in a -``lambda`` statement, you'll end up with an overly complicated expression that's -hard to read. Quick, what's the following code doing? :: - - import functools - total = functools.reduce(lambda a, b: (0, a[1] + b[1]), items)[1] - -You can figure it out, but it takes time to disentangle the expression to figure -out what's going on. Using a short nested ``def`` statements makes things a -little bit better:: - - import functools - def combine(a, b): - return 0, a[1] + b[1] - - total = functools.reduce(combine, items)[1] - -But it would be best of all if I had simply used a ``for`` loop:: - - total = 0 - for a, b in items: - total += b - -Or the :func:`sum` built-in and a generator expression:: - - total = sum(b for a, b in items) - -Many uses of :func:`functools.reduce` are clearer when written as ``for`` loops. - -Fredrik Lundh once suggested the following set of rules for refactoring uses of -``lambda``: - -1. Write a lambda function. -2. Write a comment explaining what the heck that lambda does. -3. Study the comment for a while, and think of a name that captures the essence - of the comment. -4. Convert the lambda to a def statement, using that name. -5. Remove the comment. - -I really like these rules, but you're free to disagree -about whether this lambda-free style is better. - - -Revision History and Acknowledgements -===================================== - -The author would like to thank the following people for offering suggestions, -corrections and assistance with various drafts of this article: Ian Bicking, -Nick Coghlan, Nick Efford, Raymond Hettinger, Jim Jewett, Mike Krell, Leandro -Lameiro, Jussi Salmela, Collin Winter, Blake Winton. - -Version 0.1: posted June 30 2006. - -Version 0.11: posted July 1 2006. Typo fixes. - -Version 0.2: posted July 10 2006. Merged genexp and listcomp sections into one. -Typo fixes. - -Version 0.21: Added more references suggested on the tutor mailing list. - -Version 0.30: Adds a section on the ``functional`` module written by Collin -Winter; adds short section on the operator module; a few other edits. - - -References -========== - -General -------- - -**Structure and Interpretation of Computer Programs**, by Harold Abelson and -Gerald Jay Sussman with Julie Sussman. The book can be found at -https://mitpress.mit.edu/sicp. In this classic textbook of computer science, -chapters 2 and 3 discuss the use of sequences and streams to organize the data -flow inside a program. The book uses Scheme for its examples, but many of the -design approaches described in these chapters are applicable to functional-style -Python code. - -https://www.defmacro.org/ramblings/fp.html: A general introduction to functional -programming that uses Java examples and has a lengthy historical introduction. - -https://en.wikipedia.org/wiki/Functional_programming: General Wikipedia entry -describing functional programming. - -https://en.wikipedia.org/wiki/Coroutine: Entry for coroutines. - -https://en.wikipedia.org/wiki/Currying: Entry for the concept of currying. - -Python-specific ---------------- - -https://gnosis.cx/TPiP/: The first chapter of David Mertz's book -:title-reference:`Text Processing in Python` discusses functional programming -for text processing, in the section titled "Utilizing Higher-Order Functions in -Text Processing". - -Mertz also wrote a 3-part series of articles on functional programming -for IBM's DeveloperWorks site; see -`part 1 <https://developer.ibm.com/articles/l-prog/>`__, -`part 2 <https://developer.ibm.com/tutorials/l-prog2/>`__, and -`part 3 <https://developer.ibm.com/tutorials/l-prog3/>`__, - - -Python documentation --------------------- - -Documentation for the :mod:`itertools` module. - -Documentation for the :mod:`functools` module. - -Documentation for the :mod:`operator` module. - -:pep:`289`: "Generator Expressions" - -:pep:`342`: "Coroutines via Enhanced Generators" describes the new generator -features in Python 2.5. - -.. comment - - Handy little function for printing part of an iterator -- used - while writing this document. - - import itertools - def print_iter(it): - slice = itertools.islice(it, 10) - for elem in slice[:-1]: - sys.stdout.write(str(elem)) - sys.stdout.write(', ') - print(elem[-1]) diff --git a/Doc/howto/index.rst b/Doc/howto/index.rst deleted file mode 100644 index c53c613bc5b1c6..00000000000000 --- a/Doc/howto/index.rst +++ /dev/null @@ -1,34 +0,0 @@ -*************** - Python HOWTOs -*************** - -Python HOWTOs are documents that cover a single, specific topic, -and attempt to cover it fairly completely. Modelled on the Linux -Documentation Project's HOWTO collection, this collection is an -effort to foster documentation that's more detailed than the -Python Library Reference. - -Currently, the HOWTOs are: - -.. toctree:: - :maxdepth: 1 - - pyporting.rst - cporting.rst - curses.rst - descriptor.rst - enum.rst - functional.rst - logging.rst - logging-cookbook.rst - regex.rst - sockets.rst - sorting.rst - unicode.rst - urllib2.rst - argparse.rst - ipaddress.rst - instrumentation.rst - annotations.rst - isolating-extensions.rst - diff --git a/Doc/howto/instrumentation.rst b/Doc/howto/instrumentation.rst deleted file mode 100644 index 9c99fcecce1fcb..00000000000000 --- a/Doc/howto/instrumentation.rst +++ /dev/null @@ -1,434 +0,0 @@ -.. highlight:: shell-session - -.. _instrumentation: - -=============================================== -Instrumenting CPython with DTrace and SystemTap -=============================================== - -:author: David Malcolm -:author: Łukasz Langa - -DTrace and SystemTap are monitoring tools, each providing a way to inspect -what the processes on a computer system are doing. They both use -domain-specific languages allowing a user to write scripts which: - -- filter which processes are to be observed -- gather data from the processes of interest -- generate reports on the data - -As of Python 3.6, CPython can be built with embedded "markers", also -known as "probes", that can be observed by a DTrace or SystemTap script, -making it easier to monitor what the CPython processes on a system are -doing. - -.. impl-detail:: - - DTrace markers are implementation details of the CPython interpreter. - No guarantees are made about probe compatibility between versions of - CPython. DTrace scripts can stop working or work incorrectly without - warning when changing CPython versions. - - -Enabling the static markers ---------------------------- - -macOS comes with built-in support for DTrace. On Linux, in order to -build CPython with the embedded markers for SystemTap, the SystemTap -development tools must be installed. - -On a Linux machine, this can be done via:: - - $ yum install systemtap-sdt-devel - -or:: - - $ sudo apt-get install systemtap-sdt-dev - - -CPython must then be :option:`configured with the --with-dtrace option -<--with-dtrace>`: - -.. code-block:: none - - checking for --with-dtrace... yes - -On macOS, you can list available DTrace probes by running a Python -process in the background and listing all probes made available by the -Python provider:: - - $ python3.6 -q & - $ sudo dtrace -l -P python$! # or: dtrace -l -m python3.6 - - ID PROVIDER MODULE FUNCTION NAME - 29564 python18035 python3.6 _PyEval_EvalFrameDefault function-entry - 29565 python18035 python3.6 dtrace_function_entry function-entry - 29566 python18035 python3.6 _PyEval_EvalFrameDefault function-return - 29567 python18035 python3.6 dtrace_function_return function-return - 29568 python18035 python3.6 collect gc-done - 29569 python18035 python3.6 collect gc-start - 29570 python18035 python3.6 _PyEval_EvalFrameDefault line - 29571 python18035 python3.6 maybe_dtrace_line line - -On Linux, you can verify if the SystemTap static markers are present in -the built binary by seeing if it contains a ".note.stapsdt" section. - -:: - - $ readelf -S ./python | grep .note.stapsdt - [30] .note.stapsdt NOTE 0000000000000000 00308d78 - -If you've built Python as a shared library -(with the :option:`--enable-shared` configure option), you -need to look instead within the shared library. For example:: - - $ readelf -S libpython3.3dm.so.1.0 | grep .note.stapsdt - [29] .note.stapsdt NOTE 0000000000000000 00365b68 - -Sufficiently modern readelf can print the metadata:: - - $ readelf -n ./python - - Displaying notes found at file offset 0x00000254 with length 0x00000020: - Owner Data size Description - GNU 0x00000010 NT_GNU_ABI_TAG (ABI version tag) - OS: Linux, ABI: 2.6.32 - - Displaying notes found at file offset 0x00000274 with length 0x00000024: - Owner Data size Description - GNU 0x00000014 NT_GNU_BUILD_ID (unique build ID bitstring) - Build ID: df924a2b08a7e89f6e11251d4602022977af2670 - - Displaying notes found at file offset 0x002d6c30 with length 0x00000144: - Owner Data size Description - stapsdt 0x00000031 NT_STAPSDT (SystemTap probe descriptors) - Provider: python - Name: gc__start - Location: 0x00000000004371c3, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6bf6 - Arguments: -4@%ebx - stapsdt 0x00000030 NT_STAPSDT (SystemTap probe descriptors) - Provider: python - Name: gc__done - Location: 0x00000000004374e1, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6bf8 - Arguments: -8@%rax - stapsdt 0x00000045 NT_STAPSDT (SystemTap probe descriptors) - Provider: python - Name: function__entry - Location: 0x000000000053db6c, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6be8 - Arguments: 8@%rbp 8@%r12 -4@%eax - stapsdt 0x00000046 NT_STAPSDT (SystemTap probe descriptors) - Provider: python - Name: function__return - Location: 0x000000000053dba8, Base: 0x0000000000630ce2, Semaphore: 0x00000000008d6bea - Arguments: 8@%rbp 8@%r12 -4@%eax - -The above metadata contains information for SystemTap describing how it -can patch strategically placed machine code instructions to enable the -tracing hooks used by a SystemTap script. - - -Static DTrace probes --------------------- - -The following example DTrace script can be used to show the call/return -hierarchy of a Python script, only tracing within the invocation of -a function called "start". In other words, import-time function -invocations are not going to be listed: - -.. code-block:: none - - self int indent; - - python$target:::function-entry - /copyinstr(arg1) == "start"/ - { - self->trace = 1; - } - - python$target:::function-entry - /self->trace/ - { - printf("%d\t%*s:", timestamp, 15, probename); - printf("%*s", self->indent, ""); - printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); - self->indent++; - } - - python$target:::function-return - /self->trace/ - { - self->indent--; - printf("%d\t%*s:", timestamp, 15, probename); - printf("%*s", self->indent, ""); - printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); - } - - python$target:::function-return - /copyinstr(arg1) == "start"/ - { - self->trace = 0; - } - -It can be invoked like this:: - - $ sudo dtrace -q -s call_stack.d -c "python3.6 script.py" - -The output looks like this: - -.. code-block:: none - - 156641360502280 function-entry:call_stack.py:start:23 - 156641360518804 function-entry: call_stack.py:function_1:1 - 156641360532797 function-entry: call_stack.py:function_3:9 - 156641360546807 function-return: call_stack.py:function_3:10 - 156641360563367 function-return: call_stack.py:function_1:2 - 156641360578365 function-entry: call_stack.py:function_2:5 - 156641360591757 function-entry: call_stack.py:function_1:1 - 156641360605556 function-entry: call_stack.py:function_3:9 - 156641360617482 function-return: call_stack.py:function_3:10 - 156641360629814 function-return: call_stack.py:function_1:2 - 156641360642285 function-return: call_stack.py:function_2:6 - 156641360656770 function-entry: call_stack.py:function_3:9 - 156641360669707 function-return: call_stack.py:function_3:10 - 156641360687853 function-entry: call_stack.py:function_4:13 - 156641360700719 function-return: call_stack.py:function_4:14 - 156641360719640 function-entry: call_stack.py:function_5:18 - 156641360732567 function-return: call_stack.py:function_5:21 - 156641360747370 function-return:call_stack.py:start:28 - - -Static SystemTap markers ------------------------- - -The low-level way to use the SystemTap integration is to use the static -markers directly. This requires you to explicitly state the binary file -containing them. - -For example, this SystemTap script can be used to show the call/return -hierarchy of a Python script: - -.. code-block:: none - - probe process("python").mark("function__entry") { - filename = user_string($arg1); - funcname = user_string($arg2); - lineno = $arg3; - - printf("%s => %s in %s:%d\\n", - thread_indent(1), funcname, filename, lineno); - } - - probe process("python").mark("function__return") { - filename = user_string($arg1); - funcname = user_string($arg2); - lineno = $arg3; - - printf("%s <= %s in %s:%d\\n", - thread_indent(-1), funcname, filename, lineno); - } - -It can be invoked like this:: - - $ stap \ - show-call-hierarchy.stp \ - -c "./python test.py" - -The output looks like this: - -.. code-block:: none - - 11408 python(8274): => __contains__ in Lib/_abcoll.py:362 - 11414 python(8274): => __getitem__ in Lib/os.py:425 - 11418 python(8274): => encode in Lib/os.py:490 - 11424 python(8274): <= encode in Lib/os.py:493 - 11428 python(8274): <= __getitem__ in Lib/os.py:426 - 11433 python(8274): <= __contains__ in Lib/_abcoll.py:366 - -where the columns are: - -- time in microseconds since start of script -- name of executable -- PID of process - -and the remainder indicates the call/return hierarchy as the script executes. - -For a :option:`--enable-shared` build of CPython, the markers are contained within the -libpython shared library, and the probe's dotted path needs to reflect this. For -example, this line from the above example: - -.. code-block:: none - - probe process("python").mark("function__entry") { - -should instead read: - -.. code-block:: none - - probe process("python").library("libpython3.6dm.so.1.0").mark("function__entry") { - -(assuming a :ref:`debug build <debug-build>` of CPython 3.6) - - -Available static markers ------------------------- - -.. object:: function__entry(str filename, str funcname, int lineno) - - This marker indicates that execution of a Python function has begun. - It is only triggered for pure-Python (bytecode) functions. - - The filename, function name, and line number are provided back to the - tracing script as positional arguments, which must be accessed using - ``$arg1``, ``$arg2``, ``$arg3``: - - * ``$arg1`` : ``(const char *)`` filename, accessible using ``user_string($arg1)`` - - * ``$arg2`` : ``(const char *)`` function name, accessible using - ``user_string($arg2)`` - - * ``$arg3`` : ``int`` line number - -.. object:: function__return(str filename, str funcname, int lineno) - - This marker is the converse of :c:func:`!function__entry`, and indicates that - execution of a Python function has ended (either via ``return``, or via an - exception). It is only triggered for pure-Python (bytecode) functions. - - The arguments are the same as for :c:func:`!function__entry` - -.. object:: line(str filename, str funcname, int lineno) - - This marker indicates a Python line is about to be executed. It is - the equivalent of line-by-line tracing with a Python profiler. It is - not triggered within C functions. - - The arguments are the same as for :c:func:`!function__entry`. - -.. object:: gc__start(int generation) - - Fires when the Python interpreter starts a garbage collection cycle. - ``arg0`` is the generation to scan, like :func:`gc.collect()`. - -.. object:: gc__done(long collected) - - Fires when the Python interpreter finishes a garbage collection - cycle. ``arg0`` is the number of collected objects. - -.. object:: import__find__load__start(str modulename) - - Fires before :mod:`importlib` attempts to find and load the module. - ``arg0`` is the module name. - - .. versionadded:: 3.7 - -.. object:: import__find__load__done(str modulename, int found) - - Fires after :mod:`importlib`'s find_and_load function is called. - ``arg0`` is the module name, ``arg1`` indicates if module was - successfully loaded. - - .. versionadded:: 3.7 - - -.. object:: audit(str event, void *tuple) - - Fires when :func:`sys.audit` or :c:func:`PySys_Audit` is called. - ``arg0`` is the event name as C string, ``arg1`` is a :c:type:`PyObject` - pointer to a tuple object. - - .. versionadded:: 3.8 - - -SystemTap Tapsets ------------------ - -The higher-level way to use the SystemTap integration is to use a "tapset": -SystemTap's equivalent of a library, which hides some of the lower-level -details of the static markers. - -Here is a tapset file, based on a non-shared build of CPython: - -.. code-block:: none - - /* - Provide a higher-level wrapping around the function__entry and - function__return markers: - \*/ - probe python.function.entry = process("python").mark("function__entry") - { - filename = user_string($arg1); - funcname = user_string($arg2); - lineno = $arg3; - frameptr = $arg4 - } - probe python.function.return = process("python").mark("function__return") - { - filename = user_string($arg1); - funcname = user_string($arg2); - lineno = $arg3; - frameptr = $arg4 - } - -If this file is installed in SystemTap's tapset directory (e.g. -``/usr/share/systemtap/tapset``), then these additional probepoints become -available: - -.. object:: python.function.entry(str filename, str funcname, int lineno, frameptr) - - This probe point indicates that execution of a Python function has begun. - It is only triggered for pure-Python (bytecode) functions. - -.. object:: python.function.return(str filename, str funcname, int lineno, frameptr) - - This probe point is the converse of ``python.function.return``, and - indicates that execution of a Python function has ended (either via - ``return``, or via an exception). It is only triggered for pure-Python - (bytecode) functions. - - -Examples --------- -This SystemTap script uses the tapset above to more cleanly implement the -example given above of tracing the Python function-call hierarchy, without -needing to directly name the static markers: - -.. code-block:: none - - probe python.function.entry - { - printf("%s => %s in %s:%d\n", - thread_indent(1), funcname, filename, lineno); - } - - probe python.function.return - { - printf("%s <= %s in %s:%d\n", - thread_indent(-1), funcname, filename, lineno); - } - - -The following script uses the tapset above to provide a top-like view of all -running CPython code, showing the top 20 most frequently entered bytecode -frames, each second, across the whole system: - -.. code-block:: none - - global fn_calls; - - probe python.function.entry - { - fn_calls[pid(), filename, funcname, lineno] += 1; - } - - probe timer.ms(1000) { - printf("\033[2J\033[1;1H") /* clear screen \*/ - printf("%6s %80s %6s %30s %6s\n", - "PID", "FILENAME", "LINE", "FUNCTION", "CALLS") - foreach ([pid, filename, funcname, lineno] in fn_calls- limit 20) { - printf("%6d %80s %6d %30s %6d\n", - pid, filename, lineno, funcname, - fn_calls[pid, filename, funcname, lineno]); - } - delete fn_calls; - } - diff --git a/Doc/howto/ipaddress.rst b/Doc/howto/ipaddress.rst deleted file mode 100644 index e852db98156fac..00000000000000 --- a/Doc/howto/ipaddress.rst +++ /dev/null @@ -1,340 +0,0 @@ -.. testsetup:: - - import ipaddress - -.. _ipaddress-howto: - -*************************************** -An introduction to the ipaddress module -*************************************** - -:author: Peter Moody -:author: Nick Coghlan - -.. topic:: Overview - - This document aims to provide a gentle introduction to the - :mod:`ipaddress` module. It is aimed primarily at users that aren't - already familiar with IP networking terminology, but may also be useful - to network engineers wanting an overview of how :mod:`ipaddress` - represents IP network addressing concepts. - - -Creating Address/Network/Interface objects -========================================== - -Since :mod:`ipaddress` is a module for inspecting and manipulating IP addresses, -the first thing you'll want to do is create some objects. You can use -:mod:`ipaddress` to create objects from strings and integers. - - -A Note on IP Versions ---------------------- - -For readers that aren't particularly familiar with IP addressing, it's -important to know that the Internet Protocol (IP) is currently in the process -of moving from version 4 of the protocol to version 6. This transition is -occurring largely because version 4 of the protocol doesn't provide enough -addresses to handle the needs of the whole world, especially given the -increasing number of devices with direct connections to the internet. - -Explaining the details of the differences between the two versions of the -protocol is beyond the scope of this introduction, but readers need to at -least be aware that these two versions exist, and it will sometimes be -necessary to force the use of one version or the other. - - -IP Host Addresses ------------------ - -Addresses, often referred to as "host addresses" are the most basic unit -when working with IP addressing. The simplest way to create addresses is -to use the :func:`ipaddress.ip_address` factory function, which automatically -determines whether to create an IPv4 or IPv6 address based on the passed in -value: - - >>> ipaddress.ip_address('192.0.2.1') - IPv4Address('192.0.2.1') - >>> ipaddress.ip_address('2001:DB8::1') - IPv6Address('2001:db8::1') - -Addresses can also be created directly from integers. Values that will -fit within 32 bits are assumed to be IPv4 addresses:: - - >>> ipaddress.ip_address(3221225985) - IPv4Address('192.0.2.1') - >>> ipaddress.ip_address(42540766411282592856903984951653826561) - IPv6Address('2001:db8::1') - -To force the use of IPv4 or IPv6 addresses, the relevant classes can be -invoked directly. This is particularly useful to force creation of IPv6 -addresses for small integers:: - - >>> ipaddress.ip_address(1) - IPv4Address('0.0.0.1') - >>> ipaddress.IPv4Address(1) - IPv4Address('0.0.0.1') - >>> ipaddress.IPv6Address(1) - IPv6Address('::1') - - -Defining Networks ------------------ - -Host addresses are usually grouped together into IP networks, so -:mod:`ipaddress` provides a way to create, inspect and manipulate network -definitions. IP network objects are constructed from strings that define the -range of host addresses that are part of that network. The simplest form -for that information is a "network address/network prefix" pair, where the -prefix defines the number of leading bits that are compared to determine -whether or not an address is part of the network and the network address -defines the expected value of those bits. - -As for addresses, a factory function is provided that determines the correct -IP version automatically:: - - >>> ipaddress.ip_network('192.0.2.0/24') - IPv4Network('192.0.2.0/24') - >>> ipaddress.ip_network('2001:db8::0/96') - IPv6Network('2001:db8::/96') - -Network objects cannot have any host bits set. The practical effect of this -is that ``192.0.2.1/24`` does not describe a network. Such definitions are -referred to as interface objects since the ip-on-a-network notation is -commonly used to describe network interfaces of a computer on a given network -and are described further in the next section. - -By default, attempting to create a network object with host bits set will -result in :exc:`ValueError` being raised. To request that the -additional bits instead be coerced to zero, the flag ``strict=False`` can -be passed to the constructor:: - - >>> ipaddress.ip_network('192.0.2.1/24') - Traceback (most recent call last): - ... - ValueError: 192.0.2.1/24 has host bits set - >>> ipaddress.ip_network('192.0.2.1/24', strict=False) - IPv4Network('192.0.2.0/24') - -While the string form offers significantly more flexibility, networks can -also be defined with integers, just like host addresses. In this case, the -network is considered to contain only the single address identified by the -integer, so the network prefix includes the entire network address:: - - >>> ipaddress.ip_network(3221225984) - IPv4Network('192.0.2.0/32') - >>> ipaddress.ip_network(42540766411282592856903984951653826560) - IPv6Network('2001:db8::/128') - -As with addresses, creation of a particular kind of network can be forced -by calling the class constructor directly instead of using the factory -function. - - -Host Interfaces ---------------- - -As mentioned just above, if you need to describe an address on a particular -network, neither the address nor the network classes are sufficient. -Notation like ``192.0.2.1/24`` is commonly used by network engineers and the -people who write tools for firewalls and routers as shorthand for "the host -``192.0.2.1`` on the network ``192.0.2.0/24``", Accordingly, :mod:`ipaddress` -provides a set of hybrid classes that associate an address with a particular -network. The interface for creation is identical to that for defining network -objects, except that the address portion isn't constrained to being a network -address. - - >>> ipaddress.ip_interface('192.0.2.1/24') - IPv4Interface('192.0.2.1/24') - >>> ipaddress.ip_interface('2001:db8::1/96') - IPv6Interface('2001:db8::1/96') - -Integer inputs are accepted (as with networks), and use of a particular IP -version can be forced by calling the relevant constructor directly. - - -Inspecting Address/Network/Interface Objects -============================================ - -You've gone to the trouble of creating an IPv(4|6)(Address|Network|Interface) -object, so you probably want to get information about it. :mod:`ipaddress` -tries to make doing this easy and intuitive. - -Extracting the IP version:: - - >>> addr4 = ipaddress.ip_address('192.0.2.1') - >>> addr6 = ipaddress.ip_address('2001:db8::1') - >>> addr6.version - 6 - >>> addr4.version - 4 - -Obtaining the network from an interface:: - - >>> host4 = ipaddress.ip_interface('192.0.2.1/24') - >>> host4.network - IPv4Network('192.0.2.0/24') - >>> host6 = ipaddress.ip_interface('2001:db8::1/96') - >>> host6.network - IPv6Network('2001:db8::/96') - -Finding out how many individual addresses are in a network:: - - >>> net4 = ipaddress.ip_network('192.0.2.0/24') - >>> net4.num_addresses - 256 - >>> net6 = ipaddress.ip_network('2001:db8::0/96') - >>> net6.num_addresses - 4294967296 - -Iterating through the "usable" addresses on a network:: - - >>> net4 = ipaddress.ip_network('192.0.2.0/24') - >>> for x in net4.hosts(): - ... print(x) # doctest: +ELLIPSIS - 192.0.2.1 - 192.0.2.2 - 192.0.2.3 - 192.0.2.4 - ... - 192.0.2.252 - 192.0.2.253 - 192.0.2.254 - - -Obtaining the netmask (i.e. set bits corresponding to the network prefix) or -the hostmask (any bits that are not part of the netmask): - - >>> net4 = ipaddress.ip_network('192.0.2.0/24') - >>> net4.netmask - IPv4Address('255.255.255.0') - >>> net4.hostmask - IPv4Address('0.0.0.255') - >>> net6 = ipaddress.ip_network('2001:db8::0/96') - >>> net6.netmask - IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::') - >>> net6.hostmask - IPv6Address('::ffff:ffff') - - -Exploding or compressing the address:: - - >>> addr6.exploded - '2001:0db8:0000:0000:0000:0000:0000:0001' - >>> addr6.compressed - '2001:db8::1' - >>> net6.exploded - '2001:0db8:0000:0000:0000:0000:0000:0000/96' - >>> net6.compressed - '2001:db8::/96' - -While IPv4 doesn't support explosion or compression, the associated objects -still provide the relevant properties so that version neutral code can -easily ensure the most concise or most verbose form is used for IPv6 -addresses while still correctly handling IPv4 addresses. - - -Networks as lists of Addresses -============================== - -It's sometimes useful to treat networks as lists. This means it is possible -to index them like this:: - - >>> net4[1] - IPv4Address('192.0.2.1') - >>> net4[-1] - IPv4Address('192.0.2.255') - >>> net6[1] - IPv6Address('2001:db8::1') - >>> net6[-1] - IPv6Address('2001:db8::ffff:ffff') - - -It also means that network objects lend themselves to using the list -membership test syntax like this:: - - if address in network: - # do something - -Containment testing is done efficiently based on the network prefix:: - - >>> addr4 = ipaddress.ip_address('192.0.2.1') - >>> addr4 in ipaddress.ip_network('192.0.2.0/24') - True - >>> addr4 in ipaddress.ip_network('192.0.3.0/24') - False - - -Comparisons -=========== - -:mod:`ipaddress` provides some simple, hopefully intuitive ways to compare -objects, where it makes sense:: - - >>> ipaddress.ip_address('192.0.2.1') < ipaddress.ip_address('192.0.2.2') - True - -A :exc:`TypeError` exception is raised if you try to compare objects of -different versions or different types. - - -Using IP Addresses with other modules -===================================== - -Other modules that use IP addresses (such as :mod:`socket`) usually won't -accept objects from this module directly. Instead, they must be coerced to -an integer or string that the other module will accept:: - - >>> addr4 = ipaddress.ip_address('192.0.2.1') - >>> str(addr4) - '192.0.2.1' - >>> int(addr4) - 3221225985 - - -Getting more detail when instance creation fails -================================================ - -When creating address/network/interface objects using the version-agnostic -factory functions, any errors will be reported as :exc:`ValueError` with -a generic error message that simply says the passed in value was not -recognized as an object of that type. The lack of a specific error is -because it's necessary to know whether the value is *supposed* to be IPv4 -or IPv6 in order to provide more detail on why it has been rejected. - -To support use cases where it is useful to have access to this additional -detail, the individual class constructors actually raise the -:exc:`ValueError` subclasses :exc:`ipaddress.AddressValueError` and -:exc:`ipaddress.NetmaskValueError` to indicate exactly which part of -the definition failed to parse correctly. - -The error messages are significantly more detailed when using the -class constructors directly. For example:: - - >>> ipaddress.ip_address("192.168.0.256") - Traceback (most recent call last): - ... - ValueError: '192.168.0.256' does not appear to be an IPv4 or IPv6 address - >>> ipaddress.IPv4Address("192.168.0.256") - Traceback (most recent call last): - ... - ipaddress.AddressValueError: Octet 256 (> 255) not permitted in '192.168.0.256' - - >>> ipaddress.ip_network("192.168.0.1/64") - Traceback (most recent call last): - ... - ValueError: '192.168.0.1/64' does not appear to be an IPv4 or IPv6 network - >>> ipaddress.IPv4Network("192.168.0.1/64") - Traceback (most recent call last): - ... - ipaddress.NetmaskValueError: '64' is not a valid netmask - -However, both of the module specific exceptions have :exc:`ValueError` as their -parent class, so if you're not concerned with the particular type of error, -you can still write code like the following:: - - try: - network = ipaddress.IPv4Network(address) - except ValueError: - print('address/netmask is invalid for IPv4:', address) - diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst deleted file mode 100644 index 9eb1f979cd3994..00000000000000 --- a/Doc/howto/logging-cookbook.rst +++ /dev/null @@ -1,3965 +0,0 @@ -.. _logging-cookbook: - -================ -Logging Cookbook -================ - -:Author: Vinay Sajip <vinay_sajip at red-dove dot com> - -This page contains a number of recipes related to logging, which have been found -useful in the past. For links to tutorial and reference information, please see -:ref:`cookbook-ref-links`. - -.. currentmodule:: logging - -Using logging in multiple modules ---------------------------------- - -Multiple calls to ``logging.getLogger('someLogger')`` return a reference to the -same logger object. This is true not only within the same module, but also -across modules as long as it is in the same Python interpreter process. It is -true for references to the same object; additionally, application code can -define and configure a parent logger in one module and create (but not -configure) a child logger in a separate module, and all logger calls to the -child will pass up to the parent. Here is a main module:: - - import logging - import auxiliary_module - - # create logger with 'spam_application' - logger = logging.getLogger('spam_application') - logger.setLevel(logging.DEBUG) - # create file handler which logs even debug messages - fh = logging.FileHandler('spam.log') - fh.setLevel(logging.DEBUG) - # create console handler with a higher log level - ch = logging.StreamHandler() - ch.setLevel(logging.ERROR) - # create formatter and add it to the handlers - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - fh.setFormatter(formatter) - ch.setFormatter(formatter) - # add the handlers to the logger - logger.addHandler(fh) - logger.addHandler(ch) - - logger.info('creating an instance of auxiliary_module.Auxiliary') - a = auxiliary_module.Auxiliary() - logger.info('created an instance of auxiliary_module.Auxiliary') - logger.info('calling auxiliary_module.Auxiliary.do_something') - a.do_something() - logger.info('finished auxiliary_module.Auxiliary.do_something') - logger.info('calling auxiliary_module.some_function()') - auxiliary_module.some_function() - logger.info('done with auxiliary_module.some_function()') - -Here is the auxiliary module:: - - import logging - - # create logger - module_logger = logging.getLogger('spam_application.auxiliary') - - class Auxiliary: - def __init__(self): - self.logger = logging.getLogger('spam_application.auxiliary.Auxiliary') - self.logger.info('creating an instance of Auxiliary') - - def do_something(self): - self.logger.info('doing something') - a = 1 + 1 - self.logger.info('done doing something') - - def some_function(): - module_logger.info('received a call to "some_function"') - -The output looks like this: - -.. code-block:: none - - 2005-03-23 23:47:11,663 - spam_application - INFO - - creating an instance of auxiliary_module.Auxiliary - 2005-03-23 23:47:11,665 - spam_application.auxiliary.Auxiliary - INFO - - creating an instance of Auxiliary - 2005-03-23 23:47:11,665 - spam_application - INFO - - created an instance of auxiliary_module.Auxiliary - 2005-03-23 23:47:11,668 - spam_application - INFO - - calling auxiliary_module.Auxiliary.do_something - 2005-03-23 23:47:11,668 - spam_application.auxiliary.Auxiliary - INFO - - doing something - 2005-03-23 23:47:11,669 - spam_application.auxiliary.Auxiliary - INFO - - done doing something - 2005-03-23 23:47:11,670 - spam_application - INFO - - finished auxiliary_module.Auxiliary.do_something - 2005-03-23 23:47:11,671 - spam_application - INFO - - calling auxiliary_module.some_function() - 2005-03-23 23:47:11,672 - spam_application.auxiliary - INFO - - received a call to 'some_function' - 2005-03-23 23:47:11,673 - spam_application - INFO - - done with auxiliary_module.some_function() - -Logging from multiple threads ------------------------------ - -Logging from multiple threads requires no special effort. The following example -shows logging from the main (initial) thread and another thread:: - - import logging - import threading - import time - - def worker(arg): - while not arg['stop']: - logging.debug('Hi from myfunc') - time.sleep(0.5) - - def main(): - logging.basicConfig(level=logging.DEBUG, format='%(relativeCreated)6d %(threadName)s %(message)s') - info = {'stop': False} - thread = threading.Thread(target=worker, args=(info,)) - thread.start() - while True: - try: - logging.debug('Hello from main') - time.sleep(0.75) - except KeyboardInterrupt: - info['stop'] = True - break - thread.join() - - if __name__ == '__main__': - main() - -When run, the script should print something like the following: - -.. code-block:: none - - 0 Thread-1 Hi from myfunc - 3 MainThread Hello from main - 505 Thread-1 Hi from myfunc - 755 MainThread Hello from main - 1007 Thread-1 Hi from myfunc - 1507 MainThread Hello from main - 1508 Thread-1 Hi from myfunc - 2010 Thread-1 Hi from myfunc - 2258 MainThread Hello from main - 2512 Thread-1 Hi from myfunc - 3009 MainThread Hello from main - 3013 Thread-1 Hi from myfunc - 3515 Thread-1 Hi from myfunc - 3761 MainThread Hello from main - 4017 Thread-1 Hi from myfunc - 4513 MainThread Hello from main - 4518 Thread-1 Hi from myfunc - -This shows the logging output interspersed as one might expect. This approach -works for more threads than shown here, of course. - -Multiple handlers and formatters --------------------------------- - -Loggers are plain Python objects. The :meth:`~Logger.addHandler` method has no -minimum or maximum quota for the number of handlers you may add. Sometimes it -will be beneficial for an application to log all messages of all severities to a -text file while simultaneously logging errors or above to the console. To set -this up, simply configure the appropriate handlers. The logging calls in the -application code will remain unchanged. Here is a slight modification to the -previous simple module-based configuration example:: - - import logging - - logger = logging.getLogger('simple_example') - logger.setLevel(logging.DEBUG) - # create file handler which logs even debug messages - fh = logging.FileHandler('spam.log') - fh.setLevel(logging.DEBUG) - # create console handler with a higher log level - ch = logging.StreamHandler() - ch.setLevel(logging.ERROR) - # create formatter and add it to the handlers - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - ch.setFormatter(formatter) - fh.setFormatter(formatter) - # add the handlers to logger - logger.addHandler(ch) - logger.addHandler(fh) - - # 'application' code - logger.debug('debug message') - logger.info('info message') - logger.warning('warn message') - logger.error('error message') - logger.critical('critical message') - -Notice that the 'application' code does not care about multiple handlers. All -that changed was the addition and configuration of a new handler named *fh*. - -The ability to create new handlers with higher- or lower-severity filters can be -very helpful when writing and testing an application. Instead of using many -``print`` statements for debugging, use ``logger.debug``: Unlike the print -statements, which you will have to delete or comment out later, the logger.debug -statements can remain intact in the source code and remain dormant until you -need them again. At that time, the only change that needs to happen is to -modify the severity level of the logger and/or handler to debug. - -.. _multiple-destinations: - -Logging to multiple destinations --------------------------------- - -Let's say you want to log to console and file with different message formats and -in differing circumstances. Say you want to log messages with levels of DEBUG -and higher to file, and those messages at level INFO and higher to the console. -Let's also assume that the file should contain timestamps, but the console -messages should not. Here's how you can achieve this:: - - import logging - - # set up logging to file - see previous section for more details - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', - datefmt='%m-%d %H:%M', - filename='/tmp/myapp.log', - filemode='w') - # define a Handler which writes INFO messages or higher to the sys.stderr - console = logging.StreamHandler() - console.setLevel(logging.INFO) - # set a format which is simpler for console use - formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s') - # tell the handler to use this format - console.setFormatter(formatter) - # add the handler to the root logger - logging.getLogger('').addHandler(console) - - # Now, we can log to the root logger, or any other logger. First the root... - logging.info('Jackdaws love my big sphinx of quartz.') - - # Now, define a couple of other loggers which might represent areas in your - # application: - - logger1 = logging.getLogger('myapp.area1') - logger2 = logging.getLogger('myapp.area2') - - logger1.debug('Quick zephyrs blow, vexing daft Jim.') - logger1.info('How quickly daft jumping zebras vex.') - logger2.warning('Jail zesty vixen who grabbed pay from quack.') - logger2.error('The five boxing wizards jump quickly.') - -When you run this, on the console you will see - -.. code-block:: none - - root : INFO Jackdaws love my big sphinx of quartz. - myapp.area1 : INFO How quickly daft jumping zebras vex. - myapp.area2 : WARNING Jail zesty vixen who grabbed pay from quack. - myapp.area2 : ERROR The five boxing wizards jump quickly. - -and in the file you will see something like - -.. code-block:: none - - 10-22 22:19 root INFO Jackdaws love my big sphinx of quartz. - 10-22 22:19 myapp.area1 DEBUG Quick zephyrs blow, vexing daft Jim. - 10-22 22:19 myapp.area1 INFO How quickly daft jumping zebras vex. - 10-22 22:19 myapp.area2 WARNING Jail zesty vixen who grabbed pay from quack. - 10-22 22:19 myapp.area2 ERROR The five boxing wizards jump quickly. - -As you can see, the DEBUG message only shows up in the file. The other messages -are sent to both destinations. - -This example uses console and file handlers, but you can use any number and -combination of handlers you choose. - -Note that the above choice of log filename ``/tmp/myapp.log`` implies use of a -standard location for temporary files on POSIX systems. On Windows, you may need to -choose a different directory name for the log - just ensure that the directory exists -and that you have the permissions to create and update files in it. - - -.. _custom-level-handling: - -Custom handling of levels -------------------------- - -Sometimes, you might want to do something slightly different from the standard -handling of levels in handlers, where all levels above a threshold get -processed by a handler. To do this, you need to use filters. Let's look at a -scenario where you want to arrange things as follows: - -* Send messages of severity ``INFO`` and ``WARNING`` to ``sys.stdout`` -* Send messages of severity ``ERROR`` and above to ``sys.stderr`` -* Send messages of severity ``DEBUG`` and above to file ``app.log`` - -Suppose you configure logging with the following JSON: - -.. code-block:: json - - { - "version": 1, - "disable_existing_loggers": false, - "formatters": { - "simple": { - "format": "%(levelname)-8s - %(message)s" - } - }, - "handlers": { - "stdout": { - "class": "logging.StreamHandler", - "level": "INFO", - "formatter": "simple", - "stream": "ext://sys.stdout" - }, - "stderr": { - "class": "logging.StreamHandler", - "level": "ERROR", - "formatter": "simple", - "stream": "ext://sys.stderr" - }, - "file": { - "class": "logging.FileHandler", - "formatter": "simple", - "filename": "app.log", - "mode": "w" - } - }, - "root": { - "level": "DEBUG", - "handlers": [ - "stderr", - "stdout", - "file" - ] - } - } - -This configuration does *almost* what we want, except that ``sys.stdout`` would -show messages of severity ``ERROR`` and above as well as ``INFO`` and -``WARNING`` messages. To prevent this, we can set up a filter which excludes -those messages and add it to the relevant handler. This can be configured by -adding a ``filters`` section parallel to ``formatters`` and ``handlers``: - -.. code-block:: json - - { - "filters": { - "warnings_and_below": { - "()" : "__main__.filter_maker", - "level": "WARNING" - } - } - } - -and changing the section on the ``stdout`` handler to add it: - -.. code-block:: json - - { - "stdout": { - "class": "logging.StreamHandler", - "level": "INFO", - "formatter": "simple", - "stream": "ext://sys.stdout", - "filters": ["warnings_and_below"] - } - } - -A filter is just a function, so we can define the ``filter_maker`` (a factory -function) as follows: - -.. code-block:: python - - def filter_maker(level): - level = getattr(logging, level) - - def filter(record): - return record.levelno <= level - - return filter - -This converts the string argument passed in to a numeric level, and returns a -function which only returns ``True`` if the level of the passed in record is -at or below the specified level. Note that in this example I have defined the -``filter_maker`` in a test script ``main.py`` that I run from the command line, -so its module will be ``__main__`` - hence the ``__main__.filter_maker`` in the -filter configuration. You will need to change that if you define it in a -different module. - -With the filter added, we can run ``main.py``, which in full is: - -.. code-block:: python - - import json - import logging - import logging.config - - CONFIG = ''' - { - "version": 1, - "disable_existing_loggers": false, - "formatters": { - "simple": { - "format": "%(levelname)-8s - %(message)s" - } - }, - "filters": { - "warnings_and_below": { - "()" : "__main__.filter_maker", - "level": "WARNING" - } - }, - "handlers": { - "stdout": { - "class": "logging.StreamHandler", - "level": "INFO", - "formatter": "simple", - "stream": "ext://sys.stdout", - "filters": ["warnings_and_below"] - }, - "stderr": { - "class": "logging.StreamHandler", - "level": "ERROR", - "formatter": "simple", - "stream": "ext://sys.stderr" - }, - "file": { - "class": "logging.FileHandler", - "formatter": "simple", - "filename": "app.log", - "mode": "w" - } - }, - "root": { - "level": "DEBUG", - "handlers": [ - "stderr", - "stdout", - "file" - ] - } - } - ''' - - def filter_maker(level): - level = getattr(logging, level) - - def filter(record): - return record.levelno <= level - - return filter - - logging.config.dictConfig(json.loads(CONFIG)) - logging.debug('A DEBUG message') - logging.info('An INFO message') - logging.warning('A WARNING message') - logging.error('An ERROR message') - logging.critical('A CRITICAL message') - -And after running it like this: - -.. code-block:: shell - - python main.py 2>stderr.log >stdout.log - -We can see the results are as expected: - -.. code-block:: shell - - $ more *.log - :::::::::::::: - app.log - :::::::::::::: - DEBUG - A DEBUG message - INFO - An INFO message - WARNING - A WARNING message - ERROR - An ERROR message - CRITICAL - A CRITICAL message - :::::::::::::: - stderr.log - :::::::::::::: - ERROR - An ERROR message - CRITICAL - A CRITICAL message - :::::::::::::: - stdout.log - :::::::::::::: - INFO - An INFO message - WARNING - A WARNING message - - -Configuration server example ----------------------------- - -Here is an example of a module using the logging configuration server:: - - import logging - import logging.config - import time - import os - - # read initial config file - logging.config.fileConfig('logging.conf') - - # create and start listener on port 9999 - t = logging.config.listen(9999) - t.start() - - logger = logging.getLogger('simpleExample') - - try: - # loop through logging calls to see the difference - # new configurations make, until Ctrl+C is pressed - while True: - logger.debug('debug message') - logger.info('info message') - logger.warning('warn message') - logger.error('error message') - logger.critical('critical message') - time.sleep(5) - except KeyboardInterrupt: - # cleanup - logging.config.stopListening() - t.join() - -And here is a script that takes a filename and sends that file to the server, -properly preceded with the binary-encoded length, as the new logging -configuration:: - - #!/usr/bin/env python - import socket, sys, struct - - with open(sys.argv[1], 'rb') as f: - data_to_send = f.read() - - HOST = 'localhost' - PORT = 9999 - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - print('connecting...') - s.connect((HOST, PORT)) - print('sending config...') - s.send(struct.pack('>L', len(data_to_send))) - s.send(data_to_send) - s.close() - print('complete') - - -.. _blocking-handlers: - -Dealing with handlers that block --------------------------------- - -.. currentmodule:: logging.handlers - -Sometimes you have to get your logging handlers to do their work without -blocking the thread you're logging from. This is common in web applications, -though of course it also occurs in other scenarios. - -A common culprit which demonstrates sluggish behaviour is the -:class:`SMTPHandler`: sending emails can take a long time, for a -number of reasons outside the developer's control (for example, a poorly -performing mail or network infrastructure). But almost any network-based -handler can block: Even a :class:`SocketHandler` operation may do a -DNS query under the hood which is too slow (and this query can be deep in the -socket library code, below the Python layer, and outside your control). - -One solution is to use a two-part approach. For the first part, attach only a -:class:`QueueHandler` to those loggers which are accessed from -performance-critical threads. They simply write to their queue, which can be -sized to a large enough capacity or initialized with no upper bound to their -size. The write to the queue will typically be accepted quickly, though you -will probably need to catch the :exc:`queue.Full` exception as a precaution -in your code. If you are a library developer who has performance-critical -threads in their code, be sure to document this (together with a suggestion to -attach only ``QueueHandlers`` to your loggers) for the benefit of other -developers who will use your code. - -The second part of the solution is :class:`QueueListener`, which has been -designed as the counterpart to :class:`QueueHandler`. A -:class:`QueueListener` is very simple: it's passed a queue and some handlers, -and it fires up an internal thread which listens to its queue for LogRecords -sent from ``QueueHandlers`` (or any other source of ``LogRecords``, for that -matter). The ``LogRecords`` are removed from the queue and passed to the -handlers for processing. - -The advantage of having a separate :class:`QueueListener` class is that you -can use the same instance to service multiple ``QueueHandlers``. This is more -resource-friendly than, say, having threaded versions of the existing handler -classes, which would eat up one thread per handler for no particular benefit. - -An example of using these two classes follows (imports omitted):: - - que = queue.Queue(-1) # no limit on size - queue_handler = QueueHandler(que) - handler = logging.StreamHandler() - listener = QueueListener(que, handler) - root = logging.getLogger() - root.addHandler(queue_handler) - formatter = logging.Formatter('%(threadName)s: %(message)s') - handler.setFormatter(formatter) - listener.start() - # The log output will display the thread which generated - # the event (the main thread) rather than the internal - # thread which monitors the internal queue. This is what - # you want to happen. - root.warning('Look out!') - listener.stop() - -which, when run, will produce: - -.. code-block:: none - - MainThread: Look out! - -.. note:: Although the earlier discussion wasn't specifically talking about - async code, but rather about slow logging handlers, it should be noted that - when logging from async code, network and even file handlers could lead to - problems (blocking the event loop) because some logging is done from - :mod:`asyncio` internals. It might be best, if any async code is used in an - application, to use the above approach for logging, so that any blocking code - runs only in the ``QueueListener`` thread. - -.. versionchanged:: 3.5 - Prior to Python 3.5, the :class:`QueueListener` always passed every message - received from the queue to every handler it was initialized with. (This was - because it was assumed that level filtering was all done on the other side, - where the queue is filled.) From 3.5 onwards, this behaviour can be changed - by passing a keyword argument ``respect_handler_level=True`` to the - listener's constructor. When this is done, the listener compares the level - of each message with the handler's level, and only passes a message to a - handler if it's appropriate to do so. - -.. _network-logging: - -Sending and receiving logging events across a network ------------------------------------------------------ - -Let's say you want to send logging events across a network, and handle them at -the receiving end. A simple way of doing this is attaching a -:class:`SocketHandler` instance to the root logger at the sending end:: - - import logging, logging.handlers - - rootLogger = logging.getLogger('') - rootLogger.setLevel(logging.DEBUG) - socketHandler = logging.handlers.SocketHandler('localhost', - logging.handlers.DEFAULT_TCP_LOGGING_PORT) - # don't bother with a formatter, since a socket handler sends the event as - # an unformatted pickle - rootLogger.addHandler(socketHandler) - - # Now, we can log to the root logger, or any other logger. First the root... - logging.info('Jackdaws love my big sphinx of quartz.') - - # Now, define a couple of other loggers which might represent areas in your - # application: - - logger1 = logging.getLogger('myapp.area1') - logger2 = logging.getLogger('myapp.area2') - - logger1.debug('Quick zephyrs blow, vexing daft Jim.') - logger1.info('How quickly daft jumping zebras vex.') - logger2.warning('Jail zesty vixen who grabbed pay from quack.') - logger2.error('The five boxing wizards jump quickly.') - -At the receiving end, you can set up a receiver using the :mod:`socketserver` -module. Here is a basic working example:: - - import pickle - import logging - import logging.handlers - import socketserver - import struct - - - class LogRecordStreamHandler(socketserver.StreamRequestHandler): - """Handler for a streaming logging request. - - This basically logs the record using whatever logging policy is - configured locally. - """ - - def handle(self): - """ - Handle multiple requests - each expected to be a 4-byte length, - followed by the LogRecord in pickle format. Logs the record - according to whatever policy is configured locally. - """ - while True: - chunk = self.connection.recv(4) - if len(chunk) < 4: - break - slen = struct.unpack('>L', chunk)[0] - chunk = self.connection.recv(slen) - while len(chunk) < slen: - chunk = chunk + self.connection.recv(slen - len(chunk)) - obj = self.unPickle(chunk) - record = logging.makeLogRecord(obj) - self.handleLogRecord(record) - - def unPickle(self, data): - return pickle.loads(data) - - def handleLogRecord(self, record): - # if a name is specified, we use the named logger rather than the one - # implied by the record. - if self.server.logname is not None: - name = self.server.logname - else: - name = record.name - logger = logging.getLogger(name) - # N.B. EVERY record gets logged. This is because Logger.handle - # is normally called AFTER logger-level filtering. If you want - # to do filtering, do it at the client end to save wasting - # cycles and network bandwidth! - logger.handle(record) - - class LogRecordSocketReceiver(socketserver.ThreadingTCPServer): - """ - Simple TCP socket-based logging receiver suitable for testing. - """ - - allow_reuse_address = True - - def __init__(self, host='localhost', - port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, - handler=LogRecordStreamHandler): - socketserver.ThreadingTCPServer.__init__(self, (host, port), handler) - self.abort = 0 - self.timeout = 1 - self.logname = None - - def serve_until_stopped(self): - import select - abort = 0 - while not abort: - rd, wr, ex = select.select([self.socket.fileno()], - [], [], - self.timeout) - if rd: - self.handle_request() - abort = self.abort - - def main(): - logging.basicConfig( - format='%(relativeCreated)5d %(name)-15s %(levelname)-8s %(message)s') - tcpserver = LogRecordSocketReceiver() - print('About to start TCP server...') - tcpserver.serve_until_stopped() - - if __name__ == '__main__': - main() - -First run the server, and then the client. On the client side, nothing is -printed on the console; on the server side, you should see something like: - -.. code-block:: none - - About to start TCP server... - 59 root INFO Jackdaws love my big sphinx of quartz. - 59 myapp.area1 DEBUG Quick zephyrs blow, vexing daft Jim. - 69 myapp.area1 INFO How quickly daft jumping zebras vex. - 69 myapp.area2 WARNING Jail zesty vixen who grabbed pay from quack. - 69 myapp.area2 ERROR The five boxing wizards jump quickly. - -Note that there are some security issues with pickle in some scenarios. If -these affect you, you can use an alternative serialization scheme by overriding -the :meth:`~SocketHandler.makePickle` method and implementing your -alternative there, as well as adapting the above script to use your alternative -serialization. - - -Running a logging socket listener in production -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. _socket-listener-gist: https://gist.github.com/vsajip/4b227eeec43817465ca835ca66f75e2b - -To run a logging listener in production, you may need to use a -process-management tool such as `Supervisor <http://supervisord.org/>`_. -`Here is a Gist <socket-listener-gist_>`__ -which provides the bare-bones files to run the above functionality using -Supervisor. It consists of the following files: - -+-------------------------+----------------------------------------------------+ -| File | Purpose | -+=========================+====================================================+ -| :file:`prepare.sh` | A Bash script to prepare the environment for | -| | testing | -+-------------------------+----------------------------------------------------+ -| :file:`supervisor.conf` | The Supervisor configuration file, which has | -| | entries for the listener and a multi-process web | -| | application | -+-------------------------+----------------------------------------------------+ -| :file:`ensure_app.sh` | A Bash script to ensure that Supervisor is running | -| | with the above configuration | -+-------------------------+----------------------------------------------------+ -| :file:`log_listener.py` | The socket listener program which receives log | -| | events and records them to a file | -+-------------------------+----------------------------------------------------+ -| :file:`main.py` | A simple web application which performs logging | -| | via a socket connected to the listener | -+-------------------------+----------------------------------------------------+ -| :file:`webapp.json` | A JSON configuration file for the web application | -+-------------------------+----------------------------------------------------+ -| :file:`client.py` | A Python script to exercise the web application | -+-------------------------+----------------------------------------------------+ - -The web application uses `Gunicorn <https://gunicorn.org/>`_, which is a -popular web application server that starts multiple worker processes to handle -requests. This example setup shows how the workers can write to the same log file -without conflicting with one another --- they all go through the socket listener. - -To test these files, do the following in a POSIX environment: - -#. Download `the Gist <socket-listener-gist_>`__ - as a ZIP archive using the :guilabel:`Download ZIP` button. - -#. Unzip the above files from the archive into a scratch directory. - -#. In the scratch directory, run ``bash prepare.sh`` to get things ready. - This creates a :file:`run` subdirectory to contain Supervisor-related and - log files, and a :file:`venv` subdirectory to contain a virtual environment - into which ``bottle``, ``gunicorn`` and ``supervisor`` are installed. - -#. Run ``bash ensure_app.sh`` to ensure that Supervisor is running with - the above configuration. - -#. Run ``venv/bin/python client.py`` to exercise the web application, - which will lead to records being written to the log. - -#. Inspect the log files in the :file:`run` subdirectory. You should see the - most recent log lines in files matching the pattern :file:`app.log*`. They won't be in - any particular order, since they have been handled concurrently by different - worker processes in a non-deterministic way. - -#. You can shut down the listener and the web application by running - ``venv/bin/supervisorctl -c supervisor.conf shutdown``. - -You may need to tweak the configuration files in the unlikely event that the -configured ports clash with something else in your test environment. - -.. currentmodule:: logging - -.. _context-info: - -Adding contextual information to your logging output ----------------------------------------------------- - -Sometimes you want logging output to contain contextual information in -addition to the parameters passed to the logging call. For example, in a -networked application, it may be desirable to log client-specific information -in the log (e.g. remote client's username, or IP address). Although you could -use the *extra* parameter to achieve this, it's not always convenient to pass -the information in this way. While it might be tempting to create -:class:`Logger` instances on a per-connection basis, this is not a good idea -because these instances are not garbage collected. While this is not a problem -in practice, when the number of :class:`Logger` instances is dependent on the -level of granularity you want to use in logging an application, it could -be hard to manage if the number of :class:`Logger` instances becomes -effectively unbounded. - - -Using LoggerAdapters to impart contextual information -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -An easy way in which you can pass contextual information to be output along -with logging event information is to use the :class:`LoggerAdapter` class. -This class is designed to look like a :class:`Logger`, so that you can call -:meth:`debug`, :meth:`info`, :meth:`warning`, :meth:`error`, -:meth:`exception`, :meth:`critical` and :meth:`log`. These methods have the -same signatures as their counterparts in :class:`Logger`, so you can use the -two types of instances interchangeably. - -When you create an instance of :class:`LoggerAdapter`, you pass it a -:class:`Logger` instance and a dict-like object which contains your contextual -information. When you call one of the logging methods on an instance of -:class:`LoggerAdapter`, it delegates the call to the underlying instance of -:class:`Logger` passed to its constructor, and arranges to pass the contextual -information in the delegated call. Here's a snippet from the code of -:class:`LoggerAdapter`:: - - def debug(self, msg, /, *args, **kwargs): - """ - Delegate a debug call to the underlying logger, after adding - contextual information from this adapter instance. - """ - msg, kwargs = self.process(msg, kwargs) - self.logger.debug(msg, *args, **kwargs) - -The :meth:`~LoggerAdapter.process` method of :class:`LoggerAdapter` is where the -contextual information is added to the logging output. It's passed the message -and keyword arguments of the logging call, and it passes back (potentially) -modified versions of these to use in the call to the underlying logger. The -default implementation of this method leaves the message alone, but inserts -an 'extra' key in the keyword argument whose value is the dict-like object -passed to the constructor. Of course, if you had passed an 'extra' keyword -argument in the call to the adapter, it will be silently overwritten. - -The advantage of using 'extra' is that the values in the dict-like object are -merged into the :class:`LogRecord` instance's __dict__, allowing you to use -customized strings with your :class:`Formatter` instances which know about -the keys of the dict-like object. If you need a different method, e.g. if you -want to prepend or append the contextual information to the message string, -you just need to subclass :class:`LoggerAdapter` and override -:meth:`~LoggerAdapter.process` to do what you need. Here is a simple example:: - - class CustomAdapter(logging.LoggerAdapter): - """ - This example adapter expects the passed in dict-like object to have a - 'connid' key, whose value in brackets is prepended to the log message. - """ - def process(self, msg, kwargs): - return '[%s] %s' % (self.extra['connid'], msg), kwargs - -which you can use like this:: - - logger = logging.getLogger(__name__) - adapter = CustomAdapter(logger, {'connid': some_conn_id}) - -Then any events that you log to the adapter will have the value of -``some_conn_id`` prepended to the log messages. - -Using objects other than dicts to pass contextual information -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You don't need to pass an actual dict to a :class:`LoggerAdapter` - you could -pass an instance of a class which implements ``__getitem__`` and ``__iter__`` so -that it looks like a dict to logging. This would be useful if you want to -generate values dynamically (whereas the values in a dict would be constant). - - -.. _filters-contextual: - -Using Filters to impart contextual information -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can also add contextual information to log output using a user-defined -:class:`Filter`. ``Filter`` instances are allowed to modify the ``LogRecords`` -passed to them, including adding additional attributes which can then be output -using a suitable format string, or if needed a custom :class:`Formatter`. - -For example in a web application, the request being processed (or at least, -the interesting parts of it) can be stored in a threadlocal -(:class:`threading.local`) variable, and then accessed from a ``Filter`` to -add, say, information from the request - say, the remote IP address and remote -user's username - to the ``LogRecord``, using the attribute names 'ip' and -'user' as in the ``LoggerAdapter`` example above. In that case, the same format -string can be used to get similar output to that shown above. Here's an example -script:: - - import logging - from random import choice - - class ContextFilter(logging.Filter): - """ - This is a filter which injects contextual information into the log. - - Rather than use actual contextual information, we just use random - data in this demo. - """ - - USERS = ['jim', 'fred', 'sheila'] - IPS = ['123.231.231.123', '127.0.0.1', '192.168.0.1'] - - def filter(self, record): - - record.ip = choice(ContextFilter.IPS) - record.user = choice(ContextFilter.USERS) - return True - - if __name__ == '__main__': - levels = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, logging.CRITICAL) - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)-15s %(name)-5s %(levelname)-8s IP: %(ip)-15s User: %(user)-8s %(message)s') - a1 = logging.getLogger('a.b.c') - a2 = logging.getLogger('d.e.f') - - f = ContextFilter() - a1.addFilter(f) - a2.addFilter(f) - a1.debug('A debug message') - a1.info('An info message with %s', 'some parameters') - for x in range(10): - lvl = choice(levels) - lvlname = logging.getLevelName(lvl) - a2.log(lvl, 'A message at %s level with %d %s', lvlname, 2, 'parameters') - -which, when run, produces something like: - -.. code-block:: none - - 2010-09-06 22:38:15,292 a.b.c DEBUG IP: 123.231.231.123 User: fred A debug message - 2010-09-06 22:38:15,300 a.b.c INFO IP: 192.168.0.1 User: sheila An info message with some parameters - 2010-09-06 22:38:15,300 d.e.f CRITICAL IP: 127.0.0.1 User: sheila A message at CRITICAL level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f ERROR IP: 127.0.0.1 User: jim A message at ERROR level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f DEBUG IP: 127.0.0.1 User: sheila A message at DEBUG level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f ERROR IP: 123.231.231.123 User: fred A message at ERROR level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f CRITICAL IP: 192.168.0.1 User: jim A message at CRITICAL level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f CRITICAL IP: 127.0.0.1 User: sheila A message at CRITICAL level with 2 parameters - 2010-09-06 22:38:15,300 d.e.f DEBUG IP: 192.168.0.1 User: jim A message at DEBUG level with 2 parameters - 2010-09-06 22:38:15,301 d.e.f ERROR IP: 127.0.0.1 User: sheila A message at ERROR level with 2 parameters - 2010-09-06 22:38:15,301 d.e.f DEBUG IP: 123.231.231.123 User: fred A message at DEBUG level with 2 parameters - 2010-09-06 22:38:15,301 d.e.f INFO IP: 123.231.231.123 User: fred A message at INFO level with 2 parameters - -Use of ``contextvars`` ----------------------- - -Since Python 3.7, the :mod:`contextvars` module has provided context-local storage -which works for both :mod:`threading` and :mod:`asyncio` processing needs. This type -of storage may thus be generally preferable to thread-locals. The following example -shows how, in a multi-threaded environment, logs can populated with contextual -information such as, for example, request attributes handled by web applications. - -For the purposes of illustration, say that you have different web applications, each -independent of the other but running in the same Python process and using a library -common to them. How can each of these applications have their own log, where all -logging messages from the library (and other request processing code) are directed to -the appropriate application's log file, while including in the log additional -contextual information such as client IP, HTTP request method and client username? - -Let's assume that the library can be simulated by the following code: - -.. code-block:: python - - # webapplib.py - import logging - import time - - logger = logging.getLogger(__name__) - - def useful(): - # Just a representative event logged from the library - logger.debug('Hello from webapplib!') - # Just sleep for a bit so other threads get to run - time.sleep(0.01) - -We can simulate the multiple web applications by means of two simple classes, -``Request`` and ``WebApp``. These simulate how real threaded web applications work - -each request is handled by a thread: - -.. code-block:: python - - # main.py - import argparse - from contextvars import ContextVar - import logging - import os - from random import choice - import threading - import webapplib - - logger = logging.getLogger(__name__) - root = logging.getLogger() - root.setLevel(logging.DEBUG) - - class Request: - """ - A simple dummy request class which just holds dummy HTTP request method, - client IP address and client username - """ - def __init__(self, method, ip, user): - self.method = method - self.ip = ip - self.user = user - - # A dummy set of requests which will be used in the simulation - we'll just pick - # from this list randomly. Note that all GET requests are from 192.168.2.XXX - # addresses, whereas POST requests are from 192.16.3.XXX addresses. Three users - # are represented in the sample requests. - - REQUESTS = [ - Request('GET', '192.168.2.20', 'jim'), - Request('POST', '192.168.3.20', 'fred'), - Request('GET', '192.168.2.21', 'sheila'), - Request('POST', '192.168.3.21', 'jim'), - Request('GET', '192.168.2.22', 'fred'), - Request('POST', '192.168.3.22', 'sheila'), - ] - - # Note that the format string includes references to request context information - # such as HTTP method, client IP and username - - formatter = logging.Formatter('%(threadName)-11s %(appName)s %(name)-9s %(user)-6s %(ip)s %(method)-4s %(message)s') - - # Create our context variables. These will be filled at the start of request - # processing, and used in the logging that happens during that processing - - ctx_request = ContextVar('request') - ctx_appname = ContextVar('appname') - - class InjectingFilter(logging.Filter): - """ - A filter which injects context-specific information into logs and ensures - that only information for a specific webapp is included in its log - """ - def __init__(self, app): - self.app = app - - def filter(self, record): - request = ctx_request.get() - record.method = request.method - record.ip = request.ip - record.user = request.user - record.appName = appName = ctx_appname.get() - return appName == self.app.name - - class WebApp: - """ - A dummy web application class which has its own handler and filter for a - webapp-specific log. - """ - def __init__(self, name): - self.name = name - handler = logging.FileHandler(name + '.log', 'w') - f = InjectingFilter(self) - handler.setFormatter(formatter) - handler.addFilter(f) - root.addHandler(handler) - self.num_requests = 0 - - def process_request(self, request): - """ - This is the dummy method for processing a request. It's called on a - different thread for every request. We store the context information into - the context vars before doing anything else. - """ - ctx_request.set(request) - ctx_appname.set(self.name) - self.num_requests += 1 - logger.debug('Request processing started') - webapplib.useful() - logger.debug('Request processing finished') - - def main(): - fn = os.path.splitext(os.path.basename(__file__))[0] - adhf = argparse.ArgumentDefaultsHelpFormatter - ap = argparse.ArgumentParser(formatter_class=adhf, prog=fn, - description='Simulate a couple of web ' - 'applications handling some ' - 'requests, showing how request ' - 'context can be used to ' - 'populate logs') - aa = ap.add_argument - aa('--count', '-c', type=int, default=100, help='How many requests to simulate') - options = ap.parse_args() - - # Create the dummy webapps and put them in a list which we can use to select - # from randomly - app1 = WebApp('app1') - app2 = WebApp('app2') - apps = [app1, app2] - threads = [] - # Add a common handler which will capture all events - handler = logging.FileHandler('app.log', 'w') - handler.setFormatter(formatter) - root.addHandler(handler) - - # Generate calls to process requests - for i in range(options.count): - try: - # Pick an app at random and a request for it to process - app = choice(apps) - request = choice(REQUESTS) - # Process the request in its own thread - t = threading.Thread(target=app.process_request, args=(request,)) - threads.append(t) - t.start() - except KeyboardInterrupt: - break - - # Wait for the threads to terminate - for t in threads: - t.join() - - for app in apps: - print('%s processed %s requests' % (app.name, app.num_requests)) - - if __name__ == '__main__': - main() - -If you run the above, you should find that roughly half the requests go -into :file:`app1.log` and the rest into :file:`app2.log`, and the all the requests are -logged to :file:`app.log`. Each webapp-specific log will contain only log entries for -only that webapp, and the request information will be displayed consistently in the -log (i.e. the information in each dummy request will always appear together in a log -line). This is illustrated by the following shell output: - -.. code-block:: shell - - ~/logging-contextual-webapp$ python main.py - app1 processed 51 requests - app2 processed 49 requests - ~/logging-contextual-webapp$ wc -l *.log - 153 app1.log - 147 app2.log - 300 app.log - 600 total - ~/logging-contextual-webapp$ head -3 app1.log - Thread-3 (process_request) app1 __main__ jim 192.168.3.21 POST Request processing started - Thread-3 (process_request) app1 webapplib jim 192.168.3.21 POST Hello from webapplib! - Thread-5 (process_request) app1 __main__ jim 192.168.3.21 POST Request processing started - ~/logging-contextual-webapp$ head -3 app2.log - Thread-1 (process_request) app2 __main__ sheila 192.168.2.21 GET Request processing started - Thread-1 (process_request) app2 webapplib sheila 192.168.2.21 GET Hello from webapplib! - Thread-2 (process_request) app2 __main__ jim 192.168.2.20 GET Request processing started - ~/logging-contextual-webapp$ head app.log - Thread-1 (process_request) app2 __main__ sheila 192.168.2.21 GET Request processing started - Thread-1 (process_request) app2 webapplib sheila 192.168.2.21 GET Hello from webapplib! - Thread-2 (process_request) app2 __main__ jim 192.168.2.20 GET Request processing started - Thread-3 (process_request) app1 __main__ jim 192.168.3.21 POST Request processing started - Thread-2 (process_request) app2 webapplib jim 192.168.2.20 GET Hello from webapplib! - Thread-3 (process_request) app1 webapplib jim 192.168.3.21 POST Hello from webapplib! - Thread-4 (process_request) app2 __main__ fred 192.168.2.22 GET Request processing started - Thread-5 (process_request) app1 __main__ jim 192.168.3.21 POST Request processing started - Thread-4 (process_request) app2 webapplib fred 192.168.2.22 GET Hello from webapplib! - Thread-6 (process_request) app1 __main__ jim 192.168.3.21 POST Request processing started - ~/logging-contextual-webapp$ grep app1 app1.log | wc -l - 153 - ~/logging-contextual-webapp$ grep app2 app2.log | wc -l - 147 - ~/logging-contextual-webapp$ grep app1 app.log | wc -l - 153 - ~/logging-contextual-webapp$ grep app2 app.log | wc -l - 147 - - -Imparting contextual information in handlers --------------------------------------------- - -Each :class:`~Handler` has its own chain of filters. -If you want to add contextual information to a :class:`LogRecord` without leaking -it to other handlers, you can use a filter that returns -a new :class:`~LogRecord` instead of modifying it in-place, as shown in the following script:: - - import copy - import logging - - def filter(record: logging.LogRecord): - record = copy.copy(record) - record.user = 'jim' - return record - - if __name__ == '__main__': - logger = logging.getLogger() - logger.setLevel(logging.INFO) - handler = logging.StreamHandler() - formatter = logging.Formatter('%(message)s from %(user)-8s') - handler.setFormatter(formatter) - handler.addFilter(filter) - logger.addHandler(handler) - - logger.info('A log message') - -.. _multiple-processes: - -Logging to a single file from multiple processes ------------------------------------------------- - -Although logging is thread-safe, and logging to a single file from multiple -threads in a single process *is* supported, logging to a single file from -*multiple processes* is *not* supported, because there is no standard way to -serialize access to a single file across multiple processes in Python. If you -need to log to a single file from multiple processes, one way of doing this is -to have all the processes log to a :class:`~handlers.SocketHandler`, and have a -separate process which implements a socket server which reads from the socket -and logs to file. (If you prefer, you can dedicate one thread in one of the -existing processes to perform this function.) -:ref:`This section <network-logging>` documents this approach in more detail and -includes a working socket receiver which can be used as a starting point for you -to adapt in your own applications. - -You could also write your own handler which uses the :class:`~multiprocessing.Lock` -class from the :mod:`multiprocessing` module to serialize access to the -file from your processes. The existing :class:`FileHandler` and subclasses do -not make use of :mod:`multiprocessing` at present, though they may do so in the -future. Note that at present, the :mod:`multiprocessing` module does not provide -working lock functionality on all platforms (see -https://bugs.python.org/issue3770). - -.. currentmodule:: logging.handlers - -Alternatively, you can use a ``Queue`` and a :class:`QueueHandler` to send -all logging events to one of the processes in your multi-process application. -The following example script demonstrates how you can do this; in the example -a separate listener process listens for events sent by other processes and logs -them according to its own logging configuration. Although the example only -demonstrates one way of doing it (for example, you may want to use a listener -thread rather than a separate listener process -- the implementation would be -analogous) it does allow for completely different logging configurations for -the listener and the other processes in your application, and can be used as -the basis for code meeting your own specific requirements:: - - # You'll need these imports in your own code - import logging - import logging.handlers - import multiprocessing - - # Next two import lines for this demo only - from random import choice, random - import time - - # - # Because you'll want to define the logging configurations for listener and workers, the - # listener and worker process functions take a configurer parameter which is a callable - # for configuring logging for that process. These functions are also passed the queue, - # which they use for communication. - # - # In practice, you can configure the listener however you want, but note that in this - # simple example, the listener does not apply level or filter logic to received records. - # In practice, you would probably want to do this logic in the worker processes, to avoid - # sending events which would be filtered out between processes. - # - # The size of the rotated files is made small so you can see the results easily. - def listener_configurer(): - root = logging.getLogger() - h = logging.handlers.RotatingFileHandler('mptest.log', 'a', 300, 10) - f = logging.Formatter('%(asctime)s %(processName)-10s %(name)s %(levelname)-8s %(message)s') - h.setFormatter(f) - root.addHandler(h) - - # This is the listener process top-level loop: wait for logging events - # (LogRecords)on the queue and handle them, quit when you get a None for a - # LogRecord. - def listener_process(queue, configurer): - configurer() - while True: - try: - record = queue.get() - if record is None: # We send this as a sentinel to tell the listener to quit. - break - logger = logging.getLogger(record.name) - logger.handle(record) # No level or filter logic applied - just do it! - except Exception: - import sys, traceback - print('Whoops! Problem:', file=sys.stderr) - traceback.print_exc(file=sys.stderr) - - # Arrays used for random selections in this demo - - LEVELS = [logging.DEBUG, logging.INFO, logging.WARNING, - logging.ERROR, logging.CRITICAL] - - LOGGERS = ['a.b.c', 'd.e.f'] - - MESSAGES = [ - 'Random message #1', - 'Random message #2', - 'Random message #3', - ] - - # The worker configuration is done at the start of the worker process run. - # Note that on Windows you can't rely on fork semantics, so each process - # will run the logging configuration code when it starts. - def worker_configurer(queue): - h = logging.handlers.QueueHandler(queue) # Just the one handler needed - root = logging.getLogger() - root.addHandler(h) - # send all messages, for demo; no other level or filter logic applied. - root.setLevel(logging.DEBUG) - - # This is the worker process top-level loop, which just logs ten events with - # random intervening delays before terminating. - # The print messages are just so you know it's doing something! - def worker_process(queue, configurer): - configurer(queue) - name = multiprocessing.current_process().name - print('Worker started: %s' % name) - for i in range(10): - time.sleep(random()) - logger = logging.getLogger(choice(LOGGERS)) - level = choice(LEVELS) - message = choice(MESSAGES) - logger.log(level, message) - print('Worker finished: %s' % name) - - # Here's where the demo gets orchestrated. Create the queue, create and start - # the listener, create ten workers and start them, wait for them to finish, - # then send a None to the queue to tell the listener to finish. - def main(): - queue = multiprocessing.Queue(-1) - listener = multiprocessing.Process(target=listener_process, - args=(queue, listener_configurer)) - listener.start() - workers = [] - for i in range(10): - worker = multiprocessing.Process(target=worker_process, - args=(queue, worker_configurer)) - workers.append(worker) - worker.start() - for w in workers: - w.join() - queue.put_nowait(None) - listener.join() - - if __name__ == '__main__': - main() - -A variant of the above script keeps the logging in the main process, in a -separate thread:: - - import logging - import logging.config - import logging.handlers - from multiprocessing import Process, Queue - import random - import threading - import time - - def logger_thread(q): - while True: - record = q.get() - if record is None: - break - logger = logging.getLogger(record.name) - logger.handle(record) - - - def worker_process(q): - qh = logging.handlers.QueueHandler(q) - root = logging.getLogger() - root.setLevel(logging.DEBUG) - root.addHandler(qh) - levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, - logging.CRITICAL] - loggers = ['foo', 'foo.bar', 'foo.bar.baz', - 'spam', 'spam.ham', 'spam.ham.eggs'] - for i in range(100): - lvl = random.choice(levels) - logger = logging.getLogger(random.choice(loggers)) - logger.log(lvl, 'Message no. %d', i) - - if __name__ == '__main__': - q = Queue() - d = { - 'version': 1, - 'formatters': { - 'detailed': { - 'class': 'logging.Formatter', - 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO', - }, - 'file': { - 'class': 'logging.FileHandler', - 'filename': 'mplog.log', - 'mode': 'w', - 'formatter': 'detailed', - }, - 'foofile': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-foo.log', - 'mode': 'w', - 'formatter': 'detailed', - }, - 'errors': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-errors.log', - 'mode': 'w', - 'level': 'ERROR', - 'formatter': 'detailed', - }, - }, - 'loggers': { - 'foo': { - 'handlers': ['foofile'] - } - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console', 'file', 'errors'] - }, - } - workers = [] - for i in range(5): - wp = Process(target=worker_process, name='worker %d' % (i + 1), args=(q,)) - workers.append(wp) - wp.start() - logging.config.dictConfig(d) - lp = threading.Thread(target=logger_thread, args=(q,)) - lp.start() - # At this point, the main process could do some useful work of its own - # Once it's done that, it can wait for the workers to terminate... - for wp in workers: - wp.join() - # And now tell the logging thread to finish up, too - q.put(None) - lp.join() - -This variant shows how you can e.g. apply configuration for particular loggers -- e.g. the ``foo`` logger has a special handler which stores all events in the -``foo`` subsystem in a file ``mplog-foo.log``. This will be used by the logging -machinery in the main process (even though the logging events are generated in -the worker processes) to direct the messages to the appropriate destinations. - -Using concurrent.futures.ProcessPoolExecutor -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you want to use :class:`concurrent.futures.ProcessPoolExecutor` to start -your worker processes, you need to create the queue slightly differently. -Instead of - -.. code-block:: python - - queue = multiprocessing.Queue(-1) - -you should use - -.. code-block:: python - - queue = multiprocessing.Manager().Queue(-1) # also works with the examples above - -and you can then replace the worker creation from this:: - - workers = [] - for i in range(10): - worker = multiprocessing.Process(target=worker_process, - args=(queue, worker_configurer)) - workers.append(worker) - worker.start() - for w in workers: - w.join() - -to this (remembering to first import :mod:`concurrent.futures`):: - - with concurrent.futures.ProcessPoolExecutor(max_workers=10) as executor: - for i in range(10): - executor.submit(worker_process, queue, worker_configurer) - -Deploying Web applications using Gunicorn and uWSGI -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When deploying Web applications using `Gunicorn <https://gunicorn.org/>`_ or `uWSGI -<https://uwsgi-docs.readthedocs.io/en/latest/>`_ (or similar), multiple worker -processes are created to handle client requests. In such environments, avoid creating -file-based handlers directly in your web application. Instead, use a -:class:`SocketHandler` to log from the web application to a listener in a separate -process. This can be set up using a process management tool such as Supervisor - see -`Running a logging socket listener in production`_ for more details. - - -Using file rotation -------------------- - -.. sectionauthor:: Doug Hellmann, Vinay Sajip (changes) -.. (see <https://pymotw.com/3/logging/>) - -Sometimes you want to let a log file grow to a certain size, then open a new -file and log to that. You may want to keep a certain number of these files, and -when that many files have been created, rotate the files so that the number of -files and the size of the files both remain bounded. For this usage pattern, the -logging package provides a :class:`RotatingFileHandler`:: - - import glob - import logging - import logging.handlers - - LOG_FILENAME = 'logging_rotatingfile_example.out' - - # Set up a specific logger with our desired output level - my_logger = logging.getLogger('MyLogger') - my_logger.setLevel(logging.DEBUG) - - # Add the log message handler to the logger - handler = logging.handlers.RotatingFileHandler( - LOG_FILENAME, maxBytes=20, backupCount=5) - - my_logger.addHandler(handler) - - # Log some messages - for i in range(20): - my_logger.debug('i = %d' % i) - - # See what files are created - logfiles = glob.glob('%s*' % LOG_FILENAME) - - for filename in logfiles: - print(filename) - -The result should be 6 separate files, each with part of the log history for the -application: - -.. code-block:: none - - logging_rotatingfile_example.out - logging_rotatingfile_example.out.1 - logging_rotatingfile_example.out.2 - logging_rotatingfile_example.out.3 - logging_rotatingfile_example.out.4 - logging_rotatingfile_example.out.5 - -The most current file is always :file:`logging_rotatingfile_example.out`, -and each time it reaches the size limit it is renamed with the suffix -``.1``. Each of the existing backup files is renamed to increment the suffix -(``.1`` becomes ``.2``, etc.) and the ``.6`` file is erased. - -Obviously this example sets the log length much too small as an extreme -example. You would want to set *maxBytes* to an appropriate value. - -.. currentmodule:: logging - -.. _format-styles: - -Use of alternative formatting styles ------------------------------------- - -When logging was added to the Python standard library, the only way of -formatting messages with variable content was to use the %-formatting -method. Since then, Python has gained two new formatting approaches: -:class:`string.Template` (added in Python 2.4) and :meth:`str.format` -(added in Python 2.6). - -Logging (as of 3.2) provides improved support for these two additional -formatting styles. The :class:`Formatter` class been enhanced to take an -additional, optional keyword parameter named ``style``. This defaults to -``'%'``, but other possible values are ``'{'`` and ``'$'``, which correspond -to the other two formatting styles. Backwards compatibility is maintained by -default (as you would expect), but by explicitly specifying a style parameter, -you get the ability to specify format strings which work with -:meth:`str.format` or :class:`string.Template`. Here's an example console -session to show the possibilities: - -.. code-block:: pycon - - >>> import logging - >>> root = logging.getLogger() - >>> root.setLevel(logging.DEBUG) - >>> handler = logging.StreamHandler() - >>> bf = logging.Formatter('{asctime} {name} {levelname:8s} {message}', - ... style='{') - >>> handler.setFormatter(bf) - >>> root.addHandler(handler) - >>> logger = logging.getLogger('foo.bar') - >>> logger.debug('This is a DEBUG message') - 2010-10-28 15:11:55,341 foo.bar DEBUG This is a DEBUG message - >>> logger.critical('This is a CRITICAL message') - 2010-10-28 15:12:11,526 foo.bar CRITICAL This is a CRITICAL message - >>> df = logging.Formatter('$asctime $name ${levelname} $message', - ... style='$') - >>> handler.setFormatter(df) - >>> logger.debug('This is a DEBUG message') - 2010-10-28 15:13:06,924 foo.bar DEBUG This is a DEBUG message - >>> logger.critical('This is a CRITICAL message') - 2010-10-28 15:13:11,494 foo.bar CRITICAL This is a CRITICAL message - >>> - -Note that the formatting of logging messages for final output to logs is -completely independent of how an individual logging message is constructed. -That can still use %-formatting, as shown here:: - - >>> logger.error('This is an%s %s %s', 'other,', 'ERROR,', 'message') - 2010-10-28 15:19:29,833 foo.bar ERROR This is another, ERROR, message - >>> - -Logging calls (``logger.debug()``, ``logger.info()`` etc.) only take -positional parameters for the actual logging message itself, with keyword -parameters used only for determining options for how to handle the actual -logging call (e.g. the ``exc_info`` keyword parameter to indicate that -traceback information should be logged, or the ``extra`` keyword parameter -to indicate additional contextual information to be added to the log). So -you cannot directly make logging calls using :meth:`str.format` or -:class:`string.Template` syntax, because internally the logging package -uses %-formatting to merge the format string and the variable arguments. -There would be no changing this while preserving backward compatibility, since -all logging calls which are out there in existing code will be using %-format -strings. - -There is, however, a way that you can use {}- and $- formatting to construct -your individual log messages. Recall that for a message you can use an -arbitrary object as a message format string, and that the logging package will -call ``str()`` on that object to get the actual format string. Consider the -following two classes:: - - class BraceMessage: - def __init__(self, fmt, /, *args, **kwargs): - self.fmt = fmt - self.args = args - self.kwargs = kwargs - - def __str__(self): - return self.fmt.format(*self.args, **self.kwargs) - - class DollarMessage: - def __init__(self, fmt, /, **kwargs): - self.fmt = fmt - self.kwargs = kwargs - - def __str__(self): - from string import Template - return Template(self.fmt).substitute(**self.kwargs) - -Either of these can be used in place of a format string, to allow {}- or -$-formatting to be used to build the actual "message" part which appears in the -formatted log output in place of "%(message)s" or "{message}" or "$message". -It's a little unwieldy to use the class names whenever you want to log -something, but it's quite palatable if you use an alias such as __ (double -underscore --- not to be confused with _, the single underscore used as a -synonym/alias for :func:`gettext.gettext` or its brethren). - -The above classes are not included in Python, though they're easy enough to -copy and paste into your own code. They can be used as follows (assuming that -they're declared in a module called ``wherever``): - -.. code-block:: pycon - - >>> from wherever import BraceMessage as __ - >>> print(__('Message with {0} {name}', 2, name='placeholders')) - Message with 2 placeholders - >>> class Point: pass - ... - >>> p = Point() - >>> p.x = 0.5 - >>> p.y = 0.5 - >>> print(__('Message with coordinates: ({point.x:.2f}, {point.y:.2f})', - ... point=p)) - Message with coordinates: (0.50, 0.50) - >>> from wherever import DollarMessage as __ - >>> print(__('Message with $num $what', num=2, what='placeholders')) - Message with 2 placeholders - >>> - -While the above examples use ``print()`` to show how the formatting works, you -would of course use ``logger.debug()`` or similar to actually log using this -approach. - -One thing to note is that you pay no significant performance penalty with this -approach: the actual formatting happens not when you make the logging call, but -when (and if) the logged message is actually about to be output to a log by a -handler. So the only slightly unusual thing which might trip you up is that the -parentheses go around the format string and the arguments, not just the format -string. That's because the __ notation is just syntax sugar for a constructor -call to one of the :samp:`{XXX}Message` classes. - -If you prefer, you can use a :class:`LoggerAdapter` to achieve a similar effect -to the above, as in the following example:: - - import logging - - class Message: - def __init__(self, fmt, args): - self.fmt = fmt - self.args = args - - def __str__(self): - return self.fmt.format(*self.args) - - class StyleAdapter(logging.LoggerAdapter): - def __init__(self, logger, extra=None): - super().__init__(logger, extra or {}) - - def log(self, level, msg, /, *args, **kwargs): - if self.isEnabledFor(level): - msg, kwargs = self.process(msg, kwargs) - self.logger._log(level, Message(msg, args), (), **kwargs) - - logger = StyleAdapter(logging.getLogger(__name__)) - - def main(): - logger.debug('Hello, {}', 'world!') - - if __name__ == '__main__': - logging.basicConfig(level=logging.DEBUG) - main() - -The above script should log the message ``Hello, world!`` when run with -Python 3.2 or later. - - -.. currentmodule:: logging - -.. _custom-logrecord: - -Customizing ``LogRecord`` -------------------------- - -Every logging event is represented by a :class:`LogRecord` instance. -When an event is logged and not filtered out by a logger's level, a -:class:`LogRecord` is created, populated with information about the event and -then passed to the handlers for that logger (and its ancestors, up to and -including the logger where further propagation up the hierarchy is disabled). -Before Python 3.2, there were only two places where this creation was done: - -* :meth:`Logger.makeRecord`, which is called in the normal process of - logging an event. This invoked :class:`LogRecord` directly to create an - instance. -* :func:`makeLogRecord`, which is called with a dictionary containing - attributes to be added to the LogRecord. This is typically invoked when a - suitable dictionary has been received over the network (e.g. in pickle form - via a :class:`~handlers.SocketHandler`, or in JSON form via an - :class:`~handlers.HTTPHandler`). - -This has usually meant that if you need to do anything special with a -:class:`LogRecord`, you've had to do one of the following. - -* Create your own :class:`Logger` subclass, which overrides - :meth:`Logger.makeRecord`, and set it using :func:`~logging.setLoggerClass` - before any loggers that you care about are instantiated. -* Add a :class:`Filter` to a logger or handler, which does the - necessary special manipulation you need when its - :meth:`~Filter.filter` method is called. - -The first approach would be a little unwieldy in the scenario where (say) -several different libraries wanted to do different things. Each would attempt -to set its own :class:`Logger` subclass, and the one which did this last would -win. - -The second approach works reasonably well for many cases, but does not allow -you to e.g. use a specialized subclass of :class:`LogRecord`. Library -developers can set a suitable filter on their loggers, but they would have to -remember to do this every time they introduced a new logger (which they would -do simply by adding new packages or modules and doing :: - - logger = logging.getLogger(__name__) - -at module level). It's probably one too many things to think about. Developers -could also add the filter to a :class:`~logging.NullHandler` attached to their -top-level logger, but this would not be invoked if an application developer -attached a handler to a lower-level library logger --- so output from that -handler would not reflect the intentions of the library developer. - -In Python 3.2 and later, :class:`~logging.LogRecord` creation is done through a -factory, which you can specify. The factory is just a callable you can set with -:func:`~logging.setLogRecordFactory`, and interrogate with -:func:`~logging.getLogRecordFactory`. The factory is invoked with the same -signature as the :class:`~logging.LogRecord` constructor, as :class:`LogRecord` -is the default setting for the factory. - -This approach allows a custom factory to control all aspects of LogRecord -creation. For example, you could return a subclass, or just add some additional -attributes to the record once created, using a pattern similar to this:: - - old_factory = logging.getLogRecordFactory() - - def record_factory(*args, **kwargs): - record = old_factory(*args, **kwargs) - record.custom_attribute = 0xdecafbad - return record - - logging.setLogRecordFactory(record_factory) - -This pattern allows different libraries to chain factories together, and as -long as they don't overwrite each other's attributes or unintentionally -overwrite the attributes provided as standard, there should be no surprises. -However, it should be borne in mind that each link in the chain adds run-time -overhead to all logging operations, and the technique should only be used when -the use of a :class:`Filter` does not provide the desired result. - -.. currentmodule:: logging.handlers - -.. _zeromq-handlers: - -Subclassing QueueHandler - a ZeroMQ example -------------------------------------------- - -You can use a :class:`QueueHandler` subclass to send messages to other kinds -of queues, for example a ZeroMQ 'publish' socket. In the example below,the -socket is created separately and passed to the handler (as its 'queue'):: - - import zmq # using pyzmq, the Python binding for ZeroMQ - import json # for serializing records portably - - ctx = zmq.Context() - sock = zmq.Socket(ctx, zmq.PUB) # or zmq.PUSH, or other suitable value - sock.bind('tcp://*:5556') # or wherever - - class ZeroMQSocketHandler(QueueHandler): - def enqueue(self, record): - self.queue.send_json(record.__dict__) - - - handler = ZeroMQSocketHandler(sock) - - -Of course there are other ways of organizing this, for example passing in the -data needed by the handler to create the socket:: - - class ZeroMQSocketHandler(QueueHandler): - def __init__(self, uri, socktype=zmq.PUB, ctx=None): - self.ctx = ctx or zmq.Context() - socket = zmq.Socket(self.ctx, socktype) - socket.bind(uri) - super().__init__(socket) - - def enqueue(self, record): - self.queue.send_json(record.__dict__) - - def close(self): - self.queue.close() - - -Subclassing QueueListener - a ZeroMQ example --------------------------------------------- - -You can also subclass :class:`QueueListener` to get messages from other kinds -of queues, for example a ZeroMQ 'subscribe' socket. Here's an example:: - - class ZeroMQSocketListener(QueueListener): - def __init__(self, uri, /, *handlers, **kwargs): - self.ctx = kwargs.get('ctx') or zmq.Context() - socket = zmq.Socket(self.ctx, zmq.SUB) - socket.setsockopt_string(zmq.SUBSCRIBE, '') # subscribe to everything - socket.connect(uri) - super().__init__(socket, *handlers, **kwargs) - - def dequeue(self): - msg = self.queue.recv_json() - return logging.makeLogRecord(msg) - - -.. seealso:: - - Module :mod:`logging` - API reference for the logging module. - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :ref:`A basic logging tutorial <logging-basic-tutorial>` - - :ref:`A more advanced logging tutorial <logging-advanced-tutorial>` - - -.. currentmodule:: logging - -An example dictionary-based configuration ------------------------------------------ - -Below is an example of a logging configuration dictionary - it's taken from -the `documentation on the Django project <https://docs.djangoproject.com/en/stable/topics/logging/#configuring-logging>`_. -This dictionary is passed to :func:`~config.dictConfig` to put the configuration into effect:: - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'verbose': { - 'format': '%(levelname)s %(asctime)s %(module)s %(process)d %(thread)d %(message)s' - }, - 'simple': { - 'format': '%(levelname)s %(message)s' - }, - }, - 'filters': { - 'special': { - '()': 'project.logging.SpecialFilter', - 'foo': 'bar', - } - }, - 'handlers': { - 'null': { - 'level':'DEBUG', - 'class':'django.utils.log.NullHandler', - }, - 'console':{ - 'level':'DEBUG', - 'class':'logging.StreamHandler', - 'formatter': 'simple' - }, - 'mail_admins': { - 'level': 'ERROR', - 'class': 'django.utils.log.AdminEmailHandler', - 'filters': ['special'] - } - }, - 'loggers': { - 'django': { - 'handlers':['null'], - 'propagate': True, - 'level':'INFO', - }, - 'django.request': { - 'handlers': ['mail_admins'], - 'level': 'ERROR', - 'propagate': False, - }, - 'myproject.custom': { - 'handlers': ['console', 'mail_admins'], - 'level': 'INFO', - 'filters': ['special'] - } - } - } - -For more information about this configuration, you can see the `relevant -section <https://docs.djangoproject.com/en/stable/topics/logging/#configuring-logging>`_ -of the Django documentation. - -.. _cookbook-rotator-namer: - -Using a rotator and namer to customize log rotation processing --------------------------------------------------------------- - -An example of how you can define a namer and rotator is given in the following -runnable script, which shows gzip compression of the log file:: - - import gzip - import logging - import logging.handlers - import os - import shutil - - def namer(name): - return name + ".gz" - - def rotator(source, dest): - with open(source, 'rb') as f_in: - with gzip.open(dest, 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - os.remove(source) - - - rh = logging.handlers.RotatingFileHandler('rotated.log', maxBytes=128, backupCount=5) - rh.rotator = rotator - rh.namer = namer - - root = logging.getLogger() - root.setLevel(logging.INFO) - root.addHandler(rh) - f = logging.Formatter('%(asctime)s %(message)s') - rh.setFormatter(f) - for i in range(1000): - root.info(f'Message no. {i + 1}') - -After running this, you will see six new files, five of which are compressed: - -.. code-block:: shell-session - - $ ls rotated.log* - rotated.log rotated.log.2.gz rotated.log.4.gz - rotated.log.1.gz rotated.log.3.gz rotated.log.5.gz - $ zcat rotated.log.1.gz - 2023-01-20 02:28:17,767 Message no. 996 - 2023-01-20 02:28:17,767 Message no. 997 - 2023-01-20 02:28:17,767 Message no. 998 - -A more elaborate multiprocessing example ----------------------------------------- - -The following working example shows how logging can be used with multiprocessing -using configuration files. The configurations are fairly simple, but serve to -illustrate how more complex ones could be implemented in a real multiprocessing -scenario. - -In the example, the main process spawns a listener process and some worker -processes. Each of the main process, the listener and the workers have three -separate configurations (the workers all share the same configuration). We can -see logging in the main process, how the workers log to a QueueHandler and how -the listener implements a QueueListener and a more complex logging -configuration, and arranges to dispatch events received via the queue to the -handlers specified in the configuration. Note that these configurations are -purely illustrative, but you should be able to adapt this example to your own -scenario. - -Here's the script - the docstrings and the comments hopefully explain how it -works:: - - import logging - import logging.config - import logging.handlers - from multiprocessing import Process, Queue, Event, current_process - import os - import random - import time - - class MyHandler: - """ - A simple handler for logging events. It runs in the listener process and - dispatches events to loggers based on the name in the received record, - which then get dispatched, by the logging system, to the handlers - configured for those loggers. - """ - - def handle(self, record): - if record.name == "root": - logger = logging.getLogger() - else: - logger = logging.getLogger(record.name) - - if logger.isEnabledFor(record.levelno): - # The process name is transformed just to show that it's the listener - # doing the logging to files and console - record.processName = '%s (for %s)' % (current_process().name, record.processName) - logger.handle(record) - - def listener_process(q, stop_event, config): - """ - This could be done in the main process, but is just done in a separate - process for illustrative purposes. - - This initialises logging according to the specified configuration, - starts the listener and waits for the main process to signal completion - via the event. The listener is then stopped, and the process exits. - """ - logging.config.dictConfig(config) - listener = logging.handlers.QueueListener(q, MyHandler()) - listener.start() - if os.name == 'posix': - # On POSIX, the setup logger will have been configured in the - # parent process, but should have been disabled following the - # dictConfig call. - # On Windows, since fork isn't used, the setup logger won't - # exist in the child, so it would be created and the message - # would appear - hence the "if posix" clause. - logger = logging.getLogger('setup') - logger.critical('Should not appear, because of disabled logger ...') - stop_event.wait() - listener.stop() - - def worker_process(config): - """ - A number of these are spawned for the purpose of illustration. In - practice, they could be a heterogeneous bunch of processes rather than - ones which are identical to each other. - - This initialises logging according to the specified configuration, - and logs a hundred messages with random levels to randomly selected - loggers. - - A small sleep is added to allow other processes a chance to run. This - is not strictly needed, but it mixes the output from the different - processes a bit more than if it's left out. - """ - logging.config.dictConfig(config) - levels = [logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, - logging.CRITICAL] - loggers = ['foo', 'foo.bar', 'foo.bar.baz', - 'spam', 'spam.ham', 'spam.ham.eggs'] - if os.name == 'posix': - # On POSIX, the setup logger will have been configured in the - # parent process, but should have been disabled following the - # dictConfig call. - # On Windows, since fork isn't used, the setup logger won't - # exist in the child, so it would be created and the message - # would appear - hence the "if posix" clause. - logger = logging.getLogger('setup') - logger.critical('Should not appear, because of disabled logger ...') - for i in range(100): - lvl = random.choice(levels) - logger = logging.getLogger(random.choice(loggers)) - logger.log(lvl, 'Message no. %d', i) - time.sleep(0.01) - - def main(): - q = Queue() - # The main process gets a simple configuration which prints to the console. - config_initial = { - 'version': 1, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'level': 'INFO' - } - }, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG' - } - } - # The worker process configuration is just a QueueHandler attached to the - # root logger, which allows all messages to be sent to the queue. - # We disable existing loggers to disable the "setup" logger used in the - # parent process. This is needed on POSIX because the logger will - # be there in the child following a fork(). - config_worker = { - 'version': 1, - 'disable_existing_loggers': True, - 'handlers': { - 'queue': { - 'class': 'logging.handlers.QueueHandler', - 'queue': q - } - }, - 'root': { - 'handlers': ['queue'], - 'level': 'DEBUG' - } - } - # The listener process configuration shows that the full flexibility of - # logging configuration is available to dispatch events to handlers however - # you want. - # We disable existing loggers to disable the "setup" logger used in the - # parent process. This is needed on POSIX because the logger will - # be there in the child following a fork(). - config_listener = { - 'version': 1, - 'disable_existing_loggers': True, - 'formatters': { - 'detailed': { - 'class': 'logging.Formatter', - 'format': '%(asctime)s %(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - }, - 'simple': { - 'class': 'logging.Formatter', - 'format': '%(name)-15s %(levelname)-8s %(processName)-10s %(message)s' - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'formatter': 'simple', - 'level': 'INFO' - }, - 'file': { - 'class': 'logging.FileHandler', - 'filename': 'mplog.log', - 'mode': 'w', - 'formatter': 'detailed' - }, - 'foofile': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-foo.log', - 'mode': 'w', - 'formatter': 'detailed' - }, - 'errors': { - 'class': 'logging.FileHandler', - 'filename': 'mplog-errors.log', - 'mode': 'w', - 'formatter': 'detailed', - 'level': 'ERROR' - } - }, - 'loggers': { - 'foo': { - 'handlers': ['foofile'] - } - }, - 'root': { - 'handlers': ['console', 'file', 'errors'], - 'level': 'DEBUG' - } - } - # Log some initial events, just to show that logging in the parent works - # normally. - logging.config.dictConfig(config_initial) - logger = logging.getLogger('setup') - logger.info('About to create workers ...') - workers = [] - for i in range(5): - wp = Process(target=worker_process, name='worker %d' % (i + 1), - args=(config_worker,)) - workers.append(wp) - wp.start() - logger.info('Started worker: %s', wp.name) - logger.info('About to create listener ...') - stop_event = Event() - lp = Process(target=listener_process, name='listener', - args=(q, stop_event, config_listener)) - lp.start() - logger.info('Started listener') - # We now hang around for the workers to finish their work. - for wp in workers: - wp.join() - # Workers all done, listening can now stop. - # Logging in the parent still works normally. - logger.info('Telling listener to stop ...') - stop_event.set() - lp.join() - logger.info('All done.') - - if __name__ == '__main__': - main() - - -Inserting a BOM into messages sent to a SysLogHandler ------------------------------------------------------ - -:rfc:`5424` requires that a -Unicode message be sent to a syslog daemon as a set of bytes which have the -following structure: an optional pure-ASCII component, followed by a UTF-8 Byte -Order Mark (BOM), followed by Unicode encoded using UTF-8. (See the -:rfc:`relevant section of the specification <5424#section-6>`.) - -In Python 3.1, code was added to -:class:`~logging.handlers.SysLogHandler` to insert a BOM into the message, but -unfortunately, it was implemented incorrectly, with the BOM appearing at the -beginning of the message and hence not allowing any pure-ASCII component to -appear before it. - -As this behaviour is broken, the incorrect BOM insertion code is being removed -from Python 3.2.4 and later. However, it is not being replaced, and if you -want to produce :rfc:`5424`-compliant messages which include a BOM, an optional -pure-ASCII sequence before it and arbitrary Unicode after it, encoded using -UTF-8, then you need to do the following: - -#. Attach a :class:`~logging.Formatter` instance to your - :class:`~logging.handlers.SysLogHandler` instance, with a format string - such as:: - - 'ASCII section\ufeffUnicode section' - - The Unicode code point U+FEFF, when encoded using UTF-8, will be - encoded as a UTF-8 BOM -- the byte-string ``b'\xef\xbb\xbf'``. - -#. Replace the ASCII section with whatever placeholders you like, but make sure - that the data that appears in there after substitution is always ASCII (that - way, it will remain unchanged after UTF-8 encoding). - -#. Replace the Unicode section with whatever placeholders you like; if the data - which appears there after substitution contains characters outside the ASCII - range, that's fine -- it will be encoded using UTF-8. - -The formatted message *will* be encoded using UTF-8 encoding by -``SysLogHandler``. If you follow the above rules, you should be able to produce -:rfc:`5424`-compliant messages. If you don't, logging may not complain, but your -messages will not be RFC 5424-compliant, and your syslog daemon may complain. - - -Implementing structured logging -------------------------------- - -Although most logging messages are intended for reading by humans, and thus not -readily machine-parseable, there might be circumstances where you want to output -messages in a structured format which *is* capable of being parsed by a program -(without needing complex regular expressions to parse the log message). This is -straightforward to achieve using the logging package. There are a number of -ways in which this could be achieved, but the following is a simple approach -which uses JSON to serialise the event in a machine-parseable manner:: - - import json - import logging - - class StructuredMessage: - def __init__(self, message, /, **kwargs): - self.message = message - self.kwargs = kwargs - - def __str__(self): - return '%s >>> %s' % (self.message, json.dumps(self.kwargs)) - - _ = StructuredMessage # optional, to improve readability - - logging.basicConfig(level=logging.INFO, format='%(message)s') - logging.info(_('message 1', foo='bar', bar='baz', num=123, fnum=123.456)) - -If the above script is run, it prints: - -.. code-block:: none - - message 1 >>> {"fnum": 123.456, "num": 123, "bar": "baz", "foo": "bar"} - -Note that the order of items might be different according to the version of -Python used. - -If you need more specialised processing, you can use a custom JSON encoder, -as in the following complete example:: - - import json - import logging - - - class Encoder(json.JSONEncoder): - def default(self, o): - if isinstance(o, set): - return tuple(o) - elif isinstance(o, str): - return o.encode('unicode_escape').decode('ascii') - return super().default(o) - - class StructuredMessage: - def __init__(self, message, /, **kwargs): - self.message = message - self.kwargs = kwargs - - def __str__(self): - s = Encoder().encode(self.kwargs) - return '%s >>> %s' % (self.message, s) - - _ = StructuredMessage # optional, to improve readability - - def main(): - logging.basicConfig(level=logging.INFO, format='%(message)s') - logging.info(_('message 1', set_value={1, 2, 3}, snowman='\u2603')) - - if __name__ == '__main__': - main() - -When the above script is run, it prints: - -.. code-block:: none - - message 1 >>> {"snowman": "\u2603", "set_value": [1, 2, 3]} - -Note that the order of items might be different according to the version of -Python used. - - -.. _custom-handlers: - -.. currentmodule:: logging.config - -Customizing handlers with :func:`dictConfig` --------------------------------------------- - -There are times when you want to customize logging handlers in particular ways, -and if you use :func:`dictConfig` you may be able to do this without -subclassing. As an example, consider that you may want to set the ownership of a -log file. On POSIX, this is easily done using :func:`shutil.chown`, but the file -handlers in the stdlib don't offer built-in support. You can customize handler -creation using a plain function such as:: - - def owned_file_handler(filename, mode='a', encoding=None, owner=None): - if owner: - if not os.path.exists(filename): - open(filename, 'a').close() - shutil.chown(filename, *owner) - return logging.FileHandler(filename, mode, encoding) - -You can then specify, in a logging configuration passed to :func:`dictConfig`, -that a logging handler be created by calling this function:: - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'file':{ - # The values below are popped from this dictionary and - # used to create the handler, set the handler's level and - # its formatter. - '()': owned_file_handler, - 'level':'DEBUG', - 'formatter': 'default', - # The values below are passed to the handler creator callable - # as keyword arguments. - 'owner': ['pulse', 'pulse'], - 'filename': 'chowntest.log', - 'mode': 'w', - 'encoding': 'utf-8', - }, - }, - 'root': { - 'handlers': ['file'], - 'level': 'DEBUG', - }, - } - -In this example I am setting the ownership using the ``pulse`` user and group, -just for the purposes of illustration. Putting it together into a working -script, ``chowntest.py``:: - - import logging, logging.config, os, shutil - - def owned_file_handler(filename, mode='a', encoding=None, owner=None): - if owner: - if not os.path.exists(filename): - open(filename, 'a').close() - shutil.chown(filename, *owner) - return logging.FileHandler(filename, mode, encoding) - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'default': { - 'format': '%(asctime)s %(levelname)s %(name)s %(message)s' - }, - }, - 'handlers': { - 'file':{ - # The values below are popped from this dictionary and - # used to create the handler, set the handler's level and - # its formatter. - '()': owned_file_handler, - 'level':'DEBUG', - 'formatter': 'default', - # The values below are passed to the handler creator callable - # as keyword arguments. - 'owner': ['pulse', 'pulse'], - 'filename': 'chowntest.log', - 'mode': 'w', - 'encoding': 'utf-8', - }, - }, - 'root': { - 'handlers': ['file'], - 'level': 'DEBUG', - }, - } - - logging.config.dictConfig(LOGGING) - logger = logging.getLogger('mylogger') - logger.debug('A debug message') - -To run this, you will probably need to run as ``root``: - -.. code-block:: shell-session - - $ sudo python3.3 chowntest.py - $ cat chowntest.log - 2013-11-05 09:34:51,128 DEBUG mylogger A debug message - $ ls -l chowntest.log - -rw-r--r-- 1 pulse pulse 55 2013-11-05 09:34 chowntest.log - -Note that this example uses Python 3.3 because that's where :func:`shutil.chown` -makes an appearance. This approach should work with any Python version that -supports :func:`dictConfig` - namely, Python 2.7, 3.2 or later. With pre-3.3 -versions, you would need to implement the actual ownership change using e.g. -:func:`os.chown`. - -In practice, the handler-creating function may be in a utility module somewhere -in your project. Instead of the line in the configuration:: - - '()': owned_file_handler, - -you could use e.g.:: - - '()': 'ext://project.util.owned_file_handler', - -where ``project.util`` can be replaced with the actual name of the package -where the function resides. In the above working script, using -``'ext://__main__.owned_file_handler'`` should work. Here, the actual callable -is resolved by :func:`dictConfig` from the ``ext://`` specification. - -This example hopefully also points the way to how you could implement other -types of file change - e.g. setting specific POSIX permission bits - in the -same way, using :func:`os.chmod`. - -Of course, the approach could also be extended to types of handler other than a -:class:`~logging.FileHandler` - for example, one of the rotating file handlers, -or a different type of handler altogether. - - -.. currentmodule:: logging - -.. _formatting-styles: - -Using particular formatting styles throughout your application --------------------------------------------------------------- - -In Python 3.2, the :class:`~logging.Formatter` gained a ``style`` keyword -parameter which, while defaulting to ``%`` for backward compatibility, allowed -the specification of ``{`` or ``$`` to support the formatting approaches -supported by :meth:`str.format` and :class:`string.Template`. Note that this -governs the formatting of logging messages for final output to logs, and is -completely orthogonal to how an individual logging message is constructed. - -Logging calls (:meth:`~Logger.debug`, :meth:`~Logger.info` etc.) only take -positional parameters for the actual logging message itself, with keyword -parameters used only for determining options for how to handle the logging call -(e.g. the ``exc_info`` keyword parameter to indicate that traceback information -should be logged, or the ``extra`` keyword parameter to indicate additional -contextual information to be added to the log). So you cannot directly make -logging calls using :meth:`str.format` or :class:`string.Template` syntax, -because internally the logging package uses %-formatting to merge the format -string and the variable arguments. There would be no changing this while preserving -backward compatibility, since all logging calls which are out there in existing -code will be using %-format strings. - -There have been suggestions to associate format styles with specific loggers, -but that approach also runs into backward compatibility problems because any -existing code could be using a given logger name and using %-formatting. - -For logging to work interoperably between any third-party libraries and your -code, decisions about formatting need to be made at the level of the -individual logging call. This opens up a couple of ways in which alternative -formatting styles can be accommodated. - - -Using LogRecord factories -^^^^^^^^^^^^^^^^^^^^^^^^^ - -In Python 3.2, along with the :class:`~logging.Formatter` changes mentioned -above, the logging package gained the ability to allow users to set their own -:class:`LogRecord` subclasses, using the :func:`setLogRecordFactory` function. -You can use this to set your own subclass of :class:`LogRecord`, which does the -Right Thing by overriding the :meth:`~LogRecord.getMessage` method. The base -class implementation of this method is where the ``msg % args`` formatting -happens, and where you can substitute your alternate formatting; however, you -should be careful to support all formatting styles and allow %-formatting as -the default, to ensure interoperability with other code. Care should also be -taken to call ``str(self.msg)``, just as the base implementation does. - -Refer to the reference documentation on :func:`setLogRecordFactory` and -:class:`LogRecord` for more information. - - -Using custom message objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There is another, perhaps simpler way that you can use {}- and $- formatting to -construct your individual log messages. You may recall (from -:ref:`arbitrary-object-messages`) that when logging you can use an arbitrary -object as a message format string, and that the logging package will call -:func:`str` on that object to get the actual format string. Consider the -following two classes:: - - class BraceMessage: - def __init__(self, fmt, /, *args, **kwargs): - self.fmt = fmt - self.args = args - self.kwargs = kwargs - - def __str__(self): - return self.fmt.format(*self.args, **self.kwargs) - - class DollarMessage: - def __init__(self, fmt, /, **kwargs): - self.fmt = fmt - self.kwargs = kwargs - - def __str__(self): - from string import Template - return Template(self.fmt).substitute(**self.kwargs) - -Either of these can be used in place of a format string, to allow {}- or -$-formatting to be used to build the actual "message" part which appears in the -formatted log output in place of “%(message)s” or “{message}” or “$message”. -If you find it a little unwieldy to use the class names whenever you want to log -something, you can make it more palatable if you use an alias such as ``M`` or -``_`` for the message (or perhaps ``__``, if you are using ``_`` for -localization). - -Examples of this approach are given below. Firstly, formatting with -:meth:`str.format`:: - - >>> __ = BraceMessage - >>> print(__('Message with {0} {1}', 2, 'placeholders')) - Message with 2 placeholders - >>> class Point: pass - ... - >>> p = Point() - >>> p.x = 0.5 - >>> p.y = 0.5 - >>> print(__('Message with coordinates: ({point.x:.2f}, {point.y:.2f})', point=p)) - Message with coordinates: (0.50, 0.50) - -Secondly, formatting with :class:`string.Template`:: - - >>> __ = DollarMessage - >>> print(__('Message with $num $what', num=2, what='placeholders')) - Message with 2 placeholders - >>> - -One thing to note is that you pay no significant performance penalty with this -approach: the actual formatting happens not when you make the logging call, but -when (and if) the logged message is actually about to be output to a log by a -handler. So the only slightly unusual thing which might trip you up is that the -parentheses go around the format string and the arguments, not just the format -string. That’s because the __ notation is just syntax sugar for a constructor -call to one of the :samp:`{XXX}Message` classes shown above. - - -.. _filters-dictconfig: - -.. currentmodule:: logging.config - -Configuring filters with :func:`dictConfig` -------------------------------------------- - -You *can* configure filters using :func:`~logging.config.dictConfig`, though it -might not be obvious at first glance how to do it (hence this recipe). Since -:class:`~logging.Filter` is the only filter class included in the standard -library, and it is unlikely to cater to many requirements (it's only there as a -base class), you will typically need to define your own :class:`~logging.Filter` -subclass with an overridden :meth:`~logging.Filter.filter` method. To do this, -specify the ``()`` key in the configuration dictionary for the filter, -specifying a callable which will be used to create the filter (a class is the -most obvious, but you can provide any callable which returns a -:class:`~logging.Filter` instance). Here is a complete example:: - - import logging - import logging.config - import sys - - class MyFilter(logging.Filter): - def __init__(self, param=None): - self.param = param - - def filter(self, record): - if self.param is None: - allow = True - else: - allow = self.param not in record.msg - if allow: - record.msg = 'changed: ' + record.msg - return allow - - LOGGING = { - 'version': 1, - 'filters': { - 'myfilter': { - '()': MyFilter, - 'param': 'noshow', - } - }, - 'handlers': { - 'console': { - 'class': 'logging.StreamHandler', - 'filters': ['myfilter'] - } - }, - 'root': { - 'level': 'DEBUG', - 'handlers': ['console'] - }, - } - - if __name__ == '__main__': - logging.config.dictConfig(LOGGING) - logging.debug('hello') - logging.debug('hello - noshow') - -This example shows how you can pass configuration data to the callable which -constructs the instance, in the form of keyword parameters. When run, the above -script will print: - -.. code-block:: none - - changed: hello - -which shows that the filter is working as configured. - -A couple of extra points to note: - -* If you can't refer to the callable directly in the configuration (e.g. if it - lives in a different module, and you can't import it directly where the - configuration dictionary is), you can use the form ``ext://...`` as described - in :ref:`logging-config-dict-externalobj`. For example, you could have used - the text ``'ext://__main__.MyFilter'`` instead of ``MyFilter`` in the above - example. - -* As well as for filters, this technique can also be used to configure custom - handlers and formatters. See :ref:`logging-config-dict-userdef` for more - information on how logging supports using user-defined objects in its - configuration, and see the other cookbook recipe :ref:`custom-handlers` above. - - -.. _custom-format-exception: - -Customized exception formatting -------------------------------- - -There might be times when you want to do customized exception formatting - for -argument's sake, let's say you want exactly one line per logged event, even -when exception information is present. You can do this with a custom formatter -class, as shown in the following example:: - - import logging - - class OneLineExceptionFormatter(logging.Formatter): - def formatException(self, exc_info): - """ - Format an exception so that it prints on a single line. - """ - result = super().formatException(exc_info) - return repr(result) # or format into one line however you want to - - def format(self, record): - s = super().format(record) - if record.exc_text: - s = s.replace('\n', '') + '|' - return s - - def configure_logging(): - fh = logging.FileHandler('output.txt', 'w') - f = OneLineExceptionFormatter('%(asctime)s|%(levelname)s|%(message)s|', - '%d/%m/%Y %H:%M:%S') - fh.setFormatter(f) - root = logging.getLogger() - root.setLevel(logging.DEBUG) - root.addHandler(fh) - - def main(): - configure_logging() - logging.info('Sample message') - try: - x = 1 / 0 - except ZeroDivisionError as e: - logging.exception('ZeroDivisionError: %s', e) - - if __name__ == '__main__': - main() - -When run, this produces a file with exactly two lines: - -.. code-block:: none - - 28/01/2015 07:21:23|INFO|Sample message| - 28/01/2015 07:21:23|ERROR|ZeroDivisionError: integer division or modulo by zero|'Traceback (most recent call last):\n File "logtest7.py", line 30, in main\n x = 1 / 0\nZeroDivisionError: integer division or modulo by zero'| - -While the above treatment is simplistic, it points the way to how exception -information can be formatted to your liking. The :mod:`traceback` module may be -helpful for more specialized needs. - -.. _spoken-messages: - -Speaking logging messages -------------------------- - -There might be situations when it is desirable to have logging messages rendered -in an audible rather than a visible format. This is easy to do if you have -text-to-speech (TTS) functionality available in your system, even if it doesn't have -a Python binding. Most TTS systems have a command line program you can run, and -this can be invoked from a handler using :mod:`subprocess`. It's assumed here -that TTS command line programs won't expect to interact with users or take a -long time to complete, and that the frequency of logged messages will be not so -high as to swamp the user with messages, and that it's acceptable to have the -messages spoken one at a time rather than concurrently, The example implementation -below waits for one message to be spoken before the next is processed, and this -might cause other handlers to be kept waiting. Here is a short example showing -the approach, which assumes that the ``espeak`` TTS package is available:: - - import logging - import subprocess - import sys - - class TTSHandler(logging.Handler): - def emit(self, record): - msg = self.format(record) - # Speak slowly in a female English voice - cmd = ['espeak', '-s150', '-ven+f3', msg] - p = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) - # wait for the program to finish - p.communicate() - - def configure_logging(): - h = TTSHandler() - root = logging.getLogger() - root.addHandler(h) - # the default formatter just returns the message - root.setLevel(logging.DEBUG) - - def main(): - logging.info('Hello') - logging.debug('Goodbye') - - if __name__ == '__main__': - configure_logging() - sys.exit(main()) - -When run, this script should say "Hello" and then "Goodbye" in a female voice. - -The above approach can, of course, be adapted to other TTS systems and even -other systems altogether which can process messages via external programs run -from a command line. - - -.. _buffered-logging: - -Buffering logging messages and outputting them conditionally ------------------------------------------------------------- - -There might be situations where you want to log messages in a temporary area -and only output them if a certain condition occurs. For example, you may want to -start logging debug events in a function, and if the function completes without -errors, you don't want to clutter the log with the collected debug information, -but if there is an error, you want all the debug information to be output as well -as the error. - -Here is an example which shows how you could do this using a decorator for your -functions where you want logging to behave this way. It makes use of the -:class:`logging.handlers.MemoryHandler`, which allows buffering of logged events -until some condition occurs, at which point the buffered events are ``flushed`` -- passed to another handler (the ``target`` handler) for processing. By default, -the ``MemoryHandler`` flushed when its buffer gets filled up or an event whose -level is greater than or equal to a specified threshold is seen. You can use this -recipe with a more specialised subclass of ``MemoryHandler`` if you want custom -flushing behavior. - -The example script has a simple function, ``foo``, which just cycles through -all the logging levels, writing to ``sys.stderr`` to say what level it's about -to log at, and then actually logging a message at that level. You can pass a -parameter to ``foo`` which, if true, will log at ERROR and CRITICAL levels - -otherwise, it only logs at DEBUG, INFO and WARNING levels. - -The script just arranges to decorate ``foo`` with a decorator which will do the -conditional logging that's required. The decorator takes a logger as a parameter -and attaches a memory handler for the duration of the call to the decorated -function. The decorator can be additionally parameterised using a target handler, -a level at which flushing should occur, and a capacity for the buffer (number of -records buffered). These default to a :class:`~logging.StreamHandler` which -writes to ``sys.stderr``, ``logging.ERROR`` and ``100`` respectively. - -Here's the script:: - - import logging - from logging.handlers import MemoryHandler - import sys - - logger = logging.getLogger(__name__) - logger.addHandler(logging.NullHandler()) - - def log_if_errors(logger, target_handler=None, flush_level=None, capacity=None): - if target_handler is None: - target_handler = logging.StreamHandler() - if flush_level is None: - flush_level = logging.ERROR - if capacity is None: - capacity = 100 - handler = MemoryHandler(capacity, flushLevel=flush_level, target=target_handler) - - def decorator(fn): - def wrapper(*args, **kwargs): - logger.addHandler(handler) - try: - return fn(*args, **kwargs) - except Exception: - logger.exception('call failed') - raise - finally: - super(MemoryHandler, handler).flush() - logger.removeHandler(handler) - return wrapper - - return decorator - - def write_line(s): - sys.stderr.write('%s\n' % s) - - def foo(fail=False): - write_line('about to log at DEBUG ...') - logger.debug('Actually logged at DEBUG') - write_line('about to log at INFO ...') - logger.info('Actually logged at INFO') - write_line('about to log at WARNING ...') - logger.warning('Actually logged at WARNING') - if fail: - write_line('about to log at ERROR ...') - logger.error('Actually logged at ERROR') - write_line('about to log at CRITICAL ...') - logger.critical('Actually logged at CRITICAL') - return fail - - decorated_foo = log_if_errors(logger)(foo) - - if __name__ == '__main__': - logger.setLevel(logging.DEBUG) - write_line('Calling undecorated foo with False') - assert not foo(False) - write_line('Calling undecorated foo with True') - assert foo(True) - write_line('Calling decorated foo with False') - assert not decorated_foo(False) - write_line('Calling decorated foo with True') - assert decorated_foo(True) - -When this script is run, the following output should be observed: - -.. code-block:: none - - Calling undecorated foo with False - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - Calling undecorated foo with True - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - about to log at ERROR ... - about to log at CRITICAL ... - Calling decorated foo with False - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - Calling decorated foo with True - about to log at DEBUG ... - about to log at INFO ... - about to log at WARNING ... - about to log at ERROR ... - Actually logged at DEBUG - Actually logged at INFO - Actually logged at WARNING - Actually logged at ERROR - about to log at CRITICAL ... - Actually logged at CRITICAL - -As you can see, actual logging output only occurs when an event is logged whose -severity is ERROR or greater, but in that case, any previous events at lower -severities are also logged. - -You can of course use the conventional means of decoration:: - - @log_if_errors(logger) - def foo(fail=False): - ... - - -.. _buffered-smtp: - -Sending logging messages to email, with buffering -------------------------------------------------- - -To illustrate how you can send log messages via email, so that a set number of -messages are sent per email, you can subclass -:class:`~logging.handlers.BufferingHandler`. In the following example, which you can -adapt to suit your specific needs, a simple test harness is provided which allows you -to run the script with command line arguments specifying what you typically need to -send things via SMTP. (Run the downloaded script with the ``-h`` argument to see the -required and optional arguments.) - -.. code-block:: python - - import logging - import logging.handlers - import smtplib - - class BufferingSMTPHandler(logging.handlers.BufferingHandler): - def __init__(self, mailhost, port, username, password, fromaddr, toaddrs, - subject, capacity): - logging.handlers.BufferingHandler.__init__(self, capacity) - self.mailhost = mailhost - self.mailport = port - self.username = username - self.password = password - self.fromaddr = fromaddr - if isinstance(toaddrs, str): - toaddrs = [toaddrs] - self.toaddrs = toaddrs - self.subject = subject - self.setFormatter(logging.Formatter("%(asctime)s %(levelname)-5s %(message)s")) - - def flush(self): - if len(self.buffer) > 0: - try: - smtp = smtplib.SMTP(self.mailhost, self.mailport) - smtp.starttls() - smtp.login(self.username, self.password) - msg = "From: %s\r\nTo: %s\r\nSubject: %s\r\n\r\n" % (self.fromaddr, ','.join(self.toaddrs), self.subject) - for record in self.buffer: - s = self.format(record) - msg = msg + s + "\r\n" - smtp.sendmail(self.fromaddr, self.toaddrs, msg) - smtp.quit() - except Exception: - if logging.raiseExceptions: - raise - self.buffer = [] - - if __name__ == '__main__': - import argparse - - ap = argparse.ArgumentParser() - aa = ap.add_argument - aa('host', metavar='HOST', help='SMTP server') - aa('--port', '-p', type=int, default=587, help='SMTP port') - aa('user', metavar='USER', help='SMTP username') - aa('password', metavar='PASSWORD', help='SMTP password') - aa('to', metavar='TO', help='Addressee for emails') - aa('sender', metavar='SENDER', help='Sender email address') - aa('--subject', '-s', - default='Test Logging email from Python logging module (buffering)', - help='Subject of email') - options = ap.parse_args() - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - h = BufferingSMTPHandler(options.host, options.port, options.user, - options.password, options.sender, - options.to, options.subject, 10) - logger.addHandler(h) - for i in range(102): - logger.info("Info index = %d", i) - h.flush() - h.close() - -If you run this script and your SMTP server is correctly set up, you should find that -it sends eleven emails to the addressee you specify. The first ten emails will each -have ten log messages, and the eleventh will have two messages. That makes up 102 -messages as specified in the script. - -.. _utc-formatting: - -Formatting times using UTC (GMT) via configuration --------------------------------------------------- - -Sometimes you want to format times using UTC, which can be done using a class -such as ``UTCFormatter``, shown below:: - - import logging - import time - - class UTCFormatter(logging.Formatter): - converter = time.gmtime - -and you can then use the ``UTCFormatter`` in your code instead of -:class:`~logging.Formatter`. If you want to do that via configuration, you can -use the :func:`~logging.config.dictConfig` API with an approach illustrated by -the following complete example:: - - import logging - import logging.config - import time - - class UTCFormatter(logging.Formatter): - converter = time.gmtime - - LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'utc': { - '()': UTCFormatter, - 'format': '%(asctime)s %(message)s', - }, - 'local': { - 'format': '%(asctime)s %(message)s', - } - }, - 'handlers': { - 'console1': { - 'class': 'logging.StreamHandler', - 'formatter': 'utc', - }, - 'console2': { - 'class': 'logging.StreamHandler', - 'formatter': 'local', - }, - }, - 'root': { - 'handlers': ['console1', 'console2'], - } - } - - if __name__ == '__main__': - logging.config.dictConfig(LOGGING) - logging.warning('The local time is %s', time.asctime()) - -When this script is run, it should print something like: - -.. code-block:: none - - 2015-10-17 12:53:29,501 The local time is Sat Oct 17 13:53:29 2015 - 2015-10-17 13:53:29,501 The local time is Sat Oct 17 13:53:29 2015 - -showing how the time is formatted both as local time and UTC, one for each -handler. - - -.. _context-manager: - -Using a context manager for selective logging ---------------------------------------------- - -There are times when it would be useful to temporarily change the logging -configuration and revert it back after doing something. For this, a context -manager is the most obvious way of saving and restoring the logging context. -Here is a simple example of such a context manager, which allows you to -optionally change the logging level and add a logging handler purely in the -scope of the context manager:: - - import logging - import sys - - class LoggingContext: - def __init__(self, logger, level=None, handler=None, close=True): - self.logger = logger - self.level = level - self.handler = handler - self.close = close - - def __enter__(self): - if self.level is not None: - self.old_level = self.logger.level - self.logger.setLevel(self.level) - if self.handler: - self.logger.addHandler(self.handler) - - def __exit__(self, et, ev, tb): - if self.level is not None: - self.logger.setLevel(self.old_level) - if self.handler: - self.logger.removeHandler(self.handler) - if self.handler and self.close: - self.handler.close() - # implicit return of None => don't swallow exceptions - -If you specify a level value, the logger's level is set to that value in the -scope of the with block covered by the context manager. If you specify a -handler, it is added to the logger on entry to the block and removed on exit -from the block. You can also ask the manager to close the handler for you on -block exit - you could do this if you don't need the handler any more. - -To illustrate how it works, we can add the following block of code to the -above:: - - if __name__ == '__main__': - logger = logging.getLogger('foo') - logger.addHandler(logging.StreamHandler()) - logger.setLevel(logging.INFO) - logger.info('1. This should appear just once on stderr.') - logger.debug('2. This should not appear.') - with LoggingContext(logger, level=logging.DEBUG): - logger.debug('3. This should appear once on stderr.') - logger.debug('4. This should not appear.') - h = logging.StreamHandler(sys.stdout) - with LoggingContext(logger, level=logging.DEBUG, handler=h, close=True): - logger.debug('5. This should appear twice - once on stderr and once on stdout.') - logger.info('6. This should appear just once on stderr.') - logger.debug('7. This should not appear.') - -We initially set the logger's level to ``INFO``, so message #1 appears and -message #2 doesn't. We then change the level to ``DEBUG`` temporarily in the -following ``with`` block, and so message #3 appears. After the block exits, the -logger's level is restored to ``INFO`` and so message #4 doesn't appear. In the -next ``with`` block, we set the level to ``DEBUG`` again but also add a handler -writing to ``sys.stdout``. Thus, message #5 appears twice on the console (once -via ``stderr`` and once via ``stdout``). After the ``with`` statement's -completion, the status is as it was before so message #6 appears (like message -#1) whereas message #7 doesn't (just like message #2). - -If we run the resulting script, the result is as follows: - -.. code-block:: shell-session - - $ python logctx.py - 1. This should appear just once on stderr. - 3. This should appear once on stderr. - 5. This should appear twice - once on stderr and once on stdout. - 5. This should appear twice - once on stderr and once on stdout. - 6. This should appear just once on stderr. - -If we run it again, but pipe ``stderr`` to ``/dev/null``, we see the following, -which is the only message written to ``stdout``: - -.. code-block:: shell-session - - $ python logctx.py 2>/dev/null - 5. This should appear twice - once on stderr and once on stdout. - -Once again, but piping ``stdout`` to ``/dev/null``, we get: - -.. code-block:: shell-session - - $ python logctx.py >/dev/null - 1. This should appear just once on stderr. - 3. This should appear once on stderr. - 5. This should appear twice - once on stderr and once on stdout. - 6. This should appear just once on stderr. - -In this case, the message #5 printed to ``stdout`` doesn't appear, as expected. - -Of course, the approach described here can be generalised, for example to attach -logging filters temporarily. Note that the above code works in Python 2 as well -as Python 3. - - -.. _starter-template: - -A CLI application starter template ----------------------------------- - -Here's an example which shows how you can: - -* Use a logging level based on command-line arguments -* Dispatch to multiple subcommands in separate files, all logging at the same - level in a consistent way -* Make use of simple, minimal configuration - -Suppose we have a command-line application whose job is to stop, start or -restart some services. This could be organised for the purposes of illustration -as a file ``app.py`` that is the main script for the application, with individual -commands implemented in ``start.py``, ``stop.py`` and ``restart.py``. Suppose -further that we want to control the verbosity of the application via a -command-line argument, defaulting to ``logging.INFO``. Here's one way that -``app.py`` could be written:: - - import argparse - import importlib - import logging - import os - import sys - - def main(args=None): - scriptname = os.path.basename(__file__) - parser = argparse.ArgumentParser(scriptname) - levels = ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL') - parser.add_argument('--log-level', default='INFO', choices=levels) - subparsers = parser.add_subparsers(dest='command', - help='Available commands:') - start_cmd = subparsers.add_parser('start', help='Start a service') - start_cmd.add_argument('name', metavar='NAME', - help='Name of service to start') - stop_cmd = subparsers.add_parser('stop', - help='Stop one or more services') - stop_cmd.add_argument('names', metavar='NAME', nargs='+', - help='Name of service to stop') - restart_cmd = subparsers.add_parser('restart', - help='Restart one or more services') - restart_cmd.add_argument('names', metavar='NAME', nargs='+', - help='Name of service to restart') - options = parser.parse_args() - # the code to dispatch commands could all be in this file. For the purposes - # of illustration only, we implement each command in a separate module. - try: - mod = importlib.import_module(options.command) - cmd = getattr(mod, 'command') - except (ImportError, AttributeError): - print('Unable to find the code for command \'%s\'' % options.command) - return 1 - # Could get fancy here and load configuration from file or dictionary - logging.basicConfig(level=options.log_level, - format='%(levelname)s %(name)s %(message)s') - cmd(options) - - if __name__ == '__main__': - sys.exit(main()) - -And the ``start``, ``stop`` and ``restart`` commands can be implemented in -separate modules, like so for starting:: - - # start.py - import logging - - logger = logging.getLogger(__name__) - - def command(options): - logger.debug('About to start %s', options.name) - # actually do the command processing here ... - logger.info('Started the \'%s\' service.', options.name) - -and thus for stopping:: - - # stop.py - import logging - - logger = logging.getLogger(__name__) - - def command(options): - n = len(options.names) - if n == 1: - plural = '' - services = '\'%s\'' % options.names[0] - else: - plural = 's' - services = ', '.join('\'%s\'' % name for name in options.names) - i = services.rfind(', ') - services = services[:i] + ' and ' + services[i + 2:] - logger.debug('About to stop %s', services) - # actually do the command processing here ... - logger.info('Stopped the %s service%s.', services, plural) - -and similarly for restarting:: - - # restart.py - import logging - - logger = logging.getLogger(__name__) - - def command(options): - n = len(options.names) - if n == 1: - plural = '' - services = '\'%s\'' % options.names[0] - else: - plural = 's' - services = ', '.join('\'%s\'' % name for name in options.names) - i = services.rfind(', ') - services = services[:i] + ' and ' + services[i + 2:] - logger.debug('About to restart %s', services) - # actually do the command processing here ... - logger.info('Restarted the %s service%s.', services, plural) - -If we run this application with the default log level, we get output like this: - -.. code-block:: shell-session - - $ python app.py start foo - INFO start Started the 'foo' service. - - $ python app.py stop foo bar - INFO stop Stopped the 'foo' and 'bar' services. - - $ python app.py restart foo bar baz - INFO restart Restarted the 'foo', 'bar' and 'baz' services. - -The first word is the logging level, and the second word is the module or -package name of the place where the event was logged. - -If we change the logging level, then we can change the information sent to the -log. For example, if we want more information: - -.. code-block:: shell-session - - $ python app.py --log-level DEBUG start foo - DEBUG start About to start foo - INFO start Started the 'foo' service. - - $ python app.py --log-level DEBUG stop foo bar - DEBUG stop About to stop 'foo' and 'bar' - INFO stop Stopped the 'foo' and 'bar' services. - - $ python app.py --log-level DEBUG restart foo bar baz - DEBUG restart About to restart 'foo', 'bar' and 'baz' - INFO restart Restarted the 'foo', 'bar' and 'baz' services. - -And if we want less: - -.. code-block:: shell-session - - $ python app.py --log-level WARNING start foo - $ python app.py --log-level WARNING stop foo bar - $ python app.py --log-level WARNING restart foo bar baz - -In this case, the commands don't print anything to the console, since nothing -at ``WARNING`` level or above is logged by them. - -.. _qt-gui: - -A Qt GUI for logging --------------------- - -A question that comes up from time to time is about how to log to a GUI -application. The `Qt <https://www.qt.io/>`_ framework is a popular -cross-platform UI framework with Python bindings using `PySide2 -<https://pypi.org/project/PySide2/>`_ or `PyQt5 -<https://pypi.org/project/PyQt5/>`_ libraries. - -The following example shows how to log to a Qt GUI. This introduces a simple -``QtHandler`` class which takes a callable, which should be a slot in the main -thread that does GUI updates. A worker thread is also created to show how you -can log to the GUI from both the UI itself (via a button for manual logging) -as well as a worker thread doing work in the background (here, just logging -messages at random levels with random short delays in between). - -The worker thread is implemented using Qt's ``QThread`` class rather than the -:mod:`threading` module, as there are circumstances where one has to use -``QThread``, which offers better integration with other ``Qt`` components. - -The code should work with recent releases of either ``PySide2`` or ``PyQt5``. -You should be able to adapt the approach to earlier versions of Qt. Please -refer to the comments in the code snippet for more detailed information. - -.. code-block:: python3 - - import datetime - import logging - import random - import sys - import time - - # Deal with minor differences between PySide2 and PyQt5 - try: - from PySide2 import QtCore, QtGui, QtWidgets - Signal = QtCore.Signal - Slot = QtCore.Slot - except ImportError: - from PyQt5 import QtCore, QtGui, QtWidgets - Signal = QtCore.pyqtSignal - Slot = QtCore.pyqtSlot - - - logger = logging.getLogger(__name__) - - - # - # Signals need to be contained in a QObject or subclass in order to be correctly - # initialized. - # - class Signaller(QtCore.QObject): - signal = Signal(str, logging.LogRecord) - - # - # Output to a Qt GUI is only supposed to happen on the main thread. So, this - # handler is designed to take a slot function which is set up to run in the main - # thread. In this example, the function takes a string argument which is a - # formatted log message, and the log record which generated it. The formatted - # string is just a convenience - you could format a string for output any way - # you like in the slot function itself. - # - # You specify the slot function to do whatever GUI updates you want. The handler - # doesn't know or care about specific UI elements. - # - class QtHandler(logging.Handler): - def __init__(self, slotfunc, *args, **kwargs): - super().__init__(*args, **kwargs) - self.signaller = Signaller() - self.signaller.signal.connect(slotfunc) - - def emit(self, record): - s = self.format(record) - self.signaller.signal.emit(s, record) - - # - # This example uses QThreads, which means that the threads at the Python level - # are named something like "Dummy-1". The function below gets the Qt name of the - # current thread. - # - def ctname(): - return QtCore.QThread.currentThread().objectName() - - - # - # Used to generate random levels for logging. - # - LEVELS = (logging.DEBUG, logging.INFO, logging.WARNING, logging.ERROR, - logging.CRITICAL) - - # - # This worker class represents work that is done in a thread separate to the - # main thread. The way the thread is kicked off to do work is via a button press - # that connects to a slot in the worker. - # - # Because the default threadName value in the LogRecord isn't much use, we add - # a qThreadName which contains the QThread name as computed above, and pass that - # value in an "extra" dictionary which is used to update the LogRecord with the - # QThread name. - # - # This example worker just outputs messages sequentially, interspersed with - # random delays of the order of a few seconds. - # - class Worker(QtCore.QObject): - @Slot() - def start(self): - extra = {'qThreadName': ctname() } - logger.debug('Started work', extra=extra) - i = 1 - # Let the thread run until interrupted. This allows reasonably clean - # thread termination. - while not QtCore.QThread.currentThread().isInterruptionRequested(): - delay = 0.5 + random.random() * 2 - time.sleep(delay) - level = random.choice(LEVELS) - logger.log(level, 'Message after delay of %3.1f: %d', delay, i, extra=extra) - i += 1 - - # - # Implement a simple UI for this cookbook example. This contains: - # - # * A read-only text edit window which holds formatted log messages - # * A button to start work and log stuff in a separate thread - # * A button to log something from the main thread - # * A button to clear the log window - # - class Window(QtWidgets.QWidget): - - COLORS = { - logging.DEBUG: 'black', - logging.INFO: 'blue', - logging.WARNING: 'orange', - logging.ERROR: 'red', - logging.CRITICAL: 'purple', - } - - def __init__(self, app): - super().__init__() - self.app = app - self.textedit = te = QtWidgets.QPlainTextEdit(self) - # Set whatever the default monospace font is for the platform - f = QtGui.QFont('nosuchfont') - f.setStyleHint(f.Monospace) - te.setFont(f) - te.setReadOnly(True) - PB = QtWidgets.QPushButton - self.work_button = PB('Start background work', self) - self.log_button = PB('Log a message at a random level', self) - self.clear_button = PB('Clear log window', self) - self.handler = h = QtHandler(self.update_status) - # Remember to use qThreadName rather than threadName in the format string. - fs = '%(asctime)s %(qThreadName)-12s %(levelname)-8s %(message)s' - formatter = logging.Formatter(fs) - h.setFormatter(formatter) - logger.addHandler(h) - # Set up to terminate the QThread when we exit - app.aboutToQuit.connect(self.force_quit) - - # Lay out all the widgets - layout = QtWidgets.QVBoxLayout(self) - layout.addWidget(te) - layout.addWidget(self.work_button) - layout.addWidget(self.log_button) - layout.addWidget(self.clear_button) - self.setFixedSize(900, 400) - - # Connect the non-worker slots and signals - self.log_button.clicked.connect(self.manual_update) - self.clear_button.clicked.connect(self.clear_display) - - # Start a new worker thread and connect the slots for the worker - self.start_thread() - self.work_button.clicked.connect(self.worker.start) - # Once started, the button should be disabled - self.work_button.clicked.connect(lambda : self.work_button.setEnabled(False)) - - def start_thread(self): - self.worker = Worker() - self.worker_thread = QtCore.QThread() - self.worker.setObjectName('Worker') - self.worker_thread.setObjectName('WorkerThread') # for qThreadName - self.worker.moveToThread(self.worker_thread) - # This will start an event loop in the worker thread - self.worker_thread.start() - - def kill_thread(self): - # Just tell the worker to stop, then tell it to quit and wait for that - # to happen - self.worker_thread.requestInterruption() - if self.worker_thread.isRunning(): - self.worker_thread.quit() - self.worker_thread.wait() - else: - print('worker has already exited.') - - def force_quit(self): - # For use when the window is closed - if self.worker_thread.isRunning(): - self.kill_thread() - - # The functions below update the UI and run in the main thread because - # that's where the slots are set up - - @Slot(str, logging.LogRecord) - def update_status(self, status, record): - color = self.COLORS.get(record.levelno, 'black') - s = '<pre><font color="%s">%s</font></pre>' % (color, status) - self.textedit.appendHtml(s) - - @Slot() - def manual_update(self): - # This function uses the formatted message passed in, but also uses - # information from the record to format the message in an appropriate - # color according to its severity (level). - level = random.choice(LEVELS) - extra = {'qThreadName': ctname() } - logger.log(level, 'Manually logged!', extra=extra) - - @Slot() - def clear_display(self): - self.textedit.clear() - - - def main(): - QtCore.QThread.currentThread().setObjectName('MainThread') - logging.getLogger().setLevel(logging.DEBUG) - app = QtWidgets.QApplication(sys.argv) - example = Window(app) - example.show() - sys.exit(app.exec_()) - - if __name__=='__main__': - main() - -Logging to syslog with RFC5424 support --------------------------------------- - -Although :rfc:`5424` dates from 2009, most syslog servers are configured by detault to -use the older :rfc:`3164`, which hails from 2001. When ``logging`` was added to Python -in 2003, it supported the earlier (and only existing) protocol at the time. Since -RFC5424 came out, as there has not been widespread deployment of it in syslog -servers, the :class:`~logging.handlers.SysLogHandler` functionality has not been -updated. - -RFC 5424 contains some useful features such as support for structured data, and if you -need to be able to log to a syslog server with support for it, you can do so with a -subclassed handler which looks something like this:: - - import datetime - import logging.handlers - import re - import socket - import time - - class SysLogHandler5424(logging.handlers.SysLogHandler): - - tz_offset = re.compile(r'([+-]\d{2})(\d{2})$') - escaped = re.compile(r'([\]"\\])') - - def __init__(self, *args, **kwargs): - self.msgid = kwargs.pop('msgid', None) - self.appname = kwargs.pop('appname', None) - super().__init__(*args, **kwargs) - - def format(self, record): - version = 1 - asctime = datetime.datetime.fromtimestamp(record.created).isoformat() - m = self.tz_offset.match(time.strftime('%z')) - has_offset = False - if m and time.timezone: - hrs, mins = m.groups() - if int(hrs) or int(mins): - has_offset = True - if not has_offset: - asctime += 'Z' - else: - asctime += f'{hrs}:{mins}' - try: - hostname = socket.gethostname() - except Exception: - hostname = '-' - appname = self.appname or '-' - procid = record.process - msgid = '-' - msg = super().format(record) - sdata = '-' - if hasattr(record, 'structured_data'): - sd = record.structured_data - # This should be a dict where the keys are SD-ID and the value is a - # dict mapping PARAM-NAME to PARAM-VALUE (refer to the RFC for what these - # mean) - # There's no error checking here - it's purely for illustration, and you - # can adapt this code for use in production environments - parts = [] - - def replacer(m): - g = m.groups() - return '\\' + g[0] - - for sdid, dv in sd.items(): - part = f'[{sdid}' - for k, v in dv.items(): - s = str(v) - s = self.escaped.sub(replacer, s) - part += f' {k}="{s}"' - part += ']' - parts.append(part) - sdata = ''.join(parts) - return f'{version} {asctime} {hostname} {appname} {procid} {msgid} {sdata} {msg}' - -You'll need to be familiar with RFC 5424 to fully understand the above code, and it -may be that you have slightly different needs (e.g. for how you pass structural data -to the log). Nevertheless, the above should be adaptable to your speciric needs. With -the above handler, you'd pass structured data using something like this:: - - sd = { - 'foo@12345': {'bar': 'baz', 'baz': 'bozz', 'fizz': r'buzz'}, - 'foo@54321': {'rab': 'baz', 'zab': 'bozz', 'zzif': r'buzz'} - } - extra = {'structured_data': sd} - i = 1 - logger.debug('Message %d', i, extra=extra) - -How to treat a logger like an output stream -------------------------------------------- - -Sometimes, you need to interface to a third-party API which expects a file-like -object to write to, but you want to direct the API's output to a logger. You -can do this using a class which wraps a logger with a file-like API. -Here's a short script illustrating such a class: - -.. code-block:: python - - import logging - - class LoggerWriter: - def __init__(self, logger, level): - self.logger = logger - self.level = level - - def write(self, message): - if message != '\n': # avoid printing bare newlines, if you like - self.logger.log(self.level, message) - - def flush(self): - # doesn't actually do anything, but might be expected of a file-like - # object - so optional depending on your situation - pass - - def close(self): - # doesn't actually do anything, but might be expected of a file-like - # object - so optional depending on your situation. You might want - # to set a flag so that later calls to write raise an exception - pass - - def main(): - logging.basicConfig(level=logging.DEBUG) - logger = logging.getLogger('demo') - info_fp = LoggerWriter(logger, logging.INFO) - debug_fp = LoggerWriter(logger, logging.DEBUG) - print('An INFO message', file=info_fp) - print('A DEBUG message', file=debug_fp) - - if __name__ == "__main__": - main() - -When this script is run, it prints - -.. code-block:: text - - INFO:demo:An INFO message - DEBUG:demo:A DEBUG message - -You could also use ``LoggerWriter`` to redirect ``sys.stdout`` and -``sys.stderr`` by doing something like this: - -.. code-block:: python - - import sys - - sys.stdout = LoggerWriter(logger, logging.INFO) - sys.stderr = LoggerWriter(logger, logging.WARNING) - -You should do this *after* configuring logging for your needs. In the above -example, the :func:`~logging.basicConfig` call does this (using the -``sys.stderr`` value *before* it is overwritten by a ``LoggerWriter`` -instance). Then, you'd get this kind of result: - -.. code-block:: pycon - - >>> print('Foo') - INFO:demo:Foo - >>> print('Bar', file=sys.stderr) - WARNING:demo:Bar - >>> - -Of course, the examples above show output according to the format used by -:func:`~logging.basicConfig`, but you can use a different formatter when you -configure logging. - -Note that with the above scheme, you are somewhat at the mercy of buffering and -the sequence of write calls which you are intercepting. For example, with the -definition of ``LoggerWriter`` above, if you have the snippet - -.. code-block:: python - - sys.stderr = LoggerWriter(logger, logging.WARNING) - 1 / 0 - -then running the script results in - -.. code-block:: text - - WARNING:demo:Traceback (most recent call last): - - WARNING:demo: File "/home/runner/cookbook-loggerwriter/test.py", line 53, in <module> - - WARNING:demo: - WARNING:demo:main() - WARNING:demo: File "/home/runner/cookbook-loggerwriter/test.py", line 49, in main - - WARNING:demo: - WARNING:demo:1 / 0 - WARNING:demo:ZeroDivisionError - WARNING:demo:: - WARNING:demo:division by zero - -As you can see, this output isn't ideal. That's because the underlying code -which writes to ``sys.stderr`` makes mutiple writes, each of which results in a -separate logged line (for example, the last three lines above). To get around -this problem, you need to buffer things and only output log lines when newlines -are seen. Let's use a slghtly better implementation of ``LoggerWriter``: - -.. code-block:: python - - class BufferingLoggerWriter(LoggerWriter): - def __init__(self, logger, level): - super().__init__(logger, level) - self.buffer = '' - - def write(self, message): - if '\n' not in message: - self.buffer += message - else: - parts = message.split('\n') - if self.buffer: - s = self.buffer + parts.pop(0) - self.logger.log(self.level, s) - self.buffer = parts.pop() - for part in parts: - self.logger.log(self.level, part) - -This just buffers up stuff until a newline is seen, and then logs complete -lines. With this approach, you get better output: - -.. code-block:: text - - WARNING:demo:Traceback (most recent call last): - WARNING:demo: File "/home/runner/cookbook-loggerwriter/main.py", line 55, in <module> - WARNING:demo: main() - WARNING:demo: File "/home/runner/cookbook-loggerwriter/main.py", line 52, in main - WARNING:demo: 1/0 - WARNING:demo:ZeroDivisionError: division by zero - - -.. patterns-to-avoid: - -Patterns to avoid ------------------ - -Although the preceding sections have described ways of doing things you might -need to do or deal with, it is worth mentioning some usage patterns which are -*unhelpful*, and which should therefore be avoided in most cases. The following -sections are in no particular order. - -Opening the same log file multiple times -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -On Windows, you will generally not be able to open the same file multiple times -as this will lead to a "file is in use by another process" error. However, on -POSIX platforms you'll not get any errors if you open the same file multiple -times. This could be done accidentally, for example by: - -* Adding a file handler more than once which references the same file (e.g. by - a copy/paste/forget-to-change error). - -* Opening two files that look different, as they have different names, but are - the same because one is a symbolic link to the other. - -* Forking a process, following which both parent and child have a reference to - the same file. This might be through use of the :mod:`multiprocessing` module, - for example. - -Opening a file multiple times might *appear* to work most of the time, but can -lead to a number of problems in practice: - -* Logging output can be garbled because multiple threads or processes try to - write to the same file. Although logging guards against concurrent use of the - same handler instance by multiple threads, there is no such protection if - concurrent writes are attempted by two different threads using two different - handler instances which happen to point to the same file. - -* An attempt to delete a file (e.g. during file rotation) silently fails, - because there is another reference pointing to it. This can lead to confusion - and wasted debugging time - log entries end up in unexpected places, or are - lost altogether. Or a file that was supposed to be moved remains in place, - and grows in size unexpectedly despite size-based rotation being supposedly - in place. - -Use the techniques outlined in :ref:`multiple-processes` to circumvent such -issues. - -Using loggers as attributes in a class or passing them as parameters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -While there might be unusual cases where you'll need to do this, in general -there is no point because loggers are singletons. Code can always access a -given logger instance by name using ``logging.getLogger(name)``, so passing -instances around and holding them as instance attributes is pointless. Note -that in other languages such as Java and C#, loggers are often static class -attributes. However, this pattern doesn't make sense in Python, where the -module (and not the class) is the unit of software decomposition. - -Adding handlers other than :class:`~logging.NullHandler` to a logger in a library -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Configuring logging by adding handlers, formatters and filters is the -responsibility of the application developer, not the library developer. If you -are maintaining a library, ensure that you don't add handlers to any of your -loggers other than a :class:`~logging.NullHandler` instance. - -Creating a lot of loggers -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Loggers are singletons that are never freed during a script execution, and so -creating lots of loggers will use up memory which can't then be freed. Rather -than create a logger per e.g. file processed or network connection made, use -the :ref:`existing mechanisms <context-info>` for passing contextual -information into your logs and restrict the loggers created to those describing -areas within your application (generally modules, but occasionally slightly -more fine-grained than that). - -.. _cookbook-ref-links: - -Other resources ---------------- - -.. seealso:: - - Module :mod:`logging` - API reference for the logging module. - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :ref:`Basic Tutorial <logging-basic-tutorial>` - - :ref:`Advanced Tutorial <logging-advanced-tutorial>` diff --git a/Doc/howto/logging.rst b/Doc/howto/logging.rst deleted file mode 100644 index d8ea8eb27d9550..00000000000000 --- a/Doc/howto/logging.rst +++ /dev/null @@ -1,1121 +0,0 @@ -============= -Logging HOWTO -============= - -:Author: Vinay Sajip <vinay_sajip at red-dove dot com> - -.. _logging-basic-tutorial: - -.. currentmodule:: logging - -Basic Logging Tutorial ----------------------- - -Logging is a means of tracking events that happen when some software runs. The -software's developer adds logging calls to their code to indicate that certain -events have occurred. An event is described by a descriptive message which can -optionally contain variable data (i.e. data that is potentially different for -each occurrence of the event). Events also have an importance which the -developer ascribes to the event; the importance can also be called the *level* -or *severity*. - -When to use logging -^^^^^^^^^^^^^^^^^^^ - -Logging provides a set of convenience functions for simple logging usage. These -are :func:`debug`, :func:`info`, :func:`warning`, :func:`error` and -:func:`critical`. To determine when to use logging, see the table below, which -states, for each of a set of common tasks, the best tool to use for it. - -+-------------------------------------+--------------------------------------+ -| Task you want to perform | The best tool for the task | -+=====================================+======================================+ -| Display console output for ordinary | :func:`print` | -| usage of a command line script or | | -| program | | -+-------------------------------------+--------------------------------------+ -| Report events that occur during | :func:`logging.info` (or | -| normal operation of a program (e.g. | :func:`logging.debug` for very | -| for status monitoring or fault | detailed output for diagnostic | -| investigation) | purposes) | -+-------------------------------------+--------------------------------------+ -| Issue a warning regarding a | :func:`warnings.warn` in library | -| particular runtime event | code if the issue is avoidable and | -| | the client application should be | -| | modified to eliminate the warning | -| | | -| | :func:`logging.warning` if there is | -| | nothing the client application can do| -| | about the situation, but the event | -| | should still be noted | -+-------------------------------------+--------------------------------------+ -| Report an error regarding a | Raise an exception | -| particular runtime event | | -+-------------------------------------+--------------------------------------+ -| Report suppression of an error | :func:`logging.error`, | -| without raising an exception (e.g. | :func:`logging.exception` or | -| error handler in a long-running | :func:`logging.critical` as | -| server process) | appropriate for the specific error | -| | and application domain | -+-------------------------------------+--------------------------------------+ - -The logging functions are named after the level or severity of the events -they are used to track. The standard levels and their applicability are -described below (in increasing order of severity): - -.. tabularcolumns:: |l|L| - -+--------------+---------------------------------------------+ -| Level | When it's used | -+==============+=============================================+ -| ``DEBUG`` | Detailed information, typically of interest | -| | only when diagnosing problems. | -+--------------+---------------------------------------------+ -| ``INFO`` | Confirmation that things are working as | -| | expected. | -+--------------+---------------------------------------------+ -| ``WARNING`` | An indication that something unexpected | -| | happened, or indicative of some problem in | -| | the near future (e.g. 'disk space low'). | -| | The software is still working as expected. | -+--------------+---------------------------------------------+ -| ``ERROR`` | Due to a more serious problem, the software | -| | has not been able to perform some function. | -+--------------+---------------------------------------------+ -| ``CRITICAL`` | A serious error, indicating that the program| -| | itself may be unable to continue running. | -+--------------+---------------------------------------------+ - -The default level is ``WARNING``, which means that only events of this level -and above will be tracked, unless the logging package is configured to do -otherwise. - -Events that are tracked can be handled in different ways. The simplest way of -handling tracked events is to print them to the console. Another common way -is to write them to a disk file. - - -.. _howto-minimal-example: - -A simple example -^^^^^^^^^^^^^^^^ - -A very simple example is:: - - import logging - logging.warning('Watch out!') # will print a message to the console - logging.info('I told you so') # will not print anything - -If you type these lines into a script and run it, you'll see: - -.. code-block:: none - - WARNING:root:Watch out! - -printed out on the console. The ``INFO`` message doesn't appear because the -default level is ``WARNING``. The printed message includes the indication of -the level and the description of the event provided in the logging call, i.e. -'Watch out!'. Don't worry about the 'root' part for now: it will be explained -later. The actual output can be formatted quite flexibly if you need that; -formatting options will also be explained later. - - -Logging to a file -^^^^^^^^^^^^^^^^^ - -A very common situation is that of recording logging events in a file, so let's -look at that next. Be sure to try the following in a newly started Python -interpreter, and don't just continue from the session described above:: - - import logging - logging.basicConfig(filename='example.log', encoding='utf-8', level=logging.DEBUG) - logging.debug('This message should go to the log file') - logging.info('So should this') - logging.warning('And this, too') - logging.error('And non-ASCII stuff, too, like Øresund and Malmö') - -.. versionchanged:: 3.9 - The *encoding* argument was added. In earlier Python versions, or if not - specified, the encoding used is the default value used by :func:`open`. While - not shown in the above example, an *errors* argument can also now be passed, - which determines how encoding errors are handled. For available values and - the default, see the documentation for :func:`open`. - -And now if we open the file and look at what we have, we should find the log -messages: - -.. code-block:: none - - DEBUG:root:This message should go to the log file - INFO:root:So should this - WARNING:root:And this, too - ERROR:root:And non-ASCII stuff, too, like Øresund and Malmö - -This example also shows how you can set the logging level which acts as the -threshold for tracking. In this case, because we set the threshold to -``DEBUG``, all of the messages were printed. - -If you want to set the logging level from a command-line option such as: - -.. code-block:: none - - --log=INFO - -and you have the value of the parameter passed for ``--log`` in some variable -*loglevel*, you can use:: - - getattr(logging, loglevel.upper()) - -to get the value which you'll pass to :func:`basicConfig` via the *level* -argument. You may want to error check any user input value, perhaps as in the -following example:: - - # assuming loglevel is bound to the string value obtained from the - # command line argument. Convert to upper case to allow the user to - # specify --log=DEBUG or --log=debug - numeric_level = getattr(logging, loglevel.upper(), None) - if not isinstance(numeric_level, int): - raise ValueError('Invalid log level: %s' % loglevel) - logging.basicConfig(level=numeric_level, ...) - -The call to :func:`basicConfig` should come *before* any calls to -:func:`debug`, :func:`info`, etc. Otherwise, those functions will call -:func:`basicConfig` for you with the default options. As it's intended as a -one-off simple configuration facility, only the first call will actually do -anything: subsequent calls are effectively no-ops. - -If you run the above script several times, the messages from successive runs -are appended to the file *example.log*. If you want each run to start afresh, -not remembering the messages from earlier runs, you can specify the *filemode* -argument, by changing the call in the above example to:: - - logging.basicConfig(filename='example.log', filemode='w', level=logging.DEBUG) - -The output will be the same as before, but the log file is no longer appended -to, so the messages from earlier runs are lost. - - -Logging from multiple modules -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your program consists of multiple modules, here's an example of how you -could organize logging in it:: - - # myapp.py - import logging - import mylib - - def main(): - logging.basicConfig(filename='myapp.log', level=logging.INFO) - logging.info('Started') - mylib.do_something() - logging.info('Finished') - - if __name__ == '__main__': - main() - -:: - - # mylib.py - import logging - - def do_something(): - logging.info('Doing something') - -If you run *myapp.py*, you should see this in *myapp.log*: - -.. code-block:: none - - INFO:root:Started - INFO:root:Doing something - INFO:root:Finished - -which is hopefully what you were expecting to see. You can generalize this to -multiple modules, using the pattern in *mylib.py*. Note that for this simple -usage pattern, you won't know, by looking in the log file, *where* in your -application your messages came from, apart from looking at the event -description. If you want to track the location of your messages, you'll need -to refer to the documentation beyond the tutorial level -- see -:ref:`logging-advanced-tutorial`. - - -Logging variable data -^^^^^^^^^^^^^^^^^^^^^ - -To log variable data, use a format string for the event description message and -append the variable data as arguments. For example:: - - import logging - logging.warning('%s before you %s', 'Look', 'leap!') - -will display: - -.. code-block:: none - - WARNING:root:Look before you leap! - -As you can see, merging of variable data into the event description message -uses the old, %-style of string formatting. This is for backwards -compatibility: the logging package pre-dates newer formatting options such as -:meth:`str.format` and :class:`string.Template`. These newer formatting -options *are* supported, but exploring them is outside the scope of this -tutorial: see :ref:`formatting-styles` for more information. - - -Changing the format of displayed messages -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To change the format which is used to display messages, you need to -specify the format you want to use:: - - import logging - logging.basicConfig(format='%(levelname)s:%(message)s', level=logging.DEBUG) - logging.debug('This message should appear on the console') - logging.info('So should this') - logging.warning('And this, too') - -which would print: - -.. code-block:: none - - DEBUG:This message should appear on the console - INFO:So should this - WARNING:And this, too - -Notice that the 'root' which appeared in earlier examples has disappeared. For -a full set of things that can appear in format strings, you can refer to the -documentation for :ref:`logrecord-attributes`, but for simple usage, you just -need the *levelname* (severity), *message* (event description, including -variable data) and perhaps to display when the event occurred. This is -described in the next section. - - -Displaying the date/time in messages -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To display the date and time of an event, you would place '%(asctime)s' in -your format string:: - - import logging - logging.basicConfig(format='%(asctime)s %(message)s') - logging.warning('is when this event was logged.') - -which should print something like this: - -.. code-block:: none - - 2010-12-12 11:41:42,612 is when this event was logged. - -The default format for date/time display (shown above) is like ISO8601 or -:rfc:`3339`. If you need more control over the formatting of the date/time, provide -a *datefmt* argument to ``basicConfig``, as in this example:: - - import logging - logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p') - logging.warning('is when this event was logged.') - -which would display something like this: - -.. code-block:: none - - 12/12/2010 11:46:36 AM is when this event was logged. - -The format of the *datefmt* argument is the same as supported by -:func:`time.strftime`. - - -Next Steps -^^^^^^^^^^ - -That concludes the basic tutorial. It should be enough to get you up and -running with logging. There's a lot more that the logging package offers, but -to get the best out of it, you'll need to invest a little more of your time in -reading the following sections. If you're ready for that, grab some of your -favourite beverage and carry on. - -If your logging needs are simple, then use the above examples to incorporate -logging into your own scripts, and if you run into problems or don't -understand something, please post a question on the comp.lang.python Usenet -group (available at https://groups.google.com/g/comp.lang.python) and you -should receive help before too long. - -Still here? You can carry on reading the next few sections, which provide a -slightly more advanced/in-depth tutorial than the basic one above. After that, -you can take a look at the :ref:`logging-cookbook`. - -.. _logging-advanced-tutorial: - - -Advanced Logging Tutorial -------------------------- - -The logging library takes a modular approach and offers several categories -of components: loggers, handlers, filters, and formatters. - -* Loggers expose the interface that application code directly uses. -* Handlers send the log records (created by loggers) to the appropriate - destination. -* Filters provide a finer grained facility for determining which log records - to output. -* Formatters specify the layout of log records in the final output. - -Log event information is passed between loggers, handlers, filters and -formatters in a :class:`LogRecord` instance. - -Logging is performed by calling methods on instances of the :class:`Logger` -class (hereafter called :dfn:`loggers`). Each instance has a name, and they are -conceptually arranged in a namespace hierarchy using dots (periods) as -separators. For example, a logger named 'scan' is the parent of loggers -'scan.text', 'scan.html' and 'scan.pdf'. Logger names can be anything you want, -and indicate the area of an application in which a logged message originates. - -A good convention to use when naming loggers is to use a module-level logger, -in each module which uses logging, named as follows:: - - logger = logging.getLogger(__name__) - -This means that logger names track the package/module hierarchy, and it's -intuitively obvious where events are logged just from the logger name. - -The root of the hierarchy of loggers is called the root logger. That's the -logger used by the functions :func:`debug`, :func:`info`, :func:`warning`, -:func:`error` and :func:`critical`, which just call the same-named method of -the root logger. The functions and the methods have the same signatures. The -root logger's name is printed as 'root' in the logged output. - -It is, of course, possible to log messages to different destinations. Support -is included in the package for writing log messages to files, HTTP GET/POST -locations, email via SMTP, generic sockets, queues, or OS-specific logging -mechanisms such as syslog or the Windows NT event log. Destinations are served -by :dfn:`handler` classes. You can create your own log destination class if -you have special requirements not met by any of the built-in handler classes. - -By default, no destination is set for any logging messages. You can specify -a destination (such as console or file) by using :func:`basicConfig` as in the -tutorial examples. If you call the functions :func:`debug`, :func:`info`, -:func:`warning`, :func:`error` and :func:`critical`, they will check to see -if no destination is set; and if one is not set, they will set a destination -of the console (``sys.stderr``) and a default format for the displayed -message before delegating to the root logger to do the actual message output. - -The default format set by :func:`basicConfig` for messages is: - -.. code-block:: none - - severity:logger name:message - -You can change this by passing a format string to :func:`basicConfig` with the -*format* keyword argument. For all options regarding how a format string is -constructed, see :ref:`formatter-objects`. - -Logging Flow -^^^^^^^^^^^^ - -The flow of log event information in loggers and handlers is illustrated in the -following diagram. - -.. image:: logging_flow.png - :class: invert-in-dark-mode - -Loggers -^^^^^^^ - -:class:`Logger` objects have a threefold job. First, they expose several -methods to application code so that applications can log messages at runtime. -Second, logger objects determine which log messages to act upon based upon -severity (the default filtering facility) or filter objects. Third, logger -objects pass along relevant log messages to all interested log handlers. - -The most widely used methods on logger objects fall into two categories: -configuration and message sending. - -These are the most common configuration methods: - -* :meth:`Logger.setLevel` specifies the lowest-severity log message a logger - will handle, where debug is the lowest built-in severity level and critical - is the highest built-in severity. For example, if the severity level is - INFO, the logger will handle only INFO, WARNING, ERROR, and CRITICAL messages - and will ignore DEBUG messages. - -* :meth:`Logger.addHandler` and :meth:`Logger.removeHandler` add and remove - handler objects from the logger object. Handlers are covered in more detail - in :ref:`handler-basic`. - -* :meth:`Logger.addFilter` and :meth:`Logger.removeFilter` add and remove filter - objects from the logger object. Filters are covered in more detail in - :ref:`filter`. - -You don't need to always call these methods on every logger you create. See the -last two paragraphs in this section. - -With the logger object configured, the following methods create log messages: - -* :meth:`Logger.debug`, :meth:`Logger.info`, :meth:`Logger.warning`, - :meth:`Logger.error`, and :meth:`Logger.critical` all create log records with - a message and a level that corresponds to their respective method names. The - message is actually a format string, which may contain the standard string - substitution syntax of ``%s``, ``%d``, ``%f``, and so on. The - rest of their arguments is a list of objects that correspond with the - substitution fields in the message. With regard to ``**kwargs``, the - logging methods care only about a keyword of ``exc_info`` and use it to - determine whether to log exception information. - -* :meth:`Logger.exception` creates a log message similar to - :meth:`Logger.error`. The difference is that :meth:`Logger.exception` dumps a - stack trace along with it. Call this method only from an exception handler. - -* :meth:`Logger.log` takes a log level as an explicit argument. This is a - little more verbose for logging messages than using the log level convenience - methods listed above, but this is how to log at custom log levels. - -:func:`getLogger` returns a reference to a logger instance with the specified -name if it is provided, or ``root`` if not. The names are period-separated -hierarchical structures. Multiple calls to :func:`getLogger` with the same name -will return a reference to the same logger object. Loggers that are further -down in the hierarchical list are children of loggers higher up in the list. -For example, given a logger with a name of ``foo``, loggers with names of -``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all descendants of ``foo``. - -Loggers have a concept of *effective level*. If a level is not explicitly set -on a logger, the level of its parent is used instead as its effective level. -If the parent has no explicit level set, *its* parent is examined, and so on - -all ancestors are searched until an explicitly set level is found. The root -logger always has an explicit level set (``WARNING`` by default). When deciding -whether to process an event, the effective level of the logger is used to -determine whether the event is passed to the logger's handlers. - -Child loggers propagate messages up to the handlers associated with their -ancestor loggers. Because of this, it is unnecessary to define and configure -handlers for all the loggers an application uses. It is sufficient to -configure handlers for a top-level logger and create child loggers as needed. -(You can, however, turn off propagation by setting the *propagate* -attribute of a logger to ``False``.) - - -.. _handler-basic: - -Handlers -^^^^^^^^ - -:class:`~logging.Handler` objects are responsible for dispatching the -appropriate log messages (based on the log messages' severity) to the handler's -specified destination. :class:`Logger` objects can add zero or more handler -objects to themselves with an :meth:`~Logger.addHandler` method. As an example -scenario, an application may want to send all log messages to a log file, all -log messages of error or higher to stdout, and all messages of critical to an -email address. This scenario requires three individual handlers where each -handler is responsible for sending messages of a specific severity to a specific -location. - -The standard library includes quite a few handler types (see -:ref:`useful-handlers`); the tutorials use mainly :class:`StreamHandler` and -:class:`FileHandler` in its examples. - -There are very few methods in a handler for application developers to concern -themselves with. The only handler methods that seem relevant for application -developers who are using the built-in handler objects (that is, not creating -custom handlers) are the following configuration methods: - -* The :meth:`~Handler.setLevel` method, just as in logger objects, specifies the - lowest severity that will be dispatched to the appropriate destination. Why - are there two :func:`setLevel` methods? The level set in the logger - determines which severity of messages it will pass to its handlers. The level - set in each handler determines which messages that handler will send on. - -* :meth:`~Handler.setFormatter` selects a Formatter object for this handler to - use. - -* :meth:`~Handler.addFilter` and :meth:`~Handler.removeFilter` respectively - configure and deconfigure filter objects on handlers. - -Application code should not directly instantiate and use instances of -:class:`Handler`. Instead, the :class:`Handler` class is a base class that -defines the interface that all handlers should have and establishes some -default behavior that child classes can use (or override). - - -Formatters -^^^^^^^^^^ - -Formatter objects configure the final order, structure, and contents of the log -message. Unlike the base :class:`logging.Handler` class, application code may -instantiate formatter classes, although you could likely subclass the formatter -if your application needs special behavior. The constructor takes three -optional arguments -- a message format string, a date format string and a style -indicator. - -.. method:: logging.Formatter.__init__(fmt=None, datefmt=None, style='%') - -If there is no message format string, the default is to use the -raw message. If there is no date format string, the default date format is: - -.. code-block:: none - - %Y-%m-%d %H:%M:%S - -with the milliseconds tacked on at the end. The ``style`` is one of ``'%'``, -``'{'``, or ``'$'``. If one of these is not specified, then ``'%'`` will be used. - -If the ``style`` is ``'%'``, the message format string uses -``%(<dictionary key>)s`` styled string substitution; the possible keys are -documented in :ref:`logrecord-attributes`. If the style is ``'{'``, the message -format string is assumed to be compatible with :meth:`str.format` (using -keyword arguments), while if the style is ``'$'`` then the message format string -should conform to what is expected by :meth:`string.Template.substitute`. - -.. versionchanged:: 3.2 - Added the ``style`` parameter. - -The following message format string will log the time in a human-readable -format, the severity of the message, and the contents of the message, in that -order:: - - '%(asctime)s - %(levelname)s - %(message)s' - -Formatters use a user-configurable function to convert the creation time of a -record to a tuple. By default, :func:`time.localtime` is used; to change this -for a particular formatter instance, set the ``converter`` attribute of the -instance to a function with the same signature as :func:`time.localtime` or -:func:`time.gmtime`. To change it for all formatters, for example if you want -all logging times to be shown in GMT, set the ``converter`` attribute in the -Formatter class (to ``time.gmtime`` for GMT display). - - -Configuring Logging -^^^^^^^^^^^^^^^^^^^ - -.. currentmodule:: logging.config - -Programmers can configure logging in three ways: - -1. Creating loggers, handlers, and formatters explicitly using Python - code that calls the configuration methods listed above. -2. Creating a logging config file and reading it using the :func:`fileConfig` - function. -3. Creating a dictionary of configuration information and passing it - to the :func:`dictConfig` function. - -For the reference documentation on the last two options, see -:ref:`logging-config-api`. The following example configures a very simple -logger, a console handler, and a simple formatter using Python code:: - - import logging - - # create logger - logger = logging.getLogger('simple_example') - logger.setLevel(logging.DEBUG) - - # create console handler and set level to debug - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - - # create formatter - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - - # add formatter to ch - ch.setFormatter(formatter) - - # add ch to logger - logger.addHandler(ch) - - # 'application' code - logger.debug('debug message') - logger.info('info message') - logger.warning('warn message') - logger.error('error message') - logger.critical('critical message') - -Running this module from the command line produces the following output: - -.. code-block:: shell-session - - $ python simple_logging_module.py - 2005-03-19 15:10:26,618 - simple_example - DEBUG - debug message - 2005-03-19 15:10:26,620 - simple_example - INFO - info message - 2005-03-19 15:10:26,695 - simple_example - WARNING - warn message - 2005-03-19 15:10:26,697 - simple_example - ERROR - error message - 2005-03-19 15:10:26,773 - simple_example - CRITICAL - critical message - -The following Python module creates a logger, handler, and formatter nearly -identical to those in the example listed above, with the only difference being -the names of the objects:: - - import logging - import logging.config - - logging.config.fileConfig('logging.conf') - - # create logger - logger = logging.getLogger('simpleExample') - - # 'application' code - logger.debug('debug message') - logger.info('info message') - logger.warning('warn message') - logger.error('error message') - logger.critical('critical message') - -Here is the logging.conf file: - -.. code-block:: ini - - [loggers] - keys=root,simpleExample - - [handlers] - keys=consoleHandler - - [formatters] - keys=simpleFormatter - - [logger_root] - level=DEBUG - handlers=consoleHandler - - [logger_simpleExample] - level=DEBUG - handlers=consoleHandler - qualname=simpleExample - propagate=0 - - [handler_consoleHandler] - class=StreamHandler - level=DEBUG - formatter=simpleFormatter - args=(sys.stdout,) - - [formatter_simpleFormatter] - format=%(asctime)s - %(name)s - %(levelname)s - %(message)s - -The output is nearly identical to that of the non-config-file-based example: - -.. code-block:: shell-session - - $ python simple_logging_config.py - 2005-03-19 15:38:55,977 - simpleExample - DEBUG - debug message - 2005-03-19 15:38:55,979 - simpleExample - INFO - info message - 2005-03-19 15:38:56,054 - simpleExample - WARNING - warn message - 2005-03-19 15:38:56,055 - simpleExample - ERROR - error message - 2005-03-19 15:38:56,130 - simpleExample - CRITICAL - critical message - -You can see that the config file approach has a few advantages over the Python -code approach, mainly separation of configuration and code and the ability of -noncoders to easily modify the logging properties. - -.. warning:: The :func:`fileConfig` function takes a default parameter, - ``disable_existing_loggers``, which defaults to ``True`` for reasons of - backward compatibility. This may or may not be what you want, since it - will cause any non-root loggers existing before the :func:`fileConfig` - call to be disabled unless they (or an ancestor) are explicitly named in - the configuration. Please refer to the reference documentation for more - information, and specify ``False`` for this parameter if you wish. - - The dictionary passed to :func:`dictConfig` can also specify a Boolean - value with key ``disable_existing_loggers``, which if not specified - explicitly in the dictionary also defaults to being interpreted as - ``True``. This leads to the logger-disabling behaviour described above, - which may not be what you want - in which case, provide the key - explicitly with a value of ``False``. - - -.. currentmodule:: logging - -Note that the class names referenced in config files need to be either relative -to the logging module, or absolute values which can be resolved using normal -import mechanisms. Thus, you could use either -:class:`~logging.handlers.WatchedFileHandler` (relative to the logging module) or -``mypackage.mymodule.MyHandler`` (for a class defined in package ``mypackage`` -and module ``mymodule``, where ``mypackage`` is available on the Python import -path). - -In Python 3.2, a new means of configuring logging has been introduced, using -dictionaries to hold configuration information. This provides a superset of the -functionality of the config-file-based approach outlined above, and is the -recommended configuration method for new applications and deployments. Because -a Python dictionary is used to hold configuration information, and since you -can populate that dictionary using different means, you have more options for -configuration. For example, you can use a configuration file in JSON format, -or, if you have access to YAML processing functionality, a file in YAML -format, to populate the configuration dictionary. Or, of course, you can -construct the dictionary in Python code, receive it in pickled form over a -socket, or use whatever approach makes sense for your application. - -Here's an example of the same configuration as above, in YAML format for -the new dictionary-based approach: - -.. code-block:: yaml - - version: 1 - formatters: - simple: - format: '%(asctime)s - %(name)s - %(levelname)s - %(message)s' - handlers: - console: - class: logging.StreamHandler - level: DEBUG - formatter: simple - stream: ext://sys.stdout - loggers: - simpleExample: - level: DEBUG - handlers: [console] - propagate: no - root: - level: DEBUG - handlers: [console] - -For more information about logging using a dictionary, see -:ref:`logging-config-api`. - -What happens if no configuration is provided -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If no logging configuration is provided, it is possible to have a situation -where a logging event needs to be output, but no handlers can be found to -output the event. The behaviour of the logging package in these -circumstances is dependent on the Python version. - -For versions of Python prior to 3.2, the behaviour is as follows: - -* If *logging.raiseExceptions* is ``False`` (production mode), the event is - silently dropped. - -* If *logging.raiseExceptions* is ``True`` (development mode), a message - 'No handlers could be found for logger X.Y.Z' is printed once. - -In Python 3.2 and later, the behaviour is as follows: - -* The event is output using a 'handler of last resort', stored in - ``logging.lastResort``. This internal handler is not associated with any - logger, and acts like a :class:`~logging.StreamHandler` which writes the - event description message to the current value of ``sys.stderr`` (therefore - respecting any redirections which may be in effect). No formatting is - done on the message - just the bare event description message is printed. - The handler's level is set to ``WARNING``, so all events at this and - greater severities will be output. - -To obtain the pre-3.2 behaviour, ``logging.lastResort`` can be set to ``None``. - -.. _library-config: - -Configuring Logging for a Library -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When developing a library which uses logging, you should take care to -document how the library uses logging - for example, the names of loggers -used. Some consideration also needs to be given to its logging configuration. -If the using application does not use logging, and library code makes logging -calls, then (as described in the previous section) events of severity -``WARNING`` and greater will be printed to ``sys.stderr``. This is regarded as -the best default behaviour. - -If for some reason you *don't* want these messages printed in the absence of -any logging configuration, you can attach a do-nothing handler to the top-level -logger for your library. This avoids the message being printed, since a handler -will always be found for the library's events: it just doesn't produce any -output. If the library user configures logging for application use, presumably -that configuration will add some handlers, and if levels are suitably -configured then logging calls made in library code will send output to those -handlers, as normal. - -A do-nothing handler is included in the logging package: -:class:`~logging.NullHandler` (since Python 3.1). An instance of this handler -could be added to the top-level logger of the logging namespace used by the -library (*if* you want to prevent your library's logged events being output to -``sys.stderr`` in the absence of logging configuration). If all logging by a -library *foo* is done using loggers with names matching 'foo.x', 'foo.x.y', -etc. then the code:: - - import logging - logging.getLogger('foo').addHandler(logging.NullHandler()) - -should have the desired effect. If an organisation produces a number of -libraries, then the logger name specified can be 'orgname.foo' rather than -just 'foo'. - -.. note:: It is strongly advised that you *do not log to the root logger* - in your library. Instead, use a logger with a unique and easily - identifiable name, such as the ``__name__`` for your library's top-level package - or module. Logging to the root logger will make it difficult or impossible for - the application developer to configure the logging verbosity or handlers of - your library as they wish. - -.. note:: It is strongly advised that you *do not add any handlers other - than* :class:`~logging.NullHandler` *to your library's loggers*. This is - because the configuration of handlers is the prerogative of the application - developer who uses your library. The application developer knows their - target audience and what handlers are most appropriate for their - application: if you add handlers 'under the hood', you might well interfere - with their ability to carry out unit tests and deliver logs which suit their - requirements. - - -Logging Levels --------------- - -The numeric values of logging levels are given in the following table. These are -primarily of interest if you want to define your own levels, and need them to -have specific values relative to the predefined levels. If you define a level -with the same numeric value, it overwrites the predefined value; the predefined -name is lost. - -+--------------+---------------+ -| Level | Numeric value | -+==============+===============+ -| ``CRITICAL`` | 50 | -+--------------+---------------+ -| ``ERROR`` | 40 | -+--------------+---------------+ -| ``WARNING`` | 30 | -+--------------+---------------+ -| ``INFO`` | 20 | -+--------------+---------------+ -| ``DEBUG`` | 10 | -+--------------+---------------+ -| ``NOTSET`` | 0 | -+--------------+---------------+ - -Levels can also be associated with loggers, being set either by the developer or -through loading a saved logging configuration. When a logging method is called -on a logger, the logger compares its own level with the level associated with -the method call. If the logger's level is higher than the method call's, no -logging message is actually generated. This is the basic mechanism controlling -the verbosity of logging output. - -Logging messages are encoded as instances of the :class:`~logging.LogRecord` -class. When a logger decides to actually log an event, a -:class:`~logging.LogRecord` instance is created from the logging message. - -Logging messages are subjected to a dispatch mechanism through the use of -:dfn:`handlers`, which are instances of subclasses of the :class:`Handler` -class. Handlers are responsible for ensuring that a logged message (in the form -of a :class:`LogRecord`) ends up in a particular location (or set of locations) -which is useful for the target audience for that message (such as end users, -support desk staff, system administrators, developers). Handlers are passed -:class:`LogRecord` instances intended for particular destinations. Each logger -can have zero, one or more handlers associated with it (via the -:meth:`~Logger.addHandler` method of :class:`Logger`). In addition to any -handlers directly associated with a logger, *all handlers associated with all -ancestors of the logger* are called to dispatch the message (unless the -*propagate* flag for a logger is set to a false value, at which point the -passing to ancestor handlers stops). - -Just as for loggers, handlers can have levels associated with them. A handler's -level acts as a filter in the same way as a logger's level does. If a handler -decides to actually dispatch an event, the :meth:`~Handler.emit` method is used -to send the message to its destination. Most user-defined subclasses of -:class:`Handler` will need to override this :meth:`~Handler.emit`. - -.. _custom-levels: - -Custom Levels -^^^^^^^^^^^^^ - -Defining your own levels is possible, but should not be necessary, as the -existing levels have been chosen on the basis of practical experience. -However, if you are convinced that you need custom levels, great care should -be exercised when doing this, and it is possibly *a very bad idea to define -custom levels if you are developing a library*. That's because if multiple -library authors all define their own custom levels, there is a chance that -the logging output from such multiple libraries used together will be -difficult for the using developer to control and/or interpret, because a -given numeric value might mean different things for different libraries. - -.. _useful-handlers: - -Useful Handlers ---------------- - -In addition to the base :class:`Handler` class, many useful subclasses are -provided: - -#. :class:`StreamHandler` instances send messages to streams (file-like - objects). - -#. :class:`FileHandler` instances send messages to disk files. - -#. :class:`~handlers.BaseRotatingHandler` is the base class for handlers that - rotate log files at a certain point. It is not meant to be instantiated - directly. Instead, use :class:`~handlers.RotatingFileHandler` or - :class:`~handlers.TimedRotatingFileHandler`. - -#. :class:`~handlers.RotatingFileHandler` instances send messages to disk - files, with support for maximum log file sizes and log file rotation. - -#. :class:`~handlers.TimedRotatingFileHandler` instances send messages to - disk files, rotating the log file at certain timed intervals. - -#. :class:`~handlers.SocketHandler` instances send messages to TCP/IP - sockets. Since 3.4, Unix domain sockets are also supported. - -#. :class:`~handlers.DatagramHandler` instances send messages to UDP - sockets. Since 3.4, Unix domain sockets are also supported. - -#. :class:`~handlers.SMTPHandler` instances send messages to a designated - email address. - -#. :class:`~handlers.SysLogHandler` instances send messages to a Unix - syslog daemon, possibly on a remote machine. - -#. :class:`~handlers.NTEventLogHandler` instances send messages to a - Windows NT/2000/XP event log. - -#. :class:`~handlers.MemoryHandler` instances send messages to a buffer - in memory, which is flushed whenever specific criteria are met. - -#. :class:`~handlers.HTTPHandler` instances send messages to an HTTP - server using either ``GET`` or ``POST`` semantics. - -#. :class:`~handlers.WatchedFileHandler` instances watch the file they are - logging to. If the file changes, it is closed and reopened using the file - name. This handler is only useful on Unix-like systems; Windows does not - support the underlying mechanism used. - -#. :class:`~handlers.QueueHandler` instances send messages to a queue, such as - those implemented in the :mod:`queue` or :mod:`multiprocessing` modules. - -#. :class:`NullHandler` instances do nothing with error messages. They are used - by library developers who want to use logging, but want to avoid the 'No - handlers could be found for logger *XXX*' message which can be displayed if - the library user has not configured logging. See :ref:`library-config` for - more information. - -.. versionadded:: 3.1 - The :class:`NullHandler` class. - -.. versionadded:: 3.2 - The :class:`~handlers.QueueHandler` class. - -The :class:`NullHandler`, :class:`StreamHandler` and :class:`FileHandler` -classes are defined in the core logging package. The other handlers are -defined in a sub-module, :mod:`logging.handlers`. (There is also another -sub-module, :mod:`logging.config`, for configuration functionality.) - -Logged messages are formatted for presentation through instances of the -:class:`Formatter` class. They are initialized with a format string suitable for -use with the % operator and a dictionary. - -For formatting multiple messages in a batch, instances of -:class:`~handlers.BufferingFormatter` can be used. In addition to the format -string (which is applied to each message in the batch), there is provision for -header and trailer format strings. - -When filtering based on logger level and/or handler level is not enough, -instances of :class:`Filter` can be added to both :class:`Logger` and -:class:`Handler` instances (through their :meth:`~Handler.addFilter` method). -Before deciding to process a message further, both loggers and handlers consult -all their filters for permission. If any filter returns a false value, the -message is not processed further. - -The basic :class:`Filter` functionality allows filtering by specific logger -name. If this feature is used, messages sent to the named logger and its -children are allowed through the filter, and all others dropped. - - -.. _logging-exceptions: - -Exceptions raised during logging --------------------------------- - -The logging package is designed to swallow exceptions which occur while logging -in production. This is so that errors which occur while handling logging events -- such as logging misconfiguration, network or other similar errors - do not -cause the application using logging to terminate prematurely. - -:class:`SystemExit` and :class:`KeyboardInterrupt` exceptions are never -swallowed. Other exceptions which occur during the :meth:`~Handler.emit` method -of a :class:`Handler` subclass are passed to its :meth:`~Handler.handleError` -method. - -The default implementation of :meth:`~Handler.handleError` in :class:`Handler` -checks to see if a module-level variable, :data:`raiseExceptions`, is set. If -set, a traceback is printed to :data:`sys.stderr`. If not set, the exception is -swallowed. - -.. note:: The default value of :data:`raiseExceptions` is ``True``. This is - because during development, you typically want to be notified of any - exceptions that occur. It's advised that you set :data:`raiseExceptions` to - ``False`` for production usage. - -.. currentmodule:: logging - -.. _arbitrary-object-messages: - -Using arbitrary objects as messages ------------------------------------ - -In the preceding sections and examples, it has been assumed that the message -passed when logging the event is a string. However, this is not the only -possibility. You can pass an arbitrary object as a message, and its -:meth:`~object.__str__` method will be called when the logging system needs to -convert it to a string representation. In fact, if you want to, you can avoid -computing a string representation altogether - for example, the -:class:`~handlers.SocketHandler` emits an event by pickling it and sending it -over the wire. - - -Optimization ------------- - -Formatting of message arguments is deferred until it cannot be avoided. -However, computing the arguments passed to the logging method can also be -expensive, and you may want to avoid doing it if the logger will just throw -away your event. To decide what to do, you can call the -:meth:`~Logger.isEnabledFor` method which takes a level argument and returns -true if the event would be created by the Logger for that level of call. -You can write code like this:: - - if logger.isEnabledFor(logging.DEBUG): - logger.debug('Message with %s, %s', expensive_func1(), - expensive_func2()) - -so that if the logger's threshold is set above ``DEBUG``, the calls to -:func:`expensive_func1` and :func:`expensive_func2` are never made. - -.. note:: In some cases, :meth:`~Logger.isEnabledFor` can itself be more - expensive than you'd like (e.g. for deeply nested loggers where an explicit - level is only set high up in the logger hierarchy). In such cases (or if you - want to avoid calling a method in tight loops), you can cache the result of a - call to :meth:`~Logger.isEnabledFor` in a local or instance variable, and use - that instead of calling the method each time. Such a cached value would only - need to be recomputed when the logging configuration changes dynamically - while the application is running (which is not all that common). - -There are other optimizations which can be made for specific applications which -need more precise control over what logging information is collected. Here's a -list of things you can do to avoid processing during logging which you don't -need: - -+-----------------------------------------------------+---------------------------------------------------+ -| What you don't want to collect | How to avoid collecting it | -+=====================================================+===================================================+ -| Information about where calls were made from. | Set ``logging._srcfile`` to ``None``. | -| | This avoids calling :func:`sys._getframe`, which | -| | may help to speed up your code in environments | -| | like PyPy (which can't speed up code that uses | -| | :func:`sys._getframe`). | -+-----------------------------------------------------+---------------------------------------------------+ -| Threading information. | Set ``logging.logThreads`` to ``False``. | -+-----------------------------------------------------+---------------------------------------------------+ -| Current process ID (:func:`os.getpid`) | Set ``logging.logProcesses`` to ``False``. | -+-----------------------------------------------------+---------------------------------------------------+ -| Current process name when using ``multiprocessing`` | Set ``logging.logMultiprocessing`` to ``False``. | -| to manage multiple processes. | | -+-----------------------------------------------------+---------------------------------------------------+ - -Also note that the core logging module only includes the basic handlers. If -you don't import :mod:`logging.handlers` and :mod:`logging.config`, they won't -take up any memory. - -.. seealso:: - - Module :mod:`logging` - API reference for the logging module. - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :ref:`A logging cookbook <logging-cookbook>` diff --git a/Doc/howto/logging_flow.png b/Doc/howto/logging_flow.png deleted file mode 100644 index d65e597f811db5..00000000000000 Binary files a/Doc/howto/logging_flow.png and /dev/null differ diff --git a/Doc/howto/pyporting.rst b/Doc/howto/pyporting.rst deleted file mode 100644 index 6c30a0dd7d6bcc..00000000000000 --- a/Doc/howto/pyporting.rst +++ /dev/null @@ -1,427 +0,0 @@ -.. _pyporting-howto: - -************************************* -How to port Python 2 Code to Python 3 -************************************* - -:author: Brett Cannon - -.. topic:: Abstract - - Python 2 reached its official end-of-life at the start of 2020. This means - that no new bug reports, fixes, or changes will be made to Python 2 - it's - no longer supported. - - This guide is intended to provide you with a path to Python 3 for your - code, that includes compatibility with Python 2 as a first step. - - If you are looking to port an extension module instead of pure Python code, - please see :ref:`cporting-howto`. - - The archived python-porting_ mailing list may contain some useful guidance. - - -The Short Explanation -===================== - -To achieve Python 2/3 compatibility in a single code base, the basic steps -are: - -#. Only worry about supporting Python 2.7 -#. Make sure you have good test coverage (coverage.py_ can help; - ``python -m pip install coverage``) -#. Learn the differences between Python 2 and 3 -#. Use Futurize_ (or Modernize_) to update your code (e.g. ``python -m pip install future``) -#. Use Pylint_ to help make sure you don't regress on your Python 3 support - (``python -m pip install pylint``) -#. Use caniusepython3_ to find out which of your dependencies are blocking your - use of Python 3 (``python -m pip install caniusepython3``) -#. Once your dependencies are no longer blocking you, use continuous integration - to make sure you stay compatible with Python 2 and 3 (tox_ can help test - against multiple versions of Python; ``python -m pip install tox``) -#. Consider using optional static type checking to make sure your type usage - works in both Python 2 and 3 (e.g. use mypy_ to check your typing under both - Python 2 and Python 3; ``python -m pip install mypy``). - -.. note:: - - Note: Using ``python -m pip install`` guarantees that the ``pip`` you invoke - is the one installed for the Python currently in use, whether it be - a system-wide ``pip`` or one installed within a - :ref:`virtual environment <tut-venv>`. - -Details -======= - -Even if other factors - say, dependencies over which you have no control - -still require you to support Python 2, that does not prevent you taking the -step of including Python 3 support. - -Most changes required to support Python 3 lead to cleaner code using newer -practices even in Python 2 code. - - -Different versions of Python 2 ------------------------------- - -Ideally, your code should be compatible with Python 2.7, which was the -last supported version of Python 2. - -Some of the tools mentioned in this guide will not work with Python 2.6. - -If absolutely necessary, the six_ project can help you support Python 2.5 and -3 simultaneously. Do realize, though, that nearly all the projects listed in -this guide will not be available to you. - -If you are able to skip Python 2.5 and older, the required changes to your -code will be minimal. At worst you will have to use a function instead of a -method in some instances or have to import a function instead of using a -built-in one. - - -Make sure you specify the proper version support in your ``setup.py`` file --------------------------------------------------------------------------- - -In your ``setup.py`` file you should have the proper `trove classifier`_ -specifying what versions of Python you support. As your project does not support -Python 3 yet you should at least have -``Programming Language :: Python :: 2 :: Only`` specified. Ideally you should -also specify each major/minor version of Python that you do support, e.g. -``Programming Language :: Python :: 2.7``. - - -Have good test coverage ------------------------ - -Once you have your code supporting the oldest version of Python 2 you want it -to, you will want to make sure your test suite has good coverage. A good rule of -thumb is that if you want to be confident enough in your test suite that any -failures that appear after having tools rewrite your code are actual bugs in the -tools and not in your code. If you want a number to aim for, try to get over 80% -coverage (and don't feel bad if you find it hard to get better than 90% -coverage). If you don't already have a tool to measure test coverage then -coverage.py_ is recommended. - - -Be aware of the differences between Python 2 and 3 --------------------------------------------------- - -Once you have your code well-tested you are ready to begin porting your code to -Python 3! But to fully understand how your code is going to change and what -you want to look out for while you code, you will want to learn what changes -Python 3 makes in terms of Python 2. - -Some resources for understanding the differences and their implications for you -code: - -* the :ref:`"What's New" <whatsnew-index>` doc for each release of Python 3 -* the `Porting to Python 3`_ book (which is free online) -* the handy `cheat sheet`_ from the Python-Future project. - - -Update your code ----------------- - -There are tools available that can port your code automatically. - -Futurize_ does its best to make Python 3 idioms and practices exist in Python -2, e.g. backporting the ``bytes`` type from Python 3 so that you have -semantic parity between the major versions of Python. This is the better -approach for most cases. - -Modernize_, on the other hand, is more conservative and targets a Python 2/3 -subset of Python, directly relying on six_ to help provide compatibility. - -A good approach is to run the tool over your test suite first and visually -inspect the diff to make sure the transformation is accurate. After you have -transformed your test suite and verified that all the tests still pass as -expected, then you can transform your application code knowing that any tests -which fail is a translation failure. - -Unfortunately the tools can't automate everything to make your code work under -Python 3, and you will also need to read the tools' documentation in case some -options you need are turned off by default. - -Key issues to be aware of and check for: - -Division -++++++++ - -In Python 3, ``5 / 2 == 2.5`` and not ``2`` as it was in Python 2; all -division between ``int`` values result in a ``float``. This change has -actually been planned since Python 2.2 which was released in 2002. Since then -users have been encouraged to add ``from __future__ import division`` to any -and all files which use the ``/`` and ``//`` operators or to be running the -interpreter with the ``-Q`` flag. If you have not been doing this then you -will need to go through your code and do two things: - -#. Add ``from __future__ import division`` to your files -#. Update any division operator as necessary to either use ``//`` to use floor - division or continue using ``/`` and expect a float - -The reason that ``/`` isn't simply translated to ``//`` automatically is that if -an object defines a ``__truediv__`` method but not ``__floordiv__`` then your -code would begin to fail (e.g. a user-defined class that uses ``/`` to -signify some operation but not ``//`` for the same thing or at all). - - -Text versus binary data -+++++++++++++++++++++++ - -In Python 2 you could use the ``str`` type for both text and binary data. -Unfortunately this confluence of two different concepts could lead to brittle -code which sometimes worked for either kind of data, sometimes not. It also -could lead to confusing APIs if people didn't explicitly state that something -that accepted ``str`` accepted either text or binary data instead of one -specific type. This complicated the situation especially for anyone supporting -multiple languages as APIs wouldn't bother explicitly supporting ``unicode`` -when they claimed text data support. - -Python 3 made text and binary data distinct types that cannot simply be mixed -together. For any code that deals only with text or only binary data, this -separation doesn't pose an issue. But for code that has to deal with both, it -does mean you might have to now care about when you are using text compared -to binary data, which is why this cannot be entirely automated. - -Decide which APIs take text and which take binary (it is **highly** recommended -you don't design APIs that can take both due to the difficulty of keeping the -code working; as stated earlier it is difficult to do well). In Python 2 this -means making sure the APIs that take text can work with ``unicode`` and those -that work with binary data work with the ``bytes`` type from Python 3 -(which is a subset of ``str`` in Python 2 and acts as an alias for ``bytes`` -type in Python 2). Usually the biggest issue is realizing which methods exist -on which types in Python 2 and 3 simultaneously (for text that's ``unicode`` -in Python 2 and ``str`` in Python 3, for binary that's ``str``/``bytes`` in -Python 2 and ``bytes`` in Python 3). - -The following table lists the **unique** methods of each data type across -Python 2 and 3 (e.g., the ``decode()`` method is usable on the equivalent binary -data type in either Python 2 or 3, but it can't be used by the textual data -type consistently between Python 2 and 3 because ``str`` in Python 3 doesn't -have the method). Do note that as of Python 3.5 the ``__mod__`` method was -added to the bytes type. - -======================== ===================== -**Text data** **Binary data** ------------------------- --------------------- -\ decode ------------------------- --------------------- -encode ------------------------- --------------------- -format ------------------------- --------------------- -isdecimal ------------------------- --------------------- -isnumeric -======================== ===================== - -Making the distinction easier to handle can be accomplished by encoding and -decoding between binary data and text at the edge of your code. This means that -when you receive text in binary data, you should immediately decode it. And if -your code needs to send text as binary data then encode it as late as possible. -This allows your code to work with only text internally and thus eliminates -having to keep track of what type of data you are working with. - -The next issue is making sure you know whether the string literals in your code -represent text or binary data. You should add a ``b`` prefix to any -literal that presents binary data. For text you should add a ``u`` prefix to -the text literal. (There is a :mod:`__future__` import to force all unspecified -literals to be Unicode, but usage has shown it isn't as effective as adding a -``b`` or ``u`` prefix to all literals explicitly) - -You also need to be careful about opening files. Possibly you have not always -bothered to add the ``b`` mode when opening a binary file (e.g., ``rb`` for -binary reading). Under Python 3, binary files and text files are clearly -distinct and mutually incompatible; see the :mod:`io` module for details. -Therefore, you **must** make a decision of whether a file will be used for -binary access (allowing binary data to be read and/or written) or textual access -(allowing text data to be read and/or written). You should also use :func:`io.open` -for opening files instead of the built-in :func:`open` function as the :mod:`io` -module is consistent from Python 2 to 3 while the built-in :func:`open` function -is not (in Python 3 it's actually :func:`io.open`). Do not bother with the -outdated practice of using :func:`codecs.open` as that's only necessary for -keeping compatibility with Python 2.5. - -The constructors of both ``str`` and ``bytes`` have different semantics for the -same arguments between Python 2 and 3. Passing an integer to ``bytes`` in Python 2 -will give you the string representation of the integer: ``bytes(3) == '3'``. -But in Python 3, an integer argument to ``bytes`` will give you a bytes object -as long as the integer specified, filled with null bytes: -``bytes(3) == b'\x00\x00\x00'``. A similar worry is necessary when passing a -bytes object to ``str``. In Python 2 you just get the bytes object back: -``str(b'3') == b'3'``. But in Python 3 you get the string representation of the -bytes object: ``str(b'3') == "b'3'"``. - -Finally, the indexing of binary data requires careful handling (slicing does -**not** require any special handling). In Python 2, -``b'123'[1] == b'2'`` while in Python 3 ``b'123'[1] == 50``. Because binary data -is simply a collection of binary numbers, Python 3 returns the integer value for -the byte you index on. But in Python 2 because ``bytes == str``, indexing -returns a one-item slice of bytes. The six_ project has a function -named ``six.indexbytes()`` which will return an integer like in Python 3: -``six.indexbytes(b'123', 1)``. - -To summarize: - -#. Decide which of your APIs take text and which take binary data -#. Make sure that your code that works with text also works with ``unicode`` and - code for binary data works with ``bytes`` in Python 2 (see the table above - for what methods you cannot use for each type) -#. Mark all binary literals with a ``b`` prefix, textual literals with a ``u`` - prefix -#. Decode binary data to text as soon as possible, encode text as binary data as - late as possible -#. Open files using :func:`io.open` and make sure to specify the ``b`` mode when - appropriate -#. Be careful when indexing into binary data - - -Use feature detection instead of version detection -++++++++++++++++++++++++++++++++++++++++++++++++++ - -Inevitably you will have code that has to choose what to do based on what -version of Python is running. The best way to do this is with feature detection -of whether the version of Python you're running under supports what you need. -If for some reason that doesn't work then you should make the version check be -against Python 2 and not Python 3. To help explain this, let's look at an -example. - -Let's pretend that you need access to a feature of :mod:`importlib` that -is available in Python's standard library since Python 3.3 and available for -Python 2 through importlib2_ on PyPI. You might be tempted to write code to -access e.g. the :mod:`importlib.abc` module by doing the following:: - - import sys - - if sys.version_info[0] == 3: - from importlib import abc - else: - from importlib2 import abc - -The problem with this code is what happens when Python 4 comes out? It would -be better to treat Python 2 as the exceptional case instead of Python 3 and -assume that future Python versions will be more compatible with Python 3 than -Python 2:: - - import sys - - if sys.version_info[0] > 2: - from importlib import abc - else: - from importlib2 import abc - -The best solution, though, is to do no version detection at all and instead rely -on feature detection. That avoids any potential issues of getting the version -detection wrong and helps keep you future-compatible:: - - try: - from importlib import abc - except ImportError: - from importlib2 import abc - - -Prevent compatibility regressions ---------------------------------- - -Once you have fully translated your code to be compatible with Python 3, you -will want to make sure your code doesn't regress and stop working under -Python 3. This is especially true if you have a dependency which is blocking you -from actually running under Python 3 at the moment. - -To help with staying compatible, any new modules you create should have -at least the following block of code at the top of it:: - - from __future__ import absolute_import - from __future__ import division - from __future__ import print_function - -You can also run Python 2 with the ``-3`` flag to be warned about various -compatibility issues your code triggers during execution. If you turn warnings -into errors with ``-Werror`` then you can make sure that you don't accidentally -miss a warning. - -You can also use the Pylint_ project and its ``--py3k`` flag to lint your code -to receive warnings when your code begins to deviate from Python 3 -compatibility. This also prevents you from having to run Modernize_ or Futurize_ -over your code regularly to catch compatibility regressions. This does require -you only support Python 2.7 and Python 3.4 or newer as that is Pylint's -minimum Python version support. - - -Check which dependencies block your transition ----------------------------------------------- - -**After** you have made your code compatible with Python 3 you should begin to -care about whether your dependencies have also been ported. The caniusepython3_ -project was created to help you determine which projects --- directly or indirectly -- are blocking you from supporting Python 3. There -is both a command-line tool as well as a web interface at -https://caniusepython3.com. - -The project also provides code which you can integrate into your test suite so -that you will have a failing test when you no longer have dependencies blocking -you from using Python 3. This allows you to avoid having to manually check your -dependencies and to be notified quickly when you can start running on Python 3. - - -Update your ``setup.py`` file to denote Python 3 compatibility --------------------------------------------------------------- - -Once your code works under Python 3, you should update the classifiers in -your ``setup.py`` to contain ``Programming Language :: Python :: 3`` and to not -specify sole Python 2 support. This will tell anyone using your code that you -support Python 2 **and** 3. Ideally you will also want to add classifiers for -each major/minor version of Python you now support. - - -Use continuous integration to stay compatible ---------------------------------------------- - -Once you are able to fully run under Python 3 you will want to make sure your -code always works under both Python 2 and 3. Probably the best tool for running -your tests under multiple Python interpreters is tox_. You can then integrate -tox with your continuous integration system so that you never accidentally break -Python 2 or 3 support. - -You may also want to use the ``-bb`` flag with the Python 3 interpreter to -trigger an exception when you are comparing bytes to strings or bytes to an int -(the latter is available starting in Python 3.5). By default type-differing -comparisons simply return ``False``, but if you made a mistake in your -separation of text/binary data handling or indexing on bytes you wouldn't easily -find the mistake. This flag will raise an exception when these kinds of -comparisons occur, making the mistake much easier to track down. - - -Consider using optional static type checking --------------------------------------------- - -Another way to help port your code is to use a static type checker like -mypy_ or pytype_ on your code. These tools can be used to analyze your code as -if it's being run under Python 2, then you can run the tool a second time as if -your code is running under Python 3. By running a static type checker twice like -this you can discover if you're e.g. misusing binary data type in one version -of Python compared to another. If you add optional type hints to your code you -can also explicitly state whether your APIs use textual or binary data, helping -to make sure everything functions as expected in both versions of Python. - - -.. _caniusepython3: https://pypi.org/project/caniusepython3 -.. _cheat sheet: https://python-future.org/compatible_idioms.html -.. _coverage.py: https://pypi.org/project/coverage -.. _Futurize: https://python-future.org/automatic_conversion.html -.. _importlib2: https://pypi.org/project/importlib2 -.. _Modernize: https://python-modernize.readthedocs.io/ -.. _mypy: https://mypy-lang.org/ -.. _Porting to Python 3: http://python3porting.com/ -.. _Pylint: https://pypi.org/project/pylint - -.. _Python 3 Q & A: https://ncoghlan-devs-python-notes.readthedocs.io/en/latest/python3/questions_and_answers.html - -.. _pytype: https://github.com/google/pytype -.. _python-future: https://python-future.org/ -.. _python-porting: https://mail.python.org/pipermail/python-porting/ -.. _six: https://pypi.org/project/six -.. _tox: https://pypi.org/project/tox -.. _trove classifier: https://pypi.org/classifiers - -.. _Why Python 3 exists: https://snarky.ca/why-python-3-exists diff --git a/Doc/howto/regex.rst b/Doc/howto/regex.rst deleted file mode 100644 index b69d19e212bdc9..00000000000000 --- a/Doc/howto/regex.rst +++ /dev/null @@ -1,1398 +0,0 @@ -.. _regex-howto: - -**************************** - Regular Expression HOWTO -**************************** - -:Author: A.M. Kuchling <amk@amk.ca> - -.. TODO: - Document lookbehind assertions - Better way of displaying a RE, a string, and what it matches - Mention optional argument to match.groups() - Unicode (at least a reference) - - -.. topic:: Abstract - - This document is an introductory tutorial to using regular expressions in Python - with the :mod:`re` module. It provides a gentler introduction than the - corresponding section in the Library Reference. - - -Introduction -============ - -Regular expressions (called REs, or regexes, or regex patterns) are essentially -a tiny, highly specialized programming language embedded inside Python and made -available through the :mod:`re` module. Using this little language, you specify -the rules for the set of possible strings that you want to match; this set might -contain English sentences, or e-mail addresses, or TeX commands, or anything you -like. You can then ask questions such as "Does this string match the pattern?", -or "Is there a match for the pattern anywhere in this string?". You can also -use REs to modify a string or to split it apart in various ways. - -Regular expression patterns are compiled into a series of bytecodes which are -then executed by a matching engine written in C. For advanced use, it may be -necessary to pay careful attention to how the engine will execute a given RE, -and write the RE in a certain way in order to produce bytecode that runs faster. -Optimization isn't covered in this document, because it requires that you have a -good understanding of the matching engine's internals. - -The regular expression language is relatively small and restricted, so not all -possible string processing tasks can be done using regular expressions. There -are also tasks that *can* be done with regular expressions, but the expressions -turn out to be very complicated. In these cases, you may be better off writing -Python code to do the processing; while Python code will be slower than an -elaborate regular expression, it will also probably be more understandable. - - -Simple Patterns -=============== - -We'll start by learning about the simplest possible regular expressions. Since -regular expressions are used to operate on strings, we'll begin with the most -common task: matching characters. - -For a detailed explanation of the computer science underlying regular -expressions (deterministic and non-deterministic finite automata), you can refer -to almost any textbook on writing compilers. - - -Matching Characters -------------------- - -Most letters and characters will simply match themselves. For example, the -regular expression ``test`` will match the string ``test`` exactly. (You can -enable a case-insensitive mode that would let this RE match ``Test`` or ``TEST`` -as well; more about this later.) - -There are exceptions to this rule; some characters are special -:dfn:`metacharacters`, and don't match themselves. Instead, they signal that -some out-of-the-ordinary thing should be matched, or they affect other portions -of the RE by repeating them or changing their meaning. Much of this document is -devoted to discussing various metacharacters and what they do. - -Here's a complete list of the metacharacters; their meanings will be discussed -in the rest of this HOWTO. - -.. code-block:: none - - . ^ $ * + ? { } [ ] \ | ( ) - -The first metacharacters we'll look at are ``[`` and ``]``. They're used for -specifying a character class, which is a set of characters that you wish to -match. Characters can be listed individually, or a range of characters can be -indicated by giving two characters and separating them by a ``'-'``. For -example, ``[abc]`` will match any of the characters ``a``, ``b``, or ``c``; this -is the same as ``[a-c]``, which uses a range to express the same set of -characters. If you wanted to match only lowercase letters, your RE would be -``[a-z]``. - -Metacharacters (except ``\``) are not active inside classes. For example, ``[akm$]`` will -match any of the characters ``'a'``, ``'k'``, ``'m'``, or ``'$'``; ``'$'`` is -usually a metacharacter, but inside a character class it's stripped of its -special nature. - -You can match the characters not listed within the class by :dfn:`complementing` -the set. This is indicated by including a ``'^'`` as the first character of the -class. For example, ``[^5]`` will match any character except ``'5'``. If the -caret appears elsewhere in a character class, it does not have special meaning. -For example: ``[5^]`` will match either a ``'5'`` or a ``'^'``. - -Perhaps the most important metacharacter is the backslash, ``\``. As in Python -string literals, the backslash can be followed by various characters to signal -various special sequences. It's also used to escape all the metacharacters so -you can still match them in patterns; for example, if you need to match a ``[`` -or ``\``, you can precede them with a backslash to remove their special -meaning: ``\[`` or ``\\``. - -Some of the special sequences beginning with ``'\'`` represent -predefined sets of characters that are often useful, such as the set -of digits, the set of letters, or the set of anything that isn't -whitespace. - -Let's take an example: ``\w`` matches any alphanumeric character. If -the regex pattern is expressed in bytes, this is equivalent to the -class ``[a-zA-Z0-9_]``. If the regex pattern is a string, ``\w`` will -match all the characters marked as letters in the Unicode database -provided by the :mod:`unicodedata` module. You can use the more -restricted definition of ``\w`` in a string pattern by supplying the -:const:`re.ASCII` flag when compiling the regular expression. - -The following list of special sequences isn't complete. For a complete -list of sequences and expanded class definitions for Unicode string -patterns, see the last part of :ref:`Regular Expression Syntax -<re-syntax>` in the Standard Library reference. In general, the -Unicode versions match any character that's in the appropriate -category in the Unicode database. - -``\d`` - Matches any decimal digit; this is equivalent to the class ``[0-9]``. - -``\D`` - Matches any non-digit character; this is equivalent to the class ``[^0-9]``. - -``\s`` - Matches any whitespace character; this is equivalent to the class ``[ - \t\n\r\f\v]``. - -``\S`` - Matches any non-whitespace character; this is equivalent to the class ``[^ - \t\n\r\f\v]``. - -``\w`` - Matches any alphanumeric character; this is equivalent to the class - ``[a-zA-Z0-9_]``. - -``\W`` - Matches any non-alphanumeric character; this is equivalent to the class - ``[^a-zA-Z0-9_]``. - -These sequences can be included inside a character class. For example, -``[\s,.]`` is a character class that will match any whitespace character, or -``','`` or ``'.'``. - -The final metacharacter in this section is ``.``. It matches anything except a -newline character, and there's an alternate mode (:const:`re.DOTALL`) where it will -match even a newline. ``.`` is often used where you want to match "any -character". - - -Repeating Things ----------------- - -Being able to match varying sets of characters is the first thing regular -expressions can do that isn't already possible with the methods available on -strings. However, if that was the only additional capability of regexes, they -wouldn't be much of an advance. Another capability is that you can specify that -portions of the RE must be repeated a certain number of times. - -The first metacharacter for repeating things that we'll look at is ``*``. ``*`` -doesn't match the literal character ``'*'``; instead, it specifies that the -previous character can be matched zero or more times, instead of exactly once. - -For example, ``ca*t`` will match ``'ct'`` (0 ``'a'`` characters), ``'cat'`` (1 ``'a'``), -``'caaat'`` (3 ``'a'`` characters), and so forth. - -Repetitions such as ``*`` are :dfn:`greedy`; when repeating a RE, the matching -engine will try to repeat it as many times as possible. If later portions of the -pattern don't match, the matching engine will then back up and try again with -fewer repetitions. - -A step-by-step example will make this more obvious. Let's consider the -expression ``a[bcd]*b``. This matches the letter ``'a'``, zero or more letters -from the class ``[bcd]``, and finally ends with a ``'b'``. Now imagine matching -this RE against the string ``'abcbd'``. - -+------+-----------+---------------------------------+ -| Step | Matched | Explanation | -+======+===========+=================================+ -| 1 | ``a`` | The ``a`` in the RE matches. | -+------+-----------+---------------------------------+ -| 2 | ``abcbd`` | The engine matches ``[bcd]*``, | -| | | going as far as it can, which | -| | | is to the end of the string. | -+------+-----------+---------------------------------+ -| 3 | *Failure* | The engine tries to match | -| | | ``b``, but the current position | -| | | is at the end of the string, so | -| | | it fails. | -+------+-----------+---------------------------------+ -| 4 | ``abcb`` | Back up, so that ``[bcd]*`` | -| | | matches one less character. | -+------+-----------+---------------------------------+ -| 5 | *Failure* | Try ``b`` again, but the | -| | | current position is at the last | -| | | character, which is a ``'d'``. | -+------+-----------+---------------------------------+ -| 6 | ``abc`` | Back up again, so that | -| | | ``[bcd]*`` is only matching | -| | | ``bc``. | -+------+-----------+---------------------------------+ -| 6 | ``abcb`` | Try ``b`` again. This time | -| | | the character at the | -| | | current position is ``'b'``, so | -| | | it succeeds. | -+------+-----------+---------------------------------+ - -The end of the RE has now been reached, and it has matched ``'abcb'``. This -demonstrates how the matching engine goes as far as it can at first, and if no -match is found it will then progressively back up and retry the rest of the RE -again and again. It will back up until it has tried zero matches for -``[bcd]*``, and if that subsequently fails, the engine will conclude that the -string doesn't match the RE at all. - -Another repeating metacharacter is ``+``, which matches one or more times. Pay -careful attention to the difference between ``*`` and ``+``; ``*`` matches -*zero* or more times, so whatever's being repeated may not be present at all, -while ``+`` requires at least *one* occurrence. To use a similar example, -``ca+t`` will match ``'cat'`` (1 ``'a'``), ``'caaat'`` (3 ``'a'``\ s), but won't -match ``'ct'``. - -There are two more repeating operators or quantifiers. The question mark character, ``?``, -matches either once or zero times; you can think of it as marking something as -being optional. For example, ``home-?brew`` matches either ``'homebrew'`` or -``'home-brew'``. - -The most complicated quantifier is ``{m,n}``, where *m* and *n* are -decimal integers. This quantifier means there must be at least *m* repetitions, -and at most *n*. For example, ``a/{1,3}b`` will match ``'a/b'``, ``'a//b'``, and -``'a///b'``. It won't match ``'ab'``, which has no slashes, or ``'a////b'``, which -has four. - -You can omit either *m* or *n*; in that case, a reasonable value is assumed for -the missing value. Omitting *m* is interpreted as a lower limit of 0, while -omitting *n* results in an upper bound of infinity. - -The simplest case ``{m}`` matches the preceding item exactly *m* times. -For example, ``a/{2}b`` will only match ``'a//b'``. - -Readers of a reductionist bent may notice that the three other quantifiers can -all be expressed using this notation. ``{0,}`` is the same as ``*``, ``{1,}`` -is equivalent to ``+``, and ``{0,1}`` is the same as ``?``. It's better to use -``*``, ``+``, or ``?`` when you can, simply because they're shorter and easier -to read. - - -Using Regular Expressions -========================= - -Now that we've looked at some simple regular expressions, how do we actually use -them in Python? The :mod:`re` module provides an interface to the regular -expression engine, allowing you to compile REs into objects and then perform -matches with them. - - -Compiling Regular Expressions ------------------------------ - -Regular expressions are compiled into pattern objects, which have -methods for various operations such as searching for pattern matches or -performing string substitutions. :: - - >>> import re - >>> p = re.compile('ab*') - >>> p - re.compile('ab*') - -:func:`re.compile` also accepts an optional *flags* argument, used to enable -various special features and syntax variations. We'll go over the available -settings later, but for now a single example will do:: - - >>> p = re.compile('ab*', re.IGNORECASE) - -The RE is passed to :func:`re.compile` as a string. REs are handled as strings -because regular expressions aren't part of the core Python language, and no -special syntax was created for expressing them. (There are applications that -don't need REs at all, so there's no need to bloat the language specification by -including them.) Instead, the :mod:`re` module is simply a C extension module -included with Python, just like the :mod:`socket` or :mod:`zlib` modules. - -Putting REs in strings keeps the Python language simpler, but has one -disadvantage which is the topic of the next section. - - -.. _the-backslash-plague: - -The Backslash Plague --------------------- - -As stated earlier, regular expressions use the backslash character (``'\'``) to -indicate special forms or to allow special characters to be used without -invoking their special meaning. This conflicts with Python's usage of the same -character for the same purpose in string literals. - -Let's say you want to write a RE that matches the string ``\section``, which -might be found in a LaTeX file. To figure out what to write in the program -code, start with the desired string to be matched. Next, you must escape any -backslashes and other metacharacters by preceding them with a backslash, -resulting in the string ``\\section``. The resulting string that must be passed -to :func:`re.compile` must be ``\\section``. However, to express this as a -Python string literal, both backslashes must be escaped *again*. - -+-------------------+------------------------------------------+ -| Characters | Stage | -+===================+==========================================+ -| ``\section`` | Text string to be matched | -+-------------------+------------------------------------------+ -| ``\\section`` | Escaped backslash for :func:`re.compile` | -+-------------------+------------------------------------------+ -| ``"\\\\section"`` | Escaped backslashes for a string literal | -+-------------------+------------------------------------------+ - -In short, to match a literal backslash, one has to write ``'\\\\'`` as the RE -string, because the regular expression must be ``\\``, and each backslash must -be expressed as ``\\`` inside a regular Python string literal. In REs that -feature backslashes repeatedly, this leads to lots of repeated backslashes and -makes the resulting strings difficult to understand. - -The solution is to use Python's raw string notation for regular expressions; -backslashes are not handled in any special way in a string literal prefixed with -``'r'``, so ``r"\n"`` is a two-character string containing ``'\'`` and ``'n'``, -while ``"\n"`` is a one-character string containing a newline. Regular -expressions will often be written in Python code using this raw string notation. - -In addition, special escape sequences that are valid in regular expressions, -but not valid as Python string literals, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`, -which means the sequences will be invalid if raw string notation or escaping -the backslashes isn't used. - - -+-------------------+------------------+ -| Regular String | Raw string | -+===================+==================+ -| ``"ab*"`` | ``r"ab*"`` | -+-------------------+------------------+ -| ``"\\\\section"`` | ``r"\\section"`` | -+-------------------+------------------+ -| ``"\\w+\\s+\\1"`` | ``r"\w+\s+\1"`` | -+-------------------+------------------+ - - -Performing Matches ------------------- - -Once you have an object representing a compiled regular expression, what do you -do with it? Pattern objects have several methods and attributes. -Only the most significant ones will be covered here; consult the :mod:`re` docs -for a complete listing. - -+------------------+-----------------------------------------------+ -| Method/Attribute | Purpose | -+==================+===============================================+ -| ``match()`` | Determine if the RE matches at the beginning | -| | of the string. | -+------------------+-----------------------------------------------+ -| ``search()`` | Scan through a string, looking for any | -| | location where this RE matches. | -+------------------+-----------------------------------------------+ -| ``findall()`` | Find all substrings where the RE matches, and | -| | returns them as a list. | -+------------------+-----------------------------------------------+ -| ``finditer()`` | Find all substrings where the RE matches, and | -| | returns them as an :term:`iterator`. | -+------------------+-----------------------------------------------+ - -:meth:`~re.Pattern.match` and :meth:`~re.Pattern.search` return ``None`` if no match can be found. If -they're successful, a :ref:`match object <match-objects>` instance is returned, -containing information about the match: where it starts and ends, the substring -it matched, and more. - -You can learn about this by interactively experimenting with the :mod:`re` -module. If you have :mod:`tkinter` available, you may also want to look at -:source:`Tools/demo/redemo.py`, a demonstration program included with the -Python distribution. It allows you to enter REs and strings, and displays -whether the RE matches or fails. :file:`redemo.py` can be quite useful when -trying to debug a complicated RE. - -This HOWTO uses the standard Python interpreter for its examples. First, run the -Python interpreter, import the :mod:`re` module, and compile a RE:: - - >>> import re - >>> p = re.compile('[a-z]+') - >>> p - re.compile('[a-z]+') - -Now, you can try matching various strings against the RE ``[a-z]+``. An empty -string shouldn't match at all, since ``+`` means 'one or more repetitions'. -:meth:`~re.Pattern.match` should return ``None`` in this case, which will cause the -interpreter to print no output. You can explicitly print the result of -:meth:`!match` to make this clear. :: - - >>> p.match("") - >>> print(p.match("")) - None - -Now, let's try it on a string that it should match, such as ``tempo``. In this -case, :meth:`~re.Pattern.match` will return a :ref:`match object <match-objects>`, so you -should store the result in a variable for later use. :: - - >>> m = p.match('tempo') - >>> m - <re.Match object; span=(0, 5), match='tempo'> - -Now you can query the :ref:`match object <match-objects>` for information -about the matching string. Match object instances -also have several methods and attributes; the most important ones are: - -+------------------+--------------------------------------------+ -| Method/Attribute | Purpose | -+==================+============================================+ -| ``group()`` | Return the string matched by the RE | -+------------------+--------------------------------------------+ -| ``start()`` | Return the starting position of the match | -+------------------+--------------------------------------------+ -| ``end()`` | Return the ending position of the match | -+------------------+--------------------------------------------+ -| ``span()`` | Return a tuple containing the (start, end) | -| | positions of the match | -+------------------+--------------------------------------------+ - -Trying these methods will soon clarify their meaning:: - - >>> m.group() - 'tempo' - >>> m.start(), m.end() - (0, 5) - >>> m.span() - (0, 5) - -:meth:`~re.Match.group` returns the substring that was matched by the RE. :meth:`~re.Match.start` -and :meth:`~re.Match.end` return the starting and ending index of the match. :meth:`~re.Match.span` -returns both start and end indexes in a single tuple. Since the :meth:`~re.Pattern.match` -method only checks if the RE matches at the start of a string, :meth:`!start` -will always be zero. However, the :meth:`~re.Pattern.search` method of patterns -scans through the string, so the match may not start at zero in that -case. :: - - >>> print(p.match('::: message')) - None - >>> m = p.search('::: message'); print(m) - <re.Match object; span=(4, 11), match='message'> - >>> m.group() - 'message' - >>> m.span() - (4, 11) - -In actual programs, the most common style is to store the -:ref:`match object <match-objects>` in a variable, and then check if it was -``None``. This usually looks like:: - - p = re.compile( ... ) - m = p.match( 'string goes here' ) - if m: - print('Match found: ', m.group()) - else: - print('No match') - -Two pattern methods return all of the matches for a pattern. -:meth:`~re.Pattern.findall` returns a list of matching strings:: - - >>> p = re.compile(r'\d+') - >>> p.findall('12 drummers drumming, 11 pipers piping, 10 lords a-leaping') - ['12', '11', '10'] - -The ``r`` prefix, making the literal a raw string literal, is needed in this -example because escape sequences in a normal "cooked" string literal that are -not recognized by Python, as opposed to regular expressions, now result in a -:exc:`DeprecationWarning` and will eventually become a :exc:`SyntaxError`. See -:ref:`the-backslash-plague`. - -:meth:`~re.Pattern.findall` has to create the entire list before it can be returned as the -result. The :meth:`~re.Pattern.finditer` method returns a sequence of -:ref:`match object <match-objects>` instances as an :term:`iterator`:: - - >>> iterator = p.finditer('12 drummers drumming, 11 ... 10 ...') - >>> iterator #doctest: +ELLIPSIS - <callable_iterator object at 0x...> - >>> for match in iterator: - ... print(match.span()) - ... - (0, 2) - (22, 24) - (29, 31) - - -Module-Level Functions ----------------------- - -You don't have to create a pattern object and call its methods; the -:mod:`re` module also provides top-level functions called :func:`~re.match`, -:func:`~re.search`, :func:`~re.findall`, :func:`~re.sub`, and so forth. These functions -take the same arguments as the corresponding pattern method with -the RE string added as the first argument, and still return either ``None`` or a -:ref:`match object <match-objects>` instance. :: - - >>> print(re.match(r'From\s+', 'Fromage amk')) - None - >>> re.match(r'From\s+', 'From amk Thu May 14 19:12:10 1998') #doctest: +ELLIPSIS - <re.Match object; span=(0, 5), match='From '> - -Under the hood, these functions simply create a pattern object for you -and call the appropriate method on it. They also store the compiled -object in a cache, so future calls using the same RE won't need to -parse the pattern again and again. - -Should you use these module-level functions, or should you get the -pattern and call its methods yourself? If you're accessing a regex -within a loop, pre-compiling it will save a few function calls. -Outside of loops, there's not much difference thanks to the internal -cache. - - -Compilation Flags ------------------ - -.. currentmodule:: re - -Compilation flags let you modify some aspects of how regular expressions work. -Flags are available in the :mod:`re` module under two names, a long name such as -:const:`IGNORECASE` and a short, one-letter form such as :const:`I`. (If you're -familiar with Perl's pattern modifiers, the one-letter forms use the same -letters; the short form of :const:`re.VERBOSE` is :const:`re.X`, for example.) -Multiple flags can be specified by bitwise OR-ing them; ``re.I | re.M`` sets -both the :const:`I` and :const:`M` flags, for example. - -Here's a table of the available flags, followed by a more detailed explanation -of each one. - -+---------------------------------+--------------------------------------------+ -| Flag | Meaning | -+=================================+============================================+ -| :const:`ASCII`, :const:`A` | Makes several escapes like ``\w``, ``\b``, | -| | ``\s`` and ``\d`` match only on ASCII | -| | characters with the respective property. | -+---------------------------------+--------------------------------------------+ -| :const:`DOTALL`, :const:`S` | Make ``.`` match any character, including | -| | newlines. | -+---------------------------------+--------------------------------------------+ -| :const:`IGNORECASE`, :const:`I` | Do case-insensitive matches. | -+---------------------------------+--------------------------------------------+ -| :const:`LOCALE`, :const:`L` | Do a locale-aware match. | -+---------------------------------+--------------------------------------------+ -| :const:`MULTILINE`, :const:`M` | Multi-line matching, affecting ``^`` and | -| | ``$``. | -+---------------------------------+--------------------------------------------+ -| :const:`VERBOSE`, :const:`X` | Enable verbose REs, which can be organized | -| (for 'extended') | more cleanly and understandably. | -+---------------------------------+--------------------------------------------+ - - -.. data:: I - IGNORECASE - :noindex: - - Perform case-insensitive matching; character class and literal strings will - match letters by ignoring case. For example, ``[A-Z]`` will match lowercase - letters, too. Full Unicode matching also works unless the :const:`ASCII` - flag is used to disable non-ASCII matches. When the Unicode patterns - ``[a-z]`` or ``[A-Z]`` are used in combination with the :const:`IGNORECASE` - flag, they will match the 52 ASCII letters and 4 additional non-ASCII - letters: 'İ' (U+0130, Latin capital letter I with dot above), 'ı' (U+0131, - Latin small letter dotless i), 'ſ' (U+017F, Latin small letter long s) and - 'K' (U+212A, Kelvin sign). ``Spam`` will match ``'Spam'``, ``'spam'``, - ``'spAM'``, or ``'ſpam'`` (the latter is matched only in Unicode mode). - This lowercasing doesn't take the current locale into account; - it will if you also set the :const:`LOCALE` flag. - - -.. data:: L - LOCALE - :noindex: - - Make ``\w``, ``\W``, ``\b``, ``\B`` and case-insensitive matching dependent - on the current locale instead of the Unicode database. - - Locales are a feature of the C library intended to help in writing programs - that take account of language differences. For example, if you're - processing encoded French text, you'd want to be able to write ``\w+`` to - match words, but ``\w`` only matches the character class ``[A-Za-z]`` in - bytes patterns; it won't match bytes corresponding to ``é`` or ``ç``. - If your system is configured properly and a French locale is selected, - certain C functions will tell the program that the byte corresponding to - ``é`` should also be considered a letter. - Setting the :const:`LOCALE` flag when compiling a regular expression will cause - the resulting compiled object to use these C functions for ``\w``; this is - slower, but also enables ``\w+`` to match French words as you'd expect. - The use of this flag is discouraged in Python 3 as the locale mechanism - is very unreliable, it only handles one "culture" at a time, and it only - works with 8-bit locales. Unicode matching is already enabled by default - in Python 3 for Unicode (str) patterns, and it is able to handle different - locales/languages. - - -.. data:: M - MULTILINE - :noindex: - - (``^`` and ``$`` haven't been explained yet; they'll be introduced in section - :ref:`more-metacharacters`.) - - Usually ``^`` matches only at the beginning of the string, and ``$`` matches - only at the end of the string and immediately before the newline (if any) at the - end of the string. When this flag is specified, ``^`` matches at the beginning - of the string and at the beginning of each line within the string, immediately - following each newline. Similarly, the ``$`` metacharacter matches either at - the end of the string and at the end of each line (immediately preceding each - newline). - - -.. data:: S - DOTALL - :noindex: - - Makes the ``'.'`` special character match any character at all, including a - newline; without this flag, ``'.'`` will match anything *except* a newline. - - -.. data:: A - ASCII - :noindex: - - Make ``\w``, ``\W``, ``\b``, ``\B``, ``\s`` and ``\S`` perform ASCII-only - matching instead of full Unicode matching. This is only meaningful for - Unicode patterns, and is ignored for byte patterns. - - -.. data:: X - VERBOSE - :noindex: - - This flag allows you to write regular expressions that are more readable by - granting you more flexibility in how you can format them. When this flag has - been specified, whitespace within the RE string is ignored, except when the - whitespace is in a character class or preceded by an unescaped backslash; this - lets you organize and indent the RE more clearly. This flag also lets you put - comments within a RE that will be ignored by the engine; comments are marked by - a ``'#'`` that's neither in a character class or preceded by an unescaped - backslash. - - For example, here's a RE that uses :const:`re.VERBOSE`; see how much easier it - is to read? :: - - charref = re.compile(r""" - &[#] # Start of a numeric entity reference - ( - 0[0-7]+ # Octal form - | [0-9]+ # Decimal form - | x[0-9a-fA-F]+ # Hexadecimal form - ) - ; # Trailing semicolon - """, re.VERBOSE) - - Without the verbose setting, the RE would look like this:: - - charref = re.compile("&#(0[0-7]+" - "|[0-9]+" - "|x[0-9a-fA-F]+);") - - In the above example, Python's automatic concatenation of string literals has - been used to break up the RE into smaller pieces, but it's still more difficult - to understand than the version using :const:`re.VERBOSE`. - - -More Pattern Power -================== - -So far we've only covered a part of the features of regular expressions. In -this section, we'll cover some new metacharacters, and how to use groups to -retrieve portions of the text that was matched. - - -.. _more-metacharacters: - -More Metacharacters -------------------- - -There are some metacharacters that we haven't covered yet. Most of them will be -covered in this section. - -Some of the remaining metacharacters to be discussed are :dfn:`zero-width -assertions`. They don't cause the engine to advance through the string; -instead, they consume no characters at all, and simply succeed or fail. For -example, ``\b`` is an assertion that the current position is located at a word -boundary; the position isn't changed by the ``\b`` at all. This means that -zero-width assertions should never be repeated, because if they match once at a -given location, they can obviously be matched an infinite number of times. - -``|`` - Alternation, or the "or" operator. If *A* and *B* are regular expressions, - ``A|B`` will match any string that matches either *A* or *B*. ``|`` has very - low precedence in order to make it work reasonably when you're alternating - multi-character strings. ``Crow|Servo`` will match either ``'Crow'`` or ``'Servo'``, - not ``'Cro'``, a ``'w'`` or an ``'S'``, and ``'ervo'``. - - To match a literal ``'|'``, use ``\|``, or enclose it inside a character class, - as in ``[|]``. - -``^`` - Matches at the beginning of lines. Unless the :const:`MULTILINE` flag has been - set, this will only match at the beginning of the string. In :const:`MULTILINE` - mode, this also matches immediately after each newline within the string. - - For example, if you wish to match the word ``From`` only at the beginning of a - line, the RE to use is ``^From``. :: - - >>> print(re.search('^From', 'From Here to Eternity')) #doctest: +ELLIPSIS - <re.Match object; span=(0, 4), match='From'> - >>> print(re.search('^From', 'Reciting From Memory')) - None - - To match a literal ``'^'``, use ``\^``. - -``$`` - Matches at the end of a line, which is defined as either the end of the string, - or any location followed by a newline character. :: - - >>> print(re.search('}$', '{block}')) #doctest: +ELLIPSIS - <re.Match object; span=(6, 7), match='}'> - >>> print(re.search('}$', '{block} ')) - None - >>> print(re.search('}$', '{block}\n')) #doctest: +ELLIPSIS - <re.Match object; span=(6, 7), match='}'> - - To match a literal ``'$'``, use ``\$`` or enclose it inside a character class, - as in ``[$]``. - -``\A`` - Matches only at the start of the string. When not in :const:`MULTILINE` mode, - ``\A`` and ``^`` are effectively the same. In :const:`MULTILINE` mode, they're - different: ``\A`` still matches only at the beginning of the string, but ``^`` - may match at any location inside the string that follows a newline character. - -``\Z`` - Matches only at the end of the string. - -``\b`` - Word boundary. This is a zero-width assertion that matches only at the - beginning or end of a word. A word is defined as a sequence of alphanumeric - characters, so the end of a word is indicated by whitespace or a - non-alphanumeric character. - - The following example matches ``class`` only when it's a complete word; it won't - match when it's contained inside another word. :: - - >>> p = re.compile(r'\bclass\b') - >>> print(p.search('no class at all')) - <re.Match object; span=(3, 8), match='class'> - >>> print(p.search('the declassified algorithm')) - None - >>> print(p.search('one subclass is')) - None - - There are two subtleties you should remember when using this special sequence. - First, this is the worst collision between Python's string literals and regular - expression sequences. In Python's string literals, ``\b`` is the backspace - character, ASCII value 8. If you're not using raw strings, then Python will - convert the ``\b`` to a backspace, and your RE won't match as you expect it to. - The following example looks the same as our previous RE, but omits the ``'r'`` - in front of the RE string. :: - - >>> p = re.compile('\bclass\b') - >>> print(p.search('no class at all')) - None - >>> print(p.search('\b' + 'class' + '\b')) - <re.Match object; span=(0, 7), match='\x08class\x08'> - - Second, inside a character class, where there's no use for this assertion, - ``\b`` represents the backspace character, for compatibility with Python's - string literals. - -``\B`` - Another zero-width assertion, this is the opposite of ``\b``, only matching when - the current position is not at a word boundary. - - -Grouping --------- - -Frequently you need to obtain more information than just whether the RE matched -or not. Regular expressions are often used to dissect strings by writing a RE -divided into several subgroups which match different components of interest. -For example, an RFC-822 header line is divided into a header name and a value, -separated by a ``':'``, like this: - -.. code-block:: none - - From: author@example.com - User-Agent: Thunderbird 1.5.0.9 (X11/20061227) - MIME-Version: 1.0 - To: editor@example.com - -This can be handled by writing a regular expression which matches an entire -header line, and has one group which matches the header name, and another group -which matches the header's value. - -Groups are marked by the ``'('``, ``')'`` metacharacters. ``'('`` and ``')'`` -have much the same meaning as they do in mathematical expressions; they group -together the expressions contained inside them, and you can repeat the contents -of a group with a quantifier, such as ``*``, ``+``, ``?``, or -``{m,n}``. For example, ``(ab)*`` will match zero or more repetitions of -``ab``. :: - - >>> p = re.compile('(ab)*') - >>> print(p.match('ababababab').span()) - (0, 10) - -Groups indicated with ``'('``, ``')'`` also capture the starting and ending -index of the text that they match; this can be retrieved by passing an argument -to :meth:`~re.Match.group`, :meth:`~re.Match.start`, :meth:`~re.Match.end`, and -:meth:`~re.Match.span`. Groups are -numbered starting with 0. Group 0 is always present; it's the whole RE, so -:ref:`match object <match-objects>` methods all have group 0 as their default -argument. Later we'll see how to express groups that don't capture the span -of text that they match. :: - - >>> p = re.compile('(a)b') - >>> m = p.match('ab') - >>> m.group() - 'ab' - >>> m.group(0) - 'ab' - -Subgroups are numbered from left to right, from 1 upward. Groups can be nested; -to determine the number, just count the opening parenthesis characters, going -from left to right. :: - - >>> p = re.compile('(a(b)c)d') - >>> m = p.match('abcd') - >>> m.group(0) - 'abcd' - >>> m.group(1) - 'abc' - >>> m.group(2) - 'b' - -:meth:`~re.Match.group` can be passed multiple group numbers at a time, in which case it -will return a tuple containing the corresponding values for those groups. :: - - >>> m.group(2,1,2) - ('b', 'abc', 'b') - -The :meth:`~re.Match.groups` method returns a tuple containing the strings for all the -subgroups, from 1 up to however many there are. :: - - >>> m.groups() - ('abc', 'b') - -Backreferences in a pattern allow you to specify that the contents of an earlier -capturing group must also be found at the current location in the string. For -example, ``\1`` will succeed if the exact contents of group 1 can be found at -the current position, and fails otherwise. Remember that Python's string -literals also use a backslash followed by numbers to allow including arbitrary -characters in a string, so be sure to use a raw string when incorporating -backreferences in a RE. - -For example, the following RE detects doubled words in a string. :: - - >>> p = re.compile(r'\b(\w+)\s+\1\b') - >>> p.search('Paris in the the spring').group() - 'the the' - -Backreferences like this aren't often useful for just searching through a string ---- there are few text formats which repeat data in this way --- but you'll soon -find out that they're *very* useful when performing string substitutions. - - -Non-capturing and Named Groups ------------------------------- - -Elaborate REs may use many groups, both to capture substrings of interest, and -to group and structure the RE itself. In complex REs, it becomes difficult to -keep track of the group numbers. There are two features which help with this -problem. Both of them use a common syntax for regular expression extensions, so -we'll look at that first. - -Perl 5 is well known for its powerful additions to standard regular expressions. -For these new features the Perl developers couldn't choose new single-keystroke metacharacters -or new special sequences beginning with ``\`` without making Perl's regular -expressions confusingly different from standard REs. If they chose ``&`` as a -new metacharacter, for example, old expressions would be assuming that ``&`` was -a regular character and wouldn't have escaped it by writing ``\&`` or ``[&]``. - -The solution chosen by the Perl developers was to use ``(?...)`` as the -extension syntax. ``?`` immediately after a parenthesis was a syntax error -because the ``?`` would have nothing to repeat, so this didn't introduce any -compatibility problems. The characters immediately after the ``?`` indicate -what extension is being used, so ``(?=foo)`` is one thing (a positive lookahead -assertion) and ``(?:foo)`` is something else (a non-capturing group containing -the subexpression ``foo``). - -Python supports several of Perl's extensions and adds an extension -syntax to Perl's extension syntax. If the first character after the -question mark is a ``P``, you know that it's an extension that's -specific to Python. - -Now that we've looked at the general extension syntax, we can return -to the features that simplify working with groups in complex REs. - -Sometimes you'll want to use a group to denote a part of a regular expression, -but aren't interested in retrieving the group's contents. You can make this fact -explicit by using a non-capturing group: ``(?:...)``, where you can replace the -``...`` with any other regular expression. :: - - >>> m = re.match("([abc])+", "abc") - >>> m.groups() - ('c',) - >>> m = re.match("(?:[abc])+", "abc") - >>> m.groups() - () - -Except for the fact that you can't retrieve the contents of what the group -matched, a non-capturing group behaves exactly the same as a capturing group; -you can put anything inside it, repeat it with a repetition metacharacter such -as ``*``, and nest it within other groups (capturing or non-capturing). -``(?:...)`` is particularly useful when modifying an existing pattern, since you -can add new groups without changing how all the other groups are numbered. It -should be mentioned that there's no performance difference in searching between -capturing and non-capturing groups; neither form is any faster than the other. - -A more significant feature is named groups: instead of referring to them by -numbers, groups can be referenced by a name. - -The syntax for a named group is one of the Python-specific extensions: -``(?P<name>...)``. *name* is, obviously, the name of the group. Named groups -behave exactly like capturing groups, and additionally associate a name -with a group. The :ref:`match object <match-objects>` methods that deal with -capturing groups all accept either integers that refer to the group by number -or strings that contain the desired group's name. Named groups are still -given numbers, so you can retrieve information about a group in two ways:: - - >>> p = re.compile(r'(?P<word>\b\w+\b)') - >>> m = p.search( '(((( Lots of punctuation )))' ) - >>> m.group('word') - 'Lots' - >>> m.group(1) - 'Lots' - -Additionally, you can retrieve named groups as a dictionary with -:meth:`~re.Match.groupdict`:: - - >>> m = re.match(r'(?P<first>\w+) (?P<last>\w+)', 'Jane Doe') - >>> m.groupdict() - {'first': 'Jane', 'last': 'Doe'} - -Named groups are handy because they let you use easily remembered names, instead -of having to remember numbers. Here's an example RE from the :mod:`imaplib` -module:: - - InternalDate = re.compile(r'INTERNALDATE "' - r'(?P<day>[ 123][0-9])-(?P<mon>[A-Z][a-z][a-z])-' - r'(?P<year>[0-9][0-9][0-9][0-9])' - r' (?P<hour>[0-9][0-9]):(?P<min>[0-9][0-9]):(?P<sec>[0-9][0-9])' - r' (?P<zonen>[-+])(?P<zoneh>[0-9][0-9])(?P<zonem>[0-9][0-9])' - r'"') - -It's obviously much easier to retrieve ``m.group('zonem')``, instead of having -to remember to retrieve group 9. - -The syntax for backreferences in an expression such as ``(...)\1`` refers to the -number of the group. There's naturally a variant that uses the group name -instead of the number. This is another Python extension: ``(?P=name)`` indicates -that the contents of the group called *name* should again be matched at the -current point. The regular expression for finding doubled words, -``\b(\w+)\s+\1\b`` can also be written as ``\b(?P<word>\w+)\s+(?P=word)\b``:: - - >>> p = re.compile(r'\b(?P<word>\w+)\s+(?P=word)\b') - >>> p.search('Paris in the the spring').group() - 'the the' - - -Lookahead Assertions --------------------- - -Another zero-width assertion is the lookahead assertion. Lookahead assertions -are available in both positive and negative form, and look like this: - -``(?=...)`` - Positive lookahead assertion. This succeeds if the contained regular - expression, represented here by ``...``, successfully matches at the current - location, and fails otherwise. But, once the contained expression has been - tried, the matching engine doesn't advance at all; the rest of the pattern is - tried right where the assertion started. - -``(?!...)`` - Negative lookahead assertion. This is the opposite of the positive assertion; - it succeeds if the contained expression *doesn't* match at the current position - in the string. - -To make this concrete, let's look at a case where a lookahead is useful. -Consider a simple pattern to match a filename and split it apart into a base -name and an extension, separated by a ``.``. For example, in ``news.rc``, -``news`` is the base name, and ``rc`` is the filename's extension. - -The pattern to match this is quite simple: - -``.*[.].*$`` - -Notice that the ``.`` needs to be treated specially because it's a -metacharacter, so it's inside a character class to only match that -specific character. Also notice the trailing ``$``; this is added to -ensure that all the rest of the string must be included in the -extension. This regular expression matches ``foo.bar`` and -``autoexec.bat`` and ``sendmail.cf`` and ``printers.conf``. - -Now, consider complicating the problem a bit; what if you want to match -filenames where the extension is not ``bat``? Some incorrect attempts: - -``.*[.][^b].*$`` The first attempt above tries to exclude ``bat`` by requiring -that the first character of the extension is not a ``b``. This is wrong, -because the pattern also doesn't match ``foo.bar``. - -``.*[.]([^b]..|.[^a].|..[^t])$`` - -The expression gets messier when you try to patch up the first solution by -requiring one of the following cases to match: the first character of the -extension isn't ``b``; the second character isn't ``a``; or the third character -isn't ``t``. This accepts ``foo.bar`` and rejects ``autoexec.bat``, but it -requires a three-letter extension and won't accept a filename with a two-letter -extension such as ``sendmail.cf``. We'll complicate the pattern again in an -effort to fix it. - -``.*[.]([^b].?.?|.[^a]?.?|..?[^t]?)$`` - -In the third attempt, the second and third letters are all made optional in -order to allow matching extensions shorter than three characters, such as -``sendmail.cf``. - -The pattern's getting really complicated now, which makes it hard to read and -understand. Worse, if the problem changes and you want to exclude both ``bat`` -and ``exe`` as extensions, the pattern would get even more complicated and -confusing. - -A negative lookahead cuts through all this confusion: - -``.*[.](?!bat$)[^.]*$`` The negative lookahead means: if the expression ``bat`` -doesn't match at this point, try the rest of the pattern; if ``bat$`` does -match, the whole pattern will fail. The trailing ``$`` is required to ensure -that something like ``sample.batch``, where the extension only starts with -``bat``, will be allowed. The ``[^.]*`` makes sure that the pattern works -when there are multiple dots in the filename. - -Excluding another filename extension is now easy; simply add it as an -alternative inside the assertion. The following pattern excludes filenames that -end in either ``bat`` or ``exe``: - -``.*[.](?!bat$|exe$)[^.]*$`` - - -Modifying Strings -================= - -Up to this point, we've simply performed searches against a static string. -Regular expressions are also commonly used to modify strings in various ways, -using the following pattern methods: - -+------------------+-----------------------------------------------+ -| Method/Attribute | Purpose | -+==================+===============================================+ -| ``split()`` | Split the string into a list, splitting it | -| | wherever the RE matches | -+------------------+-----------------------------------------------+ -| ``sub()`` | Find all substrings where the RE matches, and | -| | replace them with a different string | -+------------------+-----------------------------------------------+ -| ``subn()`` | Does the same thing as :meth:`!sub`, but | -| | returns the new string and the number of | -| | replacements | -+------------------+-----------------------------------------------+ - - -Splitting Strings ------------------ - -The :meth:`~re.Pattern.split` method of a pattern splits a string apart -wherever the RE matches, returning a list of the pieces. It's similar to the -:meth:`~str.split` method of strings but provides much more generality in the -delimiters that you can split by; string :meth:`!split` only supports splitting by -whitespace or by a fixed string. As you'd expect, there's a module-level -:func:`re.split` function, too. - - -.. method:: .split(string [, maxsplit=0]) - :noindex: - - Split *string* by the matches of the regular expression. If capturing - parentheses are used in the RE, then their contents will also be returned as - part of the resulting list. If *maxsplit* is nonzero, at most *maxsplit* splits - are performed. - -You can limit the number of splits made, by passing a value for *maxsplit*. -When *maxsplit* is nonzero, at most *maxsplit* splits will be made, and the -remainder of the string is returned as the final element of the list. In the -following example, the delimiter is any sequence of non-alphanumeric characters. -:: - - >>> p = re.compile(r'\W+') - >>> p.split('This is a test, short and sweet, of split().') - ['This', 'is', 'a', 'test', 'short', 'and', 'sweet', 'of', 'split', ''] - >>> p.split('This is a test, short and sweet, of split().', 3) - ['This', 'is', 'a', 'test, short and sweet, of split().'] - -Sometimes you're not only interested in what the text between delimiters is, but -also need to know what the delimiter was. If capturing parentheses are used in -the RE, then their values are also returned as part of the list. Compare the -following calls:: - - >>> p = re.compile(r'\W+') - >>> p2 = re.compile(r'(\W+)') - >>> p.split('This... is a test.') - ['This', 'is', 'a', 'test', ''] - >>> p2.split('This... is a test.') - ['This', '... ', 'is', ' ', 'a', ' ', 'test', '.', ''] - -The module-level function :func:`re.split` adds the RE to be used as the first -argument, but is otherwise the same. :: - - >>> re.split(r'[\W]+', 'Words, words, words.') - ['Words', 'words', 'words', ''] - >>> re.split(r'([\W]+)', 'Words, words, words.') - ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split(r'[\W]+', 'Words, words, words.', 1) - ['Words', 'words, words.'] - - -Search and Replace ------------------- - -Another common task is to find all the matches for a pattern, and replace them -with a different string. The :meth:`~re.Pattern.sub` method takes a replacement value, -which can be either a string or a function, and the string to be processed. - -.. method:: .sub(replacement, string[, count=0]) - :noindex: - - Returns the string obtained by replacing the leftmost non-overlapping - occurrences of the RE in *string* by the replacement *replacement*. If the - pattern isn't found, *string* is returned unchanged. - - The optional argument *count* is the maximum number of pattern occurrences to be - replaced; *count* must be a non-negative integer. The default value of 0 means - to replace all occurrences. - -Here's a simple example of using the :meth:`~re.Pattern.sub` method. It replaces colour -names with the word ``colour``:: - - >>> p = re.compile('(blue|white|red)') - >>> p.sub('colour', 'blue socks and red shoes') - 'colour socks and colour shoes' - >>> p.sub('colour', 'blue socks and red shoes', count=1) - 'colour socks and red shoes' - -The :meth:`~re.Pattern.subn` method does the same work, but returns a 2-tuple containing the -new string value and the number of replacements that were performed:: - - >>> p = re.compile('(blue|white|red)') - >>> p.subn('colour', 'blue socks and red shoes') - ('colour socks and colour shoes', 2) - >>> p.subn('colour', 'no colours at all') - ('no colours at all', 0) - -Empty matches are replaced only when they're not adjacent to a previous empty match. -:: - - >>> p = re.compile('x*') - >>> p.sub('-', 'abxd') - '-a-b--d-' - -If *replacement* is a string, any backslash escapes in it are processed. That -is, ``\n`` is converted to a single newline character, ``\r`` is converted to a -carriage return, and so forth. Unknown escapes such as ``\&`` are left alone. -Backreferences, such as ``\6``, are replaced with the substring matched by the -corresponding group in the RE. This lets you incorporate portions of the -original text in the resulting replacement string. - -This example matches the word ``section`` followed by a string enclosed in -``{``, ``}``, and changes ``section`` to ``subsection``:: - - >>> p = re.compile('section{ ( [^}]* ) }', re.VERBOSE) - >>> p.sub(r'subsection{\1}','section{First} section{second}') - 'subsection{First} subsection{second}' - -There's also a syntax for referring to named groups as defined by the -``(?P<name>...)`` syntax. ``\g<name>`` will use the substring matched by the -group named ``name``, and ``\g<number>`` uses the corresponding group number. -``\g<2>`` is therefore equivalent to ``\2``, but isn't ambiguous in a -replacement string such as ``\g<2>0``. (``\20`` would be interpreted as a -reference to group 20, not a reference to group 2 followed by the literal -character ``'0'``.) The following substitutions are all equivalent, but use all -three variations of the replacement string. :: - - >>> p = re.compile('section{ (?P<name> [^}]* ) }', re.VERBOSE) - >>> p.sub(r'subsection{\1}','section{First}') - 'subsection{First}' - >>> p.sub(r'subsection{\g<1>}','section{First}') - 'subsection{First}' - >>> p.sub(r'subsection{\g<name>}','section{First}') - 'subsection{First}' - -*replacement* can also be a function, which gives you even more control. If -*replacement* is a function, the function is called for every non-overlapping -occurrence of *pattern*. On each call, the function is passed a -:ref:`match object <match-objects>` argument for the match and can use this -information to compute the desired replacement string and return it. - -In the following example, the replacement function translates decimals into -hexadecimal:: - - >>> def hexrepl(match): - ... "Return the hex string for a decimal number" - ... value = int(match.group()) - ... return hex(value) - ... - >>> p = re.compile(r'\d+') - >>> p.sub(hexrepl, 'Call 65490 for printing, 49152 for user code.') - 'Call 0xffd2 for printing, 0xc000 for user code.' - -When using the module-level :func:`re.sub` function, the pattern is passed as -the first argument. The pattern may be provided as an object or as a string; if -you need to specify regular expression flags, you must either use a -pattern object as the first parameter, or use embedded modifiers in the -pattern string, e.g. ``sub("(?i)b+", "x", "bbbb BBBB")`` returns ``'x x'``. - - -Common Problems -=============== - -Regular expressions are a powerful tool for some applications, but in some ways -their behaviour isn't intuitive and at times they don't behave the way you may -expect them to. This section will point out some of the most common pitfalls. - - -Use String Methods ------------------- - -Sometimes using the :mod:`re` module is a mistake. If you're matching a fixed -string, or a single character class, and you're not using any :mod:`re` features -such as the :const:`~re.IGNORECASE` flag, then the full power of regular expressions -may not be required. Strings have several methods for performing operations with -fixed strings and they're usually much faster, because the implementation is a -single small C loop that's been optimized for the purpose, instead of the large, -more generalized regular expression engine. - -One example might be replacing a single fixed string with another one; for -example, you might replace ``word`` with ``deed``. :func:`re.sub` seems like the -function to use for this, but consider the :meth:`~str.replace` method. Note that -:meth:`!replace` will also replace ``word`` inside words, turning ``swordfish`` -into ``sdeedfish``, but the naive RE ``word`` would have done that, too. (To -avoid performing the substitution on parts of words, the pattern would have to -be ``\bword\b``, in order to require that ``word`` have a word boundary on -either side. This takes the job beyond :meth:`!replace`'s abilities.) - -Another common task is deleting every occurrence of a single character from a -string or replacing it with another single character. You might do this with -something like ``re.sub('\n', ' ', S)``, but :meth:`~str.translate` is capable of -doing both tasks and will be faster than any regular expression operation can -be. - -In short, before turning to the :mod:`re` module, consider whether your problem -can be solved with a faster and simpler string method. - - -match() versus search() ------------------------ - -The :func:`~re.match` function only checks if the RE matches at the beginning of the -string while :func:`~re.search` will scan forward through the string for a match. -It's important to keep this distinction in mind. Remember, :func:`!match` will -only report a successful match which will start at 0; if the match wouldn't -start at zero, :func:`!match` will *not* report it. :: - - >>> print(re.match('super', 'superstition').span()) - (0, 5) - >>> print(re.match('super', 'insuperable')) - None - -On the other hand, :func:`~re.search` will scan forward through the string, -reporting the first match it finds. :: - - >>> print(re.search('super', 'superstition').span()) - (0, 5) - >>> print(re.search('super', 'insuperable').span()) - (2, 7) - -Sometimes you'll be tempted to keep using :func:`re.match`, and just add ``.*`` -to the front of your RE. Resist this temptation and use :func:`re.search` -instead. The regular expression compiler does some analysis of REs in order to -speed up the process of looking for a match. One such analysis figures out what -the first character of a match must be; for example, a pattern starting with -``Crow`` must match starting with a ``'C'``. The analysis lets the engine -quickly scan through the string looking for the starting character, only trying -the full match if a ``'C'`` is found. - -Adding ``.*`` defeats this optimization, requiring scanning to the end of the -string and then backtracking to find a match for the rest of the RE. Use -:func:`re.search` instead. - - -Greedy versus Non-Greedy ------------------------- - -When repeating a regular expression, as in ``a*``, the resulting action is to -consume as much of the pattern as possible. This fact often bites you when -you're trying to match a pair of balanced delimiters, such as the angle brackets -surrounding an HTML tag. The naive pattern for matching a single HTML tag -doesn't work because of the greedy nature of ``.*``. :: - - >>> s = '<html><head><title>Title' - >>> len(s) - 32 - >>> print(re.match('<.*>', s).span()) - (0, 32) - >>> print(re.match('<.*>', s).group()) - Title - -The RE matches the ``'<'`` in ``''``, and the ``.*`` consumes the rest of -the string. There's still more left in the RE, though, and the ``>`` can't -match at the end of the string, so the regular expression engine has to -backtrack character by character until it finds a match for the ``>``. The -final match extends from the ``'<'`` in ``''`` to the ``'>'`` in -``''``, which isn't what you want. - -In this case, the solution is to use the non-greedy quantifiers ``*?``, ``+?``, -``??``, or ``{m,n}?``, which match as *little* text as possible. In the above -example, the ``'>'`` is tried immediately after the first ``'<'`` matches, and -when it fails, the engine advances a character at a time, retrying the ``'>'`` -at every step. This produces just the right result:: - - >>> print(re.match('<.*?>', s).group()) - - -(Note that parsing HTML or XML with regular expressions is painful. -Quick-and-dirty patterns will handle common cases, but HTML and XML have special -cases that will break the obvious regular expression; by the time you've written -a regular expression that handles all of the possible cases, the patterns will -be *very* complicated. Use an HTML or XML parser module for such tasks.) - - -Using re.VERBOSE ----------------- - -By now you've probably noticed that regular expressions are a very compact -notation, but they're not terribly readable. REs of moderate complexity can -become lengthy collections of backslashes, parentheses, and metacharacters, -making them difficult to read and understand. - -For such REs, specifying the :const:`re.VERBOSE` flag when compiling the regular -expression can be helpful, because it allows you to format the regular -expression more clearly. - -The ``re.VERBOSE`` flag has several effects. Whitespace in the regular -expression that *isn't* inside a character class is ignored. This means that an -expression such as ``dog | cat`` is equivalent to the less readable ``dog|cat``, -but ``[a b]`` will still match the characters ``'a'``, ``'b'``, or a space. In -addition, you can also put comments inside a RE; comments extend from a ``#`` -character to the next newline. When used with triple-quoted strings, this -enables REs to be formatted more neatly:: - - pat = re.compile(r""" - \s* # Skip leading whitespace - (?P
[^:]+) # Header name - \s* : # Whitespace, and a colon - (?P.*?) # The header's value -- *? used to - # lose the following trailing whitespace - \s*$ # Trailing whitespace to end-of-line - """, re.VERBOSE) - -This is far more readable than:: - - pat = re.compile(r"\s*(?P
[^:]+)\s*:(?P.*?)\s*$") - - -Feedback -======== - -Regular expressions are a complicated topic. Did this document help you -understand them? Were there parts that were unclear, or Problems you -encountered that weren't covered here? If so, please send suggestions for -improvements to the author. - -The most complete book on regular expressions is almost certainly Jeffrey -Friedl's Mastering Regular Expressions, published by O'Reilly. Unfortunately, -it exclusively concentrates on Perl and Java's flavours of regular expressions, -and doesn't contain any Python material at all, so it won't be useful as a -reference for programming in Python. (The first edition covered Python's -now-removed :mod:`!regex` module, which won't help you much.) Consider checking -it out from your library. diff --git a/Doc/howto/sockets.rst b/Doc/howto/sockets.rst deleted file mode 100644 index 0bbf97da39768d..00000000000000 --- a/Doc/howto/sockets.rst +++ /dev/null @@ -1,388 +0,0 @@ -.. _socket-howto: - -**************************** - Socket Programming HOWTO -**************************** - -:Author: Gordon McMillan - - -.. topic:: Abstract - - Sockets are used nearly everywhere, but are one of the most severely - misunderstood technologies around. This is a 10,000 foot overview of sockets. - It's not really a tutorial - you'll still have work to do in getting things - operational. It doesn't cover the fine points (and there are a lot of them), but - I hope it will give you enough background to begin using them decently. - - -Sockets -======= - -I'm only going to talk about INET (i.e. IPv4) sockets, but they account for at least 99% of -the sockets in use. And I'll only talk about STREAM (i.e. TCP) sockets - unless you really -know what you're doing (in which case this HOWTO isn't for you!), you'll get -better behavior and performance from a STREAM socket than anything else. I will -try to clear up the mystery of what a socket is, as well as some hints on how to -work with blocking and non-blocking sockets. But I'll start by talking about -blocking sockets. You'll need to know how they work before dealing with -non-blocking sockets. - -Part of the trouble with understanding these things is that "socket" can mean a -number of subtly different things, depending on context. So first, let's make a -distinction between a "client" socket - an endpoint of a conversation, and a -"server" socket, which is more like a switchboard operator. The client -application (your browser, for example) uses "client" sockets exclusively; the -web server it's talking to uses both "server" sockets and "client" sockets. - - -History -------- - -Of the various forms of :abbr:`IPC (Inter Process Communication)`, -sockets are by far the most popular. On any given platform, there are -likely to be other forms of IPC that are faster, but for -cross-platform communication, sockets are about the only game in town. - -They were invented in Berkeley as part of the BSD flavor of Unix. They spread -like wildfire with the internet. With good reason --- the combination of sockets -with INET makes talking to arbitrary machines around the world unbelievably easy -(at least compared to other schemes). - - -Creating a Socket -================= - -Roughly speaking, when you clicked on the link that brought you to this page, -your browser did something like the following:: - - # create an INET, STREAMing socket - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # now connect to the web server on port 80 - the normal http port - s.connect(("www.python.org", 80)) - -When the ``connect`` completes, the socket ``s`` can be used to send -in a request for the text of the page. The same socket will read the -reply, and then be destroyed. That's right, destroyed. Client sockets -are normally only used for one exchange (or a small set of sequential -exchanges). - -What happens in the web server is a bit more complex. First, the web server -creates a "server socket":: - - # create an INET, STREAMing socket - serversocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - # bind the socket to a public host, and a well-known port - serversocket.bind((socket.gethostname(), 80)) - # become a server socket - serversocket.listen(5) - -A couple things to notice: we used ``socket.gethostname()`` so that the socket -would be visible to the outside world. If we had used ``s.bind(('localhost', -80))`` or ``s.bind(('127.0.0.1', 80))`` we would still have a "server" socket, -but one that was only visible within the same machine. ``s.bind(('', 80))`` -specifies that the socket is reachable by any address the machine happens to -have. - -A second thing to note: low number ports are usually reserved for "well known" -services (HTTP, SNMP etc). If you're playing around, use a nice high number (4 -digits). - -Finally, the argument to ``listen`` tells the socket library that we want it to -queue up as many as 5 connect requests (the normal max) before refusing outside -connections. If the rest of the code is written properly, that should be plenty. - -Now that we have a "server" socket, listening on port 80, we can enter the -mainloop of the web server:: - - while True: - # accept connections from outside - (clientsocket, address) = serversocket.accept() - # now do something with the clientsocket - # in this case, we'll pretend this is a threaded server - ct = client_thread(clientsocket) - ct.run() - -There's actually 3 general ways in which this loop could work - dispatching a -thread to handle ``clientsocket``, create a new process to handle -``clientsocket``, or restructure this app to use non-blocking sockets, and -multiplex between our "server" socket and any active ``clientsocket``\ s using -``select``. More about that later. The important thing to understand now is -this: this is *all* a "server" socket does. It doesn't send any data. It doesn't -receive any data. It just produces "client" sockets. Each ``clientsocket`` is -created in response to some *other* "client" socket doing a ``connect()`` to the -host and port we're bound to. As soon as we've created that ``clientsocket``, we -go back to listening for more connections. The two "clients" are free to chat it -up - they are using some dynamically allocated port which will be recycled when -the conversation ends. - - -IPC ---- - -If you need fast IPC between two processes on one machine, you should look into -pipes or shared memory. If you do decide to use AF_INET sockets, bind the -"server" socket to ``'localhost'``. On most platforms, this will take a -shortcut around a couple of layers of network code and be quite a bit faster. - -.. seealso:: - The :mod:`multiprocessing` integrates cross-platform IPC into a higher-level - API. - - -Using a Socket -============== - -The first thing to note, is that the web browser's "client" socket and the web -server's "client" socket are identical beasts. That is, this is a "peer to peer" -conversation. Or to put it another way, *as the designer, you will have to -decide what the rules of etiquette are for a conversation*. Normally, the -``connect``\ ing socket starts the conversation, by sending in a request, or -perhaps a signon. But that's a design decision - it's not a rule of sockets. - -Now there are two sets of verbs to use for communication. You can use ``send`` -and ``recv``, or you can transform your client socket into a file-like beast and -use ``read`` and ``write``. The latter is the way Java presents its sockets. -I'm not going to talk about it here, except to warn you that you need to use -``flush`` on sockets. These are buffered "files", and a common mistake is to -``write`` something, and then ``read`` for a reply. Without a ``flush`` in -there, you may wait forever for the reply, because the request may still be in -your output buffer. - -Now we come to the major stumbling block of sockets - ``send`` and ``recv`` operate -on the network buffers. They do not necessarily handle all the bytes you hand -them (or expect from them), because their major focus is handling the network -buffers. In general, they return when the associated network buffers have been -filled (``send``) or emptied (``recv``). They then tell you how many bytes they -handled. It is *your* responsibility to call them again until your message has -been completely dealt with. - -When a ``recv`` returns 0 bytes, it means the other side has closed (or is in -the process of closing) the connection. You will not receive any more data on -this connection. Ever. You may be able to send data successfully; I'll talk -more about this later. - -A protocol like HTTP uses a socket for only one transfer. The client sends a -request, then reads a reply. That's it. The socket is discarded. This means that -a client can detect the end of the reply by receiving 0 bytes. - -But if you plan to reuse your socket for further transfers, you need to realize -that *there is no* :abbr:`EOT (End of Transfer)` *on a socket.* I repeat: if a socket -``send`` or ``recv`` returns after handling 0 bytes, the connection has been -broken. If the connection has *not* been broken, you may wait on a ``recv`` -forever, because the socket will *not* tell you that there's nothing more to -read (for now). Now if you think about that a bit, you'll come to realize a -fundamental truth of sockets: *messages must either be fixed length* (yuck), *or -be delimited* (shrug), *or indicate how long they are* (much better), *or end by -shutting down the connection*. The choice is entirely yours, (but some ways are -righter than others). - -Assuming you don't want to end the connection, the simplest solution is a fixed -length message:: - - class MySocket: - """demonstration class only - - coded for clarity, not efficiency - """ - - def __init__(self, sock=None): - if sock is None: - self.sock = socket.socket( - socket.AF_INET, socket.SOCK_STREAM) - else: - self.sock = sock - - def connect(self, host, port): - self.sock.connect((host, port)) - - def mysend(self, msg): - totalsent = 0 - while totalsent < MSGLEN: - sent = self.sock.send(msg[totalsent:]) - if sent == 0: - raise RuntimeError("socket connection broken") - totalsent = totalsent + sent - - def myreceive(self): - chunks = [] - bytes_recd = 0 - while bytes_recd < MSGLEN: - chunk = self.sock.recv(min(MSGLEN - bytes_recd, 2048)) - if chunk == b'': - raise RuntimeError("socket connection broken") - chunks.append(chunk) - bytes_recd = bytes_recd + len(chunk) - return b''.join(chunks) - -The sending code here is usable for almost any messaging scheme - in Python you -send strings, and you can use ``len()`` to determine its length (even if it has -embedded ``\0`` characters). It's mostly the receiving code that gets more -complex. (And in C, it's not much worse, except you can't use ``strlen`` if the -message has embedded ``\0``\ s.) - -The easiest enhancement is to make the first character of the message an -indicator of message type, and have the type determine the length. Now you have -two ``recv``\ s - the first to get (at least) that first character so you can -look up the length, and the second in a loop to get the rest. If you decide to -go the delimited route, you'll be receiving in some arbitrary chunk size, (4096 -or 8192 is frequently a good match for network buffer sizes), and scanning what -you've received for a delimiter. - -One complication to be aware of: if your conversational protocol allows multiple -messages to be sent back to back (without some kind of reply), and you pass -``recv`` an arbitrary chunk size, you may end up reading the start of a -following message. You'll need to put that aside and hold onto it, until it's -needed. - -Prefixing the message with its length (say, as 5 numeric characters) gets more -complex, because (believe it or not), you may not get all 5 characters in one -``recv``. In playing around, you'll get away with it; but in high network loads, -your code will very quickly break unless you use two ``recv`` loops - the first -to determine the length, the second to get the data part of the message. Nasty. -This is also when you'll discover that ``send`` does not always manage to get -rid of everything in one pass. And despite having read this, you will eventually -get bit by it! - -In the interests of space, building your character, (and preserving my -competitive position), these enhancements are left as an exercise for the -reader. Lets move on to cleaning up. - - -Binary Data ------------ - -It is perfectly possible to send binary data over a socket. The major problem is -that not all machines use the same formats for binary data. For example, -`network byte order `_ -is big-endian, with the most significant byte first, -so a 16 bit integer with the value ``1`` would be the two hex bytes ``00 01``. -However, most common processors (x86/AMD64, ARM, RISC-V), are little-endian, -with the least significant byte first - that same ``1`` would be ``01 00``. - -Socket libraries have calls for converting 16 and 32 bit integers - ``ntohl, -htonl, ntohs, htons`` where "n" means *network* and "h" means *host*, "s" means -*short* and "l" means *long*. Where network order is host order, these do -nothing, but where the machine is byte-reversed, these swap the bytes around -appropriately. - -In these days of 64-bit machines, the ASCII representation of binary data is -frequently smaller than the binary representation. That's because a surprising -amount of the time, most integers have the value 0, or maybe 1. -The string ``"0"`` would be two bytes, while a full 64-bit integer would be 8. -Of course, this doesn't fit well with fixed-length messages. -Decisions, decisions. - - -Disconnecting -============= - -Strictly speaking, you're supposed to use ``shutdown`` on a socket before you -``close`` it. The ``shutdown`` is an advisory to the socket at the other end. -Depending on the argument you pass it, it can mean "I'm not going to send -anymore, but I'll still listen", or "I'm not listening, good riddance!". Most -socket libraries, however, are so used to programmers neglecting to use this -piece of etiquette that normally a ``close`` is the same as ``shutdown(); -close()``. So in most situations, an explicit ``shutdown`` is not needed. - -One way to use ``shutdown`` effectively is in an HTTP-like exchange. The client -sends a request and then does a ``shutdown(1)``. This tells the server "This -client is done sending, but can still receive." The server can detect "EOF" by -a receive of 0 bytes. It can assume it has the complete request. The server -sends a reply. If the ``send`` completes successfully then, indeed, the client -was still receiving. - -Python takes the automatic shutdown a step further, and says that when a socket -is garbage collected, it will automatically do a ``close`` if it's needed. But -relying on this is a very bad habit. If your socket just disappears without -doing a ``close``, the socket at the other end may hang indefinitely, thinking -you're just being slow. *Please* ``close`` your sockets when you're done. - - -When Sockets Die ----------------- - -Probably the worst thing about using blocking sockets is what happens when the -other side comes down hard (without doing a ``close``). Your socket is likely to -hang. TCP is a reliable protocol, and it will wait a long, long time -before giving up on a connection. If you're using threads, the entire thread is -essentially dead. There's not much you can do about it. As long as you aren't -doing something dumb, like holding a lock while doing a blocking read, the -thread isn't really consuming much in the way of resources. Do *not* try to kill -the thread - part of the reason that threads are more efficient than processes -is that they avoid the overhead associated with the automatic recycling of -resources. In other words, if you do manage to kill the thread, your whole -process is likely to be screwed up. - - -Non-blocking Sockets -==================== - -If you've understood the preceding, you already know most of what you need to -know about the mechanics of using sockets. You'll still use the same calls, in -much the same ways. It's just that, if you do it right, your app will be almost -inside-out. - -In Python, you use ``socket.setblocking(False)`` to make it non-blocking. In C, it's -more complex, (for one thing, you'll need to choose between the BSD flavor -``O_NONBLOCK`` and the almost indistinguishable POSIX flavor ``O_NDELAY``, which -is completely different from ``TCP_NODELAY``), but it's the exact same idea. You -do this after creating the socket, but before using it. (Actually, if you're -nuts, you can switch back and forth.) - -The major mechanical difference is that ``send``, ``recv``, ``connect`` and -``accept`` can return without having done anything. You have (of course) a -number of choices. You can check return code and error codes and generally drive -yourself crazy. If you don't believe me, try it sometime. Your app will grow -large, buggy and suck CPU. So let's skip the brain-dead solutions and do it -right. - -Use ``select``. - -In C, coding ``select`` is fairly complex. In Python, it's a piece of cake, but -it's close enough to the C version that if you understand ``select`` in Python, -you'll have little trouble with it in C:: - - ready_to_read, ready_to_write, in_error = \ - select.select( - potential_readers, - potential_writers, - potential_errs, - timeout) - -You pass ``select`` three lists: the first contains all sockets that you might -want to try reading; the second all the sockets you might want to try writing -to, and the last (normally left empty) those that you want to check for errors. -You should note that a socket can go into more than one list. The ``select`` -call is blocking, but you can give it a timeout. This is generally a sensible -thing to do - give it a nice long timeout (say a minute) unless you have good -reason to do otherwise. - -In return, you will get three lists. They contain the sockets that are actually -readable, writable and in error. Each of these lists is a subset (possibly -empty) of the corresponding list you passed in. - -If a socket is in the output readable list, you can be -as-close-to-certain-as-we-ever-get-in-this-business that a ``recv`` on that -socket will return *something*. Same idea for the writable list. You'll be able -to send *something*. Maybe not all you want to, but *something* is better than -nothing. (Actually, any reasonably healthy socket will return as writable - it -just means outbound network buffer space is available.) - -If you have a "server" socket, put it in the potential_readers list. If it comes -out in the readable list, your ``accept`` will (almost certainly) work. If you -have created a new socket to ``connect`` to someone else, put it in the -potential_writers list. If it shows up in the writable list, you have a decent -chance that it has connected. - -Actually, ``select`` can be handy even with blocking sockets. It's one way of -determining whether you will block - the socket returns as readable when there's -something in the buffers. However, this still doesn't help with the problem of -determining whether the other end is done, or just busy with something else. - -**Portability alert**: On Unix, ``select`` works both with the sockets and -files. Don't try this on Windows. On Windows, ``select`` works with sockets -only. Also note that in C, many of the more advanced socket options are done -differently on Windows. In fact, on Windows I usually use threads (which work -very, very well) with my sockets. - - diff --git a/Doc/howto/sorting.rst b/Doc/howto/sorting.rst deleted file mode 100644 index 38dd09f0a721d2..00000000000000 --- a/Doc/howto/sorting.rst +++ /dev/null @@ -1,297 +0,0 @@ -.. _sortinghowto: - -Sorting HOW TO -************** - -:Author: Andrew Dalke and Raymond Hettinger -:Release: 0.1 - - -Python lists have a built-in :meth:`list.sort` method that modifies the list -in-place. There is also a :func:`sorted` built-in function that builds a new -sorted list from an iterable. - -In this document, we explore the various techniques for sorting data using Python. - - -Sorting Basics -============== - -A simple ascending sort is very easy: just call the :func:`sorted` function. It -returns a new sorted list: - -.. doctest:: - - >>> sorted([5, 2, 3, 1, 4]) - [1, 2, 3, 4, 5] - -You can also use the :meth:`list.sort` method. It modifies the list -in-place (and returns ``None`` to avoid confusion). Usually it's less convenient -than :func:`sorted` - but if you don't need the original list, it's slightly -more efficient. - -.. doctest:: - - >>> a = [5, 2, 3, 1, 4] - >>> a.sort() - >>> a - [1, 2, 3, 4, 5] - -Another difference is that the :meth:`list.sort` method is only defined for -lists. In contrast, the :func:`sorted` function accepts any iterable. - -.. doctest:: - - >>> sorted({1: 'D', 2: 'B', 3: 'B', 4: 'E', 5: 'A'}) - [1, 2, 3, 4, 5] - -Key Functions -============= - -Both :meth:`list.sort` and :func:`sorted` have a *key* parameter to specify a -function (or other callable) to be called on each list element prior to making -comparisons. - -For example, here's a case-insensitive string comparison: - -.. doctest:: - - >>> sorted("This is a test string from Andrew".split(), key=str.lower) - ['a', 'Andrew', 'from', 'is', 'string', 'test', 'This'] - -The value of the *key* parameter should be a function (or other callable) that -takes a single argument and returns a key to use for sorting purposes. This -technique is fast because the key function is called exactly once for each -input record. - -A common pattern is to sort complex objects using some of the object's indices -as keys. For example: - -.. doctest:: - - >>> student_tuples = [ - ... ('john', 'A', 15), - ... ('jane', 'B', 12), - ... ('dave', 'B', 10), - ... ] - >>> sorted(student_tuples, key=lambda student: student[2]) # sort by age - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -The same technique works for objects with named attributes. For example: - -.. doctest:: - - >>> class Student: - ... def __init__(self, name, grade, age): - ... self.name = name - ... self.grade = grade - ... self.age = age - ... def __repr__(self): - ... return repr((self.name, self.grade, self.age)) - - >>> student_objects = [ - ... Student('john', 'A', 15), - ... Student('jane', 'B', 12), - ... Student('dave', 'B', 10), - ... ] - >>> sorted(student_objects, key=lambda student: student.age) # sort by age - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -Operator Module Functions -========================= - -The key-function patterns shown above are very common, so Python provides -convenience functions to make accessor functions easier and faster. The -:mod:`operator` module has :func:`~operator.itemgetter`, -:func:`~operator.attrgetter`, and a :func:`~operator.methodcaller` function. - -Using those functions, the above examples become simpler and faster: - -.. doctest:: - - >>> from operator import itemgetter, attrgetter - - >>> sorted(student_tuples, key=itemgetter(2)) - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - - >>> sorted(student_objects, key=attrgetter('age')) - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -The operator module functions allow multiple levels of sorting. For example, to -sort by *grade* then by *age*: - -.. doctest:: - - >>> sorted(student_tuples, key=itemgetter(1,2)) - [('john', 'A', 15), ('dave', 'B', 10), ('jane', 'B', 12)] - - >>> sorted(student_objects, key=attrgetter('grade', 'age')) - [('john', 'A', 15), ('dave', 'B', 10), ('jane', 'B', 12)] - -Ascending and Descending -======================== - -Both :meth:`list.sort` and :func:`sorted` accept a *reverse* parameter with a -boolean value. This is used to flag descending sorts. For example, to get the -student data in reverse *age* order: - -.. doctest:: - - >>> sorted(student_tuples, key=itemgetter(2), reverse=True) - [('john', 'A', 15), ('jane', 'B', 12), ('dave', 'B', 10)] - - >>> sorted(student_objects, key=attrgetter('age'), reverse=True) - [('john', 'A', 15), ('jane', 'B', 12), ('dave', 'B', 10)] - -Sort Stability and Complex Sorts -================================ - -Sorts are guaranteed to be `stable -`_\. That means that -when multiple records have the same key, their original order is preserved. - -.. doctest:: - - >>> data = [('red', 1), ('blue', 1), ('red', 2), ('blue', 2)] - >>> sorted(data, key=itemgetter(0)) - [('blue', 1), ('blue', 2), ('red', 1), ('red', 2)] - -Notice how the two records for *blue* retain their original order so that -``('blue', 1)`` is guaranteed to precede ``('blue', 2)``. - -This wonderful property lets you build complex sorts in a series of sorting -steps. For example, to sort the student data by descending *grade* and then -ascending *age*, do the *age* sort first and then sort again using *grade*: - -.. doctest:: - - >>> s = sorted(student_objects, key=attrgetter('age')) # sort on secondary key - >>> sorted(s, key=attrgetter('grade'), reverse=True) # now sort on primary key, descending - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -This can be abstracted out into a wrapper function that can take a list and -tuples of field and order to sort them on multiple passes. - -.. doctest:: - - >>> def multisort(xs, specs): - ... for key, reverse in reversed(specs): - ... xs.sort(key=attrgetter(key), reverse=reverse) - ... return xs - - >>> multisort(list(student_objects), (('grade', True), ('age', False))) - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - -The `Timsort `_ algorithm used in Python -does multiple sorts efficiently because it can take advantage of any ordering -already present in a dataset. - -Decorate-Sort-Undecorate -======================== - -This idiom is called Decorate-Sort-Undecorate after its three steps: - -* First, the initial list is decorated with new values that control the sort order. - -* Second, the decorated list is sorted. - -* Finally, the decorations are removed, creating a list that contains only the - initial values in the new order. - -For example, to sort the student data by *grade* using the DSU approach: - - >>> decorated = [(student.grade, i, student) for i, student in enumerate(student_objects)] - >>> decorated.sort() - >>> [student for grade, i, student in decorated] # undecorate - [('john', 'A', 15), ('jane', 'B', 12), ('dave', 'B', 10)] - -This idiom works because tuples are compared lexicographically; the first items -are compared; if they are the same then the second items are compared, and so -on. - -It is not strictly necessary in all cases to include the index *i* in the -decorated list, but including it gives two benefits: - -* The sort is stable -- if two items have the same key, their order will be - preserved in the sorted list. - -* The original items do not have to be comparable because the ordering of the - decorated tuples will be determined by at most the first two items. So for - example the original list could contain complex numbers which cannot be sorted - directly. - -Another name for this idiom is -`Schwartzian transform `_\, -after Randal L. Schwartz, who popularized it among Perl programmers. - -Now that Python sorting provides key-functions, this technique is not often needed. - -Comparison Functions -==================== - -Unlike key functions that return an absolute value for sorting, a comparison -function computes the relative ordering for two inputs. - -For example, a `balance scale -`_ -compares two samples giving a relative ordering: lighter, equal, or heavier. -Likewise, a comparison function such as ``cmp(a, b)`` will return a negative -value for less-than, zero if the inputs are equal, or a positive value for -greater-than. - -It is common to encounter comparison functions when translating algorithms from -other languages. Also, some libraries provide comparison functions as part of -their API. For example, :func:`locale.strcoll` is a comparison function. - -To accommodate those situations, Python provides -:class:`functools.cmp_to_key` to wrap the comparison function -to make it usable as a key function:: - - sorted(words, key=cmp_to_key(strcoll)) # locale-aware sort order - -Odds and Ends -============= - -* For locale aware sorting, use :func:`locale.strxfrm` for a key function or - :func:`locale.strcoll` for a comparison function. This is necessary - because "alphabetical" sort orderings can vary across cultures even - if the underlying alphabet is the same. - -* The *reverse* parameter still maintains sort stability (so that records with - equal keys retain the original order). Interestingly, that effect can be - simulated without the parameter by using the builtin :func:`reversed` function - twice: - - .. doctest:: - - >>> data = [('red', 1), ('blue', 1), ('red', 2), ('blue', 2)] - >>> standard_way = sorted(data, key=itemgetter(0), reverse=True) - >>> double_reversed = list(reversed(sorted(reversed(data), key=itemgetter(0)))) - >>> assert standard_way == double_reversed - >>> standard_way - [('red', 1), ('red', 2), ('blue', 1), ('blue', 2)] - -* The sort routines use ``<`` when making comparisons - between two objects. So, it is easy to add a standard sort order to a class by - defining an :meth:`~object.__lt__` method: - - .. doctest:: - - >>> Student.__lt__ = lambda self, other: self.age < other.age - >>> sorted(student_objects) - [('dave', 'B', 10), ('jane', 'B', 12), ('john', 'A', 15)] - - However, note that ``<`` can fall back to using :meth:`~object.__gt__` if - :meth:`~object.__lt__` is not implemented (see :func:`object.__lt__`). - -* Key functions need not depend directly on the objects being sorted. A key - function can also access external resources. For instance, if the student grades - are stored in a dictionary, they can be used to sort a separate list of student - names: - - .. doctest:: - - >>> students = ['dave', 'john', 'jane'] - >>> newgrades = {'john': 'F', 'jane':'A', 'dave': 'C'} - >>> sorted(students, key=newgrades.__getitem__) - ['jane', 'dave', 'john'] diff --git a/Doc/howto/unicode.rst b/Doc/howto/unicode.rst deleted file mode 100644 index a5e0998ce8dcf9..00000000000000 --- a/Doc/howto/unicode.rst +++ /dev/null @@ -1,764 +0,0 @@ -.. _unicode-howto: - -***************** - Unicode HOWTO -***************** - -:Release: 1.12 - -This HOWTO discusses Python's support for the Unicode specification -for representing textual data, and explains various problems that -people commonly encounter when trying to work with Unicode. - - -Introduction to Unicode -======================= - -Definitions ------------ - -Today's programs need to be able to handle a wide variety of -characters. Applications are often internationalized to display -messages and output in a variety of user-selectable languages; the -same program might need to output an error message in English, French, -Japanese, Hebrew, or Russian. Web content can be written in any of -these languages and can also include a variety of emoji symbols. -Python's string type uses the Unicode Standard for representing -characters, which lets Python programs work with all these different -possible characters. - -Unicode (https://www.unicode.org/) is a specification that aims to -list every character used by human languages and give each character -its own unique code. The Unicode specifications are continually -revised and updated to add new languages and symbols. - -A **character** is the smallest possible component of a text. 'A', 'B', 'C', -etc., are all different characters. So are 'È' and 'Í'. Characters vary -depending on the language or context you're talking -about. For example, there's a character for "Roman Numeral One", 'Ⅰ', that's -separate from the uppercase letter 'I'. They'll usually look the same, -but these are two different characters that have different meanings. - -The Unicode standard describes how characters are represented by -**code points**. A code point value is an integer in the range 0 to -0x10FFFF (about 1.1 million values, the -`actual number assigned `_ -is less than that). In the standard and in this document, a code point is written -using the notation ``U+265E`` to mean the character with value -``0x265e`` (9,822 in decimal). - -The Unicode standard contains a lot of tables listing characters and -their corresponding code points: - -.. code-block:: none - - 0061 'a'; LATIN SMALL LETTER A - 0062 'b'; LATIN SMALL LETTER B - 0063 'c'; LATIN SMALL LETTER C - ... - 007B '{'; LEFT CURLY BRACKET - ... - 2167 'Ⅷ'; ROMAN NUMERAL EIGHT - 2168 'Ⅸ'; ROMAN NUMERAL NINE - ... - 265E '♞'; BLACK CHESS KNIGHT - 265F '♟'; BLACK CHESS PAWN - ... - 1F600 '😀'; GRINNING FACE - 1F609 '😉'; WINKING FACE - ... - -Strictly, these definitions imply that it's meaningless to say 'this is -character ``U+265E``'. ``U+265E`` is a code point, which represents some particular -character; in this case, it represents the character 'BLACK CHESS KNIGHT', -'♞'. In -informal contexts, this distinction between code points and characters will -sometimes be forgotten. - -A character is represented on a screen or on paper by a set of graphical -elements that's called a **glyph**. The glyph for an uppercase A, for example, -is two diagonal strokes and a horizontal stroke, though the exact details will -depend on the font being used. Most Python code doesn't need to worry about -glyphs; figuring out the correct glyph to display is generally the job of a GUI -toolkit or a terminal's font renderer. - - -Encodings ---------- - -To summarize the previous section: a Unicode string is a sequence of -code points, which are numbers from 0 through ``0x10FFFF`` (1,114,111 -decimal). This sequence of code points needs to be represented in -memory as a set of **code units**, and **code units** are then mapped -to 8-bit bytes. The rules for translating a Unicode string into a -sequence of bytes are called a **character encoding**, or just -an **encoding**. - -The first encoding you might think of is using 32-bit integers as the -code unit, and then using the CPU's representation of 32-bit integers. -In this representation, the string "Python" might look like this: - -.. code-block:: none - - P y t h o n - 0x50 00 00 00 79 00 00 00 74 00 00 00 68 00 00 00 6f 00 00 00 6e 00 00 00 - 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 - -This representation is straightforward but using it presents a number of -problems. - -1. It's not portable; different processors order the bytes differently. - -2. It's very wasteful of space. In most texts, the majority of the code points - are less than 127, or less than 255, so a lot of space is occupied by ``0x00`` - bytes. The above string takes 24 bytes compared to the 6 bytes needed for an - ASCII representation. Increased RAM usage doesn't matter too much (desktop - computers have gigabytes of RAM, and strings aren't usually that large), but - expanding our usage of disk and network bandwidth by a factor of 4 is - intolerable. - -3. It's not compatible with existing C functions such as ``strlen()``, so a new - family of wide string functions would need to be used. - -Therefore this encoding isn't used very much, and people instead choose other -encodings that are more efficient and convenient, such as UTF-8. - -UTF-8 is one of the most commonly used encodings, and Python often -defaults to using it. UTF stands for "Unicode Transformation Format", -and the '8' means that 8-bit values are used in the encoding. (There -are also UTF-16 and UTF-32 encodings, but they are less frequently -used than UTF-8.) UTF-8 uses the following rules: - -1. If the code point is < 128, it's represented by the corresponding byte value. -2. If the code point is >= 128, it's turned into a sequence of two, three, or - four bytes, where each byte of the sequence is between 128 and 255. - -UTF-8 has several convenient properties: - -1. It can handle any Unicode code point. -2. A Unicode string is turned into a sequence of bytes that contains embedded - zero bytes only where they represent the null character (U+0000). This means - that UTF-8 strings can be processed by C functions such as ``strcpy()`` and sent - through protocols that can't handle zero bytes for anything other than - end-of-string markers. -3. A string of ASCII text is also valid UTF-8 text. -4. UTF-8 is fairly compact; the majority of commonly used characters can be - represented with one or two bytes. -5. If bytes are corrupted or lost, it's possible to determine the start of the - next UTF-8-encoded code point and resynchronize. It's also unlikely that - random 8-bit data will look like valid UTF-8. -6. UTF-8 is a byte oriented encoding. The encoding specifies that each - character is represented by a specific sequence of one or more bytes. This - avoids the byte-ordering issues that can occur with integer and word oriented - encodings, like UTF-16 and UTF-32, where the sequence of bytes varies depending - on the hardware on which the string was encoded. - - -References ----------- - -The `Unicode Consortium site `_ has character charts, a -glossary, and PDF versions of the Unicode specification. Be prepared for some -difficult reading. `A chronology `_ of the -origin and development of Unicode is also available on the site. - -On the Computerphile Youtube channel, Tom Scott briefly -`discusses the history of Unicode and UTF-8 `_ -(9 minutes 36 seconds). - -To help understand the standard, Jukka Korpela has written `an introductory -guide `_ to reading the -Unicode character tables. - -Another `good introductory article `_ -was written by Joel Spolsky. -If this introduction didn't make things clear to you, you should try -reading this alternate article before continuing. - -Wikipedia entries are often helpful; see the entries for "`character encoding -`_" and `UTF-8 -`_, for example. - - -Python's Unicode Support -======================== - -Now that you've learned the rudiments of Unicode, we can look at Python's -Unicode features. - -The String Type ---------------- - -Since Python 3.0, the language's :class:`str` type contains Unicode -characters, meaning any string created using ``"unicode rocks!"``, ``'unicode -rocks!'``, or the triple-quoted string syntax is stored as Unicode. - -The default encoding for Python source code is UTF-8, so you can simply -include a Unicode character in a string literal:: - - try: - with open('/tmp/input.txt', 'r') as f: - ... - except OSError: - # 'File not found' error message. - print("Fichier non trouvé") - -Side note: Python 3 also supports using Unicode characters in identifiers:: - - répertoire = "/tmp/records.log" - with open(répertoire, "w") as f: - f.write("test\n") - -If you can't enter a particular character in your editor or want to -keep the source code ASCII-only for some reason, you can also use -escape sequences in string literals. (Depending on your system, -you may see the actual capital-delta glyph instead of a \u escape.) :: - - >>> "\N{GREEK CAPITAL LETTER DELTA}" # Using the character name - '\u0394' - >>> "\u0394" # Using a 16-bit hex value - '\u0394' - >>> "\U00000394" # Using a 32-bit hex value - '\u0394' - -In addition, one can create a string using the :func:`~bytes.decode` method of -:class:`bytes`. This method takes an *encoding* argument, such as ``UTF-8``, -and optionally an *errors* argument. - -The *errors* argument specifies the response when the input string can't be -converted according to the encoding's rules. Legal values for this argument are -``'strict'`` (raise a :exc:`UnicodeDecodeError` exception), ``'replace'`` (use -``U+FFFD``, ``REPLACEMENT CHARACTER``), ``'ignore'`` (just leave the -character out of the Unicode result), or ``'backslashreplace'`` (inserts a -``\xNN`` escape sequence). -The following examples show the differences:: - - >>> b'\x80abc'.decode("utf-8", "strict") #doctest: +NORMALIZE_WHITESPACE - Traceback (most recent call last): - ... - UnicodeDecodeError: 'utf-8' codec can't decode byte 0x80 in position 0: - invalid start byte - >>> b'\x80abc'.decode("utf-8", "replace") - '\ufffdabc' - >>> b'\x80abc'.decode("utf-8", "backslashreplace") - '\\x80abc' - >>> b'\x80abc'.decode("utf-8", "ignore") - 'abc' - -Encodings are specified as strings containing the encoding's name. Python -comes with roughly 100 different encodings; see the Python Library Reference at -:ref:`standard-encodings` for a list. Some encodings have multiple names; for -example, ``'latin-1'``, ``'iso_8859_1'`` and ``'8859``' are all synonyms for -the same encoding. - -One-character Unicode strings can also be created with the :func:`chr` -built-in function, which takes integers and returns a Unicode string of length 1 -that contains the corresponding code point. The reverse operation is the -built-in :func:`ord` function that takes a one-character Unicode string and -returns the code point value:: - - >>> chr(57344) - '\ue000' - >>> ord('\ue000') - 57344 - -Converting to Bytes -------------------- - -The opposite method of :meth:`bytes.decode` is :meth:`str.encode`, -which returns a :class:`bytes` representation of the Unicode string, encoded in the -requested *encoding*. - -The *errors* parameter is the same as the parameter of the -:meth:`~bytes.decode` method but supports a few more possible handlers. As well as -``'strict'``, ``'ignore'``, and ``'replace'`` (which in this case -inserts a question mark instead of the unencodable character), there is -also ``'xmlcharrefreplace'`` (inserts an XML character reference), -``backslashreplace`` (inserts a ``\uNNNN`` escape sequence) and -``namereplace`` (inserts a ``\N{...}`` escape sequence). - -The following example shows the different results:: - - >>> u = chr(40960) + 'abcd' + chr(1972) - >>> u.encode('utf-8') - b'\xea\x80\x80abcd\xde\xb4' - >>> u.encode('ascii') #doctest: +NORMALIZE_WHITESPACE - Traceback (most recent call last): - ... - UnicodeEncodeError: 'ascii' codec can't encode character '\ua000' in - position 0: ordinal not in range(128) - >>> u.encode('ascii', 'ignore') - b'abcd' - >>> u.encode('ascii', 'replace') - b'?abcd?' - >>> u.encode('ascii', 'xmlcharrefreplace') - b'ꀀabcd޴' - >>> u.encode('ascii', 'backslashreplace') - b'\\ua000abcd\\u07b4' - >>> u.encode('ascii', 'namereplace') - b'\\N{YI SYLLABLE IT}abcd\\u07b4' - -The low-level routines for registering and accessing the available -encodings are found in the :mod:`codecs` module. Implementing new -encodings also requires understanding the :mod:`codecs` module. -However, the encoding and decoding functions returned by this module -are usually more low-level than is comfortable, and writing new encodings -is a specialized task, so the module won't be covered in this HOWTO. - - -Unicode Literals in Python Source Code --------------------------------------- - -In Python source code, specific Unicode code points can be written using the -``\u`` escape sequence, which is followed by four hex digits giving the code -point. The ``\U`` escape sequence is similar, but expects eight hex digits, -not four:: - - >>> s = "a\xac\u1234\u20ac\U00008000" - ... # ^^^^ two-digit hex escape - ... # ^^^^^^ four-digit Unicode escape - ... # ^^^^^^^^^^ eight-digit Unicode escape - >>> [ord(c) for c in s] - [97, 172, 4660, 8364, 32768] - -Using escape sequences for code points greater than 127 is fine in small doses, -but becomes an annoyance if you're using many accented characters, as you would -in a program with messages in French or some other accent-using language. You -can also assemble strings using the :func:`chr` built-in function, but this is -even more tedious. - -Ideally, you'd want to be able to write literals in your language's natural -encoding. You could then edit Python source code with your favorite editor -which would display the accented characters naturally, and have the right -characters used at runtime. - -Python supports writing source code in UTF-8 by default, but you can use almost -any encoding if you declare the encoding being used. This is done by including -a special comment as either the first or second line of the source file:: - - #!/usr/bin/env python - # -*- coding: latin-1 -*- - - u = 'abcdé' - print(ord(u[-1])) - -The syntax is inspired by Emacs's notation for specifying variables local to a -file. Emacs supports many different variables, but Python only supports -'coding'. The ``-*-`` symbols indicate to Emacs that the comment is special; -they have no significance to Python but are a convention. Python looks for -``coding: name`` or ``coding=name`` in the comment. - -If you don't include such a comment, the default encoding used will be UTF-8 as -already mentioned. See also :pep:`263` for more information. - - -Unicode Properties ------------------- - -The Unicode specification includes a database of information about -code points. For each defined code point, the information includes -the character's name, its category, the numeric value if applicable -(for characters representing numeric concepts such as the Roman -numerals, fractions such as one-third and four-fifths, etc.). There -are also display-related properties, such as how to use the code point -in bidirectional text. - -The following program displays some information about several characters, and -prints the numeric value of one particular character:: - - import unicodedata - - u = chr(233) + chr(0x0bf2) + chr(3972) + chr(6000) + chr(13231) - - for i, c in enumerate(u): - print(i, '%04x' % ord(c), unicodedata.category(c), end=" ") - print(unicodedata.name(c)) - - # Get numeric value of second character - print(unicodedata.numeric(u[1])) - -When run, this prints: - -.. code-block:: none - - 0 00e9 Ll LATIN SMALL LETTER E WITH ACUTE - 1 0bf2 No TAMIL NUMBER ONE THOUSAND - 2 0f84 Mn TIBETAN MARK HALANTA - 3 1770 Lo TAGBANWA LETTER SA - 4 33af So SQUARE RAD OVER S SQUARED - 1000.0 - -The category codes are abbreviations describing the nature of the character. -These are grouped into categories such as "Letter", "Number", "Punctuation", or -"Symbol", which in turn are broken up into subcategories. To take the codes -from the above output, ``'Ll'`` means 'Letter, lowercase', ``'No'`` means -"Number, other", ``'Mn'`` is "Mark, nonspacing", and ``'So'`` is "Symbol, -other". See -`the General Category Values section of the Unicode Character Database documentation `_ for a -list of category codes. - - -Comparing Strings ------------------ - -Unicode adds some complication to comparing strings, because the same -set of characters can be represented by different sequences of code -points. For example, a letter like 'ê' can be represented as a single -code point U+00EA, or as U+0065 U+0302, which is the code point for -'e' followed by a code point for 'COMBINING CIRCUMFLEX ACCENT'. These -will produce the same output when printed, but one is a string of -length 1 and the other is of length 2. - -One tool for a case-insensitive comparison is the -:meth:`~str.casefold` string method that converts a string to a -case-insensitive form following an algorithm described by the Unicode -Standard. This algorithm has special handling for characters such as -the German letter 'ß' (code point U+00DF), which becomes the pair of -lowercase letters 'ss'. - -:: - - >>> street = 'Gürzenichstraße' - >>> street.casefold() - 'gürzenichstrasse' - -A second tool is the :mod:`unicodedata` module's -:func:`~unicodedata.normalize` function that converts strings to one -of several normal forms, where letters followed by a combining character are -replaced with single characters. :func:`~unicodedata.normalize` can -be used to perform string comparisons that won't falsely report -inequality if two strings use combining characters differently: - -:: - - import unicodedata - - def compare_strs(s1, s2): - def NFD(s): - return unicodedata.normalize('NFD', s) - - return NFD(s1) == NFD(s2) - - single_char = 'ê' - multiple_chars = '\N{LATIN SMALL LETTER E}\N{COMBINING CIRCUMFLEX ACCENT}' - print('length of first string=', len(single_char)) - print('length of second string=', len(multiple_chars)) - print(compare_strs(single_char, multiple_chars)) - -When run, this outputs: - -.. code-block:: shell-session - - $ python3 compare-strs.py - length of first string= 1 - length of second string= 2 - True - -The first argument to the :func:`~unicodedata.normalize` function is a -string giving the desired normalization form, which can be one of -'NFC', 'NFKC', 'NFD', and 'NFKD'. - -The Unicode Standard also specifies how to do caseless comparisons:: - - import unicodedata - - def compare_caseless(s1, s2): - def NFD(s): - return unicodedata.normalize('NFD', s) - - return NFD(NFD(s1).casefold()) == NFD(NFD(s2).casefold()) - - # Example usage - single_char = 'ê' - multiple_chars = '\N{LATIN CAPITAL LETTER E}\N{COMBINING CIRCUMFLEX ACCENT}' - - print(compare_caseless(single_char, multiple_chars)) - -This will print ``True``. (Why is :func:`!NFD` invoked twice? Because -there are a few characters that make :meth:`~str.casefold` return a -non-normalized string, so the result needs to be normalized again. See -section 3.13 of the Unicode Standard for a discussion and an example.) - - -Unicode Regular Expressions ---------------------------- - -The regular expressions supported by the :mod:`re` module can be provided -either as bytes or strings. Some of the special character sequences such as -``\d`` and ``\w`` have different meanings depending on whether -the pattern is supplied as bytes or a string. For example, -``\d`` will match the characters ``[0-9]`` in bytes but -in strings will match any character that's in the ``'Nd'`` category. - -The string in this example has the number 57 written in both Thai and -Arabic numerals:: - - import re - p = re.compile(r'\d+') - - s = "Over \u0e55\u0e57 57 flavours" - m = p.search(s) - print(repr(m.group())) - -When executed, ``\d+`` will match the Thai numerals and print them -out. If you supply the :const:`re.ASCII` flag to -:func:`~re.compile`, ``\d+`` will match the substring "57" instead. - -Similarly, ``\w`` matches a wide variety of Unicode characters but -only ``[a-zA-Z0-9_]`` in bytes or if :const:`re.ASCII` is supplied, -and ``\s`` will match either Unicode whitespace characters or -``[ \t\n\r\f\v]``. - - -References ----------- - -.. comment should these be mentioned earlier, e.g. at the start of the "introduction to Unicode" first section? - -Some good alternative discussions of Python's Unicode support are: - -* `Processing Text Files in Python 3 `_, by Nick Coghlan. -* `Pragmatic Unicode `_, a PyCon 2012 presentation by Ned Batchelder. - -The :class:`str` type is described in the Python library reference at -:ref:`textseq`. - -The documentation for the :mod:`unicodedata` module. - -The documentation for the :mod:`codecs` module. - -Marc-André Lemburg gave `a presentation titled "Python and Unicode" (PDF slides) -`_ at -EuroPython 2002. The slides are an excellent overview of the design of Python -2's Unicode features (where the Unicode string type is called ``unicode`` and -literals start with ``u``). - - -Reading and Writing Unicode Data -================================ - -Once you've written some code that works with Unicode data, the next problem is -input/output. How do you get Unicode strings into your program, and how do you -convert Unicode into a form suitable for storage or transmission? - -It's possible that you may not need to do anything depending on your input -sources and output destinations; you should check whether the libraries used in -your application support Unicode natively. XML parsers often return Unicode -data, for example. Many relational databases also support Unicode-valued -columns and can return Unicode values from an SQL query. - -Unicode data is usually converted to a particular encoding before it gets -written to disk or sent over a socket. It's possible to do all the work -yourself: open a file, read an 8-bit bytes object from it, and convert the bytes -with ``bytes.decode(encoding)``. However, the manual approach is not recommended. - -One problem is the multi-byte nature of encodings; one Unicode character can be -represented by several bytes. If you want to read the file in arbitrary-sized -chunks (say, 1024 or 4096 bytes), you need to write error-handling code to catch the case -where only part of the bytes encoding a single Unicode character are read at the -end of a chunk. One solution would be to read the entire file into memory and -then perform the decoding, but that prevents you from working with files that -are extremely large; if you need to read a 2 GiB file, you need 2 GiB of RAM. -(More, really, since for at least a moment you'd need to have both the encoded -string and its Unicode version in memory.) - -The solution would be to use the low-level decoding interface to catch the case -of partial coding sequences. The work of implementing this has already been -done for you: the built-in :func:`open` function can return a file-like object -that assumes the file's contents are in a specified encoding and accepts Unicode -parameters for methods such as :meth:`~io.TextIOBase.read` and -:meth:`~io.TextIOBase.write`. This works through :func:`open`\'s *encoding* and -*errors* parameters which are interpreted just like those in :meth:`str.encode` -and :meth:`bytes.decode`. - -Reading Unicode from a file is therefore simple:: - - with open('unicode.txt', encoding='utf-8') as f: - for line in f: - print(repr(line)) - -It's also possible to open files in update mode, allowing both reading and -writing:: - - with open('test', encoding='utf-8', mode='w+') as f: - f.write('\u4500 blah blah blah\n') - f.seek(0) - print(repr(f.readline()[:1])) - -The Unicode character ``U+FEFF`` is used as a byte-order mark (BOM), and is often -written as the first character of a file in order to assist with autodetection -of the file's byte ordering. Some encodings, such as UTF-16, expect a BOM to be -present at the start of a file; when such an encoding is used, the BOM will be -automatically written as the first character and will be silently dropped when -the file is read. There are variants of these encodings, such as 'utf-16-le' -and 'utf-16-be' for little-endian and big-endian encodings, that specify one -particular byte ordering and don't skip the BOM. - -In some areas, it is also convention to use a "BOM" at the start of UTF-8 -encoded files; the name is misleading since UTF-8 is not byte-order dependent. -The mark simply announces that the file is encoded in UTF-8. For reading such -files, use the 'utf-8-sig' codec to automatically skip the mark if present. - - -Unicode filenames ------------------ - -Most of the operating systems in common use today support filenames -that contain arbitrary Unicode characters. Usually this is -implemented by converting the Unicode string into some encoding that -varies depending on the system. Today Python is converging on using -UTF-8: Python on MacOS has used UTF-8 for several versions, and Python -3.6 switched to using UTF-8 on Windows as well. On Unix systems, -there will only be a :term:`filesystem encoding `. if you've set the ``LANG`` or ``LC_CTYPE`` environment variables; if -you haven't, the default encoding is again UTF-8. - -The :func:`sys.getfilesystemencoding` function returns the encoding to use on -your current system, in case you want to do the encoding manually, but there's -not much reason to bother. When opening a file for reading or writing, you can -usually just provide the Unicode string as the filename, and it will be -automatically converted to the right encoding for you:: - - filename = 'filename\u4500abc' - with open(filename, 'w') as f: - f.write('blah\n') - -Functions in the :mod:`os` module such as :func:`os.stat` will also accept Unicode -filenames. - -The :func:`os.listdir` function returns filenames, which raises an issue: should it return -the Unicode version of filenames, or should it return bytes containing -the encoded versions? :func:`os.listdir` can do both, depending on whether you -provided the directory path as bytes or a Unicode string. If you pass a -Unicode string as the path, filenames will be decoded using the filesystem's -encoding and a list of Unicode strings will be returned, while passing a byte -path will return the filenames as bytes. For example, -assuming the default :term:`filesystem encoding ` is UTF-8, running the following program:: - - fn = 'filename\u4500abc' - f = open(fn, 'w') - f.close() - - import os - print(os.listdir(b'.')) - print(os.listdir('.')) - -will produce the following output: - -.. code-block:: shell-session - - $ python listdir-test.py - [b'filename\xe4\x94\x80abc', ...] - ['filename\u4500abc', ...] - -The first list contains UTF-8-encoded filenames, and the second list contains -the Unicode versions. - -Note that on most occasions, you should can just stick with using -Unicode with these APIs. The bytes APIs should only be used on -systems where undecodable file names can be present; that's -pretty much only Unix systems now. - - -Tips for Writing Unicode-aware Programs ---------------------------------------- - -This section provides some suggestions on writing software that deals with -Unicode. - -The most important tip is: - - Software should only work with Unicode strings internally, decoding the input - data as soon as possible and encoding the output only at the end. - -If you attempt to write processing functions that accept both Unicode and byte -strings, you will find your program vulnerable to bugs wherever you combine the -two different kinds of strings. There is no automatic encoding or decoding: if -you do e.g. ``str + bytes``, a :exc:`TypeError` will be raised. - -When using data coming from a web browser or some other untrusted source, a -common technique is to check for illegal characters in a string before using the -string in a generated command line or storing it in a database. If you're doing -this, be careful to check the decoded string, not the encoded bytes data; -some encodings may have interesting properties, such as not being bijective -or not being fully ASCII-compatible. This is especially true if the input -data also specifies the encoding, since the attacker can then choose a -clever way to hide malicious text in the encoded bytestream. - - -Converting Between File Encodings -''''''''''''''''''''''''''''''''' - -The :class:`~codecs.StreamRecoder` class can transparently convert between -encodings, taking a stream that returns data in encoding #1 -and behaving like a stream returning data in encoding #2. - -For example, if you have an input file *f* that's in Latin-1, you -can wrap it with a :class:`~codecs.StreamRecoder` to return bytes encoded in -UTF-8:: - - new_f = codecs.StreamRecoder(f, - # en/decoder: used by read() to encode its results and - # by write() to decode its input. - codecs.getencoder('utf-8'), codecs.getdecoder('utf-8'), - - # reader/writer: used to read and write to the stream. - codecs.getreader('latin-1'), codecs.getwriter('latin-1') ) - - -Files in an Unknown Encoding -'''''''''''''''''''''''''''' - -What can you do if you need to make a change to a file, but don't know -the file's encoding? If you know the encoding is ASCII-compatible and -only want to examine or modify the ASCII parts, you can open the file -with the ``surrogateescape`` error handler:: - - with open(fname, 'r', encoding="ascii", errors="surrogateescape") as f: - data = f.read() - - # make changes to the string 'data' - - with open(fname + '.new', 'w', - encoding="ascii", errors="surrogateescape") as f: - f.write(data) - -The ``surrogateescape`` error handler will decode any non-ASCII bytes -as code points in a special range running from U+DC80 to -U+DCFF. These code points will then turn back into the -same bytes when the ``surrogateescape`` error handler is used to -encode the data and write it back out. - - -References ----------- - -One section of `Mastering Python 3 Input/Output -`_, -a PyCon 2010 talk by David Beazley, discusses text processing and binary data handling. - -The `PDF slides for Marc-André Lemburg's presentation "Writing Unicode-aware -Applications in Python" -`_ -discuss questions of character encodings as well as how to internationalize -and localize an application. These slides cover Python 2.x only. - -`The Guts of Unicode in Python -`_ -is a PyCon 2013 talk by Benjamin Peterson that discusses the internal Unicode -representation in Python 3.3. - - -Acknowledgements -================ - -The initial draft of this document was written by Andrew Kuchling. -It has since been revised further by Alexander Belopolsky, Georg Brandl, -Andrew Kuchling, and Ezio Melotti. - -Thanks to the following people who have noted errors or offered -suggestions on this article: Éric Araujo, Nicholas Bastin, Nick -Coghlan, Marius Gedminas, Kent Johnson, Ken Krugler, Marc-André -Lemburg, Martin von Löwis, Terry J. Reedy, Serhiy Storchaka, -Eryk Sun, Chad Whitacre, Graham Wideman. diff --git a/Doc/howto/urllib2.rst b/Doc/howto/urllib2.rst deleted file mode 100644 index 570435d48866d3..00000000000000 --- a/Doc/howto/urllib2.rst +++ /dev/null @@ -1,598 +0,0 @@ -.. _urllib-howto: - -*********************************************************** - HOWTO Fetch Internet Resources Using The urllib Package -*********************************************************** - -:Author: `Michael Foord `_ - - -Introduction -============ - -.. sidebar:: Related Articles - - You may also find useful the following article on fetching web resources - with Python: - - * `Basic Authentication `_ - - A tutorial on *Basic Authentication*, with examples in Python. - -**urllib.request** is a Python module for fetching URLs -(Uniform Resource Locators). It offers a very simple interface, in the form of -the *urlopen* function. This is capable of fetching URLs using a variety of -different protocols. It also offers a slightly more complex interface for -handling common situations - like basic authentication, cookies, proxies and so -on. These are provided by objects called handlers and openers. - -urllib.request supports fetching URLs for many "URL schemes" (identified by the string -before the ``":"`` in URL - for example ``"ftp"`` is the URL scheme of -``"ftp://python.org/"``) using their associated network protocols (e.g. FTP, HTTP). -This tutorial focuses on the most common case, HTTP. - -For straightforward situations *urlopen* is very easy to use. But as soon as you -encounter errors or non-trivial cases when opening HTTP URLs, you will need some -understanding of the HyperText Transfer Protocol. The most comprehensive and -authoritative reference to HTTP is :rfc:`2616`. This is a technical document and -not intended to be easy to read. This HOWTO aims to illustrate using *urllib*, -with enough detail about HTTP to help you through. It is not intended to replace -the :mod:`urllib.request` docs, but is supplementary to them. - - -Fetching URLs -============= - -The simplest way to use urllib.request is as follows:: - - import urllib.request - with urllib.request.urlopen('http://python.org/') as response: - html = response.read() - -If you wish to retrieve a resource via URL and store it in a temporary -location, you can do so via the :func:`shutil.copyfileobj` and -:func:`tempfile.NamedTemporaryFile` functions:: - - import shutil - import tempfile - import urllib.request - - with urllib.request.urlopen('http://python.org/') as response: - with tempfile.NamedTemporaryFile(delete=False) as tmp_file: - shutil.copyfileobj(response, tmp_file) - - with open(tmp_file.name) as html: - pass - -Many uses of urllib will be that simple (note that instead of an 'http:' URL we -could have used a URL starting with 'ftp:', 'file:', etc.). However, it's the -purpose of this tutorial to explain the more complicated cases, concentrating on -HTTP. - -HTTP is based on requests and responses - the client makes requests and servers -send responses. urllib.request mirrors this with a ``Request`` object which represents -the HTTP request you are making. In its simplest form you create a Request -object that specifies the URL you want to fetch. Calling ``urlopen`` with this -Request object returns a response object for the URL requested. This response is -a file-like object, which means you can for example call ``.read()`` on the -response:: - - import urllib.request - - req = urllib.request.Request('http://python.org/') - with urllib.request.urlopen(req) as response: - the_page = response.read() - -Note that urllib.request makes use of the same Request interface to handle all URL -schemes. For example, you can make an FTP request like so:: - - req = urllib.request.Request('ftp://example.com/') - -In the case of HTTP, there are two extra things that Request objects allow you -to do: First, you can pass data to be sent to the server. Second, you can pass -extra information ("metadata") *about* the data or about the request itself, to -the server - this information is sent as HTTP "headers". Let's look at each of -these in turn. - -Data ----- - -Sometimes you want to send data to a URL (often the URL will refer to a CGI -(Common Gateway Interface) script or other web application). With HTTP, -this is often done using what's known as a **POST** request. This is often what -your browser does when you submit a HTML form that you filled in on the web. Not -all POSTs have to come from forms: you can use a POST to transmit arbitrary data -to your own application. In the common case of HTML forms, the data needs to be -encoded in a standard way, and then passed to the Request object as the ``data`` -argument. The encoding is done using a function from the :mod:`urllib.parse` -library. :: - - import urllib.parse - import urllib.request - - url = 'http://www.someserver.com/cgi-bin/register.cgi' - values = {'name' : 'Michael Foord', - 'location' : 'Northampton', - 'language' : 'Python' } - - data = urllib.parse.urlencode(values) - data = data.encode('ascii') # data should be bytes - req = urllib.request.Request(url, data) - with urllib.request.urlopen(req) as response: - the_page = response.read() - -Note that other encodings are sometimes required (e.g. for file upload from HTML -forms - see `HTML Specification, Form Submission -`_ for more -details). - -If you do not pass the ``data`` argument, urllib uses a **GET** request. One -way in which GET and POST requests differ is that POST requests often have -"side-effects": they change the state of the system in some way (for example by -placing an order with the website for a hundredweight of tinned spam to be -delivered to your door). Though the HTTP standard makes it clear that POSTs are -intended to *always* cause side-effects, and GET requests *never* to cause -side-effects, nothing prevents a GET request from having side-effects, nor a -POST requests from having no side-effects. Data can also be passed in an HTTP -GET request by encoding it in the URL itself. - -This is done as follows:: - - >>> import urllib.request - >>> import urllib.parse - >>> data = {} - >>> data['name'] = 'Somebody Here' - >>> data['location'] = 'Northampton' - >>> data['language'] = 'Python' - >>> url_values = urllib.parse.urlencode(data) - >>> print(url_values) # The order may differ from below. #doctest: +SKIP - name=Somebody+Here&language=Python&location=Northampton - >>> url = 'http://www.example.com/example.cgi' - >>> full_url = url + '?' + url_values - >>> data = urllib.request.urlopen(full_url) - -Notice that the full URL is created by adding a ``?`` to the URL, followed by -the encoded values. - -Headers -------- - -We'll discuss here one particular HTTP header, to illustrate how to add headers -to your HTTP request. - -Some websites [#]_ dislike being browsed by programs, or send different versions -to different browsers [#]_. By default urllib identifies itself as -``Python-urllib/x.y`` (where ``x`` and ``y`` are the major and minor version -numbers of the Python release, -e.g. ``Python-urllib/2.5``), which may confuse the site, or just plain -not work. The way a browser identifies itself is through the -``User-Agent`` header [#]_. When you create a Request object you can -pass a dictionary of headers in. The following example makes the same -request as above, but identifies itself as a version of Internet -Explorer [#]_. :: - - import urllib.parse - import urllib.request - - url = 'http://www.someserver.com/cgi-bin/register.cgi' - user_agent = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64)' - values = {'name': 'Michael Foord', - 'location': 'Northampton', - 'language': 'Python' } - headers = {'User-Agent': user_agent} - - data = urllib.parse.urlencode(values) - data = data.encode('ascii') - req = urllib.request.Request(url, data, headers) - with urllib.request.urlopen(req) as response: - the_page = response.read() - -The response also has two useful methods. See the section on `info and geturl`_ -which comes after we have a look at what happens when things go wrong. - - -Handling Exceptions -=================== - -*urlopen* raises :exc:`~urllib.error.URLError` when it cannot handle a response (though as -usual with Python APIs, built-in exceptions such as :exc:`ValueError`, -:exc:`TypeError` etc. may also be raised). - -:exc:`~urllib.error.HTTPError` is the subclass of :exc:`~urllib.error.URLError` raised in the specific case of -HTTP URLs. - -The exception classes are exported from the :mod:`urllib.error` module. - -URLError --------- - -Often, URLError is raised because there is no network connection (no route to -the specified server), or the specified server doesn't exist. In this case, the -exception raised will have a 'reason' attribute, which is a tuple containing an -error code and a text error message. - -e.g. :: - - >>> req = urllib.request.Request('http://www.pretend_server.org') - >>> try: urllib.request.urlopen(req) - ... except urllib.error.URLError as e: - ... print(e.reason) #doctest: +SKIP - ... - (4, 'getaddrinfo failed') - - -HTTPError ---------- - -Every HTTP response from the server contains a numeric "status code". Sometimes -the status code indicates that the server is unable to fulfil the request. The -default handlers will handle some of these responses for you (for example, if -the response is a "redirection" that requests the client fetch the document from -a different URL, urllib will handle that for you). For those it can't handle, -urlopen will raise an :exc:`~urllib.error.HTTPError`. Typical errors include '404' (page not -found), '403' (request forbidden), and '401' (authentication required). - -See section 10 of :rfc:`2616` for a reference on all the HTTP error codes. - -The :exc:`~urllib.error.HTTPError` instance raised will have an integer 'code' attribute, which -corresponds to the error sent by the server. - -Error Codes -~~~~~~~~~~~ - -Because the default handlers handle redirects (codes in the 300 range), and -codes in the 100--299 range indicate success, you will usually only see error -codes in the 400--599 range. - -:attr:`http.server.BaseHTTPRequestHandler.responses` is a useful dictionary of -response codes in that shows all the response codes used by :rfc:`2616`. The -dictionary is reproduced here for convenience :: - - # Table mapping response codes to messages; entries have the - # form {code: (shortmessage, longmessage)}. - responses = { - 100: ('Continue', 'Request received, please continue'), - 101: ('Switching Protocols', - 'Switching to new protocol; obey Upgrade header'), - - 200: ('OK', 'Request fulfilled, document follows'), - 201: ('Created', 'Document created, URL follows'), - 202: ('Accepted', - 'Request accepted, processing continues off-line'), - 203: ('Non-Authoritative Information', 'Request fulfilled from cache'), - 204: ('No Content', 'Request fulfilled, nothing follows'), - 205: ('Reset Content', 'Clear input form for further input.'), - 206: ('Partial Content', 'Partial content follows.'), - - 300: ('Multiple Choices', - 'Object has several resources -- see URI list'), - 301: ('Moved Permanently', 'Object moved permanently -- see URI list'), - 302: ('Found', 'Object moved temporarily -- see URI list'), - 303: ('See Other', 'Object moved -- see Method and URL list'), - 304: ('Not Modified', - 'Document has not changed since given time'), - 305: ('Use Proxy', - 'You must use proxy specified in Location to access this ' - 'resource.'), - 307: ('Temporary Redirect', - 'Object moved temporarily -- see URI list'), - - 400: ('Bad Request', - 'Bad request syntax or unsupported method'), - 401: ('Unauthorized', - 'No permission -- see authorization schemes'), - 402: ('Payment Required', - 'No payment -- see charging schemes'), - 403: ('Forbidden', - 'Request forbidden -- authorization will not help'), - 404: ('Not Found', 'Nothing matches the given URI'), - 405: ('Method Not Allowed', - 'Specified method is invalid for this server.'), - 406: ('Not Acceptable', 'URI not available in preferred format.'), - 407: ('Proxy Authentication Required', 'You must authenticate with ' - 'this proxy before proceeding.'), - 408: ('Request Timeout', 'Request timed out; try again later.'), - 409: ('Conflict', 'Request conflict.'), - 410: ('Gone', - 'URI no longer exists and has been permanently removed.'), - 411: ('Length Required', 'Client must specify Content-Length.'), - 412: ('Precondition Failed', 'Precondition in headers is false.'), - 413: ('Request Entity Too Large', 'Entity is too large.'), - 414: ('Request-URI Too Long', 'URI is too long.'), - 415: ('Unsupported Media Type', 'Entity body in unsupported format.'), - 416: ('Requested Range Not Satisfiable', - 'Cannot satisfy request range.'), - 417: ('Expectation Failed', - 'Expect condition could not be satisfied.'), - - 500: ('Internal Server Error', 'Server got itself in trouble'), - 501: ('Not Implemented', - 'Server does not support this operation'), - 502: ('Bad Gateway', 'Invalid responses from another server/proxy.'), - 503: ('Service Unavailable', - 'The server cannot process the request due to a high load'), - 504: ('Gateway Timeout', - 'The gateway server did not receive a timely response'), - 505: ('HTTP Version Not Supported', 'Cannot fulfill request.'), - } - -When an error is raised the server responds by returning an HTTP error code -*and* an error page. You can use the :exc:`~urllib.error.HTTPError` instance as a response on the -page returned. This means that as well as the code attribute, it also has read, -geturl, and info, methods as returned by the ``urllib.response`` module:: - - >>> req = urllib.request.Request('http://www.python.org/fish.html') - >>> try: - ... urllib.request.urlopen(req) - ... except urllib.error.HTTPError as e: - ... print(e.code) - ... print(e.read()) #doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE - ... - 404 - b'\n\n\nPage Not Found\n - ... - -Wrapping it Up --------------- - -So if you want to be prepared for :exc:`~urllib.error.HTTPError` *or* :exc:`~urllib.error.URLError` there are two -basic approaches. I prefer the second approach. - -Number 1 -~~~~~~~~ - -:: - - - from urllib.request import Request, urlopen - from urllib.error import URLError, HTTPError - req = Request(someurl) - try: - response = urlopen(req) - except HTTPError as e: - print('The server couldn\'t fulfill the request.') - print('Error code: ', e.code) - except URLError as e: - print('We failed to reach a server.') - print('Reason: ', e.reason) - else: - # everything is fine - - -.. note:: - - The ``except HTTPError`` *must* come first, otherwise ``except URLError`` - will *also* catch an :exc:`~urllib.error.HTTPError`. - -Number 2 -~~~~~~~~ - -:: - - from urllib.request import Request, urlopen - from urllib.error import URLError - req = Request(someurl) - try: - response = urlopen(req) - except URLError as e: - if hasattr(e, 'reason'): - print('We failed to reach a server.') - print('Reason: ', e.reason) - elif hasattr(e, 'code'): - print('The server couldn\'t fulfill the request.') - print('Error code: ', e.code) - else: - # everything is fine - - -info and geturl -=============== - -The response returned by urlopen (or the :exc:`~urllib.error.HTTPError` instance) has two -useful methods :meth:`info` and :meth:`geturl` and is defined in the module -:mod:`urllib.response`.. - -**geturl** - this returns the real URL of the page fetched. This is useful -because ``urlopen`` (or the opener object used) may have followed a -redirect. The URL of the page fetched may not be the same as the URL requested. - -**info** - this returns a dictionary-like object that describes the page -fetched, particularly the headers sent by the server. It is currently an -:class:`http.client.HTTPMessage` instance. - -Typical headers include 'Content-length', 'Content-type', and so on. See the -`Quick Reference to HTTP Headers `_ -for a useful listing of HTTP headers with brief explanations of their meaning -and use. - - -Openers and Handlers -==================== - -When you fetch a URL you use an opener (an instance of the perhaps -confusingly named :class:`urllib.request.OpenerDirector`). Normally we have been using -the default opener - via ``urlopen`` - but you can create custom -openers. Openers use handlers. All the "heavy lifting" is done by the -handlers. Each handler knows how to open URLs for a particular URL scheme (http, -ftp, etc.), or how to handle an aspect of URL opening, for example HTTP -redirections or HTTP cookies. - -You will want to create openers if you want to fetch URLs with specific handlers -installed, for example to get an opener that handles cookies, or to get an -opener that does not handle redirections. - -To create an opener, instantiate an ``OpenerDirector``, and then call -``.add_handler(some_handler_instance)`` repeatedly. - -Alternatively, you can use ``build_opener``, which is a convenience function for -creating opener objects with a single function call. ``build_opener`` adds -several handlers by default, but provides a quick way to add more and/or -override the default handlers. - -Other sorts of handlers you might want to can handle proxies, authentication, -and other common but slightly specialised situations. - -``install_opener`` can be used to make an ``opener`` object the (global) default -opener. This means that calls to ``urlopen`` will use the opener you have -installed. - -Opener objects have an ``open`` method, which can be called directly to fetch -urls in the same way as the ``urlopen`` function: there's no need to call -``install_opener``, except as a convenience. - - -Basic Authentication -==================== - -To illustrate creating and installing a handler we will use the -``HTTPBasicAuthHandler``. For a more detailed discussion of this subject -- -including an explanation of how Basic Authentication works - see the `Basic -Authentication Tutorial -`__. - -When authentication is required, the server sends a header (as well as the 401 -error code) requesting authentication. This specifies the authentication scheme -and a 'realm'. The header looks like: ``WWW-Authenticate: SCHEME -realm="REALM"``. - -e.g. - -.. code-block:: none - - WWW-Authenticate: Basic realm="cPanel Users" - - -The client should then retry the request with the appropriate name and password -for the realm included as a header in the request. This is 'basic -authentication'. In order to simplify this process we can create an instance of -``HTTPBasicAuthHandler`` and an opener to use this handler. - -The ``HTTPBasicAuthHandler`` uses an object called a password manager to handle -the mapping of URLs and realms to passwords and usernames. If you know what the -realm is (from the authentication header sent by the server), then you can use a -``HTTPPasswordMgr``. Frequently one doesn't care what the realm is. In that -case, it is convenient to use ``HTTPPasswordMgrWithDefaultRealm``. This allows -you to specify a default username and password for a URL. This will be supplied -in the absence of you providing an alternative combination for a specific -realm. We indicate this by providing ``None`` as the realm argument to the -``add_password`` method. - -The top-level URL is the first URL that requires authentication. URLs "deeper" -than the URL you pass to .add_password() will also match. :: - - # create a password manager - password_mgr = urllib.request.HTTPPasswordMgrWithDefaultRealm() - - # Add the username and password. - # If we knew the realm, we could use it instead of None. - top_level_url = "http://example.com/foo/" - password_mgr.add_password(None, top_level_url, username, password) - - handler = urllib.request.HTTPBasicAuthHandler(password_mgr) - - # create "opener" (OpenerDirector instance) - opener = urllib.request.build_opener(handler) - - # use the opener to fetch a URL - opener.open(a_url) - - # Install the opener. - # Now all calls to urllib.request.urlopen use our opener. - urllib.request.install_opener(opener) - -.. note:: - - In the above example we only supplied our ``HTTPBasicAuthHandler`` to - ``build_opener``. By default openers have the handlers for normal situations - -- ``ProxyHandler`` (if a proxy setting such as an :envvar:`http_proxy` - environment variable is set), ``UnknownHandler``, ``HTTPHandler``, - ``HTTPDefaultErrorHandler``, ``HTTPRedirectHandler``, ``FTPHandler``, - ``FileHandler``, ``DataHandler``, ``HTTPErrorProcessor``. - -``top_level_url`` is in fact *either* a full URL (including the 'http:' scheme -component and the hostname and optionally the port number) -e.g. ``"http://example.com/"`` *or* an "authority" (i.e. the hostname, -optionally including the port number) e.g. ``"example.com"`` or ``"example.com:8080"`` -(the latter example includes a port number). The authority, if present, must -NOT contain the "userinfo" component - for example ``"joe:password@example.com"`` is -not correct. - - -Proxies -======= - -**urllib** will auto-detect your proxy settings and use those. This is through -the ``ProxyHandler``, which is part of the normal handler chain when a proxy -setting is detected. Normally that's a good thing, but there are occasions -when it may not be helpful [#]_. One way to do this is to setup our own -``ProxyHandler``, with no proxies defined. This is done using similar steps to -setting up a `Basic Authentication`_ handler: :: - - >>> proxy_support = urllib.request.ProxyHandler({}) - >>> opener = urllib.request.build_opener(proxy_support) - >>> urllib.request.install_opener(opener) - -.. note:: - - Currently ``urllib.request`` *does not* support fetching of ``https`` locations - through a proxy. However, this can be enabled by extending urllib.request as - shown in the recipe [#]_. - -.. note:: - - ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; see - the documentation on :func:`~urllib.request.getproxies`. - - -Sockets and Layers -================== - -The Python support for fetching resources from the web is layered. urllib uses -the :mod:`http.client` library, which in turn uses the socket library. - -As of Python 2.3 you can specify how long a socket should wait for a response -before timing out. This can be useful in applications which have to fetch web -pages. By default the socket module has *no timeout* and can hang. Currently, -the socket timeout is not exposed at the http.client or urllib.request levels. -However, you can set the default timeout globally for all sockets using :: - - import socket - import urllib.request - - # timeout in seconds - timeout = 10 - socket.setdefaulttimeout(timeout) - - # this call to urllib.request.urlopen now uses the default timeout - # we have set in the socket module - req = urllib.request.Request('http://www.voidspace.org.uk') - response = urllib.request.urlopen(req) - - -------- - - -Footnotes -========= - -This document was reviewed and revised by John Lee. - -.. [#] Google for example. -.. [#] Browser sniffing is a very bad practice for website design - building - sites using web standards is much more sensible. Unfortunately a lot of - sites still send different versions to different browsers. -.. [#] The user agent for MSIE 6 is - *'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)'* -.. [#] For details of more HTTP request headers, see - `Quick Reference to HTTP Headers`_. -.. [#] In my case I have to use a proxy to access the internet at work. If you - attempt to fetch *localhost* URLs through this proxy it blocks them. IE - is set to use the proxy, which urllib picks up on. In order to test - scripts with a localhost server, I have to prevent urllib from using - the proxy. -.. [#] urllib opener for SSL proxy (CONNECT method): `ASPN Cookbook Recipe - `_. - diff --git a/Doc/includes/custom.c b/Doc/includes/custom.c deleted file mode 100644 index 26ca754964733d..00000000000000 --- a/Doc/includes/custom.c +++ /dev/null @@ -1,45 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include - -typedef struct { - PyObject_HEAD - /* Type-specific fields go here. */ -} CustomObject; - -static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom.Custom", - .tp_doc = PyDoc_STR("Custom objects"), - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT, - .tp_new = PyType_GenericNew, -}; - -static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, - .m_name = "custom", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_custom(void) -{ - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - Py_INCREF(&CustomType); - if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(&CustomType); - Py_DECREF(m); - return NULL; - } - - return m; -} diff --git a/Doc/includes/custom2.c b/Doc/includes/custom2.c deleted file mode 100644 index 2a3c59f8f04c3d..00000000000000 --- a/Doc/includes/custom2.c +++ /dev/null @@ -1,138 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include -#include "structmember.h" - -typedef struct { - PyObject_HEAD - PyObject *first; /* first name */ - PyObject *last; /* last name */ - int number; -} CustomObject; - -static void -Custom_dealloc(CustomObject *self) -{ - Py_XDECREF(self->first); - Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -static PyObject * -Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CustomObject *self; - self = (CustomObject *) type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - self->number = 0; - } - return (PyObject *) self; -} - -static int -Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|OOi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_XDECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_XDECREF(tmp); - } - return 0; -} - -static PyMemberDef Custom_members[] = { - {"first", T_OBJECT_EX, offsetof(CustomObject, first), 0, - "first name"}, - {"last", T_OBJECT_EX, offsetof(CustomObject, last), 0, - "last name"}, - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) -{ - if (self->first == NULL) { - PyErr_SetString(PyExc_AttributeError, "first"); - return NULL; - } - if (self->last == NULL) { - PyErr_SetString(PyExc_AttributeError, "last"); - return NULL; - } - return PyUnicode_FromFormat("%S %S", self->first, self->last); -} - -static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ -}; - -static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom2.Custom", - .tp_doc = PyDoc_STR("Custom objects"), - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_new = Custom_new, - .tp_init = (initproc) Custom_init, - .tp_dealloc = (destructor) Custom_dealloc, - .tp_members = Custom_members, - .tp_methods = Custom_methods, -}; - -static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, - .m_name = "custom2", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_custom2(void) -{ - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - Py_INCREF(&CustomType); - if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(&CustomType); - Py_DECREF(m); - return NULL; - } - - return m; -} diff --git a/Doc/includes/custom3.c b/Doc/includes/custom3.c deleted file mode 100644 index 5a47530f0a6b0d..00000000000000 --- a/Doc/includes/custom3.c +++ /dev/null @@ -1,189 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include -#include "structmember.h" - -typedef struct { - PyObject_HEAD - PyObject *first; /* first name */ - PyObject *last; /* last name */ - int number; -} CustomObject; - -static void -Custom_dealloc(CustomObject *self) -{ - Py_XDECREF(self->first); - Py_XDECREF(self->last); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -static PyObject * -Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CustomObject *self; - self = (CustomObject *) type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - self->number = 0; - } - return (PyObject *) self; -} - -static int -Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_DECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_DECREF(tmp); - } - return 0; -} - -static PyMemberDef Custom_members[] = { - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_getfirst(CustomObject *self, void *closure) -{ - Py_INCREF(self->first); - return self->first; -} - -static int -Custom_setfirst(CustomObject *self, PyObject *value, void *closure) -{ - PyObject *tmp; - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The first attribute value must be a string"); - return -1; - } - tmp = self->first; - Py_INCREF(value); - self->first = value; - Py_DECREF(tmp); - return 0; -} - -static PyObject * -Custom_getlast(CustomObject *self, void *closure) -{ - Py_INCREF(self->last); - return self->last; -} - -static int -Custom_setlast(CustomObject *self, PyObject *value, void *closure) -{ - PyObject *tmp; - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The last attribute value must be a string"); - return -1; - } - tmp = self->last; - Py_INCREF(value); - self->last = value; - Py_DECREF(tmp); - return 0; -} - -static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, - "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, - "last name", NULL}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyUnicode_FromFormat("%S %S", self->first, self->last); -} - -static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ -}; - -static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom3.Custom", - .tp_doc = PyDoc_STR("Custom objects"), - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_new = Custom_new, - .tp_init = (initproc) Custom_init, - .tp_dealloc = (destructor) Custom_dealloc, - .tp_members = Custom_members, - .tp_methods = Custom_methods, - .tp_getset = Custom_getsetters, -}; - -static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, - .m_name = "custom3", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_custom3(void) -{ - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - Py_INCREF(&CustomType); - if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(&CustomType); - Py_DECREF(m); - return NULL; - } - - return m; -} diff --git a/Doc/includes/custom4.c b/Doc/includes/custom4.c deleted file mode 100644 index c7ee55578488ed..00000000000000 --- a/Doc/includes/custom4.c +++ /dev/null @@ -1,203 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include -#include "structmember.h" - -typedef struct { - PyObject_HEAD - PyObject *first; /* first name */ - PyObject *last; /* last name */ - int number; -} CustomObject; - -static int -Custom_traverse(CustomObject *self, visitproc visit, void *arg) -{ - Py_VISIT(self->first); - Py_VISIT(self->last); - return 0; -} - -static int -Custom_clear(CustomObject *self) -{ - Py_CLEAR(self->first); - Py_CLEAR(self->last); - return 0; -} - -static void -Custom_dealloc(CustomObject *self) -{ - PyObject_GC_UnTrack(self); - Custom_clear(self); - Py_TYPE(self)->tp_free((PyObject *) self); -} - -static PyObject * -Custom_new(PyTypeObject *type, PyObject *args, PyObject *kwds) -{ - CustomObject *self; - self = (CustomObject *) type->tp_alloc(type, 0); - if (self != NULL) { - self->first = PyUnicode_FromString(""); - if (self->first == NULL) { - Py_DECREF(self); - return NULL; - } - self->last = PyUnicode_FromString(""); - if (self->last == NULL) { - Py_DECREF(self); - return NULL; - } - self->number = 0; - } - return (PyObject *) self; -} - -static int -Custom_init(CustomObject *self, PyObject *args, PyObject *kwds) -{ - static char *kwlist[] = {"first", "last", "number", NULL}; - PyObject *first = NULL, *last = NULL, *tmp; - - if (!PyArg_ParseTupleAndKeywords(args, kwds, "|UUi", kwlist, - &first, &last, - &self->number)) - return -1; - - if (first) { - tmp = self->first; - Py_INCREF(first); - self->first = first; - Py_DECREF(tmp); - } - if (last) { - tmp = self->last; - Py_INCREF(last); - self->last = last; - Py_DECREF(tmp); - } - return 0; -} - -static PyMemberDef Custom_members[] = { - {"number", T_INT, offsetof(CustomObject, number), 0, - "custom number"}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_getfirst(CustomObject *self, void *closure) -{ - Py_INCREF(self->first); - return self->first; -} - -static int -Custom_setfirst(CustomObject *self, PyObject *value, void *closure) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the first attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The first attribute value must be a string"); - return -1; - } - Py_INCREF(value); - Py_CLEAR(self->first); - self->first = value; - return 0; -} - -static PyObject * -Custom_getlast(CustomObject *self, void *closure) -{ - Py_INCREF(self->last); - return self->last; -} - -static int -Custom_setlast(CustomObject *self, PyObject *value, void *closure) -{ - if (value == NULL) { - PyErr_SetString(PyExc_TypeError, "Cannot delete the last attribute"); - return -1; - } - if (!PyUnicode_Check(value)) { - PyErr_SetString(PyExc_TypeError, - "The last attribute value must be a string"); - return -1; - } - Py_INCREF(value); - Py_CLEAR(self->last); - self->last = value; - return 0; -} - -static PyGetSetDef Custom_getsetters[] = { - {"first", (getter) Custom_getfirst, (setter) Custom_setfirst, - "first name", NULL}, - {"last", (getter) Custom_getlast, (setter) Custom_setlast, - "last name", NULL}, - {NULL} /* Sentinel */ -}; - -static PyObject * -Custom_name(CustomObject *self, PyObject *Py_UNUSED(ignored)) -{ - return PyUnicode_FromFormat("%S %S", self->first, self->last); -} - -static PyMethodDef Custom_methods[] = { - {"name", (PyCFunction) Custom_name, METH_NOARGS, - "Return the name, combining the first and last name" - }, - {NULL} /* Sentinel */ -}; - -static PyTypeObject CustomType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "custom4.Custom", - .tp_doc = PyDoc_STR("Custom objects"), - .tp_basicsize = sizeof(CustomObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | Py_TPFLAGS_HAVE_GC, - .tp_new = Custom_new, - .tp_init = (initproc) Custom_init, - .tp_dealloc = (destructor) Custom_dealloc, - .tp_traverse = (traverseproc) Custom_traverse, - .tp_clear = (inquiry) Custom_clear, - .tp_members = Custom_members, - .tp_methods = Custom_methods, - .tp_getset = Custom_getsetters, -}; - -static PyModuleDef custommodule = { - PyModuleDef_HEAD_INIT, - .m_name = "custom4", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_custom4(void) -{ - PyObject *m; - if (PyType_Ready(&CustomType) < 0) - return NULL; - - m = PyModule_Create(&custommodule); - if (m == NULL) - return NULL; - - Py_INCREF(&CustomType); - if (PyModule_AddObject(m, "Custom", (PyObject *) &CustomType) < 0) { - Py_DECREF(&CustomType); - Py_DECREF(m); - return NULL; - } - - return m; -} diff --git a/Doc/includes/dbpickle.py b/Doc/includes/dbpickle.py deleted file mode 100644 index b88ee87d872ab4..00000000000000 --- a/Doc/includes/dbpickle.py +++ /dev/null @@ -1,87 +0,0 @@ -# Simple example presenting how persistent ID can be used to pickle -# external objects by reference. - -import pickle -import sqlite3 -from collections import namedtuple - -# Simple class representing a record in our database. -MemoRecord = namedtuple("MemoRecord", "key, task") - -class DBPickler(pickle.Pickler): - - def persistent_id(self, obj): - # Instead of pickling MemoRecord as a regular class instance, we emit a - # persistent ID. - if isinstance(obj, MemoRecord): - # Here, our persistent ID is simply a tuple, containing a tag and a - # key, which refers to a specific record in the database. - return ("MemoRecord", obj.key) - else: - # If obj does not have a persistent ID, return None. This means obj - # needs to be pickled as usual. - return None - - -class DBUnpickler(pickle.Unpickler): - - def __init__(self, file, connection): - super().__init__(file) - self.connection = connection - - def persistent_load(self, pid): - # This method is invoked whenever a persistent ID is encountered. - # Here, pid is the tuple returned by DBPickler. - cursor = self.connection.cursor() - type_tag, key_id = pid - if type_tag == "MemoRecord": - # Fetch the referenced record from the database and return it. - cursor.execute("SELECT * FROM memos WHERE key=?", (str(key_id),)) - key, task = cursor.fetchone() - return MemoRecord(key, task) - else: - # Always raises an error if you cannot return the correct object. - # Otherwise, the unpickler will think None is the object referenced - # by the persistent ID. - raise pickle.UnpicklingError("unsupported persistent object") - - -def main(): - import io - import pprint - - # Initialize and populate our database. - conn = sqlite3.connect(":memory:") - cursor = conn.cursor() - cursor.execute("CREATE TABLE memos(key INTEGER PRIMARY KEY, task TEXT)") - tasks = ( - 'give food to fish', - 'prepare group meeting', - 'fight with a zebra', - ) - for task in tasks: - cursor.execute("INSERT INTO memos VALUES(NULL, ?)", (task,)) - - # Fetch the records to be pickled. - cursor.execute("SELECT * FROM memos") - memos = [MemoRecord(key, task) for key, task in cursor] - # Save the records using our custom DBPickler. - file = io.BytesIO() - DBPickler(file).dump(memos) - - print("Pickled records:") - pprint.pprint(memos) - - # Update a record, just for good measure. - cursor.execute("UPDATE memos SET task='learn italian' WHERE key=1") - - # Load the records from the pickle data stream. - file.seek(0) - memos = DBUnpickler(file, conn).load() - - print("Unpickled records:") - pprint.pprint(memos) - - -if __name__ == '__main__': - main() diff --git a/Doc/includes/email-alternative.py b/Doc/includes/email-alternative.py deleted file mode 100644 index df7ca6f3faa332..00000000000000 --- a/Doc/includes/email-alternative.py +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env python3 - -import smtplib - -from email.message import EmailMessage -from email.headerregistry import Address -from email.utils import make_msgid - -# Create the base text message. -msg = EmailMessage() -msg['Subject'] = "Ayons asperges pour le déjeuner" -msg['From'] = Address("Pepé Le Pew", "pepe", "example.com") -msg['To'] = (Address("Penelope Pussycat", "penelope", "example.com"), - Address("Fabrette Pussycat", "fabrette", "example.com")) -msg.set_content("""\ -Salut! - -Cela ressemble à un excellent recipie[1] déjeuner. - -[1] http://www.yummly.com/recipe/Roasted-Asparagus-Epicurious-203718 - ---Pepé -""") - -# Add the html version. This converts the message into a multipart/alternative -# container, with the original text message as the first part and the new html -# message as the second part. -asparagus_cid = make_msgid() -msg.add_alternative("""\ - - - -

Salut!

-

Cela ressemble à un excellent - - recipie - déjeuner. -

- - - -""".format(asparagus_cid=asparagus_cid[1:-1]), subtype='html') -# note that we needed to peel the <> off the msgid for use in the html. - -# Now add the related image to the html part. -with open("roasted-asparagus.jpg", 'rb') as img: - msg.get_payload()[1].add_related(img.read(), 'image', 'jpeg', - cid=asparagus_cid) - -# Make a local copy of what we are going to send. -with open('outgoing.msg', 'wb') as f: - f.write(bytes(msg)) - -# Send the message via local SMTP server. -with smtplib.SMTP('localhost') as s: - s.send_message(msg) diff --git a/Doc/includes/email-dir.py b/Doc/includes/email-dir.py deleted file mode 100644 index 2fc1570e654db6..00000000000000 --- a/Doc/includes/email-dir.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 - -"""Send the contents of a directory as a MIME message.""" - -import os -import smtplib -# For guessing MIME type based on file name extension -import mimetypes - -from argparse import ArgumentParser - -from email.message import EmailMessage -from email.policy import SMTP - - -def main(): - parser = ArgumentParser(description="""\ -Send the contents of a directory as a MIME message. -Unless the -o option is given, the email is sent by forwarding to your local -SMTP server, which then does the normal delivery process. Your local machine -must be running an SMTP server. -""") - parser.add_argument('-d', '--directory', - help="""Mail the contents of the specified directory, - otherwise use the current directory. Only the regular - files in the directory are sent, and we don't recurse to - subdirectories.""") - parser.add_argument('-o', '--output', - metavar='FILE', - help="""Print the composed message to FILE instead of - sending the message to the SMTP server.""") - parser.add_argument('-s', '--sender', required=True, - help='The value of the From: header (required)') - parser.add_argument('-r', '--recipient', required=True, - action='append', metavar='RECIPIENT', - default=[], dest='recipients', - help='A To: header value (at least one required)') - args = parser.parse_args() - directory = args.directory - if not directory: - directory = '.' - # Create the message - msg = EmailMessage() - msg['Subject'] = f'Contents of directory {os.path.abspath(directory)}' - msg['To'] = ', '.join(args.recipients) - msg['From'] = args.sender - msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' - - for filename in os.listdir(directory): - path = os.path.join(directory, filename) - if not os.path.isfile(path): - continue - # Guess the content type based on the file's extension. Encoding - # will be ignored, although we should check for simple things like - # gzip'd or compressed files. - ctype, encoding = mimetypes.guess_type(path) - if ctype is None or encoding is not None: - # No guess could be made, or the file is encoded (compressed), so - # use a generic bag-of-bits type. - ctype = 'application/octet-stream' - maintype, subtype = ctype.split('/', 1) - with open(path, 'rb') as fp: - msg.add_attachment(fp.read(), - maintype=maintype, - subtype=subtype, - filename=filename) - # Now send or store the message - if args.output: - with open(args.output, 'wb') as fp: - fp.write(msg.as_bytes(policy=SMTP)) - else: - with smtplib.SMTP('localhost') as s: - s.send_message(msg) - - -if __name__ == '__main__': - main() diff --git a/Doc/includes/email-headers.py b/Doc/includes/email-headers.py deleted file mode 100644 index 2c421451a8e5a8..00000000000000 --- a/Doc/includes/email-headers.py +++ /dev/null @@ -1,24 +0,0 @@ -# Import the email modules we'll need -from email.parser import BytesParser, Parser -from email.policy import default - -# If the e-mail headers are in a file, uncomment these two lines: -# with open(messagefile, 'rb') as fp: -# headers = BytesParser(policy=default).parse(fp) - -# Or for parsing headers in a string (this is an uncommon operation), use: -headers = Parser(policy=default).parsestr( - 'From: Foo Bar \n' - 'To: \n' - 'Subject: Test message\n' - '\n' - 'Body would go here\n') - -# Now the header items can be accessed as a dictionary: -print('To: {}'.format(headers['to'])) -print('From: {}'.format(headers['from'])) -print('Subject: {}'.format(headers['subject'])) - -# You can also access the parts of the addresses: -print('Recipient username: {}'.format(headers['to'].addresses[0].username)) -print('Sender name: {}'.format(headers['from'].addresses[0].display_name)) diff --git a/Doc/includes/email-mime.py b/Doc/includes/email-mime.py deleted file mode 100644 index 34c6bdb60fff24..00000000000000 --- a/Doc/includes/email-mime.py +++ /dev/null @@ -1,26 +0,0 @@ -# Import smtplib for the actual sending function. -import smtplib - -# Here are the email package modules we'll need. -from email.message import EmailMessage - -# Create the container email message. -msg = EmailMessage() -msg['Subject'] = 'Our family reunion' -# me == the sender's email address -# family = the list of all recipients' email addresses -msg['From'] = me -msg['To'] = ', '.join(family) -msg.preamble = 'You will not see this in a MIME-aware mail reader.\n' - -# Open the files in binary mode. You can also omit the subtype -# if you want MIMEImage to guess it. -for file in pngfiles: - with open(file, 'rb') as fp: - img_data = fp.read() - msg.add_attachment(img_data, maintype='image', - subtype='png') - -# Send the email via our own SMTP server. -with smtplib.SMTP('localhost') as s: - s.send_message(msg) diff --git a/Doc/includes/email-read-alternative.py b/Doc/includes/email-read-alternative.py deleted file mode 100644 index 8d0b4e6eb6b6b5..00000000000000 --- a/Doc/includes/email-read-alternative.py +++ /dev/null @@ -1,79 +0,0 @@ -import os -import sys -import tempfile -import mimetypes -import webbrowser - -# Import the email modules we'll need -from email import policy -from email.parser import BytesParser - - -def magic_html_parser(html_text, partfiles): - """Return safety-sanitized html linked to partfiles. - - Rewrite the href="cid:...." attributes to point to the filenames in partfiles. - Though not trivial, this should be possible using html.parser. - """ - raise NotImplementedError("Add the magic needed") - - -# In a real program you'd get the filename from the arguments. -with open('outgoing.msg', 'rb') as fp: - msg = BytesParser(policy=policy.default).parse(fp) - -# Now the header items can be accessed as a dictionary, and any non-ASCII will -# be converted to unicode: -print('To:', msg['to']) -print('From:', msg['from']) -print('Subject:', msg['subject']) - -# If we want to print a preview of the message content, we can extract whatever -# the least formatted payload is and print the first three lines. Of course, -# if the message has no plain text part printing the first three lines of html -# is probably useless, but this is just a conceptual example. -simplest = msg.get_body(preferencelist=('plain', 'html')) -print() -print(''.join(simplest.get_content().splitlines(keepends=True)[:3])) - -ans = input("View full message?") -if ans.lower()[0] == 'n': - sys.exit() - -# We can extract the richest alternative in order to display it: -richest = msg.get_body() -partfiles = {} -if richest['content-type'].maintype == 'text': - if richest['content-type'].subtype == 'plain': - for line in richest.get_content().splitlines(): - print(line) - sys.exit() - elif richest['content-type'].subtype == 'html': - body = richest - else: - print("Don't know how to display {}".format(richest.get_content_type())) - sys.exit() -elif richest['content-type'].content_type == 'multipart/related': - body = richest.get_body(preferencelist=('html')) - for part in richest.iter_attachments(): - fn = part.get_filename() - if fn: - extension = os.path.splitext(part.get_filename())[1] - else: - extension = mimetypes.guess_extension(part.get_content_type()) - with tempfile.NamedTemporaryFile(suffix=extension, delete=False) as f: - f.write(part.get_content()) - # again strip the <> to go from email form of cid to html form. - partfiles[part['content-id'][1:-1]] = f.name -else: - print("Don't know how to display {}".format(richest.get_content_type())) - sys.exit() -with tempfile.NamedTemporaryFile(mode='w', delete=False) as f: - f.write(magic_html_parser(body.get_content(), partfiles)) -webbrowser.open(f.name) -os.remove(f.name) -for fn in partfiles.values(): - os.remove(fn) - -# Of course, there are lots of email messages that could break this simple -# minded program, but it will handle the most common ones. diff --git a/Doc/includes/email-simple.py b/Doc/includes/email-simple.py deleted file mode 100644 index 07dc30fd066eac..00000000000000 --- a/Doc/includes/email-simple.py +++ /dev/null @@ -1,22 +0,0 @@ -# Import smtplib for the actual sending function -import smtplib - -# Import the email modules we'll need -from email.message import EmailMessage - -# Open the plain text file whose name is in textfile for reading. -with open(textfile) as fp: - # Create a text/plain message - msg = EmailMessage() - msg.set_content(fp.read()) - -# me == the sender's email address -# you == the recipient's email address -msg['Subject'] = f'The contents of {textfile}' -msg['From'] = me -msg['To'] = you - -# Send the message via our own SMTP server. -s = smtplib.SMTP('localhost') -s.send_message(msg) -s.quit() diff --git a/Doc/includes/email-unpack.py b/Doc/includes/email-unpack.py deleted file mode 100644 index c8cb0be4560830..00000000000000 --- a/Doc/includes/email-unpack.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python3 - -"""Unpack a MIME message into a directory of files.""" - -import os -import email -import mimetypes - -from email.policy import default - -from argparse import ArgumentParser - - -def main(): - parser = ArgumentParser(description="""\ -Unpack a MIME message into a directory of files. -""") - parser.add_argument('-d', '--directory', required=True, - help="""Unpack the MIME message into the named - directory, which will be created if it doesn't already - exist.""") - parser.add_argument('msgfile') - args = parser.parse_args() - - with open(args.msgfile, 'rb') as fp: - msg = email.message_from_binary_file(fp, policy=default) - - try: - os.mkdir(args.directory) - except FileExistsError: - pass - - counter = 1 - for part in msg.walk(): - # multipart/* are just containers - if part.get_content_maintype() == 'multipart': - continue - # Applications should really sanitize the given filename so that an - # email message can't be used to overwrite important files - filename = part.get_filename() - if not filename: - ext = mimetypes.guess_extension(part.get_content_type()) - if not ext: - # Use a generic bag-of-bits extension - ext = '.bin' - filename = f'part-{counter:03d}{ext}' - counter += 1 - with open(os.path.join(args.directory, filename), 'wb') as fp: - fp.write(part.get_payload(decode=True)) - - -if __name__ == '__main__': - main() diff --git a/Doc/includes/minidom-example.py b/Doc/includes/minidom-example.py deleted file mode 100644 index 3b9e9ee2db2913..00000000000000 --- a/Doc/includes/minidom-example.py +++ /dev/null @@ -1,64 +0,0 @@ -import xml.dom.minidom - -document = """\ - -Demo slideshow -Slide title -This is a demo -Of a program for processing slides - - -Another demo slide -It is important -To have more than -one slide - - -""" - -dom = xml.dom.minidom.parseString(document) - -def getText(nodelist): - rc = [] - for node in nodelist: - if node.nodeType == node.TEXT_NODE: - rc.append(node.data) - return ''.join(rc) - -def handleSlideshow(slideshow): - print("") - handleSlideshowTitle(slideshow.getElementsByTagName("title")[0]) - slides = slideshow.getElementsByTagName("slide") - handleToc(slides) - handleSlides(slides) - print("") - -def handleSlides(slides): - for slide in slides: - handleSlide(slide) - -def handleSlide(slide): - handleSlideTitle(slide.getElementsByTagName("title")[0]) - handlePoints(slide.getElementsByTagName("point")) - -def handleSlideshowTitle(title): - print(f"{getText(title.childNodes)}") - -def handleSlideTitle(title): - print(f"

{getText(title.childNodes)}

") - -def handlePoints(points): - print("
    ") - for point in points: - handlePoint(point) - print("
") - -def handlePoint(point): - print(f"
  • {getText(point.childNodes)}
    • ") - -def handleToc(slides): - for slide in slides: - title = slide.getElementsByTagName("title")[0] - print(f"

    {getText(title.childNodes)}

    ") - -handleSlideshow(dom) diff --git a/Doc/includes/mp_newtype.py b/Doc/includes/mp_newtype.py deleted file mode 100644 index 1c796a564049ad..00000000000000 --- a/Doc/includes/mp_newtype.py +++ /dev/null @@ -1,89 +0,0 @@ -from multiprocessing import freeze_support -from multiprocessing.managers import BaseManager, BaseProxy -import operator - -## - -class Foo: - def f(self): - print('you called Foo.f()') - def g(self): - print('you called Foo.g()') - def _h(self): - print('you called Foo._h()') - -# A simple generator function -def baz(): - for i in range(10): - yield i*i - -# Proxy type for generator objects -class GeneratorProxy(BaseProxy): - _exposed_ = ['__next__'] - def __iter__(self): - return self - def __next__(self): - return self._callmethod('__next__') - -# Function to return the operator module -def get_operator_module(): - return operator - -## - -class MyManager(BaseManager): - pass - -# register the Foo class; make `f()` and `g()` accessible via proxy -MyManager.register('Foo1', Foo) - -# register the Foo class; make `g()` and `_h()` accessible via proxy -MyManager.register('Foo2', Foo, exposed=('g', '_h')) - -# register the generator function baz; use `GeneratorProxy` to make proxies -MyManager.register('baz', baz, proxytype=GeneratorProxy) - -# register get_operator_module(); make public functions accessible via proxy -MyManager.register('operator', get_operator_module) - -## - -def test(): - manager = MyManager() - manager.start() - - print('-' * 20) - - f1 = manager.Foo1() - f1.f() - f1.g() - assert not hasattr(f1, '_h') - assert sorted(f1._exposed_) == sorted(['f', 'g']) - - print('-' * 20) - - f2 = manager.Foo2() - f2.g() - f2._h() - assert not hasattr(f2, 'f') - assert sorted(f2._exposed_) == sorted(['g', '_h']) - - print('-' * 20) - - it = manager.baz() - for i in it: - print('<%d>' % i, end=' ') - print() - - print('-' * 20) - - op = manager.operator() - print('op.add(23, 45) =', op.add(23, 45)) - print('op.pow(2, 94) =', op.pow(2, 94)) - print('op._exposed_ =', op._exposed_) - -## - -if __name__ == '__main__': - freeze_support() - test() diff --git a/Doc/includes/mp_pool.py b/Doc/includes/mp_pool.py deleted file mode 100644 index 11101e1a90954e..00000000000000 --- a/Doc/includes/mp_pool.py +++ /dev/null @@ -1,153 +0,0 @@ -import multiprocessing -import time -import random -import sys - -# -# Functions used by test code -# - -def calculate(func, args): - result = func(*args) - return '%s says that %s%s = %s' % ( - multiprocessing.current_process().name, - func.__name__, args, result - ) - -def calculatestar(args): - return calculate(*args) - -def mul(a, b): - time.sleep(0.5 * random.random()) - return a * b - -def plus(a, b): - time.sleep(0.5 * random.random()) - return a + b - -def f(x): - return 1.0 / (x - 5.0) - -def pow3(x): - return x ** 3 - -def noop(x): - pass - -# -# Test code -# - -def test(): - PROCESSES = 4 - print('Creating pool with %d processes\n' % PROCESSES) - - with multiprocessing.Pool(PROCESSES) as pool: - # - # Tests - # - - TASKS = [(mul, (i, 7)) for i in range(10)] + \ - [(plus, (i, 8)) for i in range(10)] - - results = [pool.apply_async(calculate, t) for t in TASKS] - imap_it = pool.imap(calculatestar, TASKS) - imap_unordered_it = pool.imap_unordered(calculatestar, TASKS) - - print('Ordered results using pool.apply_async():') - for r in results: - print('\t', r.get()) - print() - - print('Ordered results using pool.imap():') - for x in imap_it: - print('\t', x) - print() - - print('Unordered results using pool.imap_unordered():') - for x in imap_unordered_it: - print('\t', x) - print() - - print('Ordered results using pool.map() --- will block till complete:') - for x in pool.map(calculatestar, TASKS): - print('\t', x) - print() - - # - # Test error handling - # - - print('Testing error handling:') - - try: - print(pool.apply(f, (5,))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from pool.apply()') - else: - raise AssertionError('expected ZeroDivisionError') - - try: - print(pool.map(f, list(range(10)))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from pool.map()') - else: - raise AssertionError('expected ZeroDivisionError') - - try: - print(list(pool.imap(f, list(range(10))))) - except ZeroDivisionError: - print('\tGot ZeroDivisionError as expected from list(pool.imap())') - else: - raise AssertionError('expected ZeroDivisionError') - - it = pool.imap(f, list(range(10))) - for i in range(10): - try: - x = next(it) - except ZeroDivisionError: - if i == 5: - pass - except StopIteration: - break - else: - if i == 5: - raise AssertionError('expected ZeroDivisionError') - - assert i == 9 - print('\tGot ZeroDivisionError as expected from IMapIterator.next()') - print() - - # - # Testing timeouts - # - - print('Testing ApplyResult.get() with timeout:', end=' ') - res = pool.apply_async(calculate, TASKS[0]) - while 1: - sys.stdout.flush() - try: - sys.stdout.write('\n\t%s' % res.get(0.02)) - break - except multiprocessing.TimeoutError: - sys.stdout.write('.') - print() - print() - - print('Testing IMapIterator.next() with timeout:', end=' ') - it = pool.imap(calculatestar, TASKS) - while 1: - sys.stdout.flush() - try: - sys.stdout.write('\n\t%s' % it.next(0.02)) - except StopIteration: - break - except multiprocessing.TimeoutError: - sys.stdout.write('.') - print() - print() - - -if __name__ == '__main__': - multiprocessing.freeze_support() - test() diff --git a/Doc/includes/mp_workers.py b/Doc/includes/mp_workers.py deleted file mode 100644 index 3b92269f897e93..00000000000000 --- a/Doc/includes/mp_workers.py +++ /dev/null @@ -1,77 +0,0 @@ -import time -import random - -from multiprocessing import Process, Queue, current_process, freeze_support - -# -# Function run by worker processes -# - -def worker(input, output): - for func, args in iter(input.get, 'STOP'): - result = calculate(func, args) - output.put(result) - -# -# Function used to calculate result -# - -def calculate(func, args): - result = func(*args) - return '%s says that %s%s = %s' % \ - (current_process().name, func.__name__, args, result) - -# -# Functions referenced by tasks -# - -def mul(a, b): - time.sleep(0.5*random.random()) - return a * b - -def plus(a, b): - time.sleep(0.5*random.random()) - return a + b - -# -# -# - -def test(): - NUMBER_OF_PROCESSES = 4 - TASKS1 = [(mul, (i, 7)) for i in range(20)] - TASKS2 = [(plus, (i, 8)) for i in range(10)] - - # Create queues - task_queue = Queue() - done_queue = Queue() - - # Submit tasks - for task in TASKS1: - task_queue.put(task) - - # Start worker processes - for i in range(NUMBER_OF_PROCESSES): - Process(target=worker, args=(task_queue, done_queue)).start() - - # Get and print results - print('Unordered results:') - for i in range(len(TASKS1)): - print('\t', done_queue.get()) - - # Add more tasks using `put()` - for task in TASKS2: - task_queue.put(task) - - # Get and print some more results - for i in range(len(TASKS2)): - print('\t', done_queue.get()) - - # Tell child processes to stop - for i in range(NUMBER_OF_PROCESSES): - task_queue.put('STOP') - - -if __name__ == '__main__': - freeze_support() - test() diff --git a/Doc/includes/run-func.c b/Doc/includes/run-func.c deleted file mode 100644 index 392f86d65ecc17..00000000000000 --- a/Doc/includes/run-func.c +++ /dev/null @@ -1,71 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include - -int -main(int argc, char *argv[]) -{ - PyObject *pName, *pModule, *pFunc; - PyObject *pArgs, *pValue; - int i; - - if (argc < 3) { - fprintf(stderr,"Usage: call pythonfile funcname [args]\n"); - return 1; - } - - Py_Initialize(); - pName = PyUnicode_DecodeFSDefault(argv[1]); - /* Error checking of pName left out */ - - pModule = PyImport_Import(pName); - Py_DECREF(pName); - - if (pModule != NULL) { - pFunc = PyObject_GetAttrString(pModule, argv[2]); - /* pFunc is a new reference */ - - if (pFunc && PyCallable_Check(pFunc)) { - pArgs = PyTuple_New(argc - 3); - for (i = 0; i < argc - 3; ++i) { - pValue = PyLong_FromLong(atoi(argv[i + 3])); - if (!pValue) { - Py_DECREF(pArgs); - Py_DECREF(pModule); - fprintf(stderr, "Cannot convert argument\n"); - return 1; - } - /* pValue reference stolen here: */ - PyTuple_SetItem(pArgs, i, pValue); - } - pValue = PyObject_CallObject(pFunc, pArgs); - Py_DECREF(pArgs); - if (pValue != NULL) { - printf("Result of call: %ld\n", PyLong_AsLong(pValue)); - Py_DECREF(pValue); - } - else { - Py_DECREF(pFunc); - Py_DECREF(pModule); - PyErr_Print(); - fprintf(stderr,"Call failed\n"); - return 1; - } - } - else { - if (PyErr_Occurred()) - PyErr_Print(); - fprintf(stderr, "Cannot find function \"%s\"\n", argv[2]); - } - Py_XDECREF(pFunc); - Py_DECREF(pModule); - } - else { - PyErr_Print(); - fprintf(stderr, "Failed to load \"%s\"\n", argv[1]); - return 1; - } - if (Py_FinalizeEx() < 0) { - return 120; - } - return 0; -} diff --git a/Doc/includes/setup.py b/Doc/includes/setup.py deleted file mode 100644 index a38a39de3e7c86..00000000000000 --- a/Doc/includes/setup.py +++ /dev/null @@ -1,9 +0,0 @@ -from distutils.core import setup, Extension -setup(name="noddy", version="1.0", - ext_modules=[ - Extension("noddy", ["noddy.c"]), - Extension("noddy2", ["noddy2.c"]), - Extension("noddy3", ["noddy3.c"]), - Extension("noddy4", ["noddy4.c"]), - Extension("shoddy", ["shoddy.c"]), - ]) diff --git a/Doc/includes/sqlite3/pysqlite_datetime.py b/Doc/includes/sqlite3/pysqlite_datetime.py deleted file mode 100644 index 5d843f906b3062..00000000000000 --- a/Doc/includes/sqlite3/pysqlite_datetime.py +++ /dev/null @@ -1,22 +0,0 @@ -import sqlite3 -import datetime - -con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_DECLTYPES|sqlite3.PARSE_COLNAMES) -cur = con.cursor() -cur.execute("create table test(d date, ts timestamp)") - -today = datetime.date.today() -now = datetime.datetime.now() - -cur.execute("insert into test(d, ts) values (?, ?)", (today, now)) -cur.execute("select d, ts from test") -row = cur.fetchone() -print(today, "=>", row[0], type(row[0])) -print(now, "=>", row[1], type(row[1])) - -cur.execute('select current_date as "d [date]", current_timestamp as "ts [timestamp]"') -row = cur.fetchone() -print("current_date", row[0], type(row[0])) -print("current_timestamp", row[1], type(row[1])) - -con.close() diff --git a/Doc/includes/sublist.c b/Doc/includes/sublist.c deleted file mode 100644 index b36dadf07eae87..00000000000000 --- a/Doc/includes/sublist.c +++ /dev/null @@ -1,69 +0,0 @@ -#define PY_SSIZE_T_CLEAN -#include - -typedef struct { - PyListObject list; - int state; -} SubListObject; - -static PyObject * -SubList_increment(SubListObject *self, PyObject *unused) -{ - self->state++; - return PyLong_FromLong(self->state); -} - -static PyMethodDef SubList_methods[] = { - {"increment", (PyCFunction) SubList_increment, METH_NOARGS, - PyDoc_STR("increment state counter")}, - {NULL}, -}; - -static int -SubList_init(SubListObject *self, PyObject *args, PyObject *kwds) -{ - if (PyList_Type.tp_init((PyObject *) self, args, kwds) < 0) - return -1; - self->state = 0; - return 0; -} - -static PyTypeObject SubListType = { - PyVarObject_HEAD_INIT(NULL, 0) - .tp_name = "sublist.SubList", - .tp_doc = PyDoc_STR("SubList objects"), - .tp_basicsize = sizeof(SubListObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, - .tp_init = (initproc) SubList_init, - .tp_methods = SubList_methods, -}; - -static PyModuleDef sublistmodule = { - PyModuleDef_HEAD_INIT, - .m_name = "sublist", - .m_doc = "Example module that creates an extension type.", - .m_size = -1, -}; - -PyMODINIT_FUNC -PyInit_sublist(void) -{ - PyObject *m; - SubListType.tp_base = &PyList_Type; - if (PyType_Ready(&SubListType) < 0) - return NULL; - - m = PyModule_Create(&sublistmodule); - if (m == NULL) - return NULL; - - Py_INCREF(&SubListType); - if (PyModule_AddObject(m, "SubList", (PyObject *) &SubListType) < 0) { - Py_DECREF(&SubListType); - Py_DECREF(m); - return NULL; - } - - return m; -} diff --git a/Doc/includes/test.py b/Doc/includes/test.py deleted file mode 100644 index 09ebe3fec0bdbe..00000000000000 --- a/Doc/includes/test.py +++ /dev/null @@ -1,199 +0,0 @@ -"""Test module for the custom examples - -Custom 1: - ->>> import custom ->>> c1 = custom.Custom() ->>> c2 = custom.Custom() ->>> del c1 ->>> del c2 - - -Custom 2 - ->>> import custom2 ->>> c1 = custom2.Custom('jim', 'fulton', 42) ->>> c1.first -'jim' ->>> c1.last -'fulton' ->>> c1.number -42 ->>> c1.name() -'jim fulton' ->>> c1.first = 'will' ->>> c1.name() -'will fulton' ->>> c1.last = 'tell' ->>> c1.name() -'will tell' ->>> del c1.first ->>> c1.name() -Traceback (most recent call last): -... -AttributeError: first ->>> c1.first -Traceback (most recent call last): -... -AttributeError: first ->>> c1.first = 'drew' ->>> c1.first -'drew' ->>> del c1.number -Traceback (most recent call last): -... -TypeError: can't delete numeric/char attribute ->>> c1.number=2 ->>> c1.number -2 ->>> c1.first = 42 ->>> c1.name() -'42 tell' ->>> c2 = custom2.Custom() ->>> c2.name() -' ' ->>> c2.first -'' ->>> c2.last -'' ->>> del c2.first ->>> c2.first -Traceback (most recent call last): -... -AttributeError: first ->>> c2.first -Traceback (most recent call last): -... -AttributeError: first ->>> c2.name() -Traceback (most recent call last): - File "", line 1, in ? -AttributeError: first ->>> c2.number -0 ->>> n3 = custom2.Custom('jim', 'fulton', 'waaa') -Traceback (most recent call last): - File "", line 1, in ? -TypeError: an integer is required (got type str) ->>> del c1 ->>> del c2 - - -Custom 3 - ->>> import custom3 ->>> c1 = custom3.Custom('jim', 'fulton', 42) ->>> c1 = custom3.Custom('jim', 'fulton', 42) ->>> c1.name() -'jim fulton' ->>> del c1.first -Traceback (most recent call last): - File "", line 1, in ? -TypeError: Cannot delete the first attribute ->>> c1.first = 42 -Traceback (most recent call last): - File "", line 1, in ? -TypeError: The first attribute value must be a string ->>> c1.first = 'will' ->>> c1.name() -'will fulton' ->>> c2 = custom3.Custom() ->>> c2 = custom3.Custom() ->>> c2 = custom3.Custom() ->>> n3 = custom3.Custom('jim', 'fulton', 'waaa') -Traceback (most recent call last): - File "", line 1, in ? -TypeError: an integer is required (got type str) ->>> del c1 ->>> del c2 - -Custom 4 - ->>> import custom4 ->>> c1 = custom4.Custom('jim', 'fulton', 42) ->>> c1.first -'jim' ->>> c1.last -'fulton' ->>> c1.number -42 ->>> c1.name() -'jim fulton' ->>> c1.first = 'will' ->>> c1.name() -'will fulton' ->>> c1.last = 'tell' ->>> c1.name() -'will tell' ->>> del c1.first -Traceback (most recent call last): -... -TypeError: Cannot delete the first attribute ->>> c1.name() -'will tell' ->>> c1.first = 'drew' ->>> c1.first -'drew' ->>> del c1.number -Traceback (most recent call last): -... -TypeError: can't delete numeric/char attribute ->>> c1.number=2 ->>> c1.number -2 ->>> c1.first = 42 -Traceback (most recent call last): -... -TypeError: The first attribute value must be a string ->>> c1.name() -'drew tell' ->>> c2 = custom4.Custom() ->>> c2 = custom4.Custom() ->>> c2 = custom4.Custom() ->>> c2 = custom4.Custom() ->>> c2.name() -' ' ->>> c2.first -'' ->>> c2.last -'' ->>> c2.number -0 ->>> n3 = custom4.Custom('jim', 'fulton', 'waaa') -Traceback (most recent call last): -... -TypeError: an integer is required (got type str) - - -Test cyclic gc(?) - ->>> import gc ->>> gc.disable() - ->>> class Subclass(custom4.Custom): pass -... ->>> s = Subclass() ->>> s.cycle = [s] ->>> s.cycle.append(s.cycle) ->>> x = object() ->>> s.x = x ->>> del s ->>> sys.getrefcount(x) -3 ->>> ignore = gc.collect() ->>> sys.getrefcount(x) -2 - ->>> gc.enable() -""" - -import os -import sys -from distutils.util import get_platform -PLAT_SPEC = "%s-%d.%d" % (get_platform(), *sys.version_info[:2]) -src = os.path.join("build", "lib.%s" % PLAT_SPEC) -sys.path.append(src) - -if __name__ == "__main__": - import doctest, __main__ - doctest.testmod(__main__) diff --git a/Doc/includes/typestruct.h b/Doc/includes/typestruct.h deleted file mode 100644 index 02f8ccfe4438a5..00000000000000 --- a/Doc/includes/typestruct.h +++ /dev/null @@ -1,83 +0,0 @@ -typedef struct _typeobject { - PyObject_VAR_HEAD - const char *tp_name; /* For printing, in format "." */ - Py_ssize_t tp_basicsize, tp_itemsize; /* For allocation */ - - /* Methods to implement standard operations */ - - destructor tp_dealloc; - Py_ssize_t tp_vectorcall_offset; - getattrfunc tp_getattr; - setattrfunc tp_setattr; - PyAsyncMethods *tp_as_async; /* formerly known as tp_compare (Python 2) - or tp_reserved (Python 3) */ - reprfunc tp_repr; - - /* Method suites for standard classes */ - - PyNumberMethods *tp_as_number; - PySequenceMethods *tp_as_sequence; - PyMappingMethods *tp_as_mapping; - - /* More standard operations (here for binary compatibility) */ - - hashfunc tp_hash; - ternaryfunc tp_call; - reprfunc tp_str; - getattrofunc tp_getattro; - setattrofunc tp_setattro; - - /* Functions to access object as input/output buffer */ - PyBufferProcs *tp_as_buffer; - - /* Flags to define presence of optional/expanded features */ - unsigned long tp_flags; - - const char *tp_doc; /* Documentation string */ - - /* Assigned meaning in release 2.0 */ - /* call function for all accessible objects */ - traverseproc tp_traverse; - - /* delete references to contained objects */ - inquiry tp_clear; - - /* Assigned meaning in release 2.1 */ - /* rich comparisons */ - richcmpfunc tp_richcompare; - - /* weak reference enabler */ - Py_ssize_t tp_weaklistoffset; - - /* Iterators */ - getiterfunc tp_iter; - iternextfunc tp_iternext; - - /* Attribute descriptor and subclassing stuff */ - struct PyMethodDef *tp_methods; - struct PyMemberDef *tp_members; - struct PyGetSetDef *tp_getset; - // Strong reference on a heap type, borrowed reference on a static type - struct _typeobject *tp_base; - PyObject *tp_dict; - descrgetfunc tp_descr_get; - descrsetfunc tp_descr_set; - Py_ssize_t tp_dictoffset; - initproc tp_init; - allocfunc tp_alloc; - newfunc tp_new; - freefunc tp_free; /* Low-level free-memory routine */ - inquiry tp_is_gc; /* For PyObject_IS_GC */ - PyObject *tp_bases; - PyObject *tp_mro; /* method resolution order */ - PyObject *tp_cache; - PyObject *tp_subclasses; - PyObject *tp_weaklist; - destructor tp_del; - - /* Type attribute cache version tag. Added in version 2.6 */ - unsigned int tp_version_tag; - - destructor tp_finalize; - vectorcallfunc tp_vectorcall; -} PyTypeObject; diff --git a/Doc/includes/tzinfo_examples.py b/Doc/includes/tzinfo_examples.py deleted file mode 100644 index 1fa6e615e46a76..00000000000000 --- a/Doc/includes/tzinfo_examples.py +++ /dev/null @@ -1,175 +0,0 @@ -from datetime import tzinfo, timedelta, datetime - -ZERO = timedelta(0) -HOUR = timedelta(hours=1) -SECOND = timedelta(seconds=1) - -# A class capturing the platform's idea of local time. -# (May result in wrong values on historical times in -# timezones where UTC offset and/or the DST rules had -# changed in the past.) -import time as _time - -STDOFFSET = timedelta(seconds = -_time.timezone) -if _time.daylight: - DSTOFFSET = timedelta(seconds = -_time.altzone) -else: - DSTOFFSET = STDOFFSET - -DSTDIFF = DSTOFFSET - STDOFFSET - -class LocalTimezone(tzinfo): - - def fromutc(self, dt): - assert dt.tzinfo is self - stamp = (dt - datetime(1970, 1, 1, tzinfo=self)) // SECOND - args = _time.localtime(stamp)[:6] - dst_diff = DSTDIFF // SECOND - # Detect fold - fold = (args == _time.localtime(stamp - dst_diff)) - return datetime(*args, microsecond=dt.microsecond, - tzinfo=self, fold=fold) - - def utcoffset(self, dt): - if self._isdst(dt): - return DSTOFFSET - else: - return STDOFFSET - - def dst(self, dt): - if self._isdst(dt): - return DSTDIFF - else: - return ZERO - - def tzname(self, dt): - return _time.tzname[self._isdst(dt)] - - def _isdst(self, dt): - tt = (dt.year, dt.month, dt.day, - dt.hour, dt.minute, dt.second, - dt.weekday(), 0, 0) - stamp = _time.mktime(tt) - tt = _time.localtime(stamp) - return tt.tm_isdst > 0 - -Local = LocalTimezone() - - -# A complete implementation of current DST rules for major US time zones. - -def first_sunday_on_or_after(dt): - days_to_go = 6 - dt.weekday() - if days_to_go: - dt += timedelta(days_to_go) - return dt - - -# US DST Rules -# -# This is a simplified (i.e., wrong for a few cases) set of rules for US -# DST start and end times. For a complete and up-to-date set of DST rules -# and timezone definitions, visit the Olson Database (or try pytz): -# http://www.twinsun.com/tz/tz-link.htm -# https://sourceforge.net/projects/pytz/ (might not be up-to-date) -# -# In the US, since 2007, DST starts at 2am (standard time) on the second -# Sunday in March, which is the first Sunday on or after Mar 8. -DSTSTART_2007 = datetime(1, 3, 8, 2) -# and ends at 2am (DST time) on the first Sunday of Nov. -DSTEND_2007 = datetime(1, 11, 1, 2) -# From 1987 to 2006, DST used to start at 2am (standard time) on the first -# Sunday in April and to end at 2am (DST time) on the last -# Sunday of October, which is the first Sunday on or after Oct 25. -DSTSTART_1987_2006 = datetime(1, 4, 1, 2) -DSTEND_1987_2006 = datetime(1, 10, 25, 2) -# From 1967 to 1986, DST used to start at 2am (standard time) on the last -# Sunday in April (the one on or after April 24) and to end at 2am (DST time) -# on the last Sunday of October, which is the first Sunday -# on or after Oct 25. -DSTSTART_1967_1986 = datetime(1, 4, 24, 2) -DSTEND_1967_1986 = DSTEND_1987_2006 - -def us_dst_range(year): - # Find start and end times for US DST. For years before 1967, return - # start = end for no DST. - if 2006 < year: - dststart, dstend = DSTSTART_2007, DSTEND_2007 - elif 1986 < year < 2007: - dststart, dstend = DSTSTART_1987_2006, DSTEND_1987_2006 - elif 1966 < year < 1987: - dststart, dstend = DSTSTART_1967_1986, DSTEND_1967_1986 - else: - return (datetime(year, 1, 1), ) * 2 - - start = first_sunday_on_or_after(dststart.replace(year=year)) - end = first_sunday_on_or_after(dstend.replace(year=year)) - return start, end - - -class USTimeZone(tzinfo): - - def __init__(self, hours, reprname, stdname, dstname): - self.stdoffset = timedelta(hours=hours) - self.reprname = reprname - self.stdname = stdname - self.dstname = dstname - - def __repr__(self): - return self.reprname - - def tzname(self, dt): - if self.dst(dt): - return self.dstname - else: - return self.stdname - - def utcoffset(self, dt): - return self.stdoffset + self.dst(dt) - - def dst(self, dt): - if dt is None or dt.tzinfo is None: - # An exception may be sensible here, in one or both cases. - # It depends on how you want to treat them. The default - # fromutc() implementation (called by the default astimezone() - # implementation) passes a datetime with dt.tzinfo is self. - return ZERO - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) - # Can't compare naive to aware objects, so strip the timezone from - # dt first. - dt = dt.replace(tzinfo=None) - if start + HOUR <= dt < end - HOUR: - # DST is in effect. - return HOUR - if end - HOUR <= dt < end: - # Fold (an ambiguous hour): use dt.fold to disambiguate. - return ZERO if dt.fold else HOUR - if start <= dt < start + HOUR: - # Gap (a non-existent hour): reverse the fold rule. - return HOUR if dt.fold else ZERO - # DST is off. - return ZERO - - def fromutc(self, dt): - assert dt.tzinfo is self - start, end = us_dst_range(dt.year) - start = start.replace(tzinfo=self) - end = end.replace(tzinfo=self) - std_time = dt + self.stdoffset - dst_time = std_time + HOUR - if end <= dst_time < end + HOUR: - # Repeated hour - return std_time.replace(fold=1) - if std_time < start or dst_time >= end: - # Standard time - return std_time - if start <= std_time < end - HOUR: - # Daylight saving time - return dst_time - - -Eastern = USTimeZone(-5, "Eastern", "EST", "EDT") -Central = USTimeZone(-6, "Central", "CST", "CDT") -Mountain = USTimeZone(-7, "Mountain", "MST", "MDT") -Pacific = USTimeZone(-8, "Pacific", "PST", "PDT") diff --git a/Doc/install/index.rst b/Doc/install/index.rst deleted file mode 100644 index 283070d36badfc..00000000000000 --- a/Doc/install/index.rst +++ /dev/null @@ -1,1077 +0,0 @@ -.. highlight:: none - -.. _install-index: - -******************************************** - Installing Python Modules (Legacy version) -******************************************** - -:Author: Greg Ward - -.. TODO: Fill in XXX comments - -.. note:: - - The entire ``distutils`` package has been deprecated and will be - removed in Python 3.12. This documentation is retained as a - reference only, and will be removed with the package. See the - :ref:`What's New ` entry for more information. - -.. seealso:: - - :ref:`installing-index` - The up to date module installation documentation. For regular Python - usage, you almost certainly want that document rather than this one. - -.. include:: ../distutils/_setuptools_disclaimer.rst - -.. note:: - - This guide only covers the basic tools for building and distributing - extensions that are provided as part of this version of Python. Third party - tools offer easier to use and more secure alternatives. Refer to the `quick - recommendations section `__ - in the Python Packaging User Guide for more information. - - -.. _inst-intro: - - -Introduction -============ - -In Python 2.0, the ``distutils`` API was first added to the standard library. -This provided Linux distro maintainers with a standard way of converting -Python projects into Linux distro packages, and system administrators with a -standard way of installing them directly onto target systems. - -In the many years since Python 2.0 was released, tightly coupling the build -system and package installer to the language runtime release cycle has turned -out to be problematic, and it is now recommended that projects use the -``pip`` package installer and the ``setuptools`` build system, rather than -using ``distutils`` directly. - -See :ref:`installing-index` and :ref:`distributing-index` for more details. - -This legacy documentation is being retained only until we're confident that the -``setuptools`` documentation covers everything needed. - -.. _inst-new-standard: - -Distutils based source distributions ------------------------------------- - -If you download a module source distribution, you can tell pretty quickly if it -was packaged and distributed in the standard way, i.e. using the Distutils. -First, the distribution's name and version number will be featured prominently -in the name of the downloaded archive, e.g. :file:`foo-1.0.tar.gz` or -:file:`widget-0.9.7.zip`. Next, the archive will unpack into a similarly named -directory: :file:`foo-1.0` or :file:`widget-0.9.7`. Additionally, the -distribution will contain a setup script :file:`setup.py`, and a file named -:file:`README.txt` or possibly just :file:`README`, which should explain that -building and installing the module distribution is a simple matter of running -one command from a terminal:: - - python setup.py install - -For Windows, this command should be run from a command prompt window -(:menuselection:`Start --> Accessories`):: - - setup.py install - -If all these things are true, then you already know how to build and install the -modules you've just downloaded: Run the command above. Unless you need to -install things in a non-standard way or customize the build process, you don't -really need this manual. Or rather, the above command is everything you need to -get out of this manual. - - -.. _inst-standard-install: - -Standard Build and Install -========================== - -As described in section :ref:`inst-new-standard`, building and installing a module -distribution using the Distutils is usually one simple command to run from a -terminal:: - - python setup.py install - - -.. _inst-platform-variations: - -Platform variations -------------------- - -You should always run the setup command from the distribution root directory, -i.e. the top-level subdirectory that the module source distribution unpacks -into. For example, if you've just downloaded a module source distribution -:file:`foo-1.0.tar.gz` onto a Unix system, the normal thing to do is:: - - gunzip -c foo-1.0.tar.gz | tar xf - # unpacks into directory foo-1.0 - cd foo-1.0 - python setup.py install - -On Windows, you'd probably download :file:`foo-1.0.zip`. If you downloaded the -archive file to :file:`C:\\Temp`, then it would unpack into -:file:`C:\\Temp\\foo-1.0`; you can use either an archive manipulator with a -graphical user interface (such as WinZip) or a command-line tool (such as -:program:`unzip` or :program:`pkunzip`) to unpack the archive. Then, open a -command prompt window and run:: - - cd c:\Temp\foo-1.0 - python setup.py install - - -.. _inst-splitting-up: - -Splitting the job up --------------------- - -Running ``setup.py install`` builds and installs all modules in one run. If you -prefer to work incrementally---especially useful if you want to customize the -build process, or if things are going wrong---you can use the setup script to do -one thing at a time. This is particularly helpful when the build and install -will be done by different users---for example, you might want to build a module -distribution and hand it off to a system administrator for installation (or do -it yourself, with super-user privileges). - -For example, you can build everything in one step, and then install everything -in a second step, by invoking the setup script twice:: - - python setup.py build - python setup.py install - -If you do this, you will notice that running the :command:`install` command -first runs the :command:`build` command, which---in this case---quickly notices -that it has nothing to do, since everything in the :file:`build` directory is -up-to-date. - -You may not need this ability to break things down often if all you do is -install modules downloaded off the 'net, but it's very handy for more advanced -tasks. If you get into distributing your own Python modules and extensions, -you'll run lots of individual Distutils commands on their own. - - -.. _inst-how-build-works: - -How building works ------------------- - -As implied above, the :command:`build` command is responsible for putting the -files to install into a *build directory*. By default, this is :file:`build` -under the distribution root; if you're excessively concerned with speed, or want -to keep the source tree pristine, you can change the build directory with the -:option:`!--build-base` option. For example:: - - python setup.py build --build-base=/path/to/pybuild/foo-1.0 - -(Or you could do this permanently with a directive in your system or personal -Distutils configuration file; see section :ref:`inst-config-files`.) Normally, this -isn't necessary. - -The default layout for the build tree is as follows:: - - --- build/ --- lib/ - or - --- build/ --- lib./ - temp./ - -where ```` expands to a brief description of the current OS/hardware -platform and Python version. The first form, with just a :file:`lib` directory, -is used for "pure module distributions"---that is, module distributions that -include only pure Python modules. If a module distribution contains any -extensions (modules written in C/C++), then the second form, with two ```` -directories, is used. In that case, the :file:`temp.{plat}` directory holds -temporary files generated by the compile/link process that don't actually get -installed. In either case, the :file:`lib` (or :file:`lib.{plat}`) directory -contains all Python modules (pure Python and extensions) that will be installed. - -In the future, more directories will be added to handle Python scripts, -documentation, binary executables, and whatever else is needed to handle the job -of installing Python modules and applications. - - -.. _inst-how-install-works: - -How installation works ----------------------- - -After the :command:`build` command runs (whether you run it explicitly, or the -:command:`install` command does it for you), the work of the :command:`install` -command is relatively simple: all it has to do is copy everything under -:file:`build/lib` (or :file:`build/lib.{plat}`) to your chosen installation -directory. - -If you don't choose an installation directory---i.e., if you just run ``setup.py -install``\ ---then the :command:`install` command installs to the standard -location for third-party Python modules. This location varies by platform and -by how you built/installed Python itself. On Unix (and macOS, which is also -Unix-based), it also depends on whether the module distribution being installed -is pure Python or contains extensions ("non-pure"): - -.. tabularcolumns:: |l|l|l|l| - -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Platform | Standard installation location | Default value | Notes | -+=================+=====================================================+==================================================+=======+ -| Unix (pure) | :file:`{prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Unix (non-pure) | :file:`{exec-prefix}/lib/python{X.Y}/site-packages` | :file:`/usr/local/lib/python{X.Y}/site-packages` | \(1) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ -| Windows | :file:`{prefix}\\Lib\\site-packages` | :file:`C:\\Python{XY}\\Lib\\site-packages` | \(2) | -+-----------------+-----------------------------------------------------+--------------------------------------------------+-------+ - -Notes: - -(1) - Most Linux distributions include Python as a standard part of the system, so - :file:`{prefix}` and :file:`{exec-prefix}` are usually both :file:`/usr` on - Linux. If you build Python yourself on Linux (or any Unix-like system), the - default :file:`{prefix}` and :file:`{exec-prefix}` are :file:`/usr/local`. - -(2) - The default installation directory on Windows was :file:`C:\\Program - Files\\Python` under Python 1.6a1, 1.5.2, and earlier. - -:file:`{prefix}` and :file:`{exec-prefix}` stand for the directories that Python -is installed to, and where it finds its libraries at run-time. They are always -the same under Windows, and very often the same under Unix and macOS. You -can find out what your Python installation uses for :file:`{prefix}` and -:file:`{exec-prefix}` by running Python in interactive mode and typing a few -simple commands. Under Unix, just type ``python`` at the shell prompt. Under -Windows, choose :menuselection:`Start --> Programs --> Python X.Y --> -Python (command line)`. Once the interpreter is started, you type Python code -at the prompt. For example, on my Linux system, I type the three Python -statements shown below, and get the output as shown, to find out my -:file:`{prefix}` and :file:`{exec-prefix}`: - -.. code-block:: pycon - - Python 2.4 (#26, Aug 7 2004, 17:19:02) - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.prefix - '/usr' - >>> sys.exec_prefix - '/usr' - -A few other placeholders are used in this document: :file:`{X.Y}` stands for the -version of Python, for example ``3.2``; :file:`{abiflags}` will be replaced by -the value of :data:`sys.abiflags` or the empty string for platforms which don't -define ABI flags; :file:`{distname}` will be replaced by the name of the module -distribution being installed. Dots and capitalization are important in the -paths; for example, a value that uses ``python3.2`` on UNIX will typically use -``Python32`` on Windows. - -If you don't want to install modules to the standard location, or if you don't -have permission to write there, then you need to read about alternate -installations in section :ref:`inst-alt-install`. If you want to customize your -installation directories more heavily, see section :ref:`inst-custom-install` on -custom installations. - - -.. _inst-alt-install: - -Alternate Installation -====================== - -Often, it is necessary or desirable to install modules to a location other than -the standard location for third-party Python modules. For example, on a Unix -system you might not have permission to write to the standard third-party module -directory. Or you might wish to try out a module before making it a standard -part of your local Python installation. This is especially true when upgrading -a distribution already present: you want to make sure your existing base of -scripts still works with the new version before actually upgrading. - -The Distutils :command:`install` command is designed to make installing module -distributions to an alternate location simple and painless. The basic idea is -that you supply a base directory for the installation, and the -:command:`install` command picks a set of directories (called an *installation -scheme*) under this base directory in which to install files. The details -differ across platforms, so read whichever of the following sections applies to -you. - -Note that the various alternate installation schemes are mutually exclusive: you -can pass ``--user``, or ``--home``, or ``--prefix`` and ``--exec-prefix``, or -``--install-base`` and ``--install-platbase``, but you can't mix from these -groups. - - -.. _inst-alt-install-user: - -Alternate installation: the user scheme ---------------------------------------- - -This scheme is designed to be the most convenient solution for users that don't -have write permission to the global site-packages directory or don't want to -install into it. It is enabled with a simple option:: - - python setup.py install --user - -Files will be installed into subdirectories of :const:`site.USER_BASE` (written -as :file:`{userbase}` hereafter). This scheme installs pure Python modules and -extension modules in the same location (also known as :const:`site.USER_SITE`). -Here are the values for UNIX, including macOS: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}/lib/python{X.Y}/site-packages` -scripts :file:`{userbase}/bin` -data :file:`{userbase}` -C headers :file:`{userbase}/include/python{X.Y}{abiflags}/{distname}` -=============== =========================================================== - -And here are the values used on Windows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{userbase}\\Python{XY}\\site-packages` -scripts :file:`{userbase}\\Python{XY}\\Scripts` -data :file:`{userbase}` -C headers :file:`{userbase}\\Python{XY}\\Include\\{distname}` -=============== =========================================================== - -The advantage of using this scheme compared to the other ones described below is -that the user site-packages directory is under normal conditions always included -in :data:`sys.path` (see :mod:`site` for more information), which means that -there is no additional step to perform after running the :file:`setup.py` script -to finalize the installation. - -The :command:`build_ext` command also has a ``--user`` option to add -:file:`{userbase}/include` to the compiler search path for header files and -:file:`{userbase}/lib` to the compiler search path for libraries as well as to -the runtime search path for shared C libraries (rpath). - - -.. _inst-alt-install-home: - -Alternate installation: the home scheme ---------------------------------------- - -The idea behind the "home scheme" is that you build and maintain a personal -stash of Python modules. This scheme's name is derived from the idea of a -"home" directory on Unix, since it's not unusual for a Unix user to make their -home directory have a layout similar to :file:`/usr/` or :file:`/usr/local/`. -This scheme can be used by anyone, regardless of the operating system they -are installing for. - -Installing a new module distribution is as simple as :: - - python setup.py install --home= - -where you can supply any directory you like for the :option:`!--home` option. On -Unix, lazy typists can just type a tilde (``~``); the :command:`install` command -will expand this to your home directory:: - - python setup.py install --home=~ - -To make Python find the distributions installed with this scheme, you may have -to :ref:`modify Python's search path ` or edit -:mod:`!sitecustomize` (see :mod:`site`) to call :func:`site.addsitedir` or edit -:data:`sys.path`. - -The :option:`!--home` option defines the installation base directory. Files are -installed to the following directories under the installation base as follows: - -=============== =========================================================== -Type of file Installation directory -=============== =========================================================== -modules :file:`{home}/lib/python` -scripts :file:`{home}/bin` -data :file:`{home}` -C headers :file:`{home}/include/python/{distname}` -=============== =========================================================== - -(Mentally replace slashes with backslashes if you're on Windows.) - - -.. _inst-alt-install-prefix-unix: - -Alternate installation: Unix (the prefix scheme) ------------------------------------------------- - -The "prefix scheme" is useful when you wish to use one Python installation to -perform the build/install (i.e., to run the setup script), but install modules -into the third-party module directory of a different Python installation (or -something that looks like a different Python installation). If this sounds a -trifle unusual, it is---that's why the user and home schemes come before. However, -there are at least two known cases where the prefix scheme will be useful. - -First, consider that many Linux distributions put Python in :file:`/usr`, rather -than the more traditional :file:`/usr/local`. This is entirely appropriate, -since in those cases Python is part of "the system" rather than a local add-on. -However, if you are installing Python modules from source, you probably want -them to go in :file:`/usr/local/lib/python2.{X}` rather than -:file:`/usr/lib/python2.{X}`. This can be done with :: - - /usr/bin/python setup.py install --prefix=/usr/local - -Another possibility is a network filesystem where the name used to write to a -remote directory is different from the name used to read it: for example, the -Python interpreter accessed as :file:`/usr/local/bin/python` might search for -modules in :file:`/usr/local/lib/python2.{X}`, but those modules would have to -be installed to, say, :file:`/mnt/{@server}/export/lib/python2.{X}`. This could -be done with :: - - /usr/local/bin/python setup.py install --prefix=/mnt/@server/export - -In either case, the :option:`!--prefix` option defines the installation base, and -the :option:`!--exec-prefix` option defines the platform-specific installation -base, which is used for platform-specific files. (Currently, this just means -non-pure module distributions, but could be expanded to C libraries, binary -executables, etc.) If :option:`!--exec-prefix` is not supplied, it defaults to -:option:`!--prefix`. Files are installed as follows: - -================= ========================================================== -Type of file Installation directory -================= ========================================================== -Python modules :file:`{prefix}/lib/python{X.Y}/site-packages` -extension modules :file:`{exec-prefix}/lib/python{X.Y}/site-packages` -scripts :file:`{prefix}/bin` -data :file:`{prefix}` -C headers :file:`{prefix}/include/python{X.Y}{abiflags}/{distname}` -================= ========================================================== - -There is no requirement that :option:`!--prefix` or :option:`!--exec-prefix` -actually point to an alternate Python installation; if the directories listed -above do not already exist, they are created at installation time. - -Incidentally, the real reason the prefix scheme is important is simply that a -standard Unix installation uses the prefix scheme, but with :option:`!--prefix` -and :option:`!--exec-prefix` supplied by Python itself as ``sys.prefix`` and -``sys.exec_prefix``. Thus, you might think you'll never use the prefix scheme, -but every time you run ``python setup.py install`` without any other options, -you're using it. - -Note that installing extensions to an alternate Python installation has no -effect on how those extensions are built: in particular, the Python header files -(:file:`Python.h` and friends) installed with the Python interpreter used to run -the setup script will be used in compiling extensions. It is your -responsibility to ensure that the interpreter used to run extensions installed -in this way is compatible with the interpreter used to build them. The best way -to do this is to ensure that the two interpreters are the same version of Python -(possibly different builds, or possibly copies of the same build). (Of course, -if your :option:`!--prefix` and :option:`!--exec-prefix` don't even point to an -alternate Python installation, this is immaterial.) - - -.. _inst-alt-install-prefix-windows: - -Alternate installation: Windows (the prefix scheme) ---------------------------------------------------- - -Windows has no concept of a user's home directory, and since the standard Python -installation under Windows is simpler than under Unix, the :option:`!--prefix` -option has traditionally been used to install additional packages in separate -locations on Windows. :: - - python setup.py install --prefix="\Temp\Python" - -to install modules to the :file:`\\Temp\\Python` directory on the current drive. - -The installation base is defined by the :option:`!--prefix` option; the -:option:`!--exec-prefix` option is not supported under Windows, which means that -pure Python modules and extension modules are installed into the same location. -Files are installed as follows: - -=============== ========================================================== -Type of file Installation directory -=============== ========================================================== -modules :file:`{prefix}\\Lib\\site-packages` -scripts :file:`{prefix}\\Scripts` -data :file:`{prefix}` -C headers :file:`{prefix}\\Include\\{distname}` -=============== ========================================================== - - -.. _inst-custom-install: - -Custom Installation -=================== - -Sometimes, the alternate installation schemes described in section -:ref:`inst-alt-install` just don't do what you want. You might want to tweak just -one or two directories while keeping everything under the same base directory, -or you might want to completely redefine the installation scheme. In either -case, you're creating a *custom installation scheme*. - -To create a custom installation scheme, you start with one of the alternate -schemes and override some of the installation directories used for the various -types of files, using these options: - -====================== ======================= -Type of file Override option -====================== ======================= -Python modules ``--install-purelib`` -extension modules ``--install-platlib`` -all modules ``--install-lib`` -scripts ``--install-scripts`` -data ``--install-data`` -C headers ``--install-headers`` -====================== ======================= - -These override options can be relative, absolute, -or explicitly defined in terms of one of the installation base directories. -(There are two installation base directories, and they are normally the -same---they only differ when you use the Unix "prefix scheme" and supply -different ``--prefix`` and ``--exec-prefix`` options; using ``--install-lib`` -will override values computed or given for ``--install-purelib`` and -``--install-platlib``, and is recommended for schemes that don't make a -difference between Python and extension modules.) - -For example, say you're installing a module distribution to your home directory -under Unix---but you want scripts to go in :file:`~/scripts` rather than -:file:`~/bin`. As you might expect, you can override this directory with the -:option:`!--install-scripts` option; in this case, it makes most sense to supply -a relative path, which will be interpreted relative to the installation base -directory (your home directory, in this case):: - - python setup.py install --home=~ --install-scripts=scripts - -Another Unix example: suppose your Python installation was built and installed -with a prefix of :file:`/usr/local/python`, so under a standard installation -scripts will wind up in :file:`/usr/local/python/bin`. If you want them in -:file:`/usr/local/bin` instead, you would supply this absolute directory for the -:option:`!--install-scripts` option:: - - python setup.py install --install-scripts=/usr/local/bin - -(This performs an installation using the "prefix scheme", where the prefix is -whatever your Python interpreter was installed with--- :file:`/usr/local/python` -in this case.) - -If you maintain Python on Windows, you might want third-party modules to live in -a subdirectory of :file:`{prefix}`, rather than right in :file:`{prefix}` -itself. This is almost as easy as customizing the script installation -directory---you just have to remember that there are two types of modules -to worry about, Python and extension modules, which can conveniently be both -controlled by one option:: - - python setup.py install --install-lib=Site - -The specified installation directory is relative to :file:`{prefix}`. Of -course, you also have to ensure that this directory is in Python's module -search path, such as by putting a :file:`.pth` file in a site directory (see -:mod:`site`). See section :ref:`inst-search-path` to find out how to modify -Python's search path. - -If you want to define an entire installation scheme, you just have to supply all -of the installation directory options. The recommended way to do this is to -supply relative paths; for example, if you want to maintain all Python -module-related files under :file:`python` in your home directory, and you want a -separate directory for each platform that you use your home directory from, you -might define the following installation scheme:: - - python setup.py install --home=~ \ - --install-purelib=python/lib \ - --install-platlib=python/lib.$PLAT \ - --install-scripts=python/scripts - --install-data=python/data - -or, equivalently, :: - - python setup.py install --home=~/python \ - --install-purelib=lib \ - --install-platlib='lib.$PLAT' \ - --install-scripts=scripts - --install-data=data - -``$PLAT`` is not (necessarily) an environment variable---it will be expanded by -the Distutils as it parses your command line options, just as it does when -parsing your configuration file(s). - -Obviously, specifying the entire installation scheme every time you install a -new module distribution would be very tedious. Thus, you can put these options -into your Distutils config file (see section :ref:`inst-config-files`): - -.. code-block:: ini - - [install] - install-base=$HOME - install-purelib=python/lib - install-platlib=python/lib.$PLAT - install-scripts=python/scripts - install-data=python/data - -or, equivalently, - -.. code-block:: ini - - [install] - install-base=$HOME/python - install-purelib=lib - install-platlib=lib.$PLAT - install-scripts=scripts - install-data=data - -Note that these two are *not* equivalent if you supply a different installation -base directory when you run the setup script. For example, :: - - python setup.py install --install-base=/tmp - -would install pure modules to :file:`/tmp/python/lib` in the first case, and -to :file:`/tmp/lib` in the second case. (For the second case, you probably -want to supply an installation base of :file:`/tmp/python`.) - -You probably noticed the use of ``$HOME`` and ``$PLAT`` in the sample -configuration file input. These are Distutils configuration variables, which -bear a strong resemblance to environment variables. In fact, you can use -environment variables in config files on platforms that have such a notion but -the Distutils additionally define a few extra variables that may not be in your -environment, such as ``$PLAT``. (And of course, on systems that don't have -environment variables, such as Mac OS 9, the configuration variables supplied by -the Distutils are the only ones you can use.) See section :ref:`inst-config-files` -for details. - -.. note:: When a :ref:`virtual environment ` is activated, any options - that change the installation path will be ignored from all distutils configuration - files to prevent inadvertently installing projects outside of the virtual - environment. - -.. XXX need some Windows examples---when would custom installation schemes be - needed on those platforms? - - -.. XXX Move this to Doc/using - -.. _inst-search-path: - -Modifying Python's Search Path ------------------------------- - -When the Python interpreter executes an :keyword:`import` statement, it searches -for both Python code and extension modules along a search path. A default value -for the path is configured into the Python binary when the interpreter is built. -You can determine the path by importing the :mod:`sys` module and printing the -value of ``sys.path``. :: - - $ python - Python 2.2 (#11, Oct 3 2002, 13:31:27) - [GCC 2.96 20000731 (Red Hat Linux 7.3 2.96-112)] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python2.3', '/usr/local/lib/python2.3/plat-linux2', - '/usr/local/lib/python2.3/lib-tk', '/usr/local/lib/python2.3/lib-dynload', - '/usr/local/lib/python2.3/site-packages'] - >>> - -The null string in ``sys.path`` represents the current working directory. - -The expected convention for locally installed packages is to put them in the -:file:`{...}/site-packages/` directory, but you may want to install Python -modules into some arbitrary directory. For example, your site may have a -convention of keeping all software related to the web server under :file:`/www`. -Add-on Python modules might then belong in :file:`/www/python`, and in order to -import them, this directory must be added to ``sys.path``. There are several -different ways to add the directory. - -The most convenient way is to add a path configuration file to a directory -that's already on Python's path, usually to the :file:`.../site-packages/` -directory. Path configuration files have an extension of :file:`.pth`, and each -line must contain a single path that will be appended to ``sys.path``. (Because -the new paths are appended to ``sys.path``, modules in the added directories -will not override standard modules. This means you can't use this mechanism for -installing fixed versions of standard modules.) - -Paths can be absolute or relative, in which case they're relative to the -directory containing the :file:`.pth` file. See the documentation of -the :mod:`site` module for more information. - -A slightly less convenient way is to edit the :file:`site.py` file in Python's -standard library, and modify ``sys.path``. :file:`site.py` is automatically -imported when the Python interpreter is executed, unless the :option:`-S` switch -is supplied to suppress this behaviour. So you could simply edit -:file:`site.py` and add two lines to it: - -.. code-block:: python - - import sys - sys.path.append('/www/python/') - -However, if you reinstall the same minor version of Python (perhaps when -upgrading from 2.2 to 2.2.2, for example) :file:`site.py` will be overwritten by -the stock version. You'd have to remember that it was modified and save a copy -before doing the installation. - -There are two environment variables that can modify ``sys.path``. -:envvar:`PYTHONHOME` sets an alternate value for the prefix of the Python -installation. For example, if :envvar:`PYTHONHOME` is set to ``/www/python``, -the search path will be set to ``['', '/www/python/lib/pythonX.Y/', -'/www/python/lib/pythonX.Y/plat-linux2', ...]``. - -The :envvar:`PYTHONPATH` variable can be set to a list of paths that will be -added to the beginning of ``sys.path``. For example, if :envvar:`PYTHONPATH` is -set to ``/www/python:/opt/py``, the search path will begin with -``['/www/python', '/opt/py']``. (Note that directories must exist in order to -be added to ``sys.path``; the :mod:`site` module removes paths that don't -exist.) - -Finally, ``sys.path`` is just a regular Python list, so any Python application -can modify it by adding or removing entries. - - -.. _inst-config-files: - -Distutils Configuration Files -============================= - -As mentioned above, you can use Distutils configuration files to record personal -or site preferences for any Distutils options. That is, any option to any -command can be stored in one of two or three (depending on your platform) -configuration files, which will be consulted before the command-line is parsed. -This means that configuration files will override default values, and the -command-line will in turn override configuration files. Furthermore, if -multiple configuration files apply, values from "earlier" files are overridden -by "later" files. - - -.. _inst-config-filenames: - -Location and names of config files ----------------------------------- - -The names and locations of the configuration files vary slightly across -platforms. On Unix and macOS, the three configuration files (in the order -they are processed) are: - -+--------------+----------------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+==========================================================+=======+ -| system | :file:`{prefix}/lib/python{ver}/distutils/distutils.cfg` | \(1) | -+--------------+----------------------------------------------------------+-------+ -| personal | :file:`$HOME/.pydistutils.cfg` | \(2) | -+--------------+----------------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+----------------------------------------------------------+-------+ - -And on Windows, the configuration files are: - -+--------------+-------------------------------------------------+-------+ -| Type of file | Location and filename | Notes | -+==============+=================================================+=======+ -| system | :file:`{prefix}\\Lib\\distutils\\distutils.cfg` | \(4) | -+--------------+-------------------------------------------------+-------+ -| personal | :file:`%HOME%\\pydistutils.cfg` | \(5) | -+--------------+-------------------------------------------------+-------+ -| local | :file:`setup.cfg` | \(3) | -+--------------+-------------------------------------------------+-------+ - -On all platforms, the "personal" file can be temporarily disabled by -passing the ``--no-user-cfg`` option. - -Notes: - -(1) - Strictly speaking, the system-wide configuration file lives in the directory - where the Distutils are installed; under Python 1.6 and later on Unix, this is - as shown. For Python 1.5.2, the Distutils will normally be installed to - :file:`{prefix}/lib/python1.5/site-packages/distutils`, so the system - configuration file should be put there under Python 1.5.2. - -(2) - On Unix, if the :envvar:`HOME` environment variable is not defined, the user's - home directory will be determined with the :func:`~pwd.getpwuid` function from the - standard :mod:`pwd` module. This is done by the :func:`os.path.expanduser` - function used by Distutils. - -(3) - I.e., in the current directory (usually the location of the setup script). - -(4) - (See also note (1).) Under Python 1.6 and later, Python's default "installation - prefix" is :file:`C:\\Python`, so the system configuration file is normally - :file:`C:\\Python\\Lib\\distutils\\distutils.cfg`. Under Python 1.5.2, the - default prefix was :file:`C:\\Program Files\\Python`, and the Distutils were not - part of the standard library---so the system configuration file would be - :file:`C:\\Program Files\\Python\\distutils\\distutils.cfg` in a standard Python - 1.5.2 installation under Windows. - -(5) - On Windows, if the :envvar:`HOME` environment variable is not defined, - :envvar:`USERPROFILE` then :envvar:`HOMEDRIVE` and :envvar:`HOMEPATH` will - be tried. This is done by the :func:`os.path.expanduser` function used - by Distutils. - - -.. _inst-config-syntax: - -Syntax of config files ----------------------- - -The Distutils configuration files all have the same syntax. The config files -are grouped into sections. There is one section for each Distutils command, -plus a ``global`` section for global options that affect every command. Each -section consists of one option per line, specified as ``option=value``. - -For example, the following is a complete config file that just forces all -commands to run quietly by default: - -.. code-block:: ini - - [global] - verbose=0 - -If this is installed as the system config file, it will affect all processing of -any Python module distribution by any user on the current system. If it is -installed as your personal config file (on systems that support them), it will -affect only module distributions processed by you. And if it is used as the -:file:`setup.cfg` for a particular module distribution, it affects only that -distribution. - -You could override the default "build base" directory and make the -:command:`build\*` commands always forcibly rebuild all files with the -following: - -.. code-block:: ini - - [build] - build-base=blib - force=1 - -which corresponds to the command-line arguments :: - - python setup.py build --build-base=blib --force - -except that including the :command:`build` command on the command-line means -that command will be run. Including a particular command in config files has no -such implication; it only means that if the command is run, the options in the -config file will apply. (Or if other commands that derive values from it are -run, they will use the values in the config file.) - -You can find out the complete list of options for any command using the -:option:`!--help` option, e.g.:: - - python setup.py build --help - -and you can find out the complete list of global options by using -:option:`!--help` without a command:: - - python setup.py --help - -See also the "Reference" section of the "Distributing Python Modules" manual. - - -.. _inst-building-ext: - -Building Extensions: Tips and Tricks -==================================== - -Whenever possible, the Distutils try to use the configuration information made -available by the Python interpreter used to run the :file:`setup.py` script. -For example, the same compiler and linker flags used to compile Python will also -be used for compiling extensions. Usually this will work well, but in -complicated situations this might be inappropriate. This section discusses how -to override the usual Distutils behaviour. - - -.. _inst-tweak-flags: - -Tweaking compiler/linker flags ------------------------------- - -Compiling a Python extension written in C or C++ will sometimes require -specifying custom flags for the compiler and linker in order to use a particular -library or produce a special kind of object code. This is especially true if the -extension hasn't been tested on your platform, or if you're trying to -cross-compile Python. - -In the most general case, the extension author might have foreseen that -compiling the extensions would be complicated, and provided a :file:`Setup` file -for you to edit. This will likely only be done if the module distribution -contains many separate extension modules, or if they often require elaborate -sets of compiler flags in order to work. - -A :file:`Setup` file, if present, is parsed in order to get a list of extensions -to build. Each line in a :file:`Setup` describes a single module. Lines have -the following structure:: - - module ... [sourcefile ...] [cpparg ...] [library ...] - - -Let's examine each of the fields in turn. - -* *module* is the name of the extension module to be built, and should be a - valid Python identifier. You can't just change this in order to rename a module - (edits to the source code would also be needed), so this should be left alone. - -* *sourcefile* is anything that's likely to be a source code file, at least - judging by the filename. Filenames ending in :file:`.c` are assumed to be - written in C, filenames ending in :file:`.C`, :file:`.cc`, and :file:`.c++` are - assumed to be C++, and filenames ending in :file:`.m` or :file:`.mm` are assumed - to be in Objective C. - -* *cpparg* is an argument for the C preprocessor, and is anything starting with - :option:`!-I`, :option:`!-D`, :option:`!-U` or :option:`!-C`. - -* *library* is anything ending in :file:`.a` or beginning with :option:`!-l` or - :option:`!-L`. - -If a particular platform requires a special library on your platform, you can -add it by editing the :file:`Setup` file and running ``python setup.py build``. -For example, if the module defined by the line :: - - foo foomodule.c - -must be linked with the math library :file:`libm.a` on your platform, simply add -:option:`!-lm` to the line:: - - foo foomodule.c -lm - -Arbitrary switches intended for the compiler or the linker can be supplied with -the :option:`!-Xcompiler` *arg* and :option:`!-Xlinker` *arg* options:: - - foo foomodule.c -Xcompiler -o32 -Xlinker -shared -lm - -The next option after :option:`!-Xcompiler` and :option:`!-Xlinker` will be -appended to the proper command line, so in the above example the compiler will -be passed the :option:`!-o32` option, and the linker will be passed -:option:`!-shared`. If a compiler option requires an argument, you'll have to -supply multiple :option:`!-Xcompiler` options; for example, to pass ``-x c++`` -the :file:`Setup` file would have to contain ``-Xcompiler -x -Xcompiler c++``. - -Compiler flags can also be supplied through setting the :envvar:`CFLAGS` -environment variable. If set, the contents of :envvar:`CFLAGS` will be added to -the compiler flags specified in the :file:`Setup` file. - - -.. _inst-non-ms-compilers: - -Using non-Microsoft compilers on Windows ----------------------------------------- - -.. sectionauthor:: Rene Liebscher - - - -Borland/CodeGear C++ -^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the necessary steps to use Distutils with the Borland -C++ compiler version 5.5. First you have to know that Borland's object file -format (OMF) is different from the format used by the Python version you can -download from the Python or ActiveState web site. (Python is built with -Microsoft Visual C++, which uses COFF as the object file format.) For this -reason you have to convert Python's library :file:`python25.lib` into the -Borland format. You can do this as follows: - -.. Should we mention that users have to create cfg-files for the compiler? -.. see also http://community.borland.com/article/0,1410,21205,00.html - -:: - - coff2omf python25.lib python25_bcpp.lib - -The :file:`coff2omf` program comes with the Borland compiler. The file -:file:`python25.lib` is in the :file:`Libs` directory of your Python -installation. If your extension uses other libraries (zlib, ...) you have to -convert them too. - -The converted files have to reside in the same directories as the normal -libraries. - -How does Distutils manage to use these libraries with their changed names? If -the extension needs a library (eg. :file:`foo`) Distutils checks first if it -finds a library with suffix :file:`_bcpp` (eg. :file:`foo_bcpp.lib`) and then -uses this library. In the case it doesn't find such a special library it uses -the default name (:file:`foo.lib`.) [#]_ - -To let Distutils compile your extension with Borland C++ you now have to type:: - - python setup.py build --compiler=bcpp - -If you want to use the Borland C++ compiler as the default, you could specify -this in your personal or system-wide configuration file for Distutils (see -section :ref:`inst-config-files`.) - - -.. seealso:: - - `C++Builder Compiler `_ - Information about the free C++ compiler from Borland, including links to the - download pages. - - `Creating Python Extensions Using Borland's Free Compiler `_ - Document describing how to use Borland's free command-line C++ compiler to build - Python. - - -GNU C / Cygwin / MinGW -^^^^^^^^^^^^^^^^^^^^^^ - -This section describes the necessary steps to use Distutils with the GNU C/C++ -compilers in their Cygwin and MinGW distributions. [#]_ For a Python interpreter -that was built with Cygwin, everything should work without any of these -following steps. - -Not all extensions can be built with MinGW or Cygwin, but many can. Extensions -most likely to not work are those that use C++ or depend on Microsoft Visual C -extensions. - -To let Distutils compile your extension with Cygwin you have to type:: - - python setup.py build --compiler=cygwin - -and for Cygwin in no-cygwin mode [#]_ or for MinGW type:: - - python setup.py build --compiler=mingw32 - -If you want to use any of these options/compilers as default, you should -consider writing it in your personal or system-wide configuration file for -Distutils (see section :ref:`inst-config-files`.) - -Older Versions of Python and MinGW -"""""""""""""""""""""""""""""""""" -The following instructions only apply if you're using a version of Python -inferior to 2.4.1 with a MinGW inferior to 3.0.0 (with -binutils-2.13.90-20030111-1). - -These compilers require some special libraries. This task is more complex than -for Borland's C++, because there is no program to convert the library. First -you have to create a list of symbols which the Python DLL exports. (You can find -a good program for this task at -https://sourceforge.net/projects/mingw/files/MinGW/Extension/pexports/). - -.. I don't understand what the next line means. --amk -.. (inclusive the references on data structures.) - -:: - - pexports python25.dll >python25.def - -The location of an installed :file:`python25.dll` will depend on the -installation options and the version and language of Windows. In a "just for -me" installation, it will appear in the root of the installation directory. In -a shared installation, it will be located in the system directory. - -Then you can create from these information an import library for gcc. :: - - /cygwin/bin/dlltool --dllname python25.dll --def python25.def --output-lib libpython25.a - -The resulting library has to be placed in the same directory as -:file:`python25.lib`. (Should be the :file:`libs` directory under your Python -installation directory.) - -If your extension uses other libraries (zlib,...) you might have to convert -them too. The converted files have to reside in the same directories as the -normal libraries do. - - -.. seealso:: - - `Building Python modules on MS Windows platform with MinGW `_ - Information about building the required libraries for the MinGW environment. - - -.. rubric:: Footnotes - -.. [#] This also means you could replace all existing COFF-libraries with OMF-libraries - of the same name. - -.. [#] Check https://www.sourceware.org/cygwin/ for more information - -.. [#] Then you have no POSIX emulation available, but you also don't need - :file:`cygwin1.dll`. diff --git a/Doc/installing/index.rst b/Doc/installing/index.rst deleted file mode 100644 index da4a2b3c186a77..00000000000000 --- a/Doc/installing/index.rst +++ /dev/null @@ -1,243 +0,0 @@ -.. highlight:: none - -.. _installing-index: - -************************* -Installing Python Modules -************************* - -:Email: distutils-sig@python.org - -As a popular open source development project, Python has an active -supporting community of contributors and users that also make their software -available for other Python developers to use under open source license terms. - -This allows Python users to share and collaborate effectively, benefiting -from the solutions others have already created to common (and sometimes -even rare!) problems, as well as potentially contributing their own -solutions to the common pool. - -This guide covers the installation part of the process. For a guide to -creating and sharing your own Python projects, refer to the -`Python packaging user guide`_. - -.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ - -.. note:: - - For corporate and other institutional users, be aware that many - organisations have their own policies around using and contributing to - open source software. Please take such policies into account when making - use of the distribution and installation tools provided with Python. - - -Key terms -========= - -* ``pip`` is the preferred installer program. Starting with Python 3.4, it - is included by default with the Python binary installers. -* A *virtual environment* is a semi-isolated Python environment that allows - packages to be installed for use by a particular application, rather than - being installed system wide. -* ``venv`` is the standard tool for creating virtual environments, and has - been part of Python since Python 3.3. Starting with Python 3.4, it - defaults to installing ``pip`` into all created virtual environments. -* ``virtualenv`` is a third party alternative (and predecessor) to - ``venv``. It allows virtual environments to be used on versions of - Python prior to 3.4, which either don't provide ``venv`` at all, or - aren't able to automatically install ``pip`` into created environments. -* The `Python Package Index `__ is a public - repository of open source licensed packages made available for use by - other Python users. -* the `Python Packaging Authority - `__ is the group of - developers and documentation authors responsible for the maintenance and - evolution of the standard packaging tools and the associated metadata and - file format standards. They maintain a variety of tools, documentation, - and issue trackers on both `GitHub `__ and - `Bitbucket `__. -* ``distutils`` is the original build and distribution system first added to - the Python standard library in 1998. While direct use of ``distutils`` is - being phased out, it still laid the foundation for the current packaging - and distribution infrastructure, and it not only remains part of the - standard library, but its name lives on in other ways (such as the name - of the mailing list used to coordinate Python packaging standards - development). - -.. versionchanged:: 3.5 - The use of ``venv`` is now recommended for creating virtual environments. - -.. seealso:: - - `Python Packaging User Guide: Creating and using virtual environments - `__ - - -Basic usage -=========== - -The standard packaging tools are all designed to be used from the command -line. - -The following command will install the latest version of a module and its -dependencies from the Python Package Index:: - - python -m pip install SomePackage - -.. note:: - - For POSIX users (including macOS and Linux users), the examples in - this guide assume the use of a :term:`virtual environment`. - - For Windows users, the examples in this guide assume that the option to - adjust the system PATH environment variable was selected when installing - Python. - -It's also possible to specify an exact or minimum version directly on the -command line. When using comparator operators such as ``>``, ``<`` or some other -special character which get interpreted by shell, the package name and the -version should be enclosed within double quotes:: - - python -m pip install SomePackage==1.0.4 # specific version - python -m pip install "SomePackage>=1.0.4" # minimum version - -Normally, if a suitable module is already installed, attempting to install -it again will have no effect. Upgrading existing modules must be requested -explicitly:: - - python -m pip install --upgrade SomePackage - -More information and resources regarding ``pip`` and its capabilities can be -found in the `Python Packaging User Guide `__. - -Creation of virtual environments is done through the :mod:`venv` module. -Installing packages into an active virtual environment uses the commands shown -above. - -.. seealso:: - - `Python Packaging User Guide: Installing Python Distribution Packages - `__ - - -How do I ...? -============= - -These are quick answers or links for some common tasks. - -... install ``pip`` in versions of Python prior to Python 3.4? --------------------------------------------------------------- - -Python only started bundling ``pip`` with Python 3.4. For earlier versions, -``pip`` needs to be "bootstrapped" as described in the Python Packaging -User Guide. - -.. seealso:: - - `Python Packaging User Guide: Requirements for Installing Packages - `__ - - -.. installing-per-user-installation: - -... install packages just for the current user? ------------------------------------------------ - -Passing the ``--user`` option to ``python -m pip install`` will install a -package just for the current user, rather than for all users of the system. - - -... install scientific Python packages? ---------------------------------------- - -A number of scientific Python packages have complex binary dependencies, and -aren't currently easy to install using ``pip`` directly. At this point in -time, it will often be easier for users to install these packages by -`other means `__ -rather than attempting to install them with ``pip``. - -.. seealso:: - - `Python Packaging User Guide: Installing Scientific Packages - `__ - - -... work with multiple versions of Python installed in parallel? ----------------------------------------------------------------- - -On Linux, macOS, and other POSIX systems, use the versioned Python commands -in combination with the ``-m`` switch to run the appropriate copy of -``pip``:: - - python2 -m pip install SomePackage # default Python 2 - python2.7 -m pip install SomePackage # specifically Python 2.7 - python3 -m pip install SomePackage # default Python 3 - python3.4 -m pip install SomePackage # specifically Python 3.4 - -Appropriately versioned ``pip`` commands may also be available. - -On Windows, use the ``py`` Python launcher in combination with the ``-m`` -switch:: - - py -2 -m pip install SomePackage # default Python 2 - py -2.7 -m pip install SomePackage # specifically Python 2.7 - py -3 -m pip install SomePackage # default Python 3 - py -3.4 -m pip install SomePackage # specifically Python 3.4 - -.. other questions: - - Once the Development & Deployment part of PPUG is fleshed out, some of - those sections should be linked from new questions here (most notably, - we should have a question about avoiding depending on PyPI that links to - https://packaging.python.org/en/latest/mirrors/) - - -Common installation issues -========================== - -Installing into the system Python on Linux ------------------------------------------- - -On Linux systems, a Python installation will typically be included as part -of the distribution. Installing into this Python installation requires -root access to the system, and may interfere with the operation of the -system package manager and other components of the system if a component -is unexpectedly upgraded using ``pip``. - -On such systems, it is often better to use a virtual environment or a -per-user installation when installing packages with ``pip``. - - -Pip not installed ------------------ - -It is possible that ``pip`` does not get installed by default. One potential fix is:: - - python -m ensurepip --default-pip - -There are also additional resources for `installing pip. -`__ - - -Installing binary extensions ----------------------------- - -Python has typically relied heavily on source based distribution, with end -users being expected to compile extension modules from source as part of -the installation process. - -With the introduction of support for the binary ``wheel`` format, and the -ability to publish wheels for at least Windows and macOS through the -Python Package Index, this problem is expected to diminish over time, -as users are more regularly able to install pre-built extensions rather -than needing to build them themselves. - -Some of the solutions for installing `scientific software -`__ -that are not yet available as pre-built ``wheel`` files may also help with -obtaining other binary extensions without needing to build them locally. - -.. seealso:: - - `Python Packaging User Guide: Binary Extensions - `__ diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst deleted file mode 100644 index d85ad94e9b7fe4..00000000000000 --- a/Doc/library/2to3.rst +++ /dev/null @@ -1,489 +0,0 @@ -.. _2to3-reference: - -2to3 --- Automated Python 2 to 3 code translation -================================================= - -.. sectionauthor:: Benjamin Peterson - -2to3 is a Python program that reads Python 2.x source code and applies a series -of *fixers* to transform it into valid Python 3.x code. The standard library -contains a rich set of fixers that will handle almost all code. 2to3 supporting -library :mod:`lib2to3` is, however, a flexible and generic library, so it is -possible to write your own fixers for 2to3. - -.. deprecated-removed:: 3.11 3.13 - The ``lib2to3`` module was marked pending for deprecation in Python 3.9 - (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated - in Python 3.11 (raising :exc:`DeprecationWarning`). The ``2to3`` tool is - part of that. It will be removed in Python 3.13. - -.. _2to3-using: - -Using 2to3 ----------- - -2to3 will usually be installed with the Python interpreter as a script. It is -also located in the :file:`Tools/scripts` directory of the Python root. - -2to3's basic arguments are a list of files or directories to transform. The -directories are recursively traversed for Python sources. - -Here is a sample Python 2.x source file, :file:`example.py`:: - - def greet(name): - print "Hello, {0}!".format(name) - print "What's your name?" - name = raw_input() - greet(name) - -It can be converted to Python 3.x code via 2to3 on the command line: - -.. code-block:: shell-session - - $ 2to3 example.py - -A diff against the original source file is printed. 2to3 can also write the -needed modifications right back to the source file. (A backup of the original -file is made unless :option:`!-n` is also given.) Writing the changes back is -enabled with the :option:`!-w` flag: - -.. code-block:: shell-session - - $ 2to3 -w example.py - -After transformation, :file:`example.py` looks like this:: - - def greet(name): - print("Hello, {0}!".format(name)) - print("What's your name?") - name = input() - greet(name) - -Comments and exact indentation are preserved throughout the translation process. - -By default, 2to3 runs a set of :ref:`predefined fixers <2to3-fixers>`. The -:option:`!-l` flag lists all available fixers. An explicit set of fixers to run -can be given with :option:`!-f`. Likewise the :option:`!-x` explicitly disables a -fixer. The following example runs only the ``imports`` and ``has_key`` fixers: - -.. code-block:: shell-session - - $ 2to3 -f imports -f has_key example.py - -This command runs every fixer except the ``apply`` fixer: - -.. code-block:: shell-session - - $ 2to3 -x apply example.py - -Some fixers are *explicit*, meaning they aren't run by default and must be -listed on the command line to be run. Here, in addition to the default fixers, -the ``idioms`` fixer is run: - -.. code-block:: shell-session - - $ 2to3 -f all -f idioms example.py - -Notice how passing ``all`` enables all default fixers. - -Sometimes 2to3 will find a place in your source code that needs to be changed, -but 2to3 cannot fix automatically. In this case, 2to3 will print a warning -beneath the diff for a file. You should address the warning in order to have -compliant 3.x code. - -2to3 can also refactor doctests. To enable this mode, use the :option:`!-d` -flag. Note that *only* doctests will be refactored. This also doesn't require -the module to be valid Python. For example, doctest like examples in a reST -document could also be refactored with this option. - -The :option:`!-v` option enables output of more information on the translation -process. - -Since some print statements can be parsed as function calls or statements, 2to3 -cannot always read files containing the print function. When 2to3 detects the -presence of the ``from __future__ import print_function`` compiler directive, it -modifies its internal grammar to interpret :func:`print` as a function. This -change can also be enabled manually with the :option:`!-p` flag. Use -:option:`!-p` to run fixers on code that already has had its print statements -converted. Also :option:`!-e` can be used to make :func:`exec` a function. - -The :option:`!-o` or :option:`!--output-dir` option allows specification of an -alternate directory for processed output files to be written to. The -:option:`!-n` flag is required when using this as backup files do not make sense -when not overwriting the input files. - -.. versionadded:: 3.2.3 - The :option:`!-o` option was added. - -The :option:`!-W` or :option:`!--write-unchanged-files` flag tells 2to3 to always -write output files even if no changes were required to the file. This is most -useful with :option:`!-o` so that an entire Python source tree is copied with -translation from one directory to another. -This option implies the :option:`!-w` flag as it would not make sense otherwise. - -.. versionadded:: 3.2.3 - The :option:`!-W` flag was added. - -The :option:`!--add-suffix` option specifies a string to append to all output -filenames. The :option:`!-n` flag is required when specifying this as backups -are not necessary when writing to different filenames. Example: - -.. code-block:: shell-session - - $ 2to3 -n -W --add-suffix=3 example.py - -Will cause a converted file named ``example.py3`` to be written. - -.. versionadded:: 3.2.3 - The :option:`!--add-suffix` option was added. - -To translate an entire project from one directory tree to another use: - -.. code-block:: shell-session - - $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode - - -.. _2to3-fixers: - -Fixers ------- - -Each step of transforming code is encapsulated in a fixer. The command ``2to3 --l`` lists them. As :ref:`documented above <2to3-using>`, each can be turned on -and off individually. They are described here in more detail. - - -.. 2to3fixer:: apply - - Removes usage of :func:`apply`. For example ``apply(function, *args, - **kwargs)`` is converted to ``function(*args, **kwargs)``. - -.. 2to3fixer:: asserts - - Replaces deprecated :mod:`unittest` method names with the correct ones. - - ================================ ========================================== - From To - ================================ ========================================== - ``failUnlessEqual(a, b)`` :meth:`assertEqual(a, b) - ` - ``assertEquals(a, b)`` :meth:`assertEqual(a, b) - ` - ``failIfEqual(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``assertNotEquals(a, b)`` :meth:`assertNotEqual(a, b) - ` - ``failUnless(a)`` :meth:`assertTrue(a) - ` - ``assert_(a)`` :meth:`assertTrue(a) - ` - ``failIf(a)`` :meth:`assertFalse(a) - ` - ``failUnlessRaises(exc, cal)`` :meth:`assertRaises(exc, cal) - ` - ``failUnlessAlmostEqual(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``assertAlmostEquals(a, b)`` :meth:`assertAlmostEqual(a, b) - ` - ``failIfAlmostEqual(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ``assertNotAlmostEquals(a, b)`` :meth:`assertNotAlmostEqual(a, b) - ` - ================================ ========================================== - -.. 2to3fixer:: basestring - - Converts :class:`basestring` to :class:`str`. - -.. 2to3fixer:: buffer - - Converts :class:`buffer` to :class:`memoryview`. This fixer is optional - because the :class:`memoryview` API is similar but not exactly the same as - that of :class:`buffer`. - -.. 2to3fixer:: dict - - Fixes dictionary iteration methods. :meth:`dict.iteritems` is converted to - :meth:`dict.items`, :meth:`dict.iterkeys` to :meth:`dict.keys`, and - :meth:`dict.itervalues` to :meth:`dict.values`. Similarly, - :meth:`dict.viewitems`, :meth:`dict.viewkeys` and :meth:`dict.viewvalues` are - converted respectively to :meth:`dict.items`, :meth:`dict.keys` and - :meth:`dict.values`. It also wraps existing usages of :meth:`dict.items`, - :meth:`dict.keys`, and :meth:`dict.values` in a call to :class:`list`. - -.. 2to3fixer:: except - - Converts ``except X, T`` to ``except X as T``. - -.. 2to3fixer:: exec - - Converts the ``exec`` statement to the :func:`exec` function. - -.. 2to3fixer:: execfile - - Removes usage of :func:`execfile`. The argument to :func:`execfile` is - wrapped in calls to :func:`open`, :func:`compile`, and :func:`exec`. - -.. 2to3fixer:: exitfunc - - Changes assignment of :attr:`sys.exitfunc` to use of the :mod:`atexit` - module. - -.. 2to3fixer:: filter - - Wraps :func:`filter` usage in a :class:`list` call. - -.. 2to3fixer:: funcattrs - - Fixes function attributes that have been renamed. For example, - ``my_function.func_closure`` is converted to ``my_function.__closure__``. - -.. 2to3fixer:: future - - Removes ``from __future__ import new_feature`` statements. - -.. 2to3fixer:: getcwdu - - Renames :func:`os.getcwdu` to :func:`os.getcwd`. - -.. 2to3fixer:: has_key - - Changes ``dict.has_key(key)`` to ``key in dict``. - -.. 2to3fixer:: idioms - - This optional fixer performs several transformations that make Python code - more idiomatic. Type comparisons like ``type(x) is SomeClass`` and - ``type(x) == SomeClass`` are converted to ``isinstance(x, SomeClass)``. - ``while 1`` becomes ``while True``. This fixer also tries to make use of - :func:`sorted` in appropriate places. For example, this block :: - - L = list(some_iterable) - L.sort() - - is changed to :: - - L = sorted(some_iterable) - -.. 2to3fixer:: import - - Detects sibling imports and converts them to relative imports. - -.. 2to3fixer:: imports - - Handles module renames in the standard library. - -.. 2to3fixer:: imports2 - - Handles other modules renames in the standard library. It is separate from - the :2to3fixer:`imports` fixer only because of technical limitations. - -.. 2to3fixer:: input - - Converts ``input(prompt)`` to ``eval(input(prompt))``. - -.. 2to3fixer:: intern - - Converts :func:`intern` to :func:`sys.intern`. - -.. 2to3fixer:: isinstance - - Fixes duplicate types in the second argument of :func:`isinstance`. For - example, ``isinstance(x, (int, int))`` is converted to ``isinstance(x, - int)`` and ``isinstance(x, (int, float, int))`` is converted to - ``isinstance(x, (int, float))``. - -.. 2to3fixer:: itertools_imports - - Removes imports of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap`. Imports of :func:`itertools.ifilterfalse` are also - changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: itertools - - Changes usage of :func:`itertools.ifilter`, :func:`itertools.izip`, and - :func:`itertools.imap` to their built-in equivalents. - :func:`itertools.ifilterfalse` is changed to :func:`itertools.filterfalse`. - -.. 2to3fixer:: long - - Renames :class:`long` to :class:`int`. - -.. 2to3fixer:: map - - Wraps :func:`map` in a :class:`list` call. It also changes ``map(None, x)`` - to ``list(x)``. Using ``from future_builtins import map`` disables this - fixer. - -.. 2to3fixer:: metaclass - - Converts the old metaclass syntax (``__metaclass__ = Meta`` in the class - body) to the new (``class X(metaclass=Meta)``). - -.. 2to3fixer:: methodattrs - - Fixes old method attribute names. For example, ``meth.im_func`` is converted - to ``meth.__func__``. - -.. 2to3fixer:: ne - - Converts the old not-equal syntax, ``<>``, to ``!=``. - -.. 2to3fixer:: next - - Converts the use of iterator's :meth:`~iterator.next` methods to the - :func:`next` function. It also renames :meth:`next` methods to - :meth:`~iterator.__next__`. - -.. 2to3fixer:: nonzero - - Renames definitions of methods called :meth:`__nonzero__` - to :meth:`~object.__bool__`. - -.. 2to3fixer:: numliterals - - Converts octal literals into the new syntax. - -.. 2to3fixer:: operator - - Converts calls to various functions in the :mod:`operator` module to other, - but equivalent, function calls. When needed, the appropriate ``import`` - statements are added, e.g. ``import collections.abc``. The following mapping - are made: - - ================================== ============================================= - From To - ================================== ============================================= - ``operator.isCallable(obj)`` ``callable(obj)`` - ``operator.sequenceIncludes(obj)`` ``operator.contains(obj)`` - ``operator.isSequenceType(obj)`` ``isinstance(obj, collections.abc.Sequence)`` - ``operator.isMappingType(obj)`` ``isinstance(obj, collections.abc.Mapping)`` - ``operator.isNumberType(obj)`` ``isinstance(obj, numbers.Number)`` - ``operator.repeat(obj, n)`` ``operator.mul(obj, n)`` - ``operator.irepeat(obj, n)`` ``operator.imul(obj, n)`` - ================================== ============================================= - -.. 2to3fixer:: paren - - Add extra parenthesis where they are required in list comprehensions. For - example, ``[x for x in 1, 2]`` becomes ``[x for x in (1, 2)]``. - -.. 2to3fixer:: print - - Converts the ``print`` statement to the :func:`print` function. - -.. 2to3fixer:: raise - - Converts ``raise E, V`` to ``raise E(V)``, and ``raise E, V, T`` to ``raise - E(V).with_traceback(T)``. If ``E`` is a tuple, the translation will be - incorrect because substituting tuples for exceptions has been removed in 3.0. - -.. 2to3fixer:: raw_input - - Converts :func:`raw_input` to :func:`input`. - -.. 2to3fixer:: reduce - - Handles the move of :func:`reduce` to :func:`functools.reduce`. - -.. 2to3fixer:: reload - - Converts :func:`reload` to :func:`importlib.reload`. - -.. 2to3fixer:: renames - - Changes :data:`sys.maxint` to :data:`sys.maxsize`. - -.. 2to3fixer:: repr - - Replaces backtick repr with the :func:`repr` function. - -.. 2to3fixer:: set_literal - - Replaces use of the :class:`set` constructor with set literals. This fixer - is optional. - -.. 2to3fixer:: standarderror - - Renames :exc:`StandardError` to :exc:`Exception`. - -.. 2to3fixer:: sys_exc - - Changes the deprecated :data:`sys.exc_value`, :data:`sys.exc_type`, - :data:`sys.exc_traceback` to use :func:`sys.exc_info`. - -.. 2to3fixer:: throw - - Fixes the API change in generator's :meth:`throw` method. - -.. 2to3fixer:: tuple_params - - Removes implicit tuple parameter unpacking. This fixer inserts temporary - variables. - -.. 2to3fixer:: types - - Fixes code broken from the removal of some members in the :mod:`types` - module. - -.. 2to3fixer:: unicode - - Renames :class:`unicode` to :class:`str`. - -.. 2to3fixer:: urllib - - Handles the rename of :mod:`urllib` and :mod:`urllib2` to the :mod:`urllib` - package. - -.. 2to3fixer:: ws_comma - - Removes excess whitespace from comma separated items. This fixer is - optional. - -.. 2to3fixer:: xrange - - Renames :func:`xrange` to :func:`range` and wraps existing :func:`range` - calls with :class:`list`. - -.. 2to3fixer:: xreadlines - - Changes ``for x in file.xreadlines()`` to ``for x in file``. - -.. 2to3fixer:: zip - - Wraps :func:`zip` usage in a :class:`list` call. This is disabled when - ``from future_builtins import zip`` appears. - - -:mod:`lib2to3` --- 2to3's library ---------------------------------- - -.. module:: lib2to3 - :synopsis: The 2to3 library - -.. moduleauthor:: Guido van Rossum -.. moduleauthor:: Collin Winter -.. moduleauthor:: Benjamin Peterson - -**Source code:** :source:`Lib/lib2to3/` - --------------- - -.. deprecated-removed:: 3.11 3.13 - Python 3.9 switched to a PEG parser (see :pep:`617`) while lib2to3 is - using a less flexible LL(1) parser. Python 3.10 includes new language - syntax that is not parsable by lib2to3's LL(1) parser (see :pep:`634`). - The ``lib2to3`` module was marked pending for deprecation in Python 3.9 - (raising :exc:`PendingDeprecationWarning` on import) and fully deprecated - in Python 3.11 (raising :exc:`DeprecationWarning`). - It will be removed from the standard library in Python 3.13. - Consider third-party alternatives such as `LibCST`_ or `parso`_. - -.. note:: - - The :mod:`lib2to3` API should be considered unstable and may change - drastically in the future. - -.. _LibCST: https://libcst.readthedocs.io/ -.. _parso: https://parso.readthedocs.io/ diff --git a/Doc/library/__future__.rst b/Doc/library/__future__.rst deleted file mode 100644 index 8bd23daee73977..00000000000000 --- a/Doc/library/__future__.rst +++ /dev/null @@ -1,111 +0,0 @@ -:mod:`__future__` --- Future statement definitions -================================================== - -.. module:: __future__ - :synopsis: Future statement definitions - -**Source code:** :source:`Lib/__future__.py` - --------------- - -:mod:`__future__` is a real module, and serves three purposes: - -* To avoid confusing existing tools that analyze import statements and expect to - find the modules they're importing. - -* To ensure that :ref:`future statements ` run under releases prior to - 2.1 at least yield runtime exceptions (the import of :mod:`__future__` will - fail, because there was no module of that name prior to 2.1). - -* To document when incompatible changes were introduced, and when they will be - --- or were --- made mandatory. This is a form of executable documentation, and - can be inspected programmatically via importing :mod:`__future__` and examining - its contents. - -Each statement in :file:`__future__.py` is of the form:: - - FeatureName = _Feature(OptionalRelease, MandatoryRelease, - CompilerFlag) - - -where, normally, *OptionalRelease* is less than *MandatoryRelease*, and both are -5-tuples of the same form as :data:`sys.version_info`:: - - (PY_MAJOR_VERSION, # the 2 in 2.1.0a3; an int - PY_MINOR_VERSION, # the 1; an int - PY_MICRO_VERSION, # the 0; an int - PY_RELEASE_LEVEL, # "alpha", "beta", "candidate" or "final"; string - PY_RELEASE_SERIAL # the 3; an int - ) - -*OptionalRelease* records the first release in which the feature was accepted. - -In the case of a *MandatoryRelease* that has not yet occurred, -*MandatoryRelease* predicts the release in which the feature will become part of -the language. - -Else *MandatoryRelease* records when the feature became part of the language; in -releases at or after that, modules no longer need a future statement to use the -feature in question, but may continue to use such imports. - -*MandatoryRelease* may also be ``None``, meaning that a planned feature got -dropped. - -Instances of class :class:`_Feature` have two corresponding methods, -:meth:`getOptionalRelease` and :meth:`getMandatoryRelease`. - -*CompilerFlag* is the (bitfield) flag that should be passed in the fourth -argument to the built-in function :func:`compile` to enable the feature in -dynamically compiled code. This flag is stored in the :attr:`compiler_flag` -attribute on :class:`_Feature` instances. - -No feature description will ever be deleted from :mod:`__future__`. Since its -introduction in Python 2.1 the following features have found their way into the -language using this mechanism: - -+------------------+-------------+--------------+---------------------------------------------+ -| feature | optional in | mandatory in | effect | -+==================+=============+==============+=============================================+ -| nested_scopes | 2.1.0b1 | 2.2 | :pep:`227`: | -| | | | *Statically Nested Scopes* | -+------------------+-------------+--------------+---------------------------------------------+ -| generators | 2.2.0a1 | 2.3 | :pep:`255`: | -| | | | *Simple Generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| division | 2.2.0a2 | 3.0 | :pep:`238`: | -| | | | *Changing the Division Operator* | -+------------------+-------------+--------------+---------------------------------------------+ -| absolute_import | 2.5.0a1 | 3.0 | :pep:`328`: | -| | | | *Imports: Multi-Line and Absolute/Relative* | -+------------------+-------------+--------------+---------------------------------------------+ -| with_statement | 2.5.0a1 | 2.6 | :pep:`343`: | -| | | | *The "with" Statement* | -+------------------+-------------+--------------+---------------------------------------------+ -| print_function | 2.6.0a2 | 3.0 | :pep:`3105`: | -| | | | *Make print a function* | -+------------------+-------------+--------------+---------------------------------------------+ -| unicode_literals | 2.6.0a2 | 3.0 | :pep:`3112`: | -| | | | *Bytes literals in Python 3000* | -+------------------+-------------+--------------+---------------------------------------------+ -| generator_stop | 3.5.0b1 | 3.7 | :pep:`479`: | -| | | | *StopIteration handling inside generators* | -+------------------+-------------+--------------+---------------------------------------------+ -| annotations | 3.7.0b1 | TBD [1]_ | :pep:`563`: | -| | | | *Postponed evaluation of annotations* | -+------------------+-------------+--------------+---------------------------------------------+ - -.. XXX Adding a new entry? Remember to update simple_stmts.rst, too. - -.. [1] - ``from __future__ import annotations`` was previously scheduled to - become mandatory in Python 3.10, but the Python Steering Council - twice decided to delay the change - (`announcement for Python 3.10 `__; - `announcement for Python 3.11 `__). - No final decision has been made yet. See also :pep:`563` and :pep:`649`. - - -.. seealso:: - - :ref:`future` - How the compiler treats future imports. diff --git a/Doc/library/__main__.rst b/Doc/library/__main__.rst deleted file mode 100644 index 435310d525d754..00000000000000 --- a/Doc/library/__main__.rst +++ /dev/null @@ -1,368 +0,0 @@ -:mod:`__main__` --- Top-level code environment -============================================== - -.. module:: __main__ - :synopsis: The environment where top-level code is run. Covers command-line - interfaces, import-time behavior, and ``__name__ == '__main__'``. - --------------- - -In Python, the special name ``__main__`` is used for two important constructs: - -1. the name of the top-level environment of the program, which can be - checked using the ``__name__ == '__main__'`` expression; and -2. the ``__main__.py`` file in Python packages. - -Both of these mechanisms are related to Python modules; how users interact with -them and how they interact with each other. They are explained in detail -below. If you're new to Python modules, see the tutorial section -:ref:`tut-modules` for an introduction. - - -.. _name_equals_main: - -``__name__ == '__main__'`` ---------------------------- - -When a Python module or package is imported, ``__name__`` is set to the -module's name. Usually, this is the name of the Python file itself without the -``.py`` extension:: - - >>> import configparser - >>> configparser.__name__ - 'configparser' - -If the file is part of a package, ``__name__`` will also include the parent -package's path:: - - >>> from concurrent.futures import process - >>> process.__name__ - 'concurrent.futures.process' - -However, if the module is executed in the top-level code environment, -its ``__name__`` is set to the string ``'__main__'``. - -What is the "top-level code environment"? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -``__main__`` is the name of the environment where top-level code is run. -"Top-level code" is the first user-specified Python module that starts running. -It's "top-level" because it imports all other modules that the program needs. -Sometimes "top-level code" is called an *entry point* to the application. - -The top-level code environment can be: - -* the scope of an interactive prompt:: - - >>> __name__ - '__main__' - -* the Python module passed to the Python interpreter as a file argument: - - .. code-block:: shell-session - - $ python3 helloworld.py - Hello, world! - -* the Python module or package passed to the Python interpreter with the - :option:`-m` argument: - - .. code-block:: shell-session - - $ python3 -m tarfile - usage: tarfile.py [-h] [-v] (...) - -* Python code read by the Python interpreter from standard input: - - .. code-block:: shell-session - - $ echo "import this" | python3 - The Zen of Python, by Tim Peters - - Beautiful is better than ugly. - Explicit is better than implicit. - ... - -* Python code passed to the Python interpreter with the :option:`-c` argument: - - .. code-block:: shell-session - - $ python3 -c "import this" - The Zen of Python, by Tim Peters - - Beautiful is better than ugly. - Explicit is better than implicit. - ... - -In each of these situations, the top-level module's ``__name__`` is set to -``'__main__'``. - -As a result, a module can discover whether or not it is running in the -top-level environment by checking its own ``__name__``, which allows a common -idiom for conditionally executing code when the module is not initialized from -an import statement:: - - if __name__ == '__main__': - # Execute when the module is not initialized from an import statement. - ... - -.. seealso:: - - For a more detailed look at how ``__name__`` is set in all situations, see - the tutorial section :ref:`tut-modules`. - - -Idiomatic Usage -^^^^^^^^^^^^^^^ - -Some modules contain code that is intended for script use only, like parsing -command-line arguments or fetching data from standard input. If a module -like this was imported from a different module, for example to unit test -it, the script code would unintentionally execute as well. - -This is where using the ``if __name__ == '__main__'`` code block comes in -handy. Code within this block won't run unless the module is executed in the -top-level environment. - -Putting as few statements as possible in the block below ``if __name__ == -'__main__'`` can improve code clarity and correctness. Most often, a function -named ``main`` encapsulates the program's primary behavior:: - - # echo.py - - import shlex - import sys - - def echo(phrase: str) -> None: - """A dummy wrapper around print.""" - # for demonstration purposes, you can imagine that there is some - # valuable and reusable logic inside this function - print(phrase) - - def main() -> int: - """Echo the input arguments to standard output""" - phrase = shlex.join(sys.argv) - echo(phrase) - return 0 - - if __name__ == '__main__': - sys.exit(main()) # next section explains the use of sys.exit - -Note that if the module didn't encapsulate code inside the ``main`` function -but instead put it directly within the ``if __name__ == '__main__'`` block, -the ``phrase`` variable would be global to the entire module. This is -error-prone as other functions within the module could be unintentionally using -the global variable instead of a local name. A ``main`` function solves this -problem. - -Using a ``main`` function has the added benefit of the ``echo`` function itself -being isolated and importable elsewhere. When ``echo.py`` is imported, the -``echo`` and ``main`` functions will be defined, but neither of them will be -called, because ``__name__ != '__main__'``. - - -Packaging Considerations -^^^^^^^^^^^^^^^^^^^^^^^^ - -``main`` functions are often used to create command-line tools by specifying -them as entry points for console scripts. When this is done, -`pip `_ inserts the function call into a template script, -where the return value of ``main`` is passed into :func:`sys.exit`. -For example:: - - sys.exit(main()) - -Since the call to ``main`` is wrapped in :func:`sys.exit`, the expectation is -that your function will return some value acceptable as an input to -:func:`sys.exit`; typically, an integer or ``None`` (which is implicitly -returned if your function does not have a return statement). - -By proactively following this convention ourselves, our module will have the -same behavior when run directly (i.e. ``python3 echo.py``) as it will have if -we later package it as a console script entry-point in a pip-installable -package. - -In particular, be careful about returning strings from your ``main`` function. -:func:`sys.exit` will interpret a string argument as a failure message, so -your program will have an exit code of ``1``, indicating failure, and the -string will be written to :data:`sys.stderr`. The ``echo.py`` example from -earlier exemplifies using the ``sys.exit(main())`` convention. - -.. seealso:: - - `Python Packaging User Guide `_ - contains a collection of tutorials and references on how to distribute and - install Python packages with modern tools. - - -``__main__.py`` in Python Packages ----------------------------------- - -If you are not familiar with Python packages, see section :ref:`tut-packages` -of the tutorial. Most commonly, the ``__main__.py`` file is used to provide -a command-line interface for a package. Consider the following hypothetical -package, "bandclass": - -.. code-block:: text - - bandclass - ├── __init__.py - ├── __main__.py - └── student.py - -``__main__.py`` will be executed when the package itself is invoked -directly from the command line using the :option:`-m` flag. For example: - -.. code-block:: shell-session - - $ python3 -m bandclass - -This command will cause ``__main__.py`` to run. How you utilize this mechanism -will depend on the nature of the package you are writing, but in this -hypothetical case, it might make sense to allow the teacher to search for -students:: - - # bandclass/__main__.py - - import sys - from .student import search_students - - student_name = sys.argv[2] if len(sys.argv) >= 2 else '' - print(f'Found student: {search_students(student_name)}') - -Note that ``from .student import search_students`` is an example of a relative -import. This import style can be used when referencing modules within a -package. For more details, see :ref:`intra-package-references` in the -:ref:`tut-modules` section of the tutorial. - -Idiomatic Usage -^^^^^^^^^^^^^^^ - -The content of ``__main__.py`` typically isn't fenced with an -``if __name__ == '__main__'`` block. Instead, those files are kept -short and import functions to execute from other modules. Those other modules can then be -easily unit-tested and are properly reusable. - -If used, an ``if __name__ == '__main__'`` block will still work as expected -for a ``__main__.py`` file within a package, because its ``__name__`` -attribute will include the package's path if imported:: - - >>> import asyncio.__main__ - >>> asyncio.__main__.__name__ - 'asyncio.__main__' - -This won't work for ``__main__.py`` files in the root directory of a .zip file -though. Hence, for consistency, minimal ``__main__.py`` like the :mod:`venv` -one mentioned below are preferred. - -.. seealso:: - - See :mod:`venv` for an example of a package with a minimal ``__main__.py`` - in the standard library. It doesn't contain a ``if __name__ == '__main__'`` - block. You can invoke it with ``python -m venv [directory]``. - - See :mod:`runpy` for more details on the :option:`-m` flag to the - interpreter executable. - - See :mod:`zipapp` for how to run applications packaged as *.zip* files. In - this case Python looks for a ``__main__.py`` file in the root directory of - the archive. - - - -``import __main__`` -------------------- - -Regardless of which module a Python program was started with, other modules -running within that same program can import the top-level environment's scope -(:term:`namespace`) by importing the ``__main__`` module. This doesn't import -a ``__main__.py`` file but rather whichever module that received the special -name ``'__main__'``. - -Here is an example module that consumes the ``__main__`` namespace:: - - # namely.py - - import __main__ - - def did_user_define_their_name(): - return 'my_name' in dir(__main__) - - def print_user_name(): - if not did_user_define_their_name(): - raise ValueError('Define the variable `my_name`!') - - if '__file__' in dir(__main__): - print(__main__.my_name, "found in file", __main__.__file__) - else: - print(__main__.my_name) - -Example usage of this module could be as follows:: - - # start.py - - import sys - - from namely import print_user_name - - # my_name = "Dinsdale" - - def main(): - try: - print_user_name() - except ValueError as ve: - return str(ve) - - if __name__ == "__main__": - sys.exit(main()) - -Now, if we started our program, the result would look like this: - -.. code-block:: shell-session - - $ python3 start.py - Define the variable `my_name`! - -The exit code of the program would be 1, indicating an error. Uncommenting the -line with ``my_name = "Dinsdale"`` fixes the program and now it exits with -status code 0, indicating success: - -.. code-block:: shell-session - - $ python3 start.py - Dinsdale found in file /path/to/start.py - -Note that importing ``__main__`` doesn't cause any issues with unintentionally -running top-level code meant for script use which is put in the -``if __name__ == "__main__"`` block of the ``start`` module. Why does this work? - -Python inserts an empty ``__main__`` module in :data:`sys.modules` at -interpreter startup, and populates it by running top-level code. In our example -this is the ``start`` module which runs line by line and imports ``namely``. -In turn, ``namely`` imports ``__main__`` (which is really ``start``). That's an -import cycle! Fortunately, since the partially populated ``__main__`` -module is present in :data:`sys.modules`, Python passes that to ``namely``. -See :ref:`Special considerations for __main__ ` in the -import system's reference for details on how this works. - -The Python REPL is another example of a "top-level environment", so anything -defined in the REPL becomes part of the ``__main__`` scope:: - - >>> import namely - >>> namely.did_user_define_their_name() - False - >>> namely.print_user_name() - Traceback (most recent call last): - ... - ValueError: Define the variable `my_name`! - >>> my_name = 'Jabberwocky' - >>> namely.did_user_define_their_name() - True - >>> namely.print_user_name() - Jabberwocky - -Note that in this case the ``__main__`` scope doesn't contain a ``__file__`` -attribute as it's interactive. - -The ``__main__`` scope is used in the implementation of :mod:`pdb` and -:mod:`rlcompleter`. diff --git a/Doc/library/_thread.rst b/Doc/library/_thread.rst deleted file mode 100644 index 5e506d2c37d6e0..00000000000000 --- a/Doc/library/_thread.rst +++ /dev/null @@ -1,230 +0,0 @@ -:mod:`_thread` --- Low-level threading API -========================================== - -.. module:: _thread - :synopsis: Low-level threading API. - -.. index:: - single: light-weight processes - single: processes, light-weight - single: binary semaphores - single: semaphores, binary - --------------- - -This module provides low-level primitives for working with multiple threads -(also called :dfn:`light-weight processes` or :dfn:`tasks`) --- multiple threads of -control sharing their global data space. For synchronization, simple locks -(also called :dfn:`mutexes` or :dfn:`binary semaphores`) are provided. -The :mod:`threading` module provides an easier to use and higher-level -threading API built on top of this module. - -.. index:: - single: pthreads - pair: threads; POSIX - -.. versionchanged:: 3.7 - This module used to be optional, it is now always available. - -This module defines the following constants and functions: - -.. exception:: error - - Raised on thread-specific errors. - - .. versionchanged:: 3.3 - This is now a synonym of the built-in :exc:`RuntimeError`. - - -.. data:: LockType - - This is the type of lock objects. - - -.. function:: start_new_thread(function, args[, kwargs]) - - Start a new thread and return its identifier. The thread executes the - function *function* with the argument list *args* (which must be a tuple). - The optional *kwargs* argument specifies a dictionary of keyword arguments. - - When the function returns, the thread silently exits. - - When the function terminates with an unhandled exception, - :func:`sys.unraisablehook` is called to handle the exception. The *object* - attribute of the hook argument is *function*. By default, a stack trace is - printed and then the thread exits (but other threads continue to run). - - When the function raises a :exc:`SystemExit` exception, it is silently - ignored. - - .. versionchanged:: 3.8 - :func:`sys.unraisablehook` is now used to handle unhandled exceptions. - - -.. function:: interrupt_main(signum=signal.SIGINT, /) - - Simulate the effect of a signal arriving in the main thread. - A thread can use this function to interrupt the main thread, though - there is no guarantee that the interruption will happen immediately. - - If given, *signum* is the number of the signal to simulate. - If *signum* is not given, :const:`signal.SIGINT` is simulated. - - If the given signal isn't handled by Python (it was set to - :const:`signal.SIG_DFL` or :const:`signal.SIG_IGN`), this function does - nothing. - - .. versionchanged:: 3.10 - The *signum* argument is added to customize the signal number. - - .. note:: - This does not emit the corresponding signal but schedules a call to - the associated handler (if it exists). - If you want to truly emit the signal, use :func:`signal.raise_signal`. - - -.. function:: exit() - - Raise the :exc:`SystemExit` exception. When not caught, this will cause the - thread to exit silently. - -.. - function:: exit_prog(status) - - Exit all threads and report the value of the integer argument - *status* as the exit status of the entire program. - **Caveat:** code in pending :keyword:`finally` clauses, in this thread - or in other threads, is not executed. - - -.. function:: allocate_lock() - - Return a new lock object. Methods of locks are described below. The lock is - initially unlocked. - - -.. function:: get_ident() - - Return the 'thread identifier' of the current thread. This is a nonzero - integer. Its value has no direct meaning; it is intended as a magic cookie to - be used e.g. to index a dictionary of thread-specific data. Thread identifiers - may be recycled when a thread exits and another thread is created. - - -.. function:: get_native_id() - - Return the native integral Thread ID of the current thread assigned by the kernel. - This is a non-negative integer. - Its value may be used to uniquely identify this particular thread system-wide - (until the thread terminates, after which the value may be recycled by the OS). - - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. - - .. versionadded:: 3.8 - - -.. function:: stack_size([size]) - - Return the thread stack size used when creating new threads. The optional - *size* argument specifies the stack size to be used for subsequently created - threads, and must be 0 (use platform or configured default) or a positive - integer value of at least 32,768 (32 KiB). If *size* is not specified, - 0 is used. If changing the thread stack size is - unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is - invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB - is currently the minimum supported stack size value to guarantee sufficient - stack space for the interpreter itself. Note that some platforms may have - particular restrictions on values for the stack size, such as requiring a - minimum stack size > 32 KiB or requiring allocation in multiples of the system - memory page size - platform documentation should be referred to for more - information (4 KiB pages are common; using multiples of 4096 for the stack size is - the suggested approach in the absence of more specific information). - - .. availability:: Windows, pthreads. - - Unix platforms with POSIX threads support. - - -.. data:: TIMEOUT_MAX - - The maximum value allowed for the *timeout* parameter of - :meth:`Lock.acquire `. Specifying a timeout greater - than this value will raise an :exc:`OverflowError`. - - .. versionadded:: 3.2 - - -Lock objects have the following methods: - - -.. method:: lock.acquire(blocking=True, timeout=-1) - - Without any optional argument, this method acquires the lock unconditionally, if - necessary waiting until it is released by another thread (only one thread at a - time can acquire a lock --- that's their reason for existence). - - If the *blocking* argument is present, the action depends on its - value: if it is False, the lock is only acquired if it can be acquired - immediately without waiting, while if it is True, the lock is acquired - unconditionally as above. - - If the floating-point *timeout* argument is present and positive, it - specifies the maximum wait time in seconds before returning. A negative - *timeout* argument specifies an unbounded wait. You cannot specify - a *timeout* if *blocking* is False. - - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not. - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - .. versionchanged:: 3.2 - Lock acquires can now be interrupted by signals on POSIX. - - -.. method:: lock.release() - - Releases the lock. The lock must have been acquired earlier, but not - necessarily by the same thread. - - -.. method:: lock.locked() - - Return the status of the lock: ``True`` if it has been acquired by some thread, - ``False`` if not. - -In addition to these methods, lock objects can also be used via the -:keyword:`with` statement, e.g.:: - - import _thread - - a_lock = _thread.allocate_lock() - - with a_lock: - print("a_lock is locked while this executes") - -**Caveats:** - -.. index:: pair: module; signal - -* Threads interact strangely with interrupts: the :exc:`KeyboardInterrupt` - exception will be received by an arbitrary thread. (When the :mod:`signal` - module is available, interrupts always go to the main thread.) - -* Calling :func:`sys.exit` or raising the :exc:`SystemExit` exception is - equivalent to calling :func:`_thread.exit`. - -* It is not possible to interrupt the :meth:`~threading.Lock.acquire` method on - a lock --- the :exc:`KeyboardInterrupt` exception will happen after the lock - has been acquired. - -* When the main thread exits, it is system defined whether the other threads - survive. On most systems, they are killed without executing - :keyword:`try` ... :keyword:`finally` clauses or executing object - destructors. - -* When the main thread exits, it does not do any of its usual cleanup (except - that :keyword:`try` ... :keyword:`finally` clauses are honored), and the - standard I/O files are not flushed. - diff --git a/Doc/library/abc.rst b/Doc/library/abc.rst deleted file mode 100644 index fb4f9da169c5ab..00000000000000 --- a/Doc/library/abc.rst +++ /dev/null @@ -1,359 +0,0 @@ -:mod:`abc` --- Abstract Base Classes -==================================== - -.. module:: abc - :synopsis: Abstract base classes according to :pep:`3119`. - -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Georg Brandl -.. much of the content adapted from docstrings - -**Source code:** :source:`Lib/abc.py` - --------------- - -This module provides the infrastructure for defining :term:`abstract base -classes ` (ABCs) in Python, as outlined in :pep:`3119`; -see the PEP for why this was added to Python. (See also :pep:`3141` and the -:mod:`numbers` module regarding a type hierarchy for numbers based on ABCs.) - -The :mod:`collections` module has some concrete classes that derive from -ABCs; these can, of course, be further derived. In addition, the -:mod:`collections.abc` submodule has some ABCs that can be used to test whether -a class or instance provides a particular interface, for example, if it is -:term:`hashable` or if it is a mapping. - - -This module provides the metaclass :class:`ABCMeta` for defining ABCs and -a helper class :class:`ABC` to alternatively define ABCs through inheritance: - -.. class:: ABC - - A helper class that has :class:`ABCMeta` as its metaclass. With this class, - an abstract base class can be created by simply deriving from :class:`ABC` - avoiding sometimes confusing metaclass usage, for example:: - - from abc import ABC - - class MyABC(ABC): - pass - - Note that the type of :class:`ABC` is still :class:`ABCMeta`, therefore - inheriting from :class:`ABC` requires the usual precautions regarding - metaclass usage, as multiple inheritance may lead to metaclass conflicts. - One may also define an abstract base class by passing the metaclass - keyword and using :class:`ABCMeta` directly, for example:: - - from abc import ABCMeta - - class MyABC(metaclass=ABCMeta): - pass - - .. versionadded:: 3.4 - - -.. class:: ABCMeta - - Metaclass for defining Abstract Base Classes (ABCs). - - Use this metaclass to create an ABC. An ABC can be subclassed directly, and - then acts as a mix-in class. You can also register unrelated concrete - classes (even built-in classes) and unrelated ABCs as "virtual subclasses" -- - these and their descendants will be considered subclasses of the registering - ABC by the built-in :func:`issubclass` function, but the registering ABC - won't show up in their MRO (Method Resolution Order) nor will method - implementations defined by the registering ABC be callable (not even via - :func:`super`). [#]_ - - Classes created with a metaclass of :class:`ABCMeta` have the following method: - - .. method:: register(subclass) - - Register *subclass* as a "virtual subclass" of this ABC. For - example:: - - from abc import ABC - - class MyABC(ABC): - pass - - MyABC.register(tuple) - - assert issubclass(tuple, MyABC) - assert isinstance((), MyABC) - - .. versionchanged:: 3.3 - Returns the registered subclass, to allow usage as a class decorator. - - .. versionchanged:: 3.4 - To detect calls to :meth:`register`, you can use the - :func:`get_cache_token` function. - - You can also override this method in an abstract base class: - - .. method:: __subclasshook__(subclass) - - (Must be defined as a class method.) - - Check whether *subclass* is considered a subclass of this ABC. This means - that you can customize the behavior of ``issubclass`` further without the - need to call :meth:`register` on every class you want to consider a - subclass of the ABC. (This class method is called from the - :meth:`__subclasscheck__` method of the ABC.) - - This method should return ``True``, ``False`` or ``NotImplemented``. If - it returns ``True``, the *subclass* is considered a subclass of this ABC. - If it returns ``False``, the *subclass* is not considered a subclass of - this ABC, even if it would normally be one. If it returns - ``NotImplemented``, the subclass check is continued with the usual - mechanism. - - .. XXX explain the "usual mechanism" - - - For a demonstration of these concepts, look at this example ABC definition:: - - class Foo: - def __getitem__(self, index): - ... - def __len__(self): - ... - def get_iterator(self): - return iter(self) - - class MyIterable(ABC): - - @abstractmethod - def __iter__(self): - while False: - yield None - - def get_iterator(self): - return self.__iter__() - - @classmethod - def __subclasshook__(cls, C): - if cls is MyIterable: - if any("__iter__" in B.__dict__ for B in C.__mro__): - return True - return NotImplemented - - MyIterable.register(Foo) - - The ABC ``MyIterable`` defines the standard iterable method, - :meth:`~iterator.__iter__`, as an abstract method. The implementation given - here can still be called from subclasses. The :meth:`get_iterator` method - is also part of the ``MyIterable`` abstract base class, but it does not have - to be overridden in non-abstract derived classes. - - The :meth:`__subclasshook__` class method defined here says that any class - that has an :meth:`~iterator.__iter__` method in its - :attr:`~object.__dict__` (or in that of one of its base classes, accessed - via the :attr:`~class.__mro__` list) is considered a ``MyIterable`` too. - - Finally, the last line makes ``Foo`` a virtual subclass of ``MyIterable``, - even though it does not define an :meth:`~iterator.__iter__` method (it uses - the old-style iterable protocol, defined in terms of :meth:`__len__` and - :meth:`~object.__getitem__`). Note that this will not make ``get_iterator`` - available as a method of ``Foo``, so it is provided separately. - - - - -The :mod:`abc` module also provides the following decorator: - -.. decorator:: abstractmethod - - A decorator indicating abstract methods. - - Using this decorator requires that the class's metaclass is :class:`ABCMeta` - or is derived from it. A class that has a metaclass derived from - :class:`ABCMeta` cannot be instantiated unless all of its abstract methods - and properties are overridden. The abstract methods can be called using any - of the normal 'super' call mechanisms. :func:`abstractmethod` may be used - to declare abstract methods for properties and descriptors. - - Dynamically adding abstract methods to a class, or attempting to modify the - abstraction status of a method or class once it is created, are only - supported using the :func:`update_abstractmethods` function. The - :func:`abstractmethod` only affects subclasses derived using regular - inheritance; "virtual subclasses" registered with the ABC's :meth:`register` - method are not affected. - - When :func:`abstractmethod` is applied in combination with other method - descriptors, it should be applied as the innermost decorator, as shown in - the following usage examples:: - - class C(ABC): - @abstractmethod - def my_abstract_method(self, arg1): - ... - @classmethod - @abstractmethod - def my_abstract_classmethod(cls, arg2): - ... - @staticmethod - @abstractmethod - def my_abstract_staticmethod(arg3): - ... - - @property - @abstractmethod - def my_abstract_property(self): - ... - @my_abstract_property.setter - @abstractmethod - def my_abstract_property(self, val): - ... - - @abstractmethod - def _get_x(self): - ... - @abstractmethod - def _set_x(self, val): - ... - x = property(_get_x, _set_x) - - In order to correctly interoperate with the abstract base class machinery, - the descriptor must identify itself as abstract using - :attr:`__isabstractmethod__`. In general, this attribute should be ``True`` - if any of the methods used to compose the descriptor are abstract. For - example, Python's built-in :class:`property` does the equivalent of:: - - class Descriptor: - ... - @property - def __isabstractmethod__(self): - return any(getattr(f, '__isabstractmethod__', False) for - f in (self._fget, self._fset, self._fdel)) - - .. note:: - - Unlike Java abstract methods, these abstract - methods may have an implementation. This implementation can be - called via the :func:`super` mechanism from the class that - overrides it. This could be useful as an end-point for a - super-call in a framework that uses cooperative - multiple-inheritance. - -The :mod:`abc` module also supports the following legacy decorators: - -.. decorator:: abstractclassmethod - - .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`classmethod` with - :func:`abstractmethod`, making this decorator redundant. - - A subclass of the built-in :func:`classmethod`, indicating an abstract - classmethod. Otherwise it is similar to :func:`abstractmethod`. - - This special case is deprecated, as the :func:`classmethod` decorator - is now correctly identified as abstract when applied to an abstract - method:: - - class C(ABC): - @classmethod - @abstractmethod - def my_abstract_classmethod(cls, arg): - ... - - -.. decorator:: abstractstaticmethod - - .. versionadded:: 3.2 - .. deprecated:: 3.3 - It is now possible to use :class:`staticmethod` with - :func:`abstractmethod`, making this decorator redundant. - - A subclass of the built-in :func:`staticmethod`, indicating an abstract - staticmethod. Otherwise it is similar to :func:`abstractmethod`. - - This special case is deprecated, as the :func:`staticmethod` decorator - is now correctly identified as abstract when applied to an abstract - method:: - - class C(ABC): - @staticmethod - @abstractmethod - def my_abstract_staticmethod(arg): - ... - - -.. decorator:: abstractproperty - - .. deprecated:: 3.3 - It is now possible to use :class:`property`, :meth:`property.getter`, - :meth:`property.setter` and :meth:`property.deleter` with - :func:`abstractmethod`, making this decorator redundant. - - A subclass of the built-in :func:`property`, indicating an abstract - property. - - This special case is deprecated, as the :func:`property` decorator - is now correctly identified as abstract when applied to an abstract - method:: - - class C(ABC): - @property - @abstractmethod - def my_abstract_property(self): - ... - - The above example defines a read-only property; you can also define a - read-write abstract property by appropriately marking one or more of the - underlying methods as abstract:: - - class C(ABC): - @property - def x(self): - ... - - @x.setter - @abstractmethod - def x(self, val): - ... - - If only some components are abstract, only those components need to be - updated to create a concrete property in a subclass:: - - class D(C): - @C.x.setter - def x(self, val): - ... - - -The :mod:`abc` module also provides the following functions: - -.. function:: get_cache_token() - - Returns the current abstract base class cache token. - - The token is an opaque object (that supports equality testing) identifying - the current version of the abstract base class cache for virtual subclasses. - The token changes with every call to :meth:`ABCMeta.register` on any ABC. - - .. versionadded:: 3.4 - -.. function:: update_abstractmethods(cls) - - A function to recalculate an abstract class's abstraction status. This - function should be called if a class's abstract methods have been - implemented or changed after it was created. Usually, this function should - be called from within a class decorator. - - Returns *cls*, to allow usage as a class decorator. - - If *cls* is not an instance of :class:`ABCMeta`, does nothing. - - .. note:: - - This function assumes that *cls*'s superclasses are already updated. - It does not update any subclasses. - - .. versionadded:: 3.10 - -.. rubric:: Footnotes - -.. [#] C++ programmers should note that Python's virtual base class - concept is not the same as C++'s. diff --git a/Doc/library/aifc.rst b/Doc/library/aifc.rst deleted file mode 100644 index 9f20a30193fa70..00000000000000 --- a/Doc/library/aifc.rst +++ /dev/null @@ -1,247 +0,0 @@ -:mod:`aifc` --- Read and write AIFF and AIFC files -================================================== - -.. module:: aifc - :synopsis: Read and write audio files in AIFF or AIFC format. - :deprecated: - -**Source code:** :source:`Lib/aifc.py` - -.. index:: - single: Audio Interchange File Format - single: AIFF - single: AIFF-C - - -.. deprecated-removed:: 3.11 3.13 - The :mod:`aifc` module is deprecated - (see :pep:`PEP 594 <594#aifc>` for details). - --------------- - -This module provides support for reading and writing AIFF and AIFF-C files. -AIFF is Audio Interchange File Format, a format for storing digital audio -samples in a file. AIFF-C is a newer version of the format that includes the -ability to compress the audio data. - -Audio files have a number of parameters that describe the audio data. The -sampling rate or frame rate is the number of times per second the sound is -sampled. The number of channels indicate if the audio is mono, stereo, or -quadro. Each frame consists of one sample per channel. The sample size is the -size in bytes of each sample. Thus a frame consists of -``nchannels * samplesize`` bytes, and a second's worth of audio consists of -``nchannels * samplesize * framerate`` bytes. - -For example, CD quality audio has a sample size of two bytes (16 bits), uses two -channels (stereo) and has a frame rate of 44,100 frames/second. This gives a -frame size of 4 bytes (2\*2), and a second's worth occupies 2\*2\*44100 bytes -(176,400 bytes). - -Module :mod:`aifc` defines the following function: - - -.. function:: open(file, mode=None) - - Open an AIFF or AIFF-C file and return an object instance with methods that are - described below. The argument *file* is either a string naming a file or a - :term:`file object`. *mode* must be ``'r'`` or ``'rb'`` when the file must be - opened for reading, or ``'w'`` or ``'wb'`` when the file must be opened for writing. - If omitted, ``file.mode`` is used if it exists, otherwise ``'rb'`` is used. When - used for writing, the file object should be seekable, unless you know ahead of - time how many samples you are going to write in total and use - :meth:`writeframesraw` and :meth:`setnframes`. - The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`!with` block completes, the :meth:`~aifc.close` method is called. - - .. versionchanged:: 3.4 - Support for the :keyword:`with` statement was added. - -Objects returned by :func:`.open` when a file is opened for reading have the -following methods: - - -.. method:: aifc.getnchannels() - - Return the number of audio channels (1 for mono, 2 for stereo). - - -.. method:: aifc.getsampwidth() - - Return the size in bytes of individual samples. - - -.. method:: aifc.getframerate() - - Return the sampling rate (number of audio frames per second). - - -.. method:: aifc.getnframes() - - Return the number of audio frames in the file. - - -.. method:: aifc.getcomptype() - - Return a bytes array of length 4 describing the type of compression - used in the audio file. For AIFF files, the returned value is - ``b'NONE'``. - - -.. method:: aifc.getcompname() - - Return a bytes array convertible to a human-readable description - of the type of compression used in the audio file. For AIFF files, - the returned value is ``b'not compressed'``. - - -.. method:: aifc.getparams() - - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. - - -.. method:: aifc.getmarkers() - - Return a list of markers in the audio file. A marker consists of a tuple of - three elements. The first is the mark ID (an integer), the second is the mark - position in frames from the beginning of the data (an integer), the third is the - name of the mark (a string). - - -.. method:: aifc.getmark(id) - - Return the tuple as described in :meth:`getmarkers` for the mark with the given - *id*. - - -.. method:: aifc.readframes(nframes) - - Read and return the next *nframes* frames from the audio file. The returned - data is a string containing for each frame the uncompressed samples of all - channels. - - -.. method:: aifc.rewind() - - Rewind the read pointer. The next :meth:`readframes` will start from the - beginning. - - -.. method:: aifc.setpos(pos) - - Seek to the specified frame number. - - -.. method:: aifc.tell() - - Return the current frame number. - - -.. method:: aifc.close() - - Close the AIFF file. After calling this method, the object can no longer be - used. - -Objects returned by :func:`.open` when a file is opened for writing have all the -above methods, except for :meth:`readframes` and :meth:`setpos`. In addition -the following methods exist. The :meth:`get\*` methods can only be called after -the corresponding :meth:`set\*` methods have been called. Before the first -:meth:`writeframes` or :meth:`writeframesraw`, all parameters except for the -number of frames must be filled in. - - -.. method:: aifc.aiff() - - Create an AIFF file. The default is that an AIFF-C file is created, unless the - name of the file ends in ``'.aiff'`` in which case the default is an AIFF file. - - -.. method:: aifc.aifc() - - Create an AIFF-C file. The default is that an AIFF-C file is created, unless - the name of the file ends in ``'.aiff'`` in which case the default is an AIFF - file. - - -.. method:: aifc.setnchannels(nchannels) - - Specify the number of channels in the audio file. - - -.. method:: aifc.setsampwidth(width) - - Specify the size in bytes of audio samples. - - -.. method:: aifc.setframerate(rate) - - Specify the sampling frequency in frames per second. - - -.. method:: aifc.setnframes(nframes) - - Specify the number of frames that are to be written to the audio file. If this - parameter is not set, or not set correctly, the file needs to support seeking. - - -.. method:: aifc.setcomptype(type, name) - - .. index:: - single: u-LAW - single: A-LAW - single: G.722 - - Specify the compression type. If not specified, the audio data will - not be compressed. In AIFF files, compression is not possible. - The name parameter should be a human-readable description of the - compression type as a bytes array, the type parameter should be a - bytes array of length 4. Currently the following compression types - are supported: ``b'NONE'``, ``b'ULAW'``, ``b'ALAW'``, ``b'G722'``. - - -.. method:: aifc.setparams(nchannels, sampwidth, framerate, comptype, compname) - - Set all the above parameters at once. The argument is a tuple consisting of the - various parameters. This means that it is possible to use the result of a - :meth:`getparams` call as argument to :meth:`setparams`. - - -.. method:: aifc.setmark(id, pos, name) - - Add a mark with the given id (larger than 0), and the given name at the given - position. This method can be called at any time before :meth:`close`. - - -.. method:: aifc.tell() - :noindex: - - Return the current write position in the output file. Useful in combination - with :meth:`setmark`. - - -.. method:: aifc.writeframes(data) - - Write data to the output file. This method can only be called after the audio - file parameters have been set. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: aifc.writeframesraw(data) - - Like :meth:`writeframes`, except that the header of the audio file is not - updated. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: aifc.close() - :noindex: - - Close the AIFF file. The header of the file is updated to reflect the actual - size of the audio data. After calling this method, the object can no longer be - used. - diff --git a/Doc/library/allos.rst b/Doc/library/allos.rst deleted file mode 100644 index f7105d8af8e28b..00000000000000 --- a/Doc/library/allos.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _allos: - -********************************* -Generic Operating System Services -********************************* - -The modules described in this chapter provide interfaces to operating system -features that are available on (almost) all operating systems, such as files and -a clock. The interfaces are generally modeled after the Unix or C interfaces, -but they are available on most other systems as well. Here's an overview: - - -.. toctree:: - - os.rst - io.rst - time.rst - argparse.rst - getopt.rst - logging.rst - logging.config.rst - logging.handlers.rst - getpass.rst - curses.rst - curses.ascii.rst - curses.panel.rst - platform.rst - errno.rst - ctypes.rst diff --git a/Doc/library/archiving.rst b/Doc/library/archiving.rst deleted file mode 100644 index c9284949af4972..00000000000000 --- a/Doc/library/archiving.rst +++ /dev/null @@ -1,20 +0,0 @@ -.. _archiving: - -****************************** -Data Compression and Archiving -****************************** - -The modules described in this chapter support data compression with the zlib, -gzip, bzip2 and lzma algorithms, and the creation of ZIP- and tar-format -archives. See also :ref:`archiving-operations` provided by the :mod:`shutil` -module. - - -.. toctree:: - - zlib.rst - gzip.rst - bz2.rst - lzma.rst - zipfile.rst - tarfile.rst diff --git a/Doc/library/argparse.rst b/Doc/library/argparse.rst deleted file mode 100644 index 19c510fccf050c..00000000000000 --- a/Doc/library/argparse.rst +++ /dev/null @@ -1,2279 +0,0 @@ -:mod:`argparse` --- Parser for command-line options, arguments and sub-commands -=============================================================================== - -.. module:: argparse - :synopsis: Command-line option and argument parsing library. - -.. moduleauthor:: Steven Bethard -.. sectionauthor:: Steven Bethard - -.. versionadded:: 3.2 - -**Source code:** :source:`Lib/argparse.py` - --------------- - -.. sidebar:: Tutorial - - This page contains the API reference information. For a more gentle - introduction to Python command-line parsing, have a look at the - :ref:`argparse tutorial `. - -The :mod:`argparse` module makes it easy to write user-friendly command-line -interfaces. The program defines what arguments it requires, and :mod:`argparse` -will figure out how to parse those out of :data:`sys.argv`. The :mod:`argparse` -module also automatically generates help and usage messages. The module -will also issue errors when users give the program invalid arguments. - - -Core Functionality ------------------- - -The :mod:`argparse` module's support for command-line interfaces is built -around an instance of :class:`argparse.ArgumentParser`. It is a container for -argument specifications and has options that apply the parser as whole:: - - parser = argparse.ArgumentParser( - prog='ProgramName', - description='What the program does', - epilog='Text at the bottom of help') - -The :meth:`ArgumentParser.add_argument` method attaches individual argument -specifications to the parser. It supports positional arguments, options that -accept values, and on/off flags:: - - parser.add_argument('filename') # positional argument - parser.add_argument('-c', '--count') # option that takes a value - parser.add_argument('-v', '--verbose', - action='store_true') # on/off flag - -The :meth:`ArgumentParser.parse_args` method runs the parser and places -the extracted data in a :class:`argparse.Namespace` object:: - - args = parser.parse_args() - print(args.filename, args.count, args.verbose) - - -Quick Links for add_argument() ------------------------------- - -====================== =========================================================== ========================================================================================================================== -Name Description Values -====================== =========================================================== ========================================================================================================================== -action_ Specify how an argument should be handled ``'store'``, ``'store_const'``, ``'store_true'``, ``'append'``, ``'append_const'``, ``'count'``, ``'help'``, ``'version'`` -choices_ Limit values to a specific set of choices ``['foo', 'bar']``, ``range(1, 10)``, or :class:`~collections.abc.Container` instance -const_ Store a constant value -default_ Default value used when an argument is not provided Defaults to ``None`` -dest_ Specify the attribute name used in the result namespace -help_ Help message for an argument -metavar_ Alternate display name for the argument as shown in help -nargs_ Number of times the argument can be used :class:`int`, ``'?'``, ``'*'``, or ``'+'`` -required_ Indicate whether an argument is required or optional ``True`` or ``False`` -type_ Automatically convert an argument to the given type :class:`int`, :class:`float`, ``argparse.FileType('w')``, or callable function -====================== =========================================================== ========================================================================================================================== - - -Example -------- - -The following code is a Python program that takes a list of integers and -produces either the sum or the max:: - - import argparse - - parser = argparse.ArgumentParser(description='Process some integers.') - parser.add_argument('integers', metavar='N', type=int, nargs='+', - help='an integer for the accumulator') - parser.add_argument('--sum', dest='accumulate', action='store_const', - const=sum, default=max, - help='sum the integers (default: find the max)') - - args = parser.parse_args() - print(args.accumulate(args.integers)) - -Assuming the above Python code is saved into a file called ``prog.py``, it can -be run at the command line and it provides useful help messages: - -.. code-block:: shell-session - - $ python prog.py -h - usage: prog.py [-h] [--sum] N [N ...] - - Process some integers. - - positional arguments: - N an integer for the accumulator - - options: - -h, --help show this help message and exit - --sum sum the integers (default: find the max) - -When run with the appropriate arguments, it prints either the sum or the max of -the command-line integers: - -.. code-block:: shell-session - - $ python prog.py 1 2 3 4 - 4 - - $ python prog.py 1 2 3 4 --sum - 10 - -If invalid arguments are passed in, an error will be displayed: - -.. code-block:: shell-session - - $ python prog.py a b c - usage: prog.py [-h] [--sum] N [N ...] - prog.py: error: argument N: invalid int value: 'a' - -The following sections walk you through this example. - - -Creating a parser -^^^^^^^^^^^^^^^^^ - -The first step in using the :mod:`argparse` is creating an -:class:`ArgumentParser` object:: - - >>> parser = argparse.ArgumentParser(description='Process some integers.') - -The :class:`ArgumentParser` object will hold all the information necessary to -parse the command line into Python data types. - - -Adding arguments -^^^^^^^^^^^^^^^^ - -Filling an :class:`ArgumentParser` with information about program arguments is -done by making calls to the :meth:`~ArgumentParser.add_argument` method. -Generally, these calls tell the :class:`ArgumentParser` how to take the strings -on the command line and turn them into objects. This information is stored and -used when :meth:`~ArgumentParser.parse_args` is called. For example:: - - >>> parser.add_argument('integers', metavar='N', type=int, nargs='+', - ... help='an integer for the accumulator') - >>> parser.add_argument('--sum', dest='accumulate', action='store_const', - ... const=sum, default=max, - ... help='sum the integers (default: find the max)') - -Later, calling :meth:`~ArgumentParser.parse_args` will return an object with -two attributes, ``integers`` and ``accumulate``. The ``integers`` attribute -will be a list of one or more integers, and the ``accumulate`` attribute will be -either the :func:`sum` function, if ``--sum`` was specified at the command line, -or the :func:`max` function if it was not. - - -Parsing arguments -^^^^^^^^^^^^^^^^^ - -:class:`ArgumentParser` parses arguments through the -:meth:`~ArgumentParser.parse_args` method. This will inspect the command line, -convert each argument to the appropriate type and then invoke the appropriate action. -In most cases, this means a simple :class:`Namespace` object will be built up from -attributes parsed out of the command line:: - - >>> parser.parse_args(['--sum', '7', '-1', '42']) - Namespace(accumulate=, integers=[7, -1, 42]) - -In a script, :meth:`~ArgumentParser.parse_args` will typically be called with no -arguments, and the :class:`ArgumentParser` will automatically determine the -command-line arguments from :data:`sys.argv`. - - -ArgumentParser objects ----------------------- - -.. class:: ArgumentParser(prog=None, usage=None, description=None, \ - epilog=None, parents=[], \ - formatter_class=argparse.HelpFormatter, \ - prefix_chars='-', fromfile_prefix_chars=None, \ - argument_default=None, conflict_handler='error', \ - add_help=True, allow_abbrev=True, exit_on_error=True) - - Create a new :class:`ArgumentParser` object. All parameters should be passed - as keyword arguments. Each parameter has its own more detailed description - below, but in short they are: - - * prog_ - The name of the program (default: - ``os.path.basename(sys.argv[0])``) - - * usage_ - The string describing the program usage (default: generated from - arguments added to parser) - - * description_ - Text to display before the argument help - (by default, no text) - - * epilog_ - Text to display after the argument help (by default, no text) - - * parents_ - A list of :class:`ArgumentParser` objects whose arguments should - also be included - - * formatter_class_ - A class for customizing the help output - - * prefix_chars_ - The set of characters that prefix optional arguments - (default: '-') - - * fromfile_prefix_chars_ - The set of characters that prefix files from - which additional arguments should be read (default: ``None``) - - * argument_default_ - The global default value for arguments - (default: ``None``) - - * conflict_handler_ - The strategy for resolving conflicting optionals - (usually unnecessary) - - * add_help_ - Add a ``-h/--help`` option to the parser (default: ``True``) - - * allow_abbrev_ - Allows long options to be abbreviated if the - abbreviation is unambiguous. (default: ``True``) - - * exit_on_error_ - Determines whether or not ArgumentParser exits with - error info when an error occurs. (default: ``True``) - - .. versionchanged:: 3.5 - *allow_abbrev* parameter was added. - - .. versionchanged:: 3.8 - In previous versions, *allow_abbrev* also disabled grouping of short - flags such as ``-vv`` to mean ``-v -v``. - - .. versionchanged:: 3.9 - *exit_on_error* parameter was added. - -The following sections describe how each of these are used. - - -.. _prog: - -prog -^^^^ - -By default, :class:`ArgumentParser` objects use ``sys.argv[0]`` to determine -how to display the name of the program in help messages. This default is almost -always desirable because it will make the help messages match how the program was -invoked on the command line. For example, consider a file named -``myprogram.py`` with the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -The help for this program will display ``myprogram.py`` as the program name -(regardless of where the program was invoked from): - -.. code-block:: shell-session - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help - $ cd .. - $ python subdir/myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help - -To change this default behavior, another value can be supplied using the -``prog=`` argument to :class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.print_help() - usage: myprogram [-h] - - options: - -h, --help show this help message and exit - -Note that the program name, whether determined from ``sys.argv[0]`` or from the -``prog=`` argument, is available to help messages using the ``%(prog)s`` format -specifier. - -:: - - >>> parser = argparse.ArgumentParser(prog='myprogram') - >>> parser.add_argument('--foo', help='foo of the %(prog)s program') - >>> parser.print_help() - usage: myprogram [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo of the myprogram program - - -usage -^^^^^ - -By default, :class:`ArgumentParser` calculates the usage message from the -arguments it contains:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') - >>> parser.print_help() - usage: PROG [-h] [--foo [FOO]] bar [bar ...] - - positional arguments: - bar bar help - - options: - -h, --help show this help message and exit - --foo [FOO] foo help - -The default message can be overridden with the ``usage=`` keyword argument:: - - >>> parser = argparse.ArgumentParser(prog='PROG', usage='%(prog)s [options]') - >>> parser.add_argument('--foo', nargs='?', help='foo help') - >>> parser.add_argument('bar', nargs='+', help='bar help') - >>> parser.print_help() - usage: PROG [options] - - positional arguments: - bar bar help - - options: - -h, --help show this help message and exit - --foo [FOO] foo help - -The ``%(prog)s`` format specifier is available to fill in the program name in -your usage messages. - - -.. _description: - -description -^^^^^^^^^^^ - -Most calls to the :class:`ArgumentParser` constructor will use the -``description=`` keyword argument. This argument gives a brief description of -what the program does and how it works. In help messages, the description is -displayed between the command-line usage string and the help messages for the -various arguments:: - - >>> parser = argparse.ArgumentParser(description='A foo that bars') - >>> parser.print_help() - usage: argparse.py [-h] - - A foo that bars - - options: - -h, --help show this help message and exit - -By default, the description will be line-wrapped so that it fits within the -given space. To change this behavior, see the formatter_class_ argument. - - -epilog -^^^^^^ - -Some programs like to display additional description of the program after the -description of the arguments. Such text can be specified using the ``epilog=`` -argument to :class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser( - ... description='A foo that bars', - ... epilog="And that's how you'd foo a bar") - >>> parser.print_help() - usage: argparse.py [-h] - - A foo that bars - - options: - -h, --help show this help message and exit - - And that's how you'd foo a bar - -As with the description_ argument, the ``epilog=`` text is by default -line-wrapped, but this behavior can be adjusted with the formatter_class_ -argument to :class:`ArgumentParser`. - - -parents -^^^^^^^ - -Sometimes, several parsers share a common set of arguments. Rather than -repeating the definitions of these arguments, a single parser with all the -shared arguments and passed to ``parents=`` argument to :class:`ArgumentParser` -can be used. The ``parents=`` argument takes a list of :class:`ArgumentParser` -objects, collects all the positional and optional actions from them, and adds -these actions to the :class:`ArgumentParser` object being constructed:: - - >>> parent_parser = argparse.ArgumentParser(add_help=False) - >>> parent_parser.add_argument('--parent', type=int) - - >>> foo_parser = argparse.ArgumentParser(parents=[parent_parser]) - >>> foo_parser.add_argument('foo') - >>> foo_parser.parse_args(['--parent', '2', 'XXX']) - Namespace(foo='XXX', parent=2) - - >>> bar_parser = argparse.ArgumentParser(parents=[parent_parser]) - >>> bar_parser.add_argument('--bar') - >>> bar_parser.parse_args(['--bar', 'YYY']) - Namespace(bar='YYY', parent=None) - -Note that most parent parsers will specify ``add_help=False``. Otherwise, the -:class:`ArgumentParser` will see two ``-h/--help`` options (one in the parent -and one in the child) and raise an error. - -.. note:: - You must fully initialize the parsers before passing them via ``parents=``. - If you change the parent parsers after the child parser, those changes will - not be reflected in the child. - - -.. _formatter_class: - -formatter_class -^^^^^^^^^^^^^^^ - -:class:`ArgumentParser` objects allow the help formatting to be customized by -specifying an alternate formatting class. Currently, there are four such -classes: - -.. class:: RawDescriptionHelpFormatter - RawTextHelpFormatter - ArgumentDefaultsHelpFormatter - MetavarTypeHelpFormatter - -:class:`RawDescriptionHelpFormatter` and :class:`RawTextHelpFormatter` give -more control over how textual descriptions are displayed. -By default, :class:`ArgumentParser` objects line-wrap the description_ and -epilog_ texts in command-line help messages:: - - >>> parser = argparse.ArgumentParser( - ... prog='PROG', - ... description='''this description - ... was indented weird - ... but that is okay''', - ... epilog=''' - ... likewise for this epilog whose whitespace will - ... be cleaned up and whose words will be wrapped - ... across a couple lines''') - >>> parser.print_help() - usage: PROG [-h] - - this description was indented weird but that is okay - - options: - -h, --help show this help message and exit - - likewise for this epilog whose whitespace will be cleaned up and whose words - will be wrapped across a couple lines - -Passing :class:`RawDescriptionHelpFormatter` as ``formatter_class=`` -indicates that description_ and epilog_ are already correctly formatted and -should not be line-wrapped:: - - >>> parser = argparse.ArgumentParser( - ... prog='PROG', - ... formatter_class=argparse.RawDescriptionHelpFormatter, - ... description=textwrap.dedent('''\ - ... Please do not mess up this text! - ... -------------------------------- - ... I have indented it - ... exactly the way - ... I want it - ... ''')) - >>> parser.print_help() - usage: PROG [-h] - - Please do not mess up this text! - -------------------------------- - I have indented it - exactly the way - I want it - - options: - -h, --help show this help message and exit - -:class:`RawTextHelpFormatter` maintains whitespace for all sorts of help text, -including argument descriptions. However, multiple new lines are replaced with -one. If you wish to preserve multiple blank lines, add spaces between the -newlines. - -:class:`ArgumentDefaultsHelpFormatter` automatically adds information about -default values to each of the argument help messages:: - - >>> parser = argparse.ArgumentParser( - ... prog='PROG', - ... formatter_class=argparse.ArgumentDefaultsHelpFormatter) - >>> parser.add_argument('--foo', type=int, default=42, help='FOO!') - >>> parser.add_argument('bar', nargs='*', default=[1, 2, 3], help='BAR!') - >>> parser.print_help() - usage: PROG [-h] [--foo FOO] [bar ...] - - positional arguments: - bar BAR! (default: [1, 2, 3]) - - options: - -h, --help show this help message and exit - --foo FOO FOO! (default: 42) - -:class:`MetavarTypeHelpFormatter` uses the name of the type_ argument for each -argument as the display name for its values (rather than using the dest_ -as the regular formatter does):: - - >>> parser = argparse.ArgumentParser( - ... prog='PROG', - ... formatter_class=argparse.MetavarTypeHelpFormatter) - >>> parser.add_argument('--foo', type=int) - >>> parser.add_argument('bar', type=float) - >>> parser.print_help() - usage: PROG [-h] [--foo int] float - - positional arguments: - float - - options: - -h, --help show this help message and exit - --foo int - - -prefix_chars -^^^^^^^^^^^^ - -Most command-line options will use ``-`` as the prefix, e.g. ``-f/--foo``. -Parsers that need to support different or additional prefix -characters, e.g. for options -like ``+f`` or ``/foo``, may specify them using the ``prefix_chars=`` argument -to the ArgumentParser constructor:: - - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='-+') - >>> parser.add_argument('+f') - >>> parser.add_argument('++bar') - >>> parser.parse_args('+f X ++bar Y'.split()) - Namespace(bar='Y', f='X') - -The ``prefix_chars=`` argument defaults to ``'-'``. Supplying a set of -characters that does not include ``-`` will cause ``-f/--foo`` options to be -disallowed. - - -fromfile_prefix_chars -^^^^^^^^^^^^^^^^^^^^^ - -Sometimes, when dealing with a particularly long argument list, it -may make sense to keep the list of arguments in a file rather than typing it out -at the command line. If the ``fromfile_prefix_chars=`` argument is given to the -:class:`ArgumentParser` constructor, then arguments that start with any of the -specified characters will be treated as files, and will be replaced by the -arguments they contain. For example:: - - >>> with open('args.txt', 'w') as fp: - ... fp.write('-f\nbar') - >>> parser = argparse.ArgumentParser(fromfile_prefix_chars='@') - >>> parser.add_argument('-f') - >>> parser.parse_args(['-f', 'foo', '@args.txt']) - Namespace(f='bar') - -Arguments read from a file must by default be one per line (but see also -:meth:`~ArgumentParser.convert_arg_line_to_args`) and are treated as if they -were in the same place as the original file referencing argument on the command -line. So in the example above, the expression ``['-f', 'foo', '@args.txt']`` -is considered equivalent to the expression ``['-f', 'foo', '-f', 'bar']``. - -The ``fromfile_prefix_chars=`` argument defaults to ``None``, meaning that -arguments will never be treated as file references. - - -argument_default -^^^^^^^^^^^^^^^^ - -Generally, argument defaults are specified either by passing a default to -:meth:`~ArgumentParser.add_argument` or by calling the -:meth:`~ArgumentParser.set_defaults` methods with a specific set of name-value -pairs. Sometimes however, it may be useful to specify a single parser-wide -default for arguments. This can be accomplished by passing the -``argument_default=`` keyword argument to :class:`ArgumentParser`. For example, -to globally suppress attribute creation on :meth:`~ArgumentParser.parse_args` -calls, we supply ``argument_default=SUPPRESS``:: - - >>> parser = argparse.ArgumentParser(argument_default=argparse.SUPPRESS) - >>> parser.add_argument('--foo') - >>> parser.add_argument('bar', nargs='?') - >>> parser.parse_args(['--foo', '1', 'BAR']) - Namespace(bar='BAR', foo='1') - >>> parser.parse_args([]) - Namespace() - -.. _allow_abbrev: - -allow_abbrev -^^^^^^^^^^^^ - -Normally, when you pass an argument list to the -:meth:`~ArgumentParser.parse_args` method of an :class:`ArgumentParser`, -it :ref:`recognizes abbreviations ` of long options. - -This feature can be disabled by setting ``allow_abbrev`` to ``False``:: - - >>> parser = argparse.ArgumentParser(prog='PROG', allow_abbrev=False) - >>> parser.add_argument('--foobar', action='store_true') - >>> parser.add_argument('--foonley', action='store_false') - >>> parser.parse_args(['--foon']) - usage: PROG [-h] [--foobar] [--foonley] - PROG: error: unrecognized arguments: --foon - -.. versionadded:: 3.5 - - -conflict_handler -^^^^^^^^^^^^^^^^ - -:class:`ArgumentParser` objects do not allow two actions with the same option -string. By default, :class:`ArgumentParser` objects raise an exception if an -attempt is made to create an argument with an option string that is already in -use:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-f', '--foo', help='old foo help') - >>> parser.add_argument('--foo', help='new foo help') - Traceback (most recent call last): - .. - ArgumentError: argument --foo: conflicting option string(s): --foo - -Sometimes (e.g. when using parents_) it may be useful to simply override any -older arguments with the same option string. To get this behavior, the value -``'resolve'`` can be supplied to the ``conflict_handler=`` argument of -:class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='PROG', conflict_handler='resolve') - >>> parser.add_argument('-f', '--foo', help='old foo help') - >>> parser.add_argument('--foo', help='new foo help') - >>> parser.print_help() - usage: PROG [-h] [-f FOO] [--foo FOO] - - options: - -h, --help show this help message and exit - -f FOO old foo help - --foo FOO new foo help - -Note that :class:`ArgumentParser` objects only remove an action if all of its -option strings are overridden. So, in the example above, the old ``-f/--foo`` -action is retained as the ``-f`` action, because only the ``--foo`` option -string was overridden. - - -add_help -^^^^^^^^ - -By default, ArgumentParser objects add an option which simply displays -the parser's help message. For example, consider a file named -``myprogram.py`` containing the following code:: - - import argparse - parser = argparse.ArgumentParser() - parser.add_argument('--foo', help='foo help') - args = parser.parse_args() - -If ``-h`` or ``--help`` is supplied at the command line, the ArgumentParser -help will be printed: - -.. code-block:: shell-session - - $ python myprogram.py --help - usage: myprogram.py [-h] [--foo FOO] - - options: - -h, --help show this help message and exit - --foo FOO foo help - -Occasionally, it may be useful to disable the addition of this help option. -This can be achieved by passing ``False`` as the ``add_help=`` argument to -:class:`ArgumentParser`:: - - >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) - >>> parser.add_argument('--foo', help='foo help') - >>> parser.print_help() - usage: PROG [--foo FOO] - - options: - --foo FOO foo help - -The help option is typically ``-h/--help``. The exception to this is -if the ``prefix_chars=`` is specified and does not include ``-``, in -which case ``-h`` and ``--help`` are not valid options. In -this case, the first character in ``prefix_chars`` is used to prefix -the help options:: - - >>> parser = argparse.ArgumentParser(prog='PROG', prefix_chars='+/') - >>> parser.print_help() - usage: PROG [+h] - - options: - +h, ++help show this help message and exit - - -exit_on_error -^^^^^^^^^^^^^ - -Normally, when you pass an invalid argument list to the :meth:`~ArgumentParser.parse_args` -method of an :class:`ArgumentParser`, it will exit with error info. - -If the user would like to catch errors manually, the feature can be enabled by setting -``exit_on_error`` to ``False``:: - - >>> parser = argparse.ArgumentParser(exit_on_error=False) - >>> parser.add_argument('--integers', type=int) - _StoreAction(option_strings=['--integers'], dest='integers', nargs=None, const=None, default=None, type=, choices=None, help=None, metavar=None) - >>> try: - ... parser.parse_args('--integers a'.split()) - ... except argparse.ArgumentError: - ... print('Catching an argumentError') - ... - Catching an argumentError - -.. versionadded:: 3.9 - - -The add_argument() method -------------------------- - -.. method:: ArgumentParser.add_argument(name or flags..., [action], [nargs], \ - [const], [default], [type], [choices], [required], \ - [help], [metavar], [dest]) - - Define how a single command-line argument should be parsed. Each parameter - has its own more detailed description below, but in short they are: - - * `name or flags`_ - Either a name or a list of option strings, e.g. ``foo`` - or ``-f, --foo``. - - * action_ - The basic type of action to be taken when this argument is - encountered at the command line. - - * nargs_ - The number of command-line arguments that should be consumed. - - * const_ - A constant value required by some action_ and nargs_ selections. - - * default_ - The value produced if the argument is absent from the - command line and if it is absent from the namespace object. - - * type_ - The type to which the command-line argument should be converted. - - * choices_ - A sequence of the allowable values for the argument. - - * required_ - Whether or not the command-line option may be omitted - (optionals only). - - * help_ - A brief description of what the argument does. - - * metavar_ - A name for the argument in usage messages. - - * dest_ - The name of the attribute to be added to the object returned by - :meth:`parse_args`. - -The following sections describe how each of these are used. - - -.. _name_or_flags: - -name or flags -^^^^^^^^^^^^^ - -The :meth:`~ArgumentParser.add_argument` method must know whether an optional -argument, like ``-f`` or ``--foo``, or a positional argument, like a list of -filenames, is expected. The first arguments passed to -:meth:`~ArgumentParser.add_argument` must therefore be either a series of -flags, or a simple argument name. - -For example, an optional argument could be created like:: - - >>> parser.add_argument('-f', '--foo') - -while a positional argument could be created like:: - - >>> parser.add_argument('bar') - -When :meth:`~ArgumentParser.parse_args` is called, optional arguments will be -identified by the ``-`` prefix, and the remaining arguments will be assumed to -be positional:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-f', '--foo') - >>> parser.add_argument('bar') - >>> parser.parse_args(['BAR']) - Namespace(bar='BAR', foo=None) - >>> parser.parse_args(['BAR', '--foo', 'FOO']) - Namespace(bar='BAR', foo='FOO') - >>> parser.parse_args(['--foo', 'FOO']) - usage: PROG [-h] [-f FOO] bar - PROG: error: the following arguments are required: bar - - -.. _action: - -action -^^^^^^ - -:class:`ArgumentParser` objects associate command-line arguments with actions. These -actions can do just about anything with the command-line arguments associated with -them, though most actions simply add an attribute to the object returned by -:meth:`~ArgumentParser.parse_args`. The ``action`` keyword argument specifies -how the command-line arguments should be handled. The supplied actions are: - -* ``'store'`` - This just stores the argument's value. This is the default - action. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args('--foo 1'.split()) - Namespace(foo='1') - -* ``'store_const'`` - This stores the value specified by the const_ keyword - argument; note that the const_ keyword argument defaults to ``None``. The - ``'store_const'`` action is most commonly used with optional arguments that - specify some sort of flag. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='store_const', const=42) - >>> parser.parse_args(['--foo']) - Namespace(foo=42) - -* ``'store_true'`` and ``'store_false'`` - These are special cases of - ``'store_const'`` used for storing the values ``True`` and ``False`` - respectively. In addition, they create default values of ``False`` and - ``True`` respectively. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='store_true') - >>> parser.add_argument('--bar', action='store_false') - >>> parser.add_argument('--baz', action='store_false') - >>> parser.parse_args('--foo --bar'.split()) - Namespace(foo=True, bar=False, baz=True) - -* ``'append'`` - This stores a list, and appends each argument value to the - list. It is useful to allow an option to be specified multiple times. - If the default value is non-empty, the default elements will be present - in the parsed value for the option, with any values from the - command line appended after those default values. Example usage:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='append') - >>> parser.parse_args('--foo 1 --foo 2'.split()) - Namespace(foo=['1', '2']) - -* ``'append_const'`` - This stores a list, and appends the value specified by - the const_ keyword argument to the list; note that the const_ keyword - argument defaults to ``None``. The ``'append_const'`` action is typically - useful when multiple arguments need to store constants to the same list. For - example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--str', dest='types', action='append_const', const=str) - >>> parser.add_argument('--int', dest='types', action='append_const', const=int) - >>> parser.parse_args('--str --int'.split()) - Namespace(types=[, ]) - -* ``'count'`` - This counts the number of times a keyword argument occurs. For - example, this is useful for increasing verbosity levels:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--verbose', '-v', action='count', default=0) - >>> parser.parse_args(['-vvv']) - Namespace(verbose=3) - - Note, the *default* will be ``None`` unless explicitly set to *0*. - -* ``'help'`` - This prints a complete help message for all the options in the - current parser and then exits. By default a help action is automatically - added to the parser. See :class:`ArgumentParser` for details of how the - output is created. - -* ``'version'`` - This expects a ``version=`` keyword argument in the - :meth:`~ArgumentParser.add_argument` call, and prints version information - and exits when invoked:: - - >>> import argparse - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--version', action='version', version='%(prog)s 2.0') - >>> parser.parse_args(['--version']) - PROG 2.0 - -* ``'extend'`` - This stores a list, and extends each argument value to the - list. - Example usage:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument("--foo", action="extend", nargs="+", type=str) - >>> parser.parse_args(["--foo", "f1", "--foo", "f2", "f3", "f4"]) - Namespace(foo=['f1', 'f2', 'f3', 'f4']) - - .. versionadded:: 3.8 - -You may also specify an arbitrary action by passing an Action subclass or -other object that implements the same interface. The ``BooleanOptionalAction`` -is available in ``argparse`` and adds support for boolean actions such as -``--foo`` and ``--no-foo``:: - - >>> import argparse - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=argparse.BooleanOptionalAction) - >>> parser.parse_args(['--no-foo']) - Namespace(foo=False) - -.. versionadded:: 3.9 - -The recommended way to create a custom action is to extend :class:`Action`, -overriding the ``__call__`` method and optionally the ``__init__`` and -``format_usage`` methods. - -An example of a custom action:: - - >>> class FooAction(argparse.Action): - ... def __init__(self, option_strings, dest, nargs=None, **kwargs): - ... if nargs is not None: - ... raise ValueError("nargs not allowed") - ... super().__init__(option_strings, dest, **kwargs) - ... def __call__(self, parser, namespace, values, option_string=None): - ... print('%r %r %r' % (namespace, values, option_string)) - ... setattr(namespace, self.dest, values) - ... - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action=FooAction) - >>> parser.add_argument('bar', action=FooAction) - >>> args = parser.parse_args('1 --foo 2'.split()) - Namespace(bar=None, foo=None) '1' None - Namespace(bar='1', foo=None) '2' '--foo' - >>> args - Namespace(bar='1', foo='2') - -For more details, see :class:`Action`. - - -.. _nargs: - -nargs -^^^^^ - -ArgumentParser objects usually associate a single command-line argument with a -single action to be taken. The ``nargs`` keyword argument associates a -different number of command-line arguments with a single action. The supported -values are: - -* ``N`` (an integer). ``N`` arguments from the command line will be gathered - together into a list. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', nargs=2) - >>> parser.add_argument('bar', nargs=1) - >>> parser.parse_args('c --foo a b'.split()) - Namespace(bar=['c'], foo=['a', 'b']) - - Note that ``nargs=1`` produces a list of one item. This is different from - the default, in which the item is produced by itself. - -.. index:: single: ? (question mark); in argparse module - -* ``'?'``. One argument will be consumed from the command line if possible, and - produced as a single item. If no command-line argument is present, the value from - default_ will be produced. Note that for optional arguments, there is an - additional case - the option string is present but not followed by a - command-line argument. In this case the value from const_ will be produced. Some - examples to illustrate this:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', nargs='?', const='c', default='d') - >>> parser.add_argument('bar', nargs='?', default='d') - >>> parser.parse_args(['XX', '--foo', 'YY']) - Namespace(bar='XX', foo='YY') - >>> parser.parse_args(['XX', '--foo']) - Namespace(bar='XX', foo='c') - >>> parser.parse_args([]) - Namespace(bar='d', foo='d') - - One of the more common uses of ``nargs='?'`` is to allow optional input and - output files:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('infile', nargs='?', type=argparse.FileType('r'), - ... default=sys.stdin) - >>> parser.add_argument('outfile', nargs='?', type=argparse.FileType('w'), - ... default=sys.stdout) - >>> parser.parse_args(['input.txt', 'output.txt']) - Namespace(infile=<_io.TextIOWrapper name='input.txt' encoding='UTF-8'>, - outfile=<_io.TextIOWrapper name='output.txt' encoding='UTF-8'>) - >>> parser.parse_args([]) - Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>, - outfile=<_io.TextIOWrapper name='' encoding='UTF-8'>) - -.. index:: single: * (asterisk); in argparse module - -* ``'*'``. All command-line arguments present are gathered into a list. Note that - it generally doesn't make much sense to have more than one positional argument - with ``nargs='*'``, but multiple optional arguments with ``nargs='*'`` is - possible. For example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', nargs='*') - >>> parser.add_argument('--bar', nargs='*') - >>> parser.add_argument('baz', nargs='*') - >>> parser.parse_args('a b --foo x y --bar 1 2'.split()) - Namespace(bar=['1', '2'], baz=['a', 'b'], foo=['x', 'y']) - -.. index:: single: + (plus); in argparse module - -* ``'+'``. Just like ``'*'``, all command-line args present are gathered into a - list. Additionally, an error message will be generated if there wasn't at - least one command-line argument present. For example:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('foo', nargs='+') - >>> parser.parse_args(['a', 'b']) - Namespace(foo=['a', 'b']) - >>> parser.parse_args([]) - usage: PROG [-h] foo [foo ...] - PROG: error: the following arguments are required: foo - -If the ``nargs`` keyword argument is not provided, the number of arguments consumed -is determined by the action_. Generally this means a single command-line argument -will be consumed and a single item (not a list) will be produced. - - -.. _const: - -const -^^^^^ - -The ``const`` argument of :meth:`~ArgumentParser.add_argument` is used to hold -constant values that are not read from the command line but are required for -the various :class:`ArgumentParser` actions. The two most common uses of it are: - -* When :meth:`~ArgumentParser.add_argument` is called with - ``action='store_const'`` or ``action='append_const'``. These actions add the - ``const`` value to one of the attributes of the object returned by - :meth:`~ArgumentParser.parse_args`. See the action_ description for examples. - If ``const`` is not provided to :meth:`~ArgumentParser.add_argument`, it will - receive a default value of ``None``. - - -* When :meth:`~ArgumentParser.add_argument` is called with option strings - (like ``-f`` or ``--foo``) and ``nargs='?'``. This creates an optional - argument that can be followed by zero or one command-line arguments. - When parsing the command line, if the option string is encountered with no - command-line argument following it, the value of ``const`` will be assumed to - be ``None`` instead. See the nargs_ description for examples. - -.. versionchanged:: 3.11 - ``const=None`` by default, including when ``action='append_const'`` or - ``action='store_const'``. - -.. _default: - -default -^^^^^^^ - -All optional arguments and some positional arguments may be omitted at the -command line. The ``default`` keyword argument of -:meth:`~ArgumentParser.add_argument`, whose value defaults to ``None``, -specifies what value should be used if the command-line argument is not present. -For optional arguments, the ``default`` value is used when the option string -was not present at the command line:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', default=42) - >>> parser.parse_args(['--foo', '2']) - Namespace(foo='2') - >>> parser.parse_args([]) - Namespace(foo=42) - -If the target namespace already has an attribute set, the action *default* -will not over write it:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', default=42) - >>> parser.parse_args([], namespace=argparse.Namespace(foo=101)) - Namespace(foo=101) - -If the ``default`` value is a string, the parser parses the value as if it -were a command-line argument. In particular, the parser applies any type_ -conversion argument, if provided, before setting the attribute on the -:class:`Namespace` return value. Otherwise, the parser uses the value as is:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--length', default='10', type=int) - >>> parser.add_argument('--width', default=10.5, type=int) - >>> parser.parse_args() - Namespace(length=10, width=10.5) - -For positional arguments with nargs_ equal to ``?`` or ``*``, the ``default`` value -is used when no command-line argument was present:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('foo', nargs='?', default=42) - >>> parser.parse_args(['a']) - Namespace(foo='a') - >>> parser.parse_args([]) - Namespace(foo=42) - - -Providing ``default=argparse.SUPPRESS`` causes no attribute to be added if the -command-line argument was not present:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', default=argparse.SUPPRESS) - >>> parser.parse_args([]) - Namespace() - >>> parser.parse_args(['--foo', '1']) - Namespace(foo='1') - - -.. _type: - -type -^^^^ - -By default, the parser reads command-line arguments in as simple -strings. However, quite often the command-line string should instead be -interpreted as another type, such as a :class:`float` or :class:`int`. The -``type`` keyword for :meth:`~ArgumentParser.add_argument` allows any -necessary type-checking and type conversions to be performed. - -If the type_ keyword is used with the default_ keyword, the type converter -is only applied if the default is a string. - -The argument to ``type`` can be any callable that accepts a single string. -If the function raises :exc:`ArgumentTypeError`, :exc:`TypeError`, or -:exc:`ValueError`, the exception is caught and a nicely formatted error -message is displayed. No other exception types are handled. - -Common built-in types and functions can be used as type converters: - -.. testcode:: - - import argparse - import pathlib - - parser = argparse.ArgumentParser() - parser.add_argument('count', type=int) - parser.add_argument('distance', type=float) - parser.add_argument('street', type=ascii) - parser.add_argument('code_point', type=ord) - parser.add_argument('source_file', type=open) - parser.add_argument('dest_file', type=argparse.FileType('w', encoding='latin-1')) - parser.add_argument('datapath', type=pathlib.Path) - -User defined functions can be used as well: - -.. doctest:: - - >>> def hyphenated(string): - ... return '-'.join([word[:4] for word in string.casefold().split()]) - ... - >>> parser = argparse.ArgumentParser() - >>> _ = parser.add_argument('short_title', type=hyphenated) - >>> parser.parse_args(['"The Tale of Two Cities"']) - Namespace(short_title='"the-tale-of-two-citi') - -The :func:`bool` function is not recommended as a type converter. All it does -is convert empty strings to ``False`` and non-empty strings to ``True``. -This is usually not what is desired. - -In general, the ``type`` keyword is a convenience that should only be used for -simple conversions that can only raise one of the three supported exceptions. -Anything with more interesting error-handling or resource management should be -done downstream after the arguments are parsed. - -For example, JSON or YAML conversions have complex error cases that require -better reporting than can be given by the ``type`` keyword. A -:exc:`~json.JSONDecodeError` would not be well formatted and a -:exc:`FileNotFoundError` exception would not be handled at all. - -Even :class:`~argparse.FileType` has its limitations for use with the ``type`` -keyword. If one argument uses *FileType* and then a subsequent argument fails, -an error is reported but the file is not automatically closed. In this case, it -would be better to wait until after the parser has run and then use the -:keyword:`with`-statement to manage the files. - -For type checkers that simply check against a fixed set of values, consider -using the choices_ keyword instead. - - -.. _choices: - -choices -^^^^^^^ - -Some command-line arguments should be selected from a restricted set of values. -These can be handled by passing a sequence object as the *choices* keyword -argument to :meth:`~ArgumentParser.add_argument`. When the command line is -parsed, argument values will be checked, and an error message will be displayed -if the argument was not one of the acceptable values:: - - >>> parser = argparse.ArgumentParser(prog='game.py') - >>> parser.add_argument('move', choices=['rock', 'paper', 'scissors']) - >>> parser.parse_args(['rock']) - Namespace(move='rock') - >>> parser.parse_args(['fire']) - usage: game.py [-h] {rock,paper,scissors} - game.py: error: argument move: invalid choice: 'fire' (choose from 'rock', - 'paper', 'scissors') - -Note that inclusion in the *choices* sequence is checked after any type_ -conversions have been performed, so the type of the objects in the *choices* -sequence should match the type_ specified:: - - >>> parser = argparse.ArgumentParser(prog='doors.py') - >>> parser.add_argument('door', type=int, choices=range(1, 4)) - >>> print(parser.parse_args(['3'])) - Namespace(door=3) - >>> parser.parse_args(['4']) - usage: doors.py [-h] {1,2,3} - doors.py: error: argument door: invalid choice: 4 (choose from 1, 2, 3) - -Any sequence can be passed as the *choices* value, so :class:`list` objects, -:class:`tuple` objects, and custom sequences are all supported. - -Use of :class:`enum.Enum` is not recommended because it is difficult to -control its appearance in usage, help, and error messages. - -Formatted choices override the default *metavar* which is normally derived -from *dest*. This is usually what you want because the user never sees the -*dest* parameter. If this display isn't desirable (perhaps because there are -many choices), just specify an explicit metavar_. - - -.. _required: - -required -^^^^^^^^ - -In general, the :mod:`argparse` module assumes that flags like ``-f`` and ``--bar`` -indicate *optional* arguments, which can always be omitted at the command line. -To make an option *required*, ``True`` can be specified for the ``required=`` -keyword argument to :meth:`~ArgumentParser.add_argument`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', required=True) - >>> parser.parse_args(['--foo', 'BAR']) - Namespace(foo='BAR') - >>> parser.parse_args([]) - usage: [-h] --foo FOO - : error: the following arguments are required: --foo - -As the example shows, if an option is marked as ``required``, -:meth:`~ArgumentParser.parse_args` will report an error if that option is not -present at the command line. - -.. note:: - - Required options are generally considered bad form because users expect - *options* to be *optional*, and thus they should be avoided when possible. - - -.. _help: - -help -^^^^ - -The ``help`` value is a string containing a brief description of the argument. -When a user requests help (usually by using ``-h`` or ``--help`` at the -command line), these ``help`` descriptions will be displayed with each -argument:: - - >>> parser = argparse.ArgumentParser(prog='frobble') - >>> parser.add_argument('--foo', action='store_true', - ... help='foo the bars before frobbling') - >>> parser.add_argument('bar', nargs='+', - ... help='one of the bars to be frobbled') - >>> parser.parse_args(['-h']) - usage: frobble [-h] [--foo] bar [bar ...] - - positional arguments: - bar one of the bars to be frobbled - - options: - -h, --help show this help message and exit - --foo foo the bars before frobbling - -The ``help`` strings can include various format specifiers to avoid repetition -of things like the program name or the argument default_. The available -specifiers include the program name, ``%(prog)s`` and most keyword arguments to -:meth:`~ArgumentParser.add_argument`, e.g. ``%(default)s``, ``%(type)s``, etc.:: - - >>> parser = argparse.ArgumentParser(prog='frobble') - >>> parser.add_argument('bar', nargs='?', type=int, default=42, - ... help='the bar to %(prog)s (default: %(default)s)') - >>> parser.print_help() - usage: frobble [-h] [bar] - - positional arguments: - bar the bar to frobble (default: 42) - - options: - -h, --help show this help message and exit - -As the help string supports %-formatting, if you want a literal ``%`` to appear -in the help string, you must escape it as ``%%``. - -:mod:`argparse` supports silencing the help entry for certain options, by -setting the ``help`` value to ``argparse.SUPPRESS``:: - - >>> parser = argparse.ArgumentParser(prog='frobble') - >>> parser.add_argument('--foo', help=argparse.SUPPRESS) - >>> parser.print_help() - usage: frobble [-h] - - options: - -h, --help show this help message and exit - - -.. _metavar: - -metavar -^^^^^^^ - -When :class:`ArgumentParser` generates help messages, it needs some way to refer -to each expected argument. By default, ArgumentParser objects use the dest_ -value as the "name" of each object. By default, for positional argument -actions, the dest_ value is used directly, and for optional argument actions, -the dest_ value is uppercased. So, a single positional argument with -``dest='bar'`` will be referred to as ``bar``. A single -optional argument ``--foo`` that should be followed by a single command-line argument -will be referred to as ``FOO``. An example:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.add_argument('bar') - >>> parser.parse_args('X --foo Y'.split()) - Namespace(bar='X', foo='Y') - >>> parser.print_help() - usage: [-h] [--foo FOO] bar - - positional arguments: - bar - - options: - -h, --help show this help message and exit - --foo FOO - -An alternative name can be specified with ``metavar``:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', metavar='YYY') - >>> parser.add_argument('bar', metavar='XXX') - >>> parser.parse_args('X --foo Y'.split()) - Namespace(bar='X', foo='Y') - >>> parser.print_help() - usage: [-h] [--foo YYY] XXX - - positional arguments: - XXX - - options: - -h, --help show this help message and exit - --foo YYY - -Note that ``metavar`` only changes the *displayed* name - the name of the -attribute on the :meth:`~ArgumentParser.parse_args` object is still determined -by the dest_ value. - -Different values of ``nargs`` may cause the metavar to be used multiple times. -Providing a tuple to ``metavar`` specifies a different display for each of the -arguments:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-x', nargs=2) - >>> parser.add_argument('--foo', nargs=2, metavar=('bar', 'baz')) - >>> parser.print_help() - usage: PROG [-h] [-x X X] [--foo bar baz] - - options: - -h, --help show this help message and exit - -x X X - --foo bar baz - - -.. _dest: - -dest -^^^^ - -Most :class:`ArgumentParser` actions add some value as an attribute of the -object returned by :meth:`~ArgumentParser.parse_args`. The name of this -attribute is determined by the ``dest`` keyword argument of -:meth:`~ArgumentParser.add_argument`. For positional argument actions, -``dest`` is normally supplied as the first argument to -:meth:`~ArgumentParser.add_argument`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('bar') - >>> parser.parse_args(['XXX']) - Namespace(bar='XXX') - -For optional argument actions, the value of ``dest`` is normally inferred from -the option strings. :class:`ArgumentParser` generates the value of ``dest`` by -taking the first long option string and stripping away the initial ``--`` -string. If no long option strings were supplied, ``dest`` will be derived from -the first short option string by stripping the initial ``-`` character. Any -internal ``-`` characters will be converted to ``_`` characters to make sure -the string is a valid attribute name. The examples below illustrate this -behavior:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('-f', '--foo-bar', '--foo') - >>> parser.add_argument('-x', '-y') - >>> parser.parse_args('-f 1 -x 2'.split()) - Namespace(foo_bar='1', x='2') - >>> parser.parse_args('--foo 1 -y 2'.split()) - Namespace(foo_bar='1', x='2') - -``dest`` allows a custom attribute name to be provided:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', dest='bar') - >>> parser.parse_args('--foo XXX'.split()) - Namespace(bar='XXX') - -Action classes -^^^^^^^^^^^^^^ - -Action classes implement the Action API, a callable which returns a callable -which processes arguments from the command-line. Any object which follows -this API may be passed as the ``action`` parameter to -:meth:`~ArgumentParser.add_argument`. - -.. class:: Action(option_strings, dest, nargs=None, const=None, default=None, \ - type=None, choices=None, required=False, help=None, \ - metavar=None) - -Action objects are used by an ArgumentParser to represent the information -needed to parse a single argument from one or more strings from the -command line. The Action class must accept the two positional arguments -plus any keyword arguments passed to :meth:`ArgumentParser.add_argument` -except for the ``action`` itself. - -Instances of Action (or return value of any callable to the ``action`` -parameter) should have attributes "dest", "option_strings", "default", "type", -"required", "help", etc. defined. The easiest way to ensure these attributes -are defined is to call ``Action.__init__``. - -Action instances should be callable, so subclasses must override the -``__call__`` method, which should accept four parameters: - -* ``parser`` - The ArgumentParser object which contains this action. - -* ``namespace`` - The :class:`Namespace` object that will be returned by - :meth:`~ArgumentParser.parse_args`. Most actions add an attribute to this - object using :func:`setattr`. - -* ``values`` - The associated command-line arguments, with any type conversions - applied. Type conversions are specified with the type_ keyword argument to - :meth:`~ArgumentParser.add_argument`. - -* ``option_string`` - The option string that was used to invoke this action. - The ``option_string`` argument is optional, and will be absent if the action - is associated with a positional argument. - -The ``__call__`` method may perform arbitrary actions, but will typically set -attributes on the ``namespace`` based on ``dest`` and ``values``. - -Action subclasses can define a ``format_usage`` method that takes no argument -and return a string which will be used when printing the usage of the program. -If such method is not provided, a sensible default will be used. - -The parse_args() method ------------------------ - -.. method:: ArgumentParser.parse_args(args=None, namespace=None) - - Convert argument strings to objects and assign them as attributes of the - namespace. Return the populated namespace. - - Previous calls to :meth:`add_argument` determine exactly what objects are - created and how they are assigned. See the documentation for - :meth:`add_argument` for details. - - * args_ - List of strings to parse. The default is taken from - :data:`sys.argv`. - - * namespace_ - An object to take the attributes. The default is a new empty - :class:`Namespace` object. - - -Option value syntax -^^^^^^^^^^^^^^^^^^^ - -The :meth:`~ArgumentParser.parse_args` method supports several ways of -specifying the value of an option (if it takes one). In the simplest case, the -option and its value are passed as two separate arguments:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-x') - >>> parser.add_argument('--foo') - >>> parser.parse_args(['-x', 'X']) - Namespace(foo=None, x='X') - >>> parser.parse_args(['--foo', 'FOO']) - Namespace(foo='FOO', x=None) - -For long options (options with names longer than a single character), the option -and value can also be passed as a single command-line argument, using ``=`` to -separate them:: - - >>> parser.parse_args(['--foo=FOO']) - Namespace(foo='FOO', x=None) - -For short options (options only one character long), the option and its value -can be concatenated:: - - >>> parser.parse_args(['-xX']) - Namespace(foo=None, x='X') - -Several short options can be joined together, using only a single ``-`` prefix, -as long as only the last option (or none of them) requires a value:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-x', action='store_true') - >>> parser.add_argument('-y', action='store_true') - >>> parser.add_argument('-z') - >>> parser.parse_args(['-xyzZ']) - Namespace(x=True, y=True, z='Z') - - -Invalid arguments -^^^^^^^^^^^^^^^^^ - -While parsing the command line, :meth:`~ArgumentParser.parse_args` checks for a -variety of errors, including ambiguous options, invalid types, invalid options, -wrong number of positional arguments, etc. When it encounters such an error, -it exits and prints the error along with a usage message:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', type=int) - >>> parser.add_argument('bar', nargs='?') - - >>> # invalid type - >>> parser.parse_args(['--foo', 'spam']) - usage: PROG [-h] [--foo FOO] [bar] - PROG: error: argument --foo: invalid int value: 'spam' - - >>> # invalid option - >>> parser.parse_args(['--bar']) - usage: PROG [-h] [--foo FOO] [bar] - PROG: error: no such option: --bar - - >>> # wrong number of arguments - >>> parser.parse_args(['spam', 'badger']) - usage: PROG [-h] [--foo FOO] [bar] - PROG: error: extra arguments found: badger - - -Arguments containing ``-`` -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :meth:`~ArgumentParser.parse_args` method attempts to give errors whenever -the user has clearly made a mistake, but some situations are inherently -ambiguous. For example, the command-line argument ``-1`` could either be an -attempt to specify an option or an attempt to provide a positional argument. -The :meth:`~ArgumentParser.parse_args` method is cautious here: positional -arguments may only begin with ``-`` if they look like negative numbers and -there are no options in the parser that look like negative numbers:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-x') - >>> parser.add_argument('foo', nargs='?') - - >>> # no negative number options, so -1 is a positional argument - >>> parser.parse_args(['-x', '-1']) - Namespace(foo=None, x='-1') - - >>> # no negative number options, so -1 and -5 are positional arguments - >>> parser.parse_args(['-x', '-1', '-5']) - Namespace(foo='-5', x='-1') - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-1', dest='one') - >>> parser.add_argument('foo', nargs='?') - - >>> # negative number options present, so -1 is an option - >>> parser.parse_args(['-1', 'X']) - Namespace(foo=None, one='X') - - >>> # negative number options present, so -2 is an option - >>> parser.parse_args(['-2']) - usage: PROG [-h] [-1 ONE] [foo] - PROG: error: no such option: -2 - - >>> # negative number options present, so both -1s are options - >>> parser.parse_args(['-1', '-1']) - usage: PROG [-h] [-1 ONE] [foo] - PROG: error: argument -1: expected one argument - -If you have positional arguments that must begin with ``-`` and don't look -like negative numbers, you can insert the pseudo-argument ``'--'`` which tells -:meth:`~ArgumentParser.parse_args` that everything after that is a positional -argument:: - - >>> parser.parse_args(['--', '-f']) - Namespace(foo='-f', one=None) - -.. _prefix-matching: - -Argument abbreviations (prefix matching) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :meth:`~ArgumentParser.parse_args` method :ref:`by default ` -allows long options to be abbreviated to a prefix, if the abbreviation is -unambiguous (the prefix matches a unique option):: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('-bacon') - >>> parser.add_argument('-badger') - >>> parser.parse_args('-bac MMM'.split()) - Namespace(bacon='MMM', badger=None) - >>> parser.parse_args('-bad WOOD'.split()) - Namespace(bacon=None, badger='WOOD') - >>> parser.parse_args('-ba BA'.split()) - usage: PROG [-h] [-bacon BACON] [-badger BADGER] - PROG: error: ambiguous option: -ba could match -badger, -bacon - -An error is produced for arguments that could produce more than one options. -This feature can be disabled by setting :ref:`allow_abbrev` to ``False``. - -.. _args: - -Beyond ``sys.argv`` -^^^^^^^^^^^^^^^^^^^ - -Sometimes it may be useful to have an ArgumentParser parse arguments other than those -of :data:`sys.argv`. This can be accomplished by passing a list of strings to -:meth:`~ArgumentParser.parse_args`. This is useful for testing at the -interactive prompt:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument( - ... 'integers', metavar='int', type=int, choices=range(10), - ... nargs='+', help='an integer in the range 0..9') - >>> parser.add_argument( - ... '--sum', dest='accumulate', action='store_const', const=sum, - ... default=max, help='sum the integers (default: find the max)') - >>> parser.parse_args(['1', '2', '3', '4']) - Namespace(accumulate=, integers=[1, 2, 3, 4]) - >>> parser.parse_args(['1', '2', '3', '4', '--sum']) - Namespace(accumulate=, integers=[1, 2, 3, 4]) - -.. _namespace: - -The Namespace object -^^^^^^^^^^^^^^^^^^^^ - -.. class:: Namespace - - Simple class used by default by :meth:`~ArgumentParser.parse_args` to create - an object holding attributes and return it. - -This class is deliberately simple, just an :class:`object` subclass with a -readable string representation. If you prefer to have dict-like view of the -attributes, you can use the standard Python idiom, :func:`vars`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> args = parser.parse_args(['--foo', 'BAR']) - >>> vars(args) - {'foo': 'BAR'} - -It may also be useful to have an :class:`ArgumentParser` assign attributes to an -already existing object, rather than a new :class:`Namespace` object. This can -be achieved by specifying the ``namespace=`` keyword argument:: - - >>> class C: - ... pass - ... - >>> c = C() - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.parse_args(args=['--foo', 'BAR'], namespace=c) - >>> c.foo - 'BAR' - - -Other utilities ---------------- - -Sub-commands -^^^^^^^^^^^^ - -.. method:: ArgumentParser.add_subparsers([title], [description], [prog], \ - [parser_class], [action], \ - [option_strings], [dest], [required], \ - [help], [metavar]) - - Many programs split up their functionality into a number of sub-commands, - for example, the ``svn`` program can invoke sub-commands like ``svn - checkout``, ``svn update``, and ``svn commit``. Splitting up functionality - this way can be a particularly good idea when a program performs several - different functions which require different kinds of command-line arguments. - :class:`ArgumentParser` supports the creation of such sub-commands with the - :meth:`add_subparsers` method. The :meth:`add_subparsers` method is normally - called with no arguments and returns a special action object. This object - has a single method, :meth:`~_SubParsersAction.add_parser`, which takes a - command name and any :class:`ArgumentParser` constructor arguments, and - returns an :class:`ArgumentParser` object that can be modified as usual. - - Description of parameters: - - * title - title for the sub-parser group in help output; by default - "subcommands" if description is provided, otherwise uses title for - positional arguments - - * description - description for the sub-parser group in help output, by - default ``None`` - - * prog - usage information that will be displayed with sub-command help, - by default the name of the program and any positional arguments before the - subparser argument - - * parser_class - class which will be used to create sub-parser instances, by - default the class of the current parser (e.g. ArgumentParser) - - * action_ - the basic type of action to be taken when this argument is - encountered at the command line - - * dest_ - name of the attribute under which sub-command name will be - stored; by default ``None`` and no value is stored - - * required_ - Whether or not a subcommand must be provided, by default - ``False`` (added in 3.7) - - * help_ - help for sub-parser group in help output, by default ``None`` - - * metavar_ - string presenting available sub-commands in help; by default it - is ``None`` and presents sub-commands in form {cmd1, cmd2, ..} - - Some example usage:: - - >>> # create the top-level parser - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> parser.add_argument('--foo', action='store_true', help='foo help') - >>> subparsers = parser.add_subparsers(help='sub-command help') - >>> - >>> # create the parser for the "a" command - >>> parser_a = subparsers.add_parser('a', help='a help') - >>> parser_a.add_argument('bar', type=int, help='bar help') - >>> - >>> # create the parser for the "b" command - >>> parser_b = subparsers.add_parser('b', help='b help') - >>> parser_b.add_argument('--baz', choices='XYZ', help='baz help') - >>> - >>> # parse some argument lists - >>> parser.parse_args(['a', '12']) - Namespace(bar=12, foo=False) - >>> parser.parse_args(['--foo', 'b', '--baz', 'Z']) - Namespace(baz='Z', foo=True) - - Note that the object returned by :meth:`parse_args` will only contain - attributes for the main parser and the subparser that was selected by the - command line (and not any other subparsers). So in the example above, when - the ``a`` command is specified, only the ``foo`` and ``bar`` attributes are - present, and when the ``b`` command is specified, only the ``foo`` and - ``baz`` attributes are present. - - Similarly, when a help message is requested from a subparser, only the help - for that particular parser will be printed. The help message will not - include parent parser or sibling parser messages. (A help message for each - subparser command, however, can be given by supplying the ``help=`` argument - to :meth:`~_SubParsersAction.add_parser` as above.) - - :: - - >>> parser.parse_args(['--help']) - usage: PROG [-h] [--foo] {a,b} ... - - positional arguments: - {a,b} sub-command help - a a help - b b help - - options: - -h, --help show this help message and exit - --foo foo help - - >>> parser.parse_args(['a', '--help']) - usage: PROG a [-h] bar - - positional arguments: - bar bar help - - options: - -h, --help show this help message and exit - - >>> parser.parse_args(['b', '--help']) - usage: PROG b [-h] [--baz {X,Y,Z}] - - options: - -h, --help show this help message and exit - --baz {X,Y,Z} baz help - - The :meth:`add_subparsers` method also supports ``title`` and ``description`` - keyword arguments. When either is present, the subparser's commands will - appear in their own group in the help output. For example:: - - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers(title='subcommands', - ... description='valid subcommands', - ... help='additional help') - >>> subparsers.add_parser('foo') - >>> subparsers.add_parser('bar') - >>> parser.parse_args(['-h']) - usage: [-h] {foo,bar} ... - - options: - -h, --help show this help message and exit - - subcommands: - valid subcommands - - {foo,bar} additional help - - Furthermore, ``add_parser`` supports an additional ``aliases`` argument, - which allows multiple strings to refer to the same subparser. This example, - like ``svn``, aliases ``co`` as a shorthand for ``checkout``:: - - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers() - >>> checkout = subparsers.add_parser('checkout', aliases=['co']) - >>> checkout.add_argument('foo') - >>> parser.parse_args(['co', 'bar']) - Namespace(foo='bar') - - One particularly effective way of handling sub-commands is to combine the use - of the :meth:`add_subparsers` method with calls to :meth:`set_defaults` so - that each subparser knows which Python function it should execute. For - example:: - - >>> # sub-command functions - >>> def foo(args): - ... print(args.x * args.y) - ... - >>> def bar(args): - ... print('((%s))' % args.z) - ... - >>> # create the top-level parser - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers(required=True) - >>> - >>> # create the parser for the "foo" command - >>> parser_foo = subparsers.add_parser('foo') - >>> parser_foo.add_argument('-x', type=int, default=1) - >>> parser_foo.add_argument('y', type=float) - >>> parser_foo.set_defaults(func=foo) - >>> - >>> # create the parser for the "bar" command - >>> parser_bar = subparsers.add_parser('bar') - >>> parser_bar.add_argument('z') - >>> parser_bar.set_defaults(func=bar) - >>> - >>> # parse the args and call whatever function was selected - >>> args = parser.parse_args('foo 1 -x 2'.split()) - >>> args.func(args) - 2.0 - >>> - >>> # parse the args and call whatever function was selected - >>> args = parser.parse_args('bar XYZYX'.split()) - >>> args.func(args) - ((XYZYX)) - - This way, you can let :meth:`parse_args` do the job of calling the - appropriate function after argument parsing is complete. Associating - functions with actions like this is typically the easiest way to handle the - different actions for each of your subparsers. However, if it is necessary - to check the name of the subparser that was invoked, the ``dest`` keyword - argument to the :meth:`add_subparsers` call will work:: - - >>> parser = argparse.ArgumentParser() - >>> subparsers = parser.add_subparsers(dest='subparser_name') - >>> subparser1 = subparsers.add_parser('1') - >>> subparser1.add_argument('-x') - >>> subparser2 = subparsers.add_parser('2') - >>> subparser2.add_argument('y') - >>> parser.parse_args(['2', 'frobble']) - Namespace(subparser_name='2', y='frobble') - - .. versionchanged:: 3.7 - New *required* keyword argument. - - -FileType objects -^^^^^^^^^^^^^^^^ - -.. class:: FileType(mode='r', bufsize=-1, encoding=None, errors=None) - - The :class:`FileType` factory creates objects that can be passed to the type - argument of :meth:`ArgumentParser.add_argument`. Arguments that have - :class:`FileType` objects as their type will open command-line arguments as - files with the requested modes, buffer sizes, encodings and error handling - (see the :func:`open` function for more details):: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--raw', type=argparse.FileType('wb', 0)) - >>> parser.add_argument('out', type=argparse.FileType('w', encoding='UTF-8')) - >>> parser.parse_args(['--raw', 'raw.dat', 'file.txt']) - Namespace(out=<_io.TextIOWrapper name='file.txt' mode='w' encoding='UTF-8'>, raw=<_io.FileIO name='raw.dat' mode='wb'>) - - FileType objects understand the pseudo-argument ``'-'`` and automatically - convert this into :data:`sys.stdin` for readable :class:`FileType` objects and - :data:`sys.stdout` for writable :class:`FileType` objects:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('infile', type=argparse.FileType('r')) - >>> parser.parse_args(['-']) - Namespace(infile=<_io.TextIOWrapper name='' encoding='UTF-8'>) - - .. versionadded:: 3.4 - The *encodings* and *errors* keyword arguments. - - -Argument groups -^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.add_argument_group(title=None, description=None) - - By default, :class:`ArgumentParser` groups command-line arguments into - "positional arguments" and "options" when displaying help - messages. When there is a better conceptual grouping of arguments than this - default one, appropriate groups can be created using the - :meth:`add_argument_group` method:: - - >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) - >>> group = parser.add_argument_group('group') - >>> group.add_argument('--foo', help='foo help') - >>> group.add_argument('bar', help='bar help') - >>> parser.print_help() - usage: PROG [--foo FOO] bar - - group: - bar bar help - --foo FOO foo help - - The :meth:`add_argument_group` method returns an argument group object which - has an :meth:`~ArgumentParser.add_argument` method just like a regular - :class:`ArgumentParser`. When an argument is added to the group, the parser - treats it just like a normal argument, but displays the argument in a - separate group for help messages. The :meth:`add_argument_group` method - accepts *title* and *description* arguments which can be used to - customize this display:: - - >>> parser = argparse.ArgumentParser(prog='PROG', add_help=False) - >>> group1 = parser.add_argument_group('group1', 'group1 description') - >>> group1.add_argument('foo', help='foo help') - >>> group2 = parser.add_argument_group('group2', 'group2 description') - >>> group2.add_argument('--bar', help='bar help') - >>> parser.print_help() - usage: PROG [--bar BAR] foo - - group1: - group1 description - - foo foo help - - group2: - group2 description - - --bar BAR bar help - - Note that any arguments not in your user-defined groups will end up back - in the usual "positional arguments" and "optional arguments" sections. - - .. versionchanged:: 3.11 - Calling :meth:`add_argument_group` on an argument group is deprecated. - This feature was never supported and does not always work correctly. - The function exists on the API by accident through inheritance and - will be removed in the future. - - -Mutual exclusion -^^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.add_mutually_exclusive_group(required=False) - - Create a mutually exclusive group. :mod:`argparse` will make sure that only - one of the arguments in the mutually exclusive group was present on the - command line:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> group = parser.add_mutually_exclusive_group() - >>> group.add_argument('--foo', action='store_true') - >>> group.add_argument('--bar', action='store_false') - >>> parser.parse_args(['--foo']) - Namespace(bar=True, foo=True) - >>> parser.parse_args(['--bar']) - Namespace(bar=False, foo=False) - >>> parser.parse_args(['--foo', '--bar']) - usage: PROG [-h] [--foo | --bar] - PROG: error: argument --bar: not allowed with argument --foo - - The :meth:`add_mutually_exclusive_group` method also accepts a *required* - argument, to indicate that at least one of the mutually exclusive arguments - is required:: - - >>> parser = argparse.ArgumentParser(prog='PROG') - >>> group = parser.add_mutually_exclusive_group(required=True) - >>> group.add_argument('--foo', action='store_true') - >>> group.add_argument('--bar', action='store_false') - >>> parser.parse_args([]) - usage: PROG [-h] (--foo | --bar) - PROG: error: one of the arguments --foo --bar is required - - Note that currently mutually exclusive argument groups do not support the - *title* and *description* arguments of - :meth:`~ArgumentParser.add_argument_group`. - - .. versionchanged:: 3.11 - Calling :meth:`add_argument_group` or :meth:`add_mutually_exclusive_group` - on a mutually exclusive group is deprecated. These features were never - supported and do not always work correctly. The functions exist on the - API by accident through inheritance and will be removed in the future. - - -Parser defaults -^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.set_defaults(**kwargs) - - Most of the time, the attributes of the object returned by :meth:`parse_args` - will be fully determined by inspecting the command-line arguments and the argument - actions. :meth:`set_defaults` allows some additional - attributes that are determined without any inspection of the command line to - be added:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('foo', type=int) - >>> parser.set_defaults(bar=42, baz='badger') - >>> parser.parse_args(['736']) - Namespace(bar=42, baz='badger', foo=736) - - Note that parser-level defaults always override argument-level defaults:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', default='bar') - >>> parser.set_defaults(foo='spam') - >>> parser.parse_args([]) - Namespace(foo='spam') - - Parser-level defaults can be particularly useful when working with multiple - parsers. See the :meth:`~ArgumentParser.add_subparsers` method for an - example of this type. - -.. method:: ArgumentParser.get_default(dest) - - Get the default value for a namespace attribute, as set by either - :meth:`~ArgumentParser.add_argument` or by - :meth:`~ArgumentParser.set_defaults`:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', default='badger') - >>> parser.get_default('foo') - 'badger' - - -Printing help -^^^^^^^^^^^^^ - -In most typical applications, :meth:`~ArgumentParser.parse_args` will take -care of formatting and printing any usage or error messages. However, several -formatting methods are available: - -.. method:: ArgumentParser.print_usage(file=None) - - Print a brief description of how the :class:`ArgumentParser` should be - invoked on the command line. If *file* is ``None``, :data:`sys.stdout` is - assumed. - -.. method:: ArgumentParser.print_help(file=None) - - Print a help message, including the program usage and information about the - arguments registered with the :class:`ArgumentParser`. If *file* is - ``None``, :data:`sys.stdout` is assumed. - -There are also variants of these methods that simply return a string instead of -printing it: - -.. method:: ArgumentParser.format_usage() - - Return a string containing a brief description of how the - :class:`ArgumentParser` should be invoked on the command line. - -.. method:: ArgumentParser.format_help() - - Return a string containing a help message, including the program usage and - information about the arguments registered with the :class:`ArgumentParser`. - - -Partial parsing -^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.parse_known_args(args=None, namespace=None) - -Sometimes a script may only parse a few of the command-line arguments, passing -the remaining arguments on to another script or program. In these cases, the -:meth:`~ArgumentParser.parse_known_args` method can be useful. It works much like -:meth:`~ArgumentParser.parse_args` except that it does not produce an error when -extra arguments are present. Instead, it returns a two item tuple containing -the populated namespace and the list of remaining argument strings. - -:: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo', action='store_true') - >>> parser.add_argument('bar') - >>> parser.parse_known_args(['--foo', '--badger', 'BAR', 'spam']) - (Namespace(bar='BAR', foo=True), ['--badger', 'spam']) - -.. warning:: - :ref:`Prefix matching ` rules apply to - :meth:`~ArgumentParser.parse_known_args`. The parser may consume an option even if it's just - a prefix of one of its known options, instead of leaving it in the remaining - arguments list. - - -Customizing file parsing -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.convert_arg_line_to_args(arg_line) - - Arguments that are read from a file (see the *fromfile_prefix_chars* - keyword argument to the :class:`ArgumentParser` constructor) are read one - argument per line. :meth:`convert_arg_line_to_args` can be overridden for - fancier reading. - - This method takes a single argument *arg_line* which is a string read from - the argument file. It returns a list of arguments parsed from this string. - The method is called once per line read from the argument file, in order. - - A useful override of this method is one that treats each space-separated word - as an argument. The following example demonstrates how to do this:: - - class MyArgumentParser(argparse.ArgumentParser): - def convert_arg_line_to_args(self, arg_line): - return arg_line.split() - - -Exiting methods -^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.exit(status=0, message=None) - - This method terminates the program, exiting with the specified *status* - and, if given, it prints a *message* before that. The user can override - this method to handle these steps differently:: - - class ErrorCatchingArgumentParser(argparse.ArgumentParser): - def exit(self, status=0, message=None): - if status: - raise Exception(f'Exiting because of an error: {message}') - exit(status) - -.. method:: ArgumentParser.error(message) - - This method prints a usage message including the *message* to the - standard error and terminates the program with a status code of 2. - - -Intermixed parsing -^^^^^^^^^^^^^^^^^^ - -.. method:: ArgumentParser.parse_intermixed_args(args=None, namespace=None) -.. method:: ArgumentParser.parse_known_intermixed_args(args=None, namespace=None) - -A number of Unix commands allow the user to intermix optional arguments with -positional arguments. The :meth:`~ArgumentParser.parse_intermixed_args` -and :meth:`~ArgumentParser.parse_known_intermixed_args` methods -support this parsing style. - -These parsers do not support all the argparse features, and will raise -exceptions if unsupported features are used. In particular, subparsers, -and mutually exclusive groups that include both -optionals and positionals are not supported. - -The following example shows the difference between -:meth:`~ArgumentParser.parse_known_args` and -:meth:`~ArgumentParser.parse_intermixed_args`: the former returns ``['2', -'3']`` as unparsed arguments, while the latter collects all the positionals -into ``rest``. :: - - >>> parser = argparse.ArgumentParser() - >>> parser.add_argument('--foo') - >>> parser.add_argument('cmd') - >>> parser.add_argument('rest', nargs='*', type=int) - >>> parser.parse_known_args('doit 1 --foo bar 2 3'.split()) - (Namespace(cmd='doit', foo='bar', rest=[1]), ['2', '3']) - >>> parser.parse_intermixed_args('doit 1 --foo bar 2 3'.split()) - Namespace(cmd='doit', foo='bar', rest=[1, 2, 3]) - -:meth:`~ArgumentParser.parse_known_intermixed_args` returns a two item tuple -containing the populated namespace and the list of remaining argument strings. -:meth:`~ArgumentParser.parse_intermixed_args` raises an error if there are any -remaining unparsed argument strings. - -.. versionadded:: 3.7 - -.. _upgrading-optparse-code: - -Upgrading optparse code ------------------------ - -Originally, the :mod:`argparse` module had attempted to maintain compatibility -with :mod:`optparse`. However, :mod:`optparse` was difficult to extend -transparently, particularly with the changes required to support the new -``nargs=`` specifiers and better usage messages. When most everything in -:mod:`optparse` had either been copy-pasted over or monkey-patched, it no -longer seemed practical to try to maintain the backwards compatibility. - -The :mod:`argparse` module improves on the standard library :mod:`optparse` -module in a number of ways including: - -* Handling positional arguments. -* Supporting sub-commands. -* Allowing alternative option prefixes like ``+`` and ``/``. -* Handling zero-or-more and one-or-more style arguments. -* Producing more informative usage messages. -* Providing a much simpler interface for custom ``type`` and ``action``. - -A partial upgrade path from :mod:`optparse` to :mod:`argparse`: - -* Replace all :meth:`optparse.OptionParser.add_option` calls with - :meth:`ArgumentParser.add_argument` calls. - -* Replace ``(options, args) = parser.parse_args()`` with ``args = - parser.parse_args()`` and add additional :meth:`ArgumentParser.add_argument` - calls for the positional arguments. Keep in mind that what was previously - called ``options``, now in the :mod:`argparse` context is called ``args``. - -* Replace :meth:`optparse.OptionParser.disable_interspersed_args` - by using :meth:`~ArgumentParser.parse_intermixed_args` instead of - :meth:`~ArgumentParser.parse_args`. - -* Replace callback actions and the ``callback_*`` keyword arguments with - ``type`` or ``action`` arguments. - -* Replace string names for ``type`` keyword arguments with the corresponding - type objects (e.g. int, float, complex, etc). - -* Replace :class:`optparse.Values` with :class:`Namespace` and - :exc:`optparse.OptionError` and :exc:`optparse.OptionValueError` with - :exc:`ArgumentError`. - -* Replace strings with implicit arguments such as ``%default`` or ``%prog`` with - the standard Python syntax to use dictionaries to format strings, that is, - ``%(default)s`` and ``%(prog)s``. - -* Replace the OptionParser constructor ``version`` argument with a call to - ``parser.add_argument('--version', action='version', version='')``. - -Exceptions ----------- - -.. exception:: ArgumentError - - An error from creating or using an argument (optional or positional). - - The string value of this exception is the message, augmented with - information about the argument that caused it. - -.. exception:: ArgumentTypeError - - Raised when something goes wrong converting a command line string to a type. diff --git a/Doc/library/array.rst b/Doc/library/array.rst deleted file mode 100644 index d6cb8c623adbf1..00000000000000 --- a/Doc/library/array.rst +++ /dev/null @@ -1,269 +0,0 @@ -:mod:`array` --- Efficient arrays of numeric values -=================================================== - -.. module:: array - :synopsis: Space efficient arrays of uniformly typed numeric values. - -.. index:: single: arrays - --------------- - -This module defines an object type which can compactly represent an array of -basic values: characters, integers, floating point numbers. Arrays are sequence -types and behave very much like lists, except that the type of objects stored in -them is constrained. The type is specified at object creation time by using a -:dfn:`type code`, which is a single character. The following type codes are -defined: - -+-----------+--------------------+-------------------+-----------------------+-------+ -| Type code | C Type | Python Type | Minimum size in bytes | Notes | -+===========+====================+===================+=======================+=======+ -| ``'b'`` | signed char | int | 1 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'B'`` | unsigned char | int | 1 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'u'`` | wchar_t | Unicode character | 2 | \(1) | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'h'`` | signed short | int | 2 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'H'`` | unsigned short | int | 2 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'i'`` | signed int | int | 2 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'I'`` | unsigned int | int | 2 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'l'`` | signed long | int | 4 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'L'`` | unsigned long | int | 4 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'q'`` | signed long long | int | 8 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'Q'`` | unsigned long long | int | 8 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'f'`` | float | float | 4 | | -+-----------+--------------------+-------------------+-----------------------+-------+ -| ``'d'`` | double | float | 8 | | -+-----------+--------------------+-------------------+-----------------------+-------+ - -Notes: - -(1) - It can be 16 bits or 32 bits depending on the platform. - - .. versionchanged:: 3.9 - ``array('u')`` now uses :c:type:`wchar_t` as C type instead of deprecated - ``Py_UNICODE``. This change doesn't affect its behavior because - ``Py_UNICODE`` is alias of :c:type:`wchar_t` since Python 3.3. - - .. deprecated-removed:: 3.3 4.0 - - -The actual representation of values is determined by the machine architecture -(strictly speaking, by the C implementation). The actual size can be accessed -through the :attr:`array.itemsize` attribute. - -The module defines the following item: - - -.. data:: typecodes - - A string with all available type codes. - - -The module defines the following type: - - -.. class:: array(typecode[, initializer]) - - A new array whose items are restricted by *typecode*, and initialized - from the optional *initializer* value, which must be a list, a - :term:`bytes-like object`, or iterable over elements of the - appropriate type. - - If given a list or string, the initializer is passed to the new array's - :meth:`fromlist`, :meth:`frombytes`, or :meth:`fromunicode` method (see below) - to add initial items to the array. Otherwise, the iterable initializer is - passed to the :meth:`extend` method. - - Array objects support the ordinary sequence operations of indexing, slicing, - concatenation, and multiplication. When using slice assignment, the assigned - value must be an array object with the same type code; in all other cases, - :exc:`TypeError` is raised. Array objects also implement the buffer interface, - and may be used wherever :term:`bytes-like objects ` are supported. - - .. audit-event:: array.__new__ typecode,initializer array.array - - - .. attribute:: typecode - - The typecode character used to create the array. - - - .. attribute:: itemsize - - The length in bytes of one array item in the internal representation. - - - .. method:: append(x) - - Append a new item with value *x* to the end of the array. - - - .. method:: buffer_info() - - Return a tuple ``(address, length)`` giving the current memory address and the - length in elements of the buffer used to hold array's contents. The size of the - memory buffer in bytes can be computed as ``array.buffer_info()[1] * - array.itemsize``. This is occasionally useful when working with low-level (and - inherently unsafe) I/O interfaces that require memory addresses, such as certain - :c:func:`!ioctl` operations. The returned numbers are valid as long as the array - exists and no length-changing operations are applied to it. - - .. note:: - - When using array objects from code written in C or C++ (the only way to - effectively make use of this information), it makes more sense to use the buffer - interface supported by array objects. This method is maintained for backward - compatibility and should be avoided in new code. The buffer interface is - documented in :ref:`bufferobjects`. - - - .. method:: byteswap() - - "Byteswap" all items of the array. This is only supported for values which are - 1, 2, 4, or 8 bytes in size; for other types of values, :exc:`RuntimeError` is - raised. It is useful when reading data from a file written on a machine with a - different byte order. - - - .. method:: count(x) - - Return the number of occurrences of *x* in the array. - - - .. method:: extend(iterable) - - Append items from *iterable* to the end of the array. If *iterable* is another - array, it must have *exactly* the same type code; if not, :exc:`TypeError` will - be raised. If *iterable* is not an array, it must be iterable and its elements - must be the right type to be appended to the array. - - - .. method:: frombytes(s) - - Appends items from the string, interpreting the string as an array of machine - values (as if it had been read from a file using the :meth:`fromfile` method). - - .. versionadded:: 3.2 - :meth:`!fromstring` is renamed to :meth:`frombytes` for clarity. - - - .. method:: fromfile(f, n) - - Read *n* items (as machine values) from the :term:`file object` *f* and append - them to the end of the array. If less than *n* items are available, - :exc:`EOFError` is raised, but the items that were available are still - inserted into the array. - - - .. method:: fromlist(list) - - Append items from the list. This is equivalent to ``for x in list: - a.append(x)`` except that if there is a type error, the array is unchanged. - - - .. method:: fromunicode(s) - - Extends this array with data from the given unicode string. The array must - be a type ``'u'`` array; otherwise a :exc:`ValueError` is raised. Use - ``array.frombytes(unicodestring.encode(enc))`` to append Unicode data to an - array of some other type. - - - .. method:: index(x[, start[, stop]]) - - Return the smallest *i* such that *i* is the index of the first occurrence of - *x* in the array. The optional arguments *start* and *stop* can be - specified to search for *x* within a subsection of the array. Raise - :exc:`ValueError` if *x* is not found. - - .. versionchanged:: 3.10 - Added optional *start* and *stop* parameters. - - - .. method:: insert(i, x) - - Insert a new item with value *x* in the array before position *i*. Negative - values are treated as being relative to the end of the array. - - - .. method:: pop([i]) - - Removes the item with the index *i* from the array and returns it. The optional - argument defaults to ``-1``, so that by default the last item is removed and - returned. - - - .. method:: remove(x) - - Remove the first occurrence of *x* from the array. - - - .. method:: reverse() - - Reverse the order of the items in the array. - - - .. method:: tobytes() - - Convert the array to an array of machine values and return the bytes - representation (the same sequence of bytes that would be written to a file by - the :meth:`tofile` method.) - - .. versionadded:: 3.2 - :meth:`!tostring` is renamed to :meth:`tobytes` for clarity. - - - .. method:: tofile(f) - - Write all items (as machine values) to the :term:`file object` *f*. - - - .. method:: tolist() - - Convert the array to an ordinary list with the same items. - - - .. method:: tounicode() - - Convert the array to a unicode string. The array must be a type ``'u'`` array; - otherwise a :exc:`ValueError` is raised. Use ``array.tobytes().decode(enc)`` to - obtain a unicode string from an array of some other type. - - -When an array object is printed or converted to a string, it is represented as -``array(typecode, initializer)``. The *initializer* is omitted if the array is -empty, otherwise it is a string if the *typecode* is ``'u'``, otherwise it is a -list of numbers. The string is guaranteed to be able to be converted back to an -array with the same type and value using :func:`eval`, so long as the -:class:`~array.array` class has been imported using ``from array import array``. -Examples:: - - array('l') - array('u', 'hello \u2641') - array('l', [1, 2, 3, 4, 5]) - array('d', [1.0, 2.0, 3.14]) - - -.. seealso:: - - Module :mod:`struct` - Packing and unpacking of heterogeneous binary data. - - Module :mod:`xdrlib` - Packing and unpacking of External Data Representation (XDR) data as used in some - remote procedure call systems. - - `NumPy `_ - The NumPy package defines another array type. - diff --git a/Doc/library/ast.rst b/Doc/library/ast.rst deleted file mode 100644 index e95e18c1778069..00000000000000 --- a/Doc/library/ast.rst +++ /dev/null @@ -1,2389 +0,0 @@ -:mod:`ast` --- Abstract Syntax Trees -==================================== - -.. module:: ast - :synopsis: Abstract Syntax Tree classes and manipulation. - -.. sectionauthor:: Martin v. Löwis -.. sectionauthor:: Georg Brandl - -.. testsetup:: - - import ast - -**Source code:** :source:`Lib/ast.py` - --------------- - -The :mod:`ast` module helps Python applications to process trees of the Python -abstract syntax grammar. The abstract syntax itself might change with each -Python release; this module helps to find out programmatically what the current -grammar looks like. - -An abstract syntax tree can be generated by passing :data:`ast.PyCF_ONLY_AST` as -a flag to the :func:`compile` built-in function, or using the :func:`parse` -helper provided in this module. The result will be a tree of objects whose -classes all inherit from :class:`ast.AST`. An abstract syntax tree can be -compiled into a Python code object using the built-in :func:`compile` function. - - -.. _abstract-grammar: - -Abstract Grammar ----------------- - -The abstract grammar is currently defined as follows: - -.. literalinclude:: ../../Parser/Python.asdl - :language: asdl - - -Node classes ------------- - -.. class:: AST - - This is the base of all AST node classes. The actual node classes are - derived from the :file:`Parser/Python.asdl` file, which is reproduced - :ref:`above `. They are defined in the :mod:`_ast` C - module and re-exported in :mod:`ast`. - - There is one class defined for each left-hand side symbol in the abstract - grammar (for example, :class:`ast.stmt` or :class:`ast.expr`). In addition, - there is one class defined for each constructor on the right-hand side; these - classes inherit from the classes for the left-hand side trees. For example, - :class:`ast.BinOp` inherits from :class:`ast.expr`. For production rules - with alternatives (aka "sums"), the left-hand side class is abstract: only - instances of specific constructor nodes are ever created. - - .. index:: single: ? (question mark); in AST grammar - .. index:: single: * (asterisk); in AST grammar - - .. attribute:: _fields - - Each concrete class has an attribute :attr:`_fields` which gives the names - of all child nodes. - - Each instance of a concrete class has one attribute for each child node, - of the type as defined in the grammar. For example, :class:`ast.BinOp` - instances have an attribute :attr:`left` of type :class:`ast.expr`. - - If these attributes are marked as optional in the grammar (using a - question mark), the value might be ``None``. If the attributes can have - zero-or-more values (marked with an asterisk), the values are represented - as Python lists. All possible attributes must be present and have valid - values when compiling an AST with :func:`compile`. - - .. attribute:: lineno - col_offset - end_lineno - end_col_offset - - Instances of :class:`ast.expr` and :class:`ast.stmt` subclasses have - :attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, and - :attr:`end_col_offset` attributes. The :attr:`lineno` and :attr:`end_lineno` - are the first and last line numbers of source text span (1-indexed so the - first line is line 1) and the :attr:`col_offset` and :attr:`end_col_offset` - are the corresponding UTF-8 byte offsets of the first and last tokens that - generated the node. The UTF-8 offset is recorded because the parser uses - UTF-8 internally. - - Note that the end positions are not required by the compiler and are - therefore optional. The end offset is *after* the last symbol, for example - one can get the source segment of a one-line expression node using - ``source_line[node.col_offset : node.end_col_offset]``. - - The constructor of a class :class:`ast.T` parses its arguments as follows: - - * If there are positional arguments, there must be as many as there are items - in :attr:`T._fields`; they will be assigned as attributes of these names. - * If there are keyword arguments, they will set the attributes of the same - names to the given values. - - For example, to create and populate an :class:`ast.UnaryOp` node, you could - use :: - - node = ast.UnaryOp() - node.op = ast.USub() - node.operand = ast.Constant() - node.operand.value = 5 - node.operand.lineno = 0 - node.operand.col_offset = 0 - node.lineno = 0 - node.col_offset = 0 - - or the more compact :: - - node = ast.UnaryOp(ast.USub(), ast.Constant(5, lineno=0, col_offset=0), - lineno=0, col_offset=0) - -.. versionchanged:: 3.8 - - Class :class:`ast.Constant` is now used for all constants. - -.. versionchanged:: 3.9 - - Simple indices are represented by their value, extended slices are - represented as tuples. - -.. deprecated:: 3.8 - - Old classes :class:`ast.Num`, :class:`ast.Str`, :class:`ast.Bytes`, - :class:`ast.NameConstant` and :class:`ast.Ellipsis` are still available, - but they will be removed in future Python releases. In the meantime, - instantiating them will return an instance of a different class. - -.. deprecated:: 3.9 - - Old classes :class:`ast.Index` and :class:`ast.ExtSlice` are still - available, but they will be removed in future Python releases. - In the meantime, instantiating them will return an instance of - a different class. - -.. note:: - The descriptions of the specific node classes displayed here - were initially adapted from the fantastic `Green Tree - Snakes `__ project and - all its contributors. - - -.. _ast-root-nodes: - -Root nodes -^^^^^^^^^^ - -.. class:: Module(body, type_ignores) - - A Python module, as with :ref:`file input `. - Node type generated by :func:`ast.parse` in the default ``"exec"`` *mode*. - - *body* is a :class:`list` of the module's :ref:`ast-statements`. - - *type_ignores* is a :class:`list` of the module's type ignore comments; - see :func:`ast.parse` for more details. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x = 1'), indent=4)) - Module( - body=[ - Assign( - targets=[ - Name(id='x', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) - - -.. class:: Expression(body) - - A single Python :ref:`expression input `. - Node type generated by :func:`ast.parse` when *mode* is ``"eval"``. - - *body* is a single node, - one of the :ref:`expression types `. - - .. doctest:: - - >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) - Expression( - body=Constant(value=123)) - - -.. class:: Interactive(body) - - A single :ref:`interactive input `, like in :ref:`tut-interac`. - Node type generated by :func:`ast.parse` when *mode* is ``"single"``. - - *body* is a :class:`list` of :ref:`statement nodes `. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x = 1; y = 2', mode='single'), indent=4)) - Interactive( - body=[ - Assign( - targets=[ - Name(id='x', ctx=Store())], - value=Constant(value=1)), - Assign( - targets=[ - Name(id='y', ctx=Store())], - value=Constant(value=2))]) - - -.. class:: FunctionType(argtypes, returns) - - A representation of an old-style type comments for functions, - as Python versions prior to 3.5 didn't support :pep:`484` annotations. - Node type generated by :func:`ast.parse` when *mode* is ``"func_type"``. - - Such type comments would look like this:: - - def sum_two_number(a, b): - # type: (int, int) -> int - return a + b - - *argtypes* is a :class:`list` of :ref:`expression nodes `. - - *returns* is a single :ref:`expression node `. - - .. doctest:: - - >>> print(ast.dump(ast.parse('(int, str) -> List[int]', mode='func_type'), indent=4)) - FunctionType( - argtypes=[ - Name(id='int', ctx=Load()), - Name(id='str', ctx=Load())], - returns=Subscript( - value=Name(id='List', ctx=Load()), - slice=Name(id='int', ctx=Load()), - ctx=Load())) - - .. versionadded:: 3.8 - - -Literals -^^^^^^^^ - -.. class:: Constant(value) - - A constant value. The ``value`` attribute of the ``Constant`` literal contains the - Python object it represents. The values represented can be simple types - such as a number, string or ``None``, but also immutable container types - (tuples and frozensets) if all of their elements are constant. - - .. doctest:: - - >>> print(ast.dump(ast.parse('123', mode='eval'), indent=4)) - Expression( - body=Constant(value=123)) - - -.. class:: FormattedValue(value, conversion, format_spec) - - Node representing a single formatting field in an f-string. If the string - contains a single formatting field and nothing else the node can be - isolated otherwise it appears in :class:`JoinedStr`. - - * ``value`` is any expression node (such as a literal, a variable, or a - function call). - * ``conversion`` is an integer: - - * -1: no formatting - * 115: ``!s`` string formatting - * 114: ``!r`` repr formatting - * 97: ``!a`` ascii formatting - - * ``format_spec`` is a :class:`JoinedStr` node representing the formatting - of the value, or ``None`` if no format was specified. Both - ``conversion`` and ``format_spec`` can be set at the same time. - - -.. class:: JoinedStr(values) - - An f-string, comprising a series of :class:`FormattedValue` and :class:`Constant` - nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('f"sin({a}) is {sin(a):.3}"', mode='eval'), indent=4)) - Expression( - body=JoinedStr( - values=[ - Constant(value='sin('), - FormattedValue( - value=Name(id='a', ctx=Load()), - conversion=-1), - Constant(value=') is '), - FormattedValue( - value=Call( - func=Name(id='sin', ctx=Load()), - args=[ - Name(id='a', ctx=Load())], - keywords=[]), - conversion=-1, - format_spec=JoinedStr( - values=[ - Constant(value='.3')]))])) - - -.. class:: List(elts, ctx) - Tuple(elts, ctx) - - A list or tuple. ``elts`` holds a list of nodes representing the elements. - ``ctx`` is :class:`Store` if the container is an assignment target (i.e. - ``(x,y)=something``), and :class:`Load` otherwise. - - .. doctest:: - - >>> print(ast.dump(ast.parse('[1, 2, 3]', mode='eval'), indent=4)) - Expression( - body=List( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)], - ctx=Load())) - >>> print(ast.dump(ast.parse('(1, 2, 3)', mode='eval'), indent=4)) - Expression( - body=Tuple( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)], - ctx=Load())) - - -.. class:: Set(elts) - - A set. ``elts`` holds a list of nodes representing the set's elements. - - .. doctest:: - - >>> print(ast.dump(ast.parse('{1, 2, 3}', mode='eval'), indent=4)) - Expression( - body=Set( - elts=[ - Constant(value=1), - Constant(value=2), - Constant(value=3)])) - - -.. class:: Dict(keys, values) - - A dictionary. ``keys`` and ``values`` hold lists of nodes representing the - keys and the values respectively, in matching order (what would be returned - when calling :code:`dictionary.keys()` and :code:`dictionary.values()`). - - When doing dictionary unpacking using dictionary literals the expression to be - expanded goes in the ``values`` list, with a ``None`` at the corresponding - position in ``keys``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('{"a":1, **d}', mode='eval'), indent=4)) - Expression( - body=Dict( - keys=[ - Constant(value='a'), - None], - values=[ - Constant(value=1), - Name(id='d', ctx=Load())])) - - -Variables -^^^^^^^^^ - -.. class:: Name(id, ctx) - - A variable name. ``id`` holds the name as a string, and ``ctx`` is one of - the following types. - - -.. class:: Load() - Store() - Del() - - Variable references can be used to load the value of a variable, to assign - a new value to it, or to delete it. Variable references are given a context - to distinguish these cases. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a'), indent=4)) - Module( - body=[ - Expr( - value=Name(id='a', ctx=Load()))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a = 1'), indent=4)) - Module( - body=[ - Assign( - targets=[ - Name(id='a', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('del a'), indent=4)) - Module( - body=[ - Delete( - targets=[ - Name(id='a', ctx=Del())])], - type_ignores=[]) - - -.. class:: Starred(value, ctx) - - A ``*var`` variable reference. ``value`` holds the variable, typically a - :class:`Name` node. This type must be used when building a :class:`Call` - node with ``*args``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a, *b = it'), indent=4)) - Module( - body=[ - Assign( - targets=[ - Tuple( - elts=[ - Name(id='a', ctx=Store()), - Starred( - value=Name(id='b', ctx=Store()), - ctx=Store())], - ctx=Store())], - value=Name(id='it', ctx=Load()))], - type_ignores=[]) - - -.. _ast-expressions: - -Expressions -^^^^^^^^^^^ - -.. class:: Expr(value) - - When an expression, such as a function call, appears as a statement by itself - with its return value not used or stored, it is wrapped in this container. - ``value`` holds one of the other nodes in this section, a :class:`Constant`, a - :class:`Name`, a :class:`Lambda`, a :class:`Yield` or :class:`YieldFrom` node. - - .. doctest:: - - >>> print(ast.dump(ast.parse('-a'), indent=4)) - Module( - body=[ - Expr( - value=UnaryOp( - op=USub(), - operand=Name(id='a', ctx=Load())))], - type_ignores=[]) - - -.. class:: UnaryOp(op, operand) - - A unary operation. ``op`` is the operator, and ``operand`` any expression - node. - - -.. class:: UAdd - USub - Not - Invert - - Unary operator tokens. :class:`Not` is the ``not`` keyword, :class:`Invert` - is the ``~`` operator. - - .. doctest:: - - >>> print(ast.dump(ast.parse('not x', mode='eval'), indent=4)) - Expression( - body=UnaryOp( - op=Not(), - operand=Name(id='x', ctx=Load()))) - - -.. class:: BinOp(left, op, right) - - A binary operation (like addition or division). ``op`` is the operator, and - ``left`` and ``right`` are any expression nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x + y', mode='eval'), indent=4)) - Expression( - body=BinOp( - left=Name(id='x', ctx=Load()), - op=Add(), - right=Name(id='y', ctx=Load()))) - - -.. class:: Add - Sub - Mult - Div - FloorDiv - Mod - Pow - LShift - RShift - BitOr - BitXor - BitAnd - MatMult - - Binary operator tokens. - - -.. class:: BoolOp(op, values) - - A boolean operation, 'or' or 'and'. ``op`` is :class:`Or` or :class:`And`. - ``values`` are the values involved. Consecutive operations with the same - operator, such as ``a or b or c``, are collapsed into one node with several - values. - - This doesn't include ``not``, which is a :class:`UnaryOp`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x or y', mode='eval'), indent=4)) - Expression( - body=BoolOp( - op=Or(), - values=[ - Name(id='x', ctx=Load()), - Name(id='y', ctx=Load())])) - - -.. class:: And - Or - - Boolean operator tokens. - - -.. class:: Compare(left, ops, comparators) - - A comparison of two or more values. ``left`` is the first value in the - comparison, ``ops`` the list of operators, and ``comparators`` the list - of values after the first element in the comparison. - - .. doctest:: - - >>> print(ast.dump(ast.parse('1 <= a < 10', mode='eval'), indent=4)) - Expression( - body=Compare( - left=Constant(value=1), - ops=[ - LtE(), - Lt()], - comparators=[ - Name(id='a', ctx=Load()), - Constant(value=10)])) - - -.. class:: Eq - NotEq - Lt - LtE - Gt - GtE - Is - IsNot - In - NotIn - - Comparison operator tokens. - - -.. class:: Call(func, args, keywords) - - A function call. ``func`` is the function, which will often be a - :class:`Name` or :class:`Attribute` object. Of the arguments: - - * ``args`` holds a list of the arguments passed by position. - * ``keywords`` holds a list of :class:`.keyword` objects representing - arguments passed by keyword. - - When creating a ``Call`` node, ``args`` and ``keywords`` are required, but - they can be empty lists. - - .. doctest:: - - >>> print(ast.dump(ast.parse('func(a, b=c, *d, **e)', mode='eval'), indent=4)) - Expression( - body=Call( - func=Name(id='func', ctx=Load()), - args=[ - Name(id='a', ctx=Load()), - Starred( - value=Name(id='d', ctx=Load()), - ctx=Load())], - keywords=[ - keyword( - arg='b', - value=Name(id='c', ctx=Load())), - keyword( - value=Name(id='e', ctx=Load()))])) - - -.. class:: keyword(arg, value) - - A keyword argument to a function call or class definition. ``arg`` is a raw - string of the parameter name, ``value`` is a node to pass in. - - -.. class:: IfExp(test, body, orelse) - - An expression such as ``a if b else c``. Each field holds a single node, so - in the following example, all three are :class:`Name` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a if b else c', mode='eval'), indent=4)) - Expression( - body=IfExp( - test=Name(id='b', ctx=Load()), - body=Name(id='a', ctx=Load()), - orelse=Name(id='c', ctx=Load()))) - - -.. class:: Attribute(value, attr, ctx) - - Attribute access, e.g. ``d.keys``. ``value`` is a node, typically a - :class:`Name`. ``attr`` is a bare string giving the name of the attribute, - and ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` according to how - the attribute is acted on. - - .. doctest:: - - >>> print(ast.dump(ast.parse('snake.colour', mode='eval'), indent=4)) - Expression( - body=Attribute( - value=Name(id='snake', ctx=Load()), - attr='colour', - ctx=Load())) - - -.. class:: NamedExpr(target, value) - - A named expression. This AST node is produced by the assignment expressions - operator (also known as the walrus operator). As opposed to the :class:`Assign` - node in which the first argument can be multiple nodes, in this case both - ``target`` and ``value`` must be single nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('(x := 4)', mode='eval'), indent=4)) - Expression( - body=NamedExpr( - target=Name(id='x', ctx=Store()), - value=Constant(value=4))) - - -Subscripting -~~~~~~~~~~~~ - -.. class:: Subscript(value, slice, ctx) - - A subscript, such as ``l[1]``. ``value`` is the subscripted object - (usually sequence or mapping). ``slice`` is an index, slice or key. - It can be a :class:`Tuple` and contain a :class:`Slice`. - ``ctx`` is :class:`Load`, :class:`Store` or :class:`Del` - according to the action performed with the subscript. - - .. doctest:: - - >>> print(ast.dump(ast.parse('l[1:2, 3]', mode='eval'), indent=4)) - Expression( - body=Subscript( - value=Name(id='l', ctx=Load()), - slice=Tuple( - elts=[ - Slice( - lower=Constant(value=1), - upper=Constant(value=2)), - Constant(value=3)], - ctx=Load()), - ctx=Load())) - - -.. class:: Slice(lower, upper, step) - - Regular slicing (on the form ``lower:upper`` or ``lower:upper:step``). - Can occur only inside the *slice* field of :class:`Subscript`, either - directly or as an element of :class:`Tuple`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('l[1:2]', mode='eval'), indent=4)) - Expression( - body=Subscript( - value=Name(id='l', ctx=Load()), - slice=Slice( - lower=Constant(value=1), - upper=Constant(value=2)), - ctx=Load())) - - -Comprehensions -~~~~~~~~~~~~~~ - -.. class:: ListComp(elt, generators) - SetComp(elt, generators) - GeneratorExp(elt, generators) - DictComp(key, value, generators) - - List and set comprehensions, generator expressions, and dictionary - comprehensions. ``elt`` (or ``key`` and ``value``) is a single node - representing the part that will be evaluated for each item. - - ``generators`` is a list of :class:`comprehension` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('[x for x in numbers]', mode='eval'), indent=4)) - Expression( - body=ListComp( - elt=Name(id='x', ctx=Load()), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - >>> print(ast.dump(ast.parse('{x: x**2 for x in numbers}', mode='eval'), indent=4)) - Expression( - body=DictComp( - key=Name(id='x', ctx=Load()), - value=BinOp( - left=Name(id='x', ctx=Load()), - op=Pow(), - right=Constant(value=2)), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - >>> print(ast.dump(ast.parse('{x for x in numbers}', mode='eval'), indent=4)) - Expression( - body=SetComp( - elt=Name(id='x', ctx=Load()), - generators=[ - comprehension( - target=Name(id='x', ctx=Store()), - iter=Name(id='numbers', ctx=Load()), - ifs=[], - is_async=0)])) - - -.. class:: comprehension(target, iter, ifs, is_async) - - One ``for`` clause in a comprehension. ``target`` is the reference to use for - each element - typically a :class:`Name` or :class:`Tuple` node. ``iter`` - is the object to iterate over. ``ifs`` is a list of test expressions: each - ``for`` clause can have multiple ``ifs``. - - ``is_async`` indicates a comprehension is asynchronous (using an - ``async for`` instead of ``for``). The value is an integer (0 or 1). - - .. doctest:: - - >>> print(ast.dump(ast.parse('[ord(c) for line in file for c in line]', mode='eval'), - ... indent=4)) # Multiple comprehensions in one. - Expression( - body=ListComp( - elt=Call( - func=Name(id='ord', ctx=Load()), - args=[ - Name(id='c', ctx=Load())], - keywords=[]), - generators=[ - comprehension( - target=Name(id='line', ctx=Store()), - iter=Name(id='file', ctx=Load()), - ifs=[], - is_async=0), - comprehension( - target=Name(id='c', ctx=Store()), - iter=Name(id='line', ctx=Load()), - ifs=[], - is_async=0)])) - - >>> print(ast.dump(ast.parse('(n**2 for n in it if n>5 if n<10)', mode='eval'), - ... indent=4)) # generator comprehension - Expression( - body=GeneratorExp( - elt=BinOp( - left=Name(id='n', ctx=Load()), - op=Pow(), - right=Constant(value=2)), - generators=[ - comprehension( - target=Name(id='n', ctx=Store()), - iter=Name(id='it', ctx=Load()), - ifs=[ - Compare( - left=Name(id='n', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=5)]), - Compare( - left=Name(id='n', ctx=Load()), - ops=[ - Lt()], - comparators=[ - Constant(value=10)])], - is_async=0)])) - - >>> print(ast.dump(ast.parse('[i async for i in soc]', mode='eval'), - ... indent=4)) # Async comprehension - Expression( - body=ListComp( - elt=Name(id='i', ctx=Load()), - generators=[ - comprehension( - target=Name(id='i', ctx=Store()), - iter=Name(id='soc', ctx=Load()), - ifs=[], - is_async=1)])) - - -.. _ast-statements: - -Statements -^^^^^^^^^^ - -.. class:: Assign(targets, value, type_comment) - - An assignment. ``targets`` is a list of nodes, and ``value`` is a single node. - - Multiple nodes in ``targets`` represents assigning the same value to each. - Unpacking is represented by putting a :class:`Tuple` or :class:`List` - within ``targets``. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - .. doctest:: - - >>> print(ast.dump(ast.parse('a = b = 1'), indent=4)) # Multiple assignment - Module( - body=[ - Assign( - targets=[ - Name(id='a', ctx=Store()), - Name(id='b', ctx=Store())], - value=Constant(value=1))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a,b = c'), indent=4)) # Unpacking - Module( - body=[ - Assign( - targets=[ - Tuple( - elts=[ - Name(id='a', ctx=Store()), - Name(id='b', ctx=Store())], - ctx=Store())], - value=Name(id='c', ctx=Load()))], - type_ignores=[]) - - -.. class:: AnnAssign(target, annotation, value, simple) - - An assignment with a type annotation. ``target`` is a single node and can - be a :class:`Name`, a :class:`Attribute` or a :class:`Subscript`. - ``annotation`` is the annotation, such as a :class:`Constant` or :class:`Name` - node. ``value`` is a single optional node. ``simple`` is a boolean integer - set to True for a :class:`Name` node in ``target`` that do not appear in - between parenthesis and are hence pure names and not expressions. - - .. doctest:: - - >>> print(ast.dump(ast.parse('c: int'), indent=4)) - Module( - body=[ - AnnAssign( - target=Name(id='c', ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=1)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('(a): int = 1'), indent=4)) # Annotation with parenthesis - Module( - body=[ - AnnAssign( - target=Name(id='a', ctx=Store()), - annotation=Name(id='int', ctx=Load()), - value=Constant(value=1), - simple=0)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a.b: int'), indent=4)) # Attribute annotation - Module( - body=[ - AnnAssign( - target=Attribute( - value=Name(id='a', ctx=Load()), - attr='b', - ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('a[1]: int'), indent=4)) # Subscript annotation - Module( - body=[ - AnnAssign( - target=Subscript( - value=Name(id='a', ctx=Load()), - slice=Constant(value=1), - ctx=Store()), - annotation=Name(id='int', ctx=Load()), - simple=0)], - type_ignores=[]) - - -.. class:: AugAssign(target, op, value) - - Augmented assignment, such as ``a += 1``. In the following example, - ``target`` is a :class:`Name` node for ``x`` (with the :class:`Store` - context), ``op`` is :class:`Add`, and ``value`` is a :class:`Constant` with - value for 1. - - The ``target`` attribute cannot be of class :class:`Tuple` or :class:`List`, - unlike the targets of :class:`Assign`. - - .. doctest:: - - >>> print(ast.dump(ast.parse('x += 2'), indent=4)) - Module( - body=[ - AugAssign( - target=Name(id='x', ctx=Store()), - op=Add(), - value=Constant(value=2))], - type_ignores=[]) - - -.. class:: Raise(exc, cause) - - A ``raise`` statement. ``exc`` is the exception object to be raised, normally a - :class:`Call` or :class:`Name`, or ``None`` for a standalone ``raise``. - ``cause`` is the optional part for ``y`` in ``raise x from y``. - - .. doctest:: - - >>> print(ast.dump(ast.parse('raise x from y'), indent=4)) - Module( - body=[ - Raise( - exc=Name(id='x', ctx=Load()), - cause=Name(id='y', ctx=Load()))], - type_ignores=[]) - - -.. class:: Assert(test, msg) - - An assertion. ``test`` holds the condition, such as a :class:`Compare` node. - ``msg`` holds the failure message. - - .. doctest:: - - >>> print(ast.dump(ast.parse('assert x,y'), indent=4)) - Module( - body=[ - Assert( - test=Name(id='x', ctx=Load()), - msg=Name(id='y', ctx=Load()))], - type_ignores=[]) - - -.. class:: Delete(targets) - - Represents a ``del`` statement. ``targets`` is a list of nodes, such as - :class:`Name`, :class:`Attribute` or :class:`Subscript` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('del x,y,z'), indent=4)) - Module( - body=[ - Delete( - targets=[ - Name(id='x', ctx=Del()), - Name(id='y', ctx=Del()), - Name(id='z', ctx=Del())])], - type_ignores=[]) - - -.. class:: Pass() - - A ``pass`` statement. - - .. doctest:: - - >>> print(ast.dump(ast.parse('pass'), indent=4)) - Module( - body=[ - Pass()], - type_ignores=[]) - - -Other statements which are only applicable inside functions or loops are -described in other sections. - -Imports -~~~~~~~ - -.. class:: Import(names) - - An import statement. ``names`` is a list of :class:`alias` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse('import x,y,z'), indent=4)) - Module( - body=[ - Import( - names=[ - alias(name='x'), - alias(name='y'), - alias(name='z')])], - type_ignores=[]) - - -.. class:: ImportFrom(module, names, level) - - Represents ``from x import y``. ``module`` is a raw string of the 'from' name, - without any leading dots, or ``None`` for statements such as ``from . import foo``. - ``level`` is an integer holding the level of the relative import (0 means - absolute import). - - .. doctest:: - - >>> print(ast.dump(ast.parse('from y import x,y,z'), indent=4)) - Module( - body=[ - ImportFrom( - module='y', - names=[ - alias(name='x'), - alias(name='y'), - alias(name='z')], - level=0)], - type_ignores=[]) - - -.. class:: alias(name, asname) - - Both parameters are raw strings of the names. ``asname`` can be ``None`` if - the regular name is to be used. - - .. doctest:: - - >>> print(ast.dump(ast.parse('from ..foo.bar import a as b, c'), indent=4)) - Module( - body=[ - ImportFrom( - module='foo.bar', - names=[ - alias(name='a', asname='b'), - alias(name='c')], - level=2)], - type_ignores=[]) - -Control flow -^^^^^^^^^^^^ - -.. note:: - Optional clauses such as ``else`` are stored as an empty list if they're - not present. - -.. class:: If(test, body, orelse) - - An ``if`` statement. ``test`` holds a single node, such as a :class:`Compare` - node. ``body`` and ``orelse`` each hold a list of nodes. - - ``elif`` clauses don't have a special representation in the AST, but rather - appear as extra :class:`If` nodes within the ``orelse`` section of the - previous one. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... if x: - ... ... - ... elif y: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - If( - test=Name(id='x', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - If( - test=Name(id='y', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -.. class:: For(target, iter, body, orelse, type_comment) - - A ``for`` loop. ``target`` holds the variable(s) the loop assigns to, as a - single :class:`Name`, :class:`Tuple` or :class:`List` node. ``iter`` holds - the item to be looped over, again as a single node. ``body`` and ``orelse`` - contain lists of nodes to execute. Those in ``orelse`` are executed if the - loop finishes normally, rather than via a ``break`` statement. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... for x in y: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - For( - target=Name(id='x', ctx=Store()), - iter=Name(id='y', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: While(test, body, orelse) - - A ``while`` loop. ``test`` holds the condition, such as a :class:`Compare` - node. - - .. doctest:: - - >> print(ast.dump(ast.parse(""" - ... while x: - ... ... - ... else: - ... ... - ... """), indent=4)) - Module( - body=[ - While( - test=Name(id='x', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))], - orelse=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: Break - Continue - - The ``break`` and ``continue`` statements. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... for a in b: - ... if a > 5: - ... break - ... else: - ... continue - ... - ... """), indent=4)) - Module( - body=[ - For( - target=Name(id='a', ctx=Store()), - iter=Name(id='b', ctx=Load()), - body=[ - If( - test=Compare( - left=Name(id='a', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=5)]), - body=[ - Break()], - orelse=[ - Continue()])], - orelse=[])], - type_ignores=[]) - - -.. class:: Try(body, handlers, orelse, finalbody) - - ``try`` blocks. All attributes are list of nodes to execute, except for - ``handlers``, which is a list of :class:`ExceptHandler` nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... try: - ... ... - ... except Exception: - ... ... - ... except OtherException as e: - ... ... - ... else: - ... ... - ... finally: - ... ... - ... """), indent=4)) - Module( - body=[ - Try( - body=[ - Expr( - value=Constant(value=Ellipsis))], - handlers=[ - ExceptHandler( - type=Name(id='Exception', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - ExceptHandler( - type=Name(id='OtherException', ctx=Load()), - name='e', - body=[ - Expr( - value=Constant(value=Ellipsis))])], - orelse=[ - Expr( - value=Constant(value=Ellipsis))], - finalbody=[ - Expr( - value=Constant(value=Ellipsis))])], - type_ignores=[]) - - -.. class:: TryStar(body, handlers, orelse, finalbody) - - ``try`` blocks which are followed by ``except*`` clauses. The attributes are the - same as for :class:`Try` but the :class:`ExceptHandler` nodes in ``handlers`` - are interpreted as ``except*`` blocks rather then ``except``. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... try: - ... ... - ... except* Exception: - ... ... - ... """), indent=4)) - Module( - body=[ - TryStar( - body=[ - Expr( - value=Constant(value=Ellipsis))], - handlers=[ - ExceptHandler( - type=Name(id='Exception', ctx=Load()), - body=[ - Expr( - value=Constant(value=Ellipsis))])], - orelse=[], - finalbody=[])], - type_ignores=[]) - - -.. class:: ExceptHandler(type, name, body) - - A single ``except`` clause. ``type`` is the exception type it will match, - typically a :class:`Name` node (or ``None`` for a catch-all ``except:`` clause). - ``name`` is a raw string for the name to hold the exception, or ``None`` if - the clause doesn't have ``as foo``. ``body`` is a list of nodes. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... try: - ... a + 1 - ... except TypeError: - ... pass - ... """), indent=4)) - Module( - body=[ - Try( - body=[ - Expr( - value=BinOp( - left=Name(id='a', ctx=Load()), - op=Add(), - right=Constant(value=1)))], - handlers=[ - ExceptHandler( - type=Name(id='TypeError', ctx=Load()), - body=[ - Pass()])], - orelse=[], - finalbody=[])], - type_ignores=[]) - - -.. class:: With(items, body, type_comment) - - A ``with`` block. ``items`` is a list of :class:`withitem` nodes representing - the context managers, and ``body`` is the indented block inside the context. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - -.. class:: withitem(context_expr, optional_vars) - - A single context manager in a ``with`` block. ``context_expr`` is the context - manager, often a :class:`Call` node. ``optional_vars`` is a :class:`Name`, - :class:`Tuple` or :class:`List` for the ``as foo`` part, or ``None`` if that - isn't used. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... with a as b, c as d: - ... something(b, d) - ... """), indent=4)) - Module( - body=[ - With( - items=[ - withitem( - context_expr=Name(id='a', ctx=Load()), - optional_vars=Name(id='b', ctx=Store())), - withitem( - context_expr=Name(id='c', ctx=Load()), - optional_vars=Name(id='d', ctx=Store()))], - body=[ - Expr( - value=Call( - func=Name(id='something', ctx=Load()), - args=[ - Name(id='b', ctx=Load()), - Name(id='d', ctx=Load())], - keywords=[]))])], - type_ignores=[]) - - -Pattern matching -^^^^^^^^^^^^^^^^ - - -.. class:: Match(subject, cases) - - A ``match`` statement. ``subject`` holds the subject of the match (the object - that is being matched against the cases) and ``cases`` contains an iterable of - :class:`match_case` nodes with the different cases. - -.. class:: match_case(pattern, guard, body) - - A single case pattern in a ``match`` statement. ``pattern`` contains the - match pattern that the subject will be matched against. Note that the - :class:`AST` nodes produced for patterns differ from those produced for - expressions, even when they share the same syntax. - - The ``guard`` attribute contains an expression that will be evaluated if - the pattern matches the subject. - - ``body`` contains a list of nodes to execute if the pattern matches and - the result of evaluating the guard expression is true. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] if x>0: - ... ... - ... case tuple(): - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchSequence( - patterns=[ - MatchAs(name='x')]), - guard=Compare( - left=Name(id='x', ctx=Load()), - ops=[ - Gt()], - comparators=[ - Constant(value=0)]), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - match_case( - pattern=MatchClass( - cls=Name(id='tuple', ctx=Load()), - patterns=[], - kwd_attrs=[], - kwd_patterns=[]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchValue(value) - - A match literal or value pattern that compares by equality. ``value`` is - an expression node. Permitted value nodes are restricted as described in - the match statement documentation. This pattern succeeds if the match - subject is equal to the evaluated value. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case "Relevant": - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchValue( - value=Constant(value='Relevant')), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchSingleton(value) - - A match literal pattern that compares by identity. ``value`` is the - singleton to be compared against: ``None``, ``True``, or ``False``. This - pattern succeeds if the match subject is the given constant. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case None: - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchSingleton(value=None), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchSequence(patterns) - - A match sequence pattern. ``patterns`` contains the patterns to be matched - against the subject elements if the subject is a sequence. Matches a variable - length sequence if one of the subpatterns is a ``MatchStar`` node, otherwise - matches a fixed length sequence. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [1, 2]: - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchSequence( - patterns=[ - MatchValue( - value=Constant(value=1)), - MatchValue( - value=Constant(value=2))]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchStar(name) - - Matches the rest of the sequence in a variable length match sequence pattern. - If ``name`` is not ``None``, a list containing the remaining sequence - elements is bound to that name if the overall sequence pattern is successful. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [1, 2, *rest]: - ... ... - ... case [*_]: - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchSequence( - patterns=[ - MatchValue( - value=Constant(value=1)), - MatchValue( - value=Constant(value=2)), - MatchStar(name='rest')]), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - match_case( - pattern=MatchSequence( - patterns=[ - MatchStar()]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchMapping(keys, patterns, rest) - - A match mapping pattern. ``keys`` is a sequence of expression nodes. - ``patterns`` is a corresponding sequence of pattern nodes. ``rest`` is an - optional name that can be specified to capture the remaining mapping elements. - Permitted key expressions are restricted as described in the match statement - documentation. - - This pattern succeeds if the subject is a mapping, all evaluated key - expressions are present in the mapping, and the value corresponding to each - key matches the corresponding subpattern. If ``rest`` is not ``None``, a dict - containing the remaining mapping elements is bound to that name if the overall - mapping pattern is successful. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case {1: _, 2: _}: - ... ... - ... case {**rest}: - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchMapping( - keys=[ - Constant(value=1), - Constant(value=2)], - patterns=[ - MatchAs(), - MatchAs()]), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - match_case( - pattern=MatchMapping(keys=[], patterns=[], rest='rest'), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchClass(cls, patterns, kwd_attrs, kwd_patterns) - - A match class pattern. ``cls`` is an expression giving the nominal class to - be matched. ``patterns`` is a sequence of pattern nodes to be matched against - the class defined sequence of pattern matching attributes. ``kwd_attrs`` is a - sequence of additional attributes to be matched (specified as keyword arguments - in the class pattern), ``kwd_patterns`` are the corresponding patterns - (specified as keyword values in the class pattern). - - This pattern succeeds if the subject is an instance of the nominated class, - all positional patterns match the corresponding class-defined attributes, and - any specified keyword attributes match their corresponding pattern. - - Note: classes may define a property that returns self in order to match a - pattern node against the instance being matched. Several builtin types are - also matched that way, as described in the match statement documentation. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case Point2D(0, 0): - ... ... - ... case Point3D(x=0, y=0, z=0): - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchClass( - cls=Name(id='Point2D', ctx=Load()), - patterns=[ - MatchValue( - value=Constant(value=0)), - MatchValue( - value=Constant(value=0))], - kwd_attrs=[], - kwd_patterns=[]), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - match_case( - pattern=MatchClass( - cls=Name(id='Point3D', ctx=Load()), - patterns=[], - kwd_attrs=[ - 'x', - 'y', - 'z'], - kwd_patterns=[ - MatchValue( - value=Constant(value=0)), - MatchValue( - value=Constant(value=0)), - MatchValue( - value=Constant(value=0))]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchAs(pattern, name) - - A match "as-pattern", capture pattern or wildcard pattern. ``pattern`` - contains the match pattern that the subject will be matched against. - If the pattern is ``None``, the node represents a capture pattern (i.e a - bare name) and will always succeed. - - The ``name`` attribute contains the name that will be bound if the pattern - is successful. If ``name`` is ``None``, ``pattern`` must also be ``None`` - and the node represents the wildcard pattern. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] as y: - ... ... - ... case _: - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchAs( - pattern=MatchSequence( - patterns=[ - MatchAs(name='x')]), - name='y'), - body=[ - Expr( - value=Constant(value=Ellipsis))]), - match_case( - pattern=MatchAs(), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - -.. class:: MatchOr(patterns) - - A match "or-pattern". An or-pattern matches each of its subpatterns in turn - to the subject, until one succeeds. The or-pattern is then deemed to - succeed. If none of the subpatterns succeed the or-pattern fails. The - ``patterns`` attribute contains a list of match pattern nodes that will be - matched against the subject. - - .. doctest:: - - >>> print(ast.dump(ast.parse(""" - ... match x: - ... case [x] | (y): - ... ... - ... """), indent=4)) - Module( - body=[ - Match( - subject=Name(id='x', ctx=Load()), - cases=[ - match_case( - pattern=MatchOr( - patterns=[ - MatchSequence( - patterns=[ - MatchAs(name='x')]), - MatchAs(name='y')]), - body=[ - Expr( - value=Constant(value=Ellipsis))])])], - type_ignores=[]) - - -Function and class definitions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. class:: FunctionDef(name, args, body, decorator_list, returns, type_comment) - - A function definition. - - * ``name`` is a raw string of the function name. - * ``args`` is an :class:`arguments` node. - * ``body`` is the list of nodes inside the function. - * ``decorator_list`` is the list of decorators to be applied, stored outermost - first (i.e. the first in the list will be applied last). - * ``returns`` is the return annotation. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment. - - -.. class:: Lambda(args, body) - - ``lambda`` is a minimal function definition that can be used inside an - expression. Unlike :class:`FunctionDef`, ``body`` holds a single node. - - .. doctest:: - - >>> print(ast.dump(ast.parse('lambda x,y: ...'), indent=4)) - Module( - body=[ - Expr( - value=Lambda( - args=arguments( - posonlyargs=[], - args=[ - arg(arg='x'), - arg(arg='y')], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=Constant(value=Ellipsis)))], - type_ignores=[]) - - -.. class:: arguments(posonlyargs, args, vararg, kwonlyargs, kw_defaults, kwarg, defaults) - - The arguments for a function. - - * ``posonlyargs``, ``args`` and ``kwonlyargs`` are lists of :class:`arg` nodes. - * ``vararg`` and ``kwarg`` are single :class:`arg` nodes, referring to the - ``*args, **kwargs`` parameters. - * ``kw_defaults`` is a list of default values for keyword-only arguments. If - one is ``None``, the corresponding argument is required. - * ``defaults`` is a list of default values for arguments that can be passed - positionally. If there are fewer defaults, they correspond to the last n - arguments. - - -.. class:: arg(arg, annotation, type_comment) - - A single argument in a list. ``arg`` is a raw string of the argument - name, ``annotation`` is its annotation, such as a :class:`Str` or - :class:`Name` node. - - .. attribute:: type_comment - - ``type_comment`` is an optional string with the type annotation as a comment - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... @decorator1 - ... @decorator2 - ... def f(a: 'annotation', b=1, c=2, *d, e, f=3, **g) -> 'return annotation': - ... pass - ... """), indent=4)) - Module( - body=[ - FunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[ - arg( - arg='a', - annotation=Constant(value='annotation')), - arg(arg='b'), - arg(arg='c')], - vararg=arg(arg='d'), - kwonlyargs=[ - arg(arg='e'), - arg(arg='f')], - kw_defaults=[ - None, - Constant(value=3)], - kwarg=arg(arg='g'), - defaults=[ - Constant(value=1), - Constant(value=2)]), - body=[ - Pass()], - decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())], - returns=Constant(value='return annotation'))], - type_ignores=[]) - - -.. class:: Return(value) - - A ``return`` statement. - - .. doctest:: - - >>> print(ast.dump(ast.parse('return 4'), indent=4)) - Module( - body=[ - Return( - value=Constant(value=4))], - type_ignores=[]) - - -.. class:: Yield(value) - YieldFrom(value) - - A ``yield`` or ``yield from`` expression. Because these are expressions, they - must be wrapped in a :class:`Expr` node if the value sent back is not used. - - .. doctest:: - - >>> print(ast.dump(ast.parse('yield x'), indent=4)) - Module( - body=[ - Expr( - value=Yield( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('yield from x'), indent=4)) - Module( - body=[ - Expr( - value=YieldFrom( - value=Name(id='x', ctx=Load())))], - type_ignores=[]) - - -.. class:: Global(names) - Nonlocal(names) - - ``global`` and ``nonlocal`` statements. ``names`` is a list of raw strings. - - .. doctest:: - - >>> print(ast.dump(ast.parse('global x,y,z'), indent=4)) - Module( - body=[ - Global( - names=[ - 'x', - 'y', - 'z'])], - type_ignores=[]) - - >>> print(ast.dump(ast.parse('nonlocal x,y,z'), indent=4)) - Module( - body=[ - Nonlocal( - names=[ - 'x', - 'y', - 'z'])], - type_ignores=[]) - - -.. class:: ClassDef(name, bases, keywords, body, decorator_list) - - A class definition. - - * ``name`` is a raw string for the class name - * ``bases`` is a list of nodes for explicitly specified base classes. - * ``keywords`` is a list of :class:`.keyword` nodes, principally for 'metaclass'. - Other keywords will be passed to the metaclass, as per `PEP-3115 - `_. - * ``body`` is a list of nodes representing the code within the class - definition. - * ``decorator_list`` is a list of nodes, as in :class:`FunctionDef`. - - .. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... @decorator1 - ... @decorator2 - ... class Foo(base1, base2, metaclass=meta): - ... pass - ... """), indent=4)) - Module( - body=[ - ClassDef( - name='Foo', - bases=[ - Name(id='base1', ctx=Load()), - Name(id='base2', ctx=Load())], - keywords=[ - keyword( - arg='metaclass', - value=Name(id='meta', ctx=Load()))], - body=[ - Pass()], - decorator_list=[ - Name(id='decorator1', ctx=Load()), - Name(id='decorator2', ctx=Load())])], - type_ignores=[]) - -Async and await -^^^^^^^^^^^^^^^ - -.. class:: AsyncFunctionDef(name, args, body, decorator_list, returns, type_comment) - - An ``async def`` function definition. Has the same fields as - :class:`FunctionDef`. - - -.. class:: Await(value) - - An ``await`` expression. ``value`` is what it waits for. - Only valid in the body of an :class:`AsyncFunctionDef`. - -.. doctest:: - - >>> print(ast.dump(ast.parse("""\ - ... async def f(): - ... await other_func() - ... """), indent=4)) - Module( - body=[ - AsyncFunctionDef( - name='f', - args=arguments( - posonlyargs=[], - args=[], - kwonlyargs=[], - kw_defaults=[], - defaults=[]), - body=[ - Expr( - value=Await( - value=Call( - func=Name(id='other_func', ctx=Load()), - args=[], - keywords=[])))], - decorator_list=[])], - type_ignores=[]) - - -.. class:: AsyncFor(target, iter, body, orelse, type_comment) - AsyncWith(items, body, type_comment) - - ``async for`` loops and ``async with`` context managers. They have the same - fields as :class:`For` and :class:`With`, respectively. Only valid in the - body of an :class:`AsyncFunctionDef`. - -.. note:: - When a string is parsed by :func:`ast.parse`, operator nodes (subclasses - of :class:`ast.operator`, :class:`ast.unaryop`, :class:`ast.cmpop`, - :class:`ast.boolop` and :class:`ast.expr_context`) on the returned tree - will be singletons. Changes to one will be reflected in all other - occurrences of the same value (e.g. :class:`ast.Add`). - - -:mod:`ast` Helpers ------------------- - -Apart from the node classes, the :mod:`ast` module defines these utility functions -and classes for traversing abstract syntax trees: - -.. function:: parse(source, filename='', mode='exec', *, type_comments=False, feature_version=None) - - Parse the source into an AST node. Equivalent to ``compile(source, - filename, mode, ast.PyCF_ONLY_AST)``. - - If ``type_comments=True`` is given, the parser is modified to check - and return type comments as specified by :pep:`484` and :pep:`526`. - This is equivalent to adding :data:`ast.PyCF_TYPE_COMMENTS` to the - flags passed to :func:`compile()`. This will report syntax errors - for misplaced type comments. Without this flag, type comments will - be ignored, and the ``type_comment`` field on selected AST nodes - will always be ``None``. In addition, the locations of ``# type: - ignore`` comments will be returned as the ``type_ignores`` - attribute of :class:`Module` (otherwise it is always an empty list). - - In addition, if ``mode`` is ``'func_type'``, the input syntax is - modified to correspond to :pep:`484` "signature type comments", - e.g. ``(str, int) -> List[str]``. - - Also, setting ``feature_version`` to a tuple ``(major, minor)`` - will attempt to parse using that Python version's grammar. - Currently ``major`` must equal to ``3``. For example, setting - ``feature_version=(3, 4)`` will allow the use of ``async`` and - ``await`` as variable names. The lowest supported version is - ``(3, 4)``; the highest is ``sys.version_info[0:2]``. - - If source contains a null character ('\0'), :exc:`ValueError` is raised. - - .. warning:: - Note that successfully parsing source code into an AST object doesn't - guarantee that the source code provided is valid Python code that can - be executed as the compilation step can raise further :exc:`SyntaxError` - exceptions. For instance, the source ``return 42`` generates a valid - AST node for a return statement, but it cannot be compiled alone (it needs - to be inside a function node). - - In particular, :func:`ast.parse` won't do any scoping checks, which the - compilation step does. - - .. warning:: - It is possible to crash the Python interpreter with a - sufficiently large/complex string due to stack depth limitations - in Python's AST compiler. - - .. versionchanged:: 3.8 - Added ``type_comments``, ``mode='func_type'`` and ``feature_version``. - - -.. function:: unparse(ast_obj) - - Unparse an :class:`ast.AST` object and generate a string with code - that would produce an equivalent :class:`ast.AST` object if parsed - back with :func:`ast.parse`. - - .. warning:: - The produced code string will not necessarily be equal to the original - code that generated the :class:`ast.AST` object (without any compiler - optimizations, such as constant tuples/frozensets). - - .. warning:: - Trying to unparse a highly complex expression would result with - :exc:`RecursionError`. - - .. versionadded:: 3.9 - - -.. function:: literal_eval(node_or_string) - - Evaluate an expression node or a string containing only a Python literal or - container display. The string or node provided may only consist of the - following Python literal structures: strings, bytes, numbers, tuples, lists, - dicts, sets, booleans, ``None`` and ``Ellipsis``. - - This can be used for evaluating strings containing Python values without the - need to parse the values oneself. It is not capable of evaluating - arbitrarily complex expressions, for example involving operators or - indexing. - - This function had been documented as "safe" in the past without defining - what that meant. That was misleading. This is specifically designed not to - execute Python code, unlike the more general :func:`eval`. There is no - namespace, no name lookups, or ability to call out. But it is not free from - attack: A relatively small input can lead to memory exhaustion or to C stack - exhaustion, crashing the process. There is also the possibility for - excessive CPU consumption denial of service on some inputs. Calling it on - untrusted data is thus not recommended. - - .. warning:: - It is possible to crash the Python interpreter due to stack depth - limitations in Python's AST compiler. - - It can raise :exc:`ValueError`, :exc:`TypeError`, :exc:`SyntaxError`, - :exc:`MemoryError` and :exc:`RecursionError` depending on the malformed - input. - - .. versionchanged:: 3.2 - Now allows bytes and set literals. - - .. versionchanged:: 3.9 - Now supports creating empty sets with ``'set()'``. - - .. versionchanged:: 3.10 - For string inputs, leading spaces and tabs are now stripped. - - -.. function:: get_docstring(node, clean=True) - - Return the docstring of the given *node* (which must be a - :class:`FunctionDef`, :class:`AsyncFunctionDef`, :class:`ClassDef`, - or :class:`Module` node), or ``None`` if it has no docstring. - If *clean* is true, clean up the docstring's indentation with - :func:`inspect.cleandoc`. - - .. versionchanged:: 3.5 - :class:`AsyncFunctionDef` is now supported. - - -.. function:: get_source_segment(source, node, *, padded=False) - - Get source code segment of the *source* that generated *node*. - If some location information (:attr:`lineno`, :attr:`end_lineno`, - :attr:`col_offset`, or :attr:`end_col_offset`) is missing, return ``None``. - - If *padded* is ``True``, the first line of a multi-line statement will - be padded with spaces to match its original position. - - .. versionadded:: 3.8 - - -.. function:: fix_missing_locations(node) - - When you compile a node tree with :func:`compile`, the compiler expects - :attr:`lineno` and :attr:`col_offset` attributes for every node that supports - them. This is rather tedious to fill in for generated nodes, so this helper - adds these attributes recursively where not already set, by setting them to - the values of the parent node. It works recursively starting at *node*. - - -.. function:: increment_lineno(node, n=1) - - Increment the line number and end line number of each node in the tree - starting at *node* by *n*. This is useful to "move code" to a different - location in a file. - - -.. function:: copy_location(new_node, old_node) - - Copy source location (:attr:`lineno`, :attr:`col_offset`, :attr:`end_lineno`, - and :attr:`end_col_offset`) from *old_node* to *new_node* if possible, - and return *new_node*. - - -.. function:: iter_fields(node) - - Yield a tuple of ``(fieldname, value)`` for each field in ``node._fields`` - that is present on *node*. - - -.. function:: iter_child_nodes(node) - - Yield all direct child nodes of *node*, that is, all fields that are nodes - and all items of fields that are lists of nodes. - - -.. function:: walk(node) - - Recursively yield all descendant nodes in the tree starting at *node* - (including *node* itself), in no specified order. This is useful if you only - want to modify nodes in place and don't care about the context. - - -.. class:: NodeVisitor() - - A node visitor base class that walks the abstract syntax tree and calls a - visitor function for every node found. This function may return a value - which is forwarded by the :meth:`visit` method. - - This class is meant to be subclassed, with the subclass adding visitor - methods. - - .. method:: visit(node) - - Visit a node. The default implementation calls the method called - :samp:`self.visit_{classname}` where *classname* is the name of the node - class, or :meth:`generic_visit` if that method doesn't exist. - - .. method:: generic_visit(node) - - This visitor calls :meth:`visit` on all children of the node. - - Note that child nodes of nodes that have a custom visitor method won't be - visited unless the visitor calls :meth:`generic_visit` or visits them - itself. - - Don't use the :class:`NodeVisitor` if you want to apply changes to nodes - during traversal. For this a special visitor exists - (:class:`NodeTransformer`) that allows modifications. - - .. deprecated:: 3.8 - - Methods :meth:`visit_Num`, :meth:`visit_Str`, :meth:`visit_Bytes`, - :meth:`visit_NameConstant` and :meth:`visit_Ellipsis` are deprecated - now and will not be called in future Python versions. Add the - :meth:`visit_Constant` method to handle all constant nodes. - - -.. class:: NodeTransformer() - - A :class:`NodeVisitor` subclass that walks the abstract syntax tree and - allows modification of nodes. - - The :class:`NodeTransformer` will walk the AST and use the return value of - the visitor methods to replace or remove the old node. If the return value - of the visitor method is ``None``, the node will be removed from its - location, otherwise it is replaced with the return value. The return value - may be the original node in which case no replacement takes place. - - Here is an example transformer that rewrites all occurrences of name lookups - (``foo``) to ``data['foo']``:: - - class RewriteName(NodeTransformer): - - def visit_Name(self, node): - return Subscript( - value=Name(id='data', ctx=Load()), - slice=Constant(value=node.id), - ctx=node.ctx - ) - - Keep in mind that if the node you're operating on has child nodes you must - either transform the child nodes yourself or call the :meth:`generic_visit` - method for the node first. - - For nodes that were part of a collection of statements (that applies to all - statement nodes), the visitor may also return a list of nodes rather than - just a single node. - - If :class:`NodeTransformer` introduces new nodes (that weren't part of - original tree) without giving them location information (such as - :attr:`lineno`), :func:`fix_missing_locations` should be called with - the new sub-tree to recalculate the location information:: - - tree = ast.parse('foo', mode='eval') - new_tree = fix_missing_locations(RewriteName().visit(tree)) - - Usually you use the transformer like this:: - - node = YourTransformer().visit(node) - - -.. function:: dump(node, annotate_fields=True, include_attributes=False, *, indent=None) - - Return a formatted dump of the tree in *node*. This is mainly useful for - debugging purposes. If *annotate_fields* is true (by default), - the returned string will show the names and the values for fields. - If *annotate_fields* is false, the result string will be more compact by - omitting unambiguous field names. Attributes such as line - numbers and column offsets are not dumped by default. If this is wanted, - *include_attributes* can be set to true. - - If *indent* is a non-negative integer or string, then the tree will be - pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the single line representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. - - .. versionchanged:: 3.9 - Added the *indent* option. - - -.. _ast-compiler-flags: - -Compiler Flags --------------- - -The following flags may be passed to :func:`compile` in order to change -effects on the compilation of a program: - -.. data:: PyCF_ALLOW_TOP_LEVEL_AWAIT - - Enables support for top-level ``await``, ``async for``, ``async with`` - and async comprehensions. - - .. versionadded:: 3.8 - -.. data:: PyCF_ONLY_AST - - Generates and returns an abstract syntax tree instead of returning a - compiled code object. - -.. data:: PyCF_TYPE_COMMENTS - - Enables support for :pep:`484` and :pep:`526` style type comments - (``# type: ``, ``# type: ignore ``). - - .. versionadded:: 3.8 - - -.. _ast-cli: - -Command-Line Usage ------------------- - -.. versionadded:: 3.9 - -The :mod:`ast` module can be executed as a script from the command line. -It is as simple as: - -.. code-block:: sh - - python -m ast [-m ] [-a] [infile] - -The following options are accepted: - -.. program:: ast - -.. option:: -h, --help - - Show the help message and exit. - -.. option:: -m - --mode - - Specify what kind of code must be compiled, like the *mode* argument - in :func:`parse`. - -.. option:: --no-type-comments - - Don't parse type comments. - -.. option:: -a, --include-attributes - - Include attributes such as line numbers and column offsets. - -.. option:: -i - --indent - - Indentation of nodes in AST (number of spaces). - -If :file:`infile` is specified its contents are parsed to AST and dumped -to stdout. Otherwise, the content is read from stdin. - - -.. seealso:: - - `Green Tree Snakes `_, an external - documentation resource, has good details on working with Python ASTs. - - `ASTTokens `_ - annotates Python ASTs with the positions of tokens and text in the source - code that generated them. This is helpful for tools that make source code - transformations. - - `leoAst.py `_ unifies the - token-based and parse-tree-based views of python programs by inserting - two-way links between tokens and ast nodes. - - `LibCST `_ parses code as a Concrete Syntax - Tree that looks like an ast tree and keeps all formatting details. It's - useful for building automated refactoring (codemod) applications and - linters. - - `Parso `_ is a Python parser that supports - error recovery and round-trip parsing for different Python versions (in - multiple Python versions). Parso is also able to list multiple syntax errors - in your python file. diff --git a/Doc/library/asynchat.rst b/Doc/library/asynchat.rst deleted file mode 100644 index 32e04ad6d19533..00000000000000 --- a/Doc/library/asynchat.rst +++ /dev/null @@ -1,217 +0,0 @@ -:mod:`asynchat` --- Asynchronous socket command/response handler -================================================================ - -.. module:: asynchat - :synopsis: Support for asynchronous command/response protocols. - :deprecated: - -.. moduleauthor:: Sam Rushing -.. sectionauthor:: Steve Holden - -**Source code:** :source:`Lib/asynchat.py` - -.. deprecated-removed:: 3.6 3.12 - The :mod:`asynchat` module is deprecated - (see :pep:`PEP 594 <594#asynchat>` for details). - Please use :mod:`asyncio` instead. - --------------- - -.. note:: - - This module exists for backwards compatibility only. For new code we - recommend using :mod:`asyncio`. - -This module builds on the :mod:`asyncore` infrastructure, simplifying -asynchronous clients and servers and making it easier to handle protocols -whose elements are terminated by arbitrary strings, or are of variable length. -:mod:`asynchat` defines the abstract class :class:`async_chat` that you -subclass, providing implementations of the :meth:`collect_incoming_data` and -:meth:`found_terminator` methods. It uses the same asynchronous loop as -:mod:`asyncore`, and the two types of channel, :class:`asyncore.dispatcher` -and :class:`asynchat.async_chat`, can freely be mixed in the channel map. -Typically an :class:`asyncore.dispatcher` server channel generates new -:class:`asynchat.async_chat` channel objects as it receives incoming -connection requests. - -.. include:: ../includes/wasm-notavail.rst - -.. class:: async_chat() - - This class is an abstract subclass of :class:`asyncore.dispatcher`. To make - practical use of the code you must subclass :class:`async_chat`, providing - meaningful :meth:`collect_incoming_data` and :meth:`found_terminator` - methods. - The :class:`asyncore.dispatcher` methods can be used, although not all make - sense in a message/response context. - - Like :class:`asyncore.dispatcher`, :class:`async_chat` defines a set of - events that are generated by an analysis of socket conditions after a - :c:func:`select` call. Once the polling loop has been started the - :class:`async_chat` object's methods are called by the event-processing - framework with no action on the part of the programmer. - - Two class attributes can be modified, to improve performance, or possibly - even to conserve memory. - - - .. data:: ac_in_buffer_size - - The asynchronous input buffer size (default ``4096``). - - - .. data:: ac_out_buffer_size - - The asynchronous output buffer size (default ``4096``). - - Unlike :class:`asyncore.dispatcher`, :class:`async_chat` allows you to - define a :abbr:`FIFO (first-in, first-out)` queue of *producers*. A producer need - have only one method, :meth:`more`, which should return data to be - transmitted on the channel. - The producer indicates exhaustion (*i.e.* that it contains no more data) by - having its :meth:`more` method return the empty bytes object. At this point - the :class:`async_chat` object removes the producer from the queue and starts - using the next producer, if any. When the producer queue is empty the - :meth:`handle_write` method does nothing. You use the channel object's - :meth:`set_terminator` method to describe how to recognize the end of, or - an important breakpoint in, an incoming transmission from the remote - endpoint. - - To build a functioning :class:`async_chat` subclass your input methods - :meth:`collect_incoming_data` and :meth:`found_terminator` must handle the - data that the channel receives asynchronously. The methods are described - below. - - -.. method:: async_chat.close_when_done() - - Pushes a ``None`` on to the producer queue. When this producer is popped off - the queue it causes the channel to be closed. - - -.. method:: async_chat.collect_incoming_data(data) - - Called with *data* holding an arbitrary amount of received data. The - default method, which must be overridden, raises a - :exc:`NotImplementedError` exception. - - -.. method:: async_chat.discard_buffers() - - In emergencies this method will discard any data held in the input and/or - output buffers and the producer queue. - - -.. method:: async_chat.found_terminator() - - Called when the incoming data stream matches the termination condition set - by :meth:`set_terminator`. The default method, which must be overridden, - raises a :exc:`NotImplementedError` exception. The buffered input data - should be available via an instance attribute. - - -.. method:: async_chat.get_terminator() - - Returns the current terminator for the channel. - - -.. method:: async_chat.push(data) - - Pushes data on to the channel's queue to ensure its transmission. - This is all you need to do to have the channel write the data out to the - network, although it is possible to use your own producers in more complex - schemes to implement encryption and chunking, for example. - - -.. method:: async_chat.push_with_producer(producer) - - Takes a producer object and adds it to the producer queue associated with - the channel. When all currently pushed producers have been exhausted the - channel will consume this producer's data by calling its :meth:`more` - method and send the data to the remote endpoint. - - -.. method:: async_chat.set_terminator(term) - - Sets the terminating condition to be recognized on the channel. ``term`` - may be any of three types of value, corresponding to three different ways - to handle incoming protocol data. - - +-----------+---------------------------------------------+ - | term | Description | - +===========+=============================================+ - | *string* | Will call :meth:`found_terminator` when the | - | | string is found in the input stream | - +-----------+---------------------------------------------+ - | *integer* | Will call :meth:`found_terminator` when the | - | | indicated number of characters have been | - | | received | - +-----------+---------------------------------------------+ - | ``None`` | The channel continues to collect data | - | | forever | - +-----------+---------------------------------------------+ - - Note that any data following the terminator will be available for reading - by the channel after :meth:`found_terminator` is called. - - -.. _asynchat-example: - -asynchat Example ----------------- - -The following partial example shows how HTTP requests can be read with -:class:`async_chat`. A web server might create an -:class:`http_request_handler` object for each incoming client connection. -Notice that initially the channel terminator is set to match the blank line at -the end of the HTTP headers, and a flag indicates that the headers are being -read. - -Once the headers have been read, if the request is of type POST (indicating -that further data are present in the input stream) then the -``Content-Length:`` header is used to set a numeric terminator to read the -right amount of data from the channel. - -The :meth:`handle_request` method is called once all relevant input has been -marshalled, after setting the channel terminator to ``None`` to ensure that -any extraneous data sent by the web client are ignored. :: - - - import asynchat - - class http_request_handler(asynchat.async_chat): - - def __init__(self, sock, addr, sessions, log): - asynchat.async_chat.__init__(self, sock=sock) - self.addr = addr - self.sessions = sessions - self.ibuffer = [] - self.obuffer = b"" - self.set_terminator(b"\r\n\r\n") - self.reading_headers = True - self.handling = False - self.cgi_data = None - self.log = log - - def collect_incoming_data(self, data): - """Buffer the data""" - self.ibuffer.append(data) - - def found_terminator(self): - if self.reading_headers: - self.reading_headers = False - self.parse_headers(b"".join(self.ibuffer)) - self.ibuffer = [] - if self.op.upper() == b"POST": - clen = self.headers.getheader("content-length") - self.set_terminator(int(clen)) - else: - self.handling = True - self.set_terminator(None) - self.handle_request() - elif not self.handling: - self.set_terminator(None) # browsers sometimes over-send - self.cgi_data = parse(self.headers, b"".join(self.ibuffer)) - self.handling = True - self.ibuffer = [] - self.handle_request() diff --git a/Doc/library/asyncio-api-index.rst b/Doc/library/asyncio-api-index.rst deleted file mode 100644 index ad475150fe7d91..00000000000000 --- a/Doc/library/asyncio-api-index.rst +++ /dev/null @@ -1,235 +0,0 @@ -.. currentmodule:: asyncio - - -==================== -High-level API Index -==================== - -This page lists all high-level async/await enabled asyncio APIs. - - -Tasks -===== - -Utilities to run asyncio programs, create Tasks, and -await on multiple things with timeouts. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :func:`run` - - Create event loop, run a coroutine, close the loop. - - * - :class:`Runner` - - A context manager that simplifies multiple async function calls. - - * - :class:`Task` - - Task object. - - * - :class:`TaskGroup` - - A context manager that holds a group of tasks. Provides - a convenient and reliable way to wait for all tasks in the group to - finish. - - * - :func:`create_task` - - Start an asyncio Task, then returns it. - - * - :func:`current_task` - - Return the current Task. - - * - :func:`all_tasks` - - Return all tasks that are not yet finished for an event loop. - - * - ``await`` :func:`sleep` - - Sleep for a number of seconds. - - * - ``await`` :func:`gather` - - Schedule and wait for things concurrently. - - * - ``await`` :func:`wait_for` - - Run with a timeout. - - * - ``await`` :func:`shield` - - Shield from cancellation. - - * - ``await`` :func:`wait` - - Monitor for completion. - - * - :func:`timeout` - - Run with a timeout. Useful in cases when ``wait_for`` is not suitable. - - * - :func:`to_thread` - - Asynchronously run a function in a separate OS thread. - - * - :func:`run_coroutine_threadsafe` - - Schedule a coroutine from another OS thread. - - * - ``for in`` :func:`as_completed` - - Monitor for completion with a ``for`` loop. - - -.. rubric:: Examples - -* :ref:`Using asyncio.gather() to run things in parallel - `. - -* :ref:`Using asyncio.wait_for() to enforce a timeout - `. - -* :ref:`Cancellation `. - -* :ref:`Using asyncio.sleep() `. - -* See also the main :ref:`Tasks documentation page `. - - -Queues -====== - -Queues should be used to distribute work amongst multiple asyncio Tasks, -implement connection pools, and pub/sub patterns. - - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :class:`Queue` - - A FIFO queue. - - * - :class:`PriorityQueue` - - A priority queue. - - * - :class:`LifoQueue` - - A LIFO queue. - - -.. rubric:: Examples - -* :ref:`Using asyncio.Queue to distribute workload between several - Tasks `. - -* See also the :ref:`Queues documentation page `. - - -Subprocesses -============ - -Utilities to spawn subprocesses and run shell commands. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :func:`create_subprocess_exec` - - Create a subprocess. - - * - ``await`` :func:`create_subprocess_shell` - - Run a shell command. - - -.. rubric:: Examples - -* :ref:`Executing a shell command `. - -* See also the :ref:`subprocess APIs ` - documentation. - - -Streams -======= - -High-level APIs to work with network IO. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :func:`open_connection` - - Establish a TCP connection. - - * - ``await`` :func:`open_unix_connection` - - Establish a Unix socket connection. - - * - ``await`` :func:`start_server` - - Start a TCP server. - - * - ``await`` :func:`start_unix_server` - - Start a Unix socket server. - - * - :class:`StreamReader` - - High-level async/await object to receive network data. - - * - :class:`StreamWriter` - - High-level async/await object to send network data. - - -.. rubric:: Examples - -* :ref:`Example TCP client `. - -* See also the :ref:`streams APIs ` - documentation. - - -Synchronization -=============== - -Threading-like synchronization primitives that can be used in Tasks. - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :class:`Lock` - - A mutex lock. - - * - :class:`Event` - - An event object. - - * - :class:`Condition` - - A condition object. - - * - :class:`Semaphore` - - A semaphore. - - * - :class:`BoundedSemaphore` - - A bounded semaphore. - - * - :class:`Barrier` - - A barrier object. - - -.. rubric:: Examples - -* :ref:`Using asyncio.Event `. - -* :ref:`Using asyncio.Barrier `. - -* See also the documentation of asyncio - :ref:`synchronization primitives `. - - -Exceptions -========== - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - - * - :exc:`asyncio.CancelledError` - - Raised when a Task is cancelled. See also :meth:`Task.cancel`. - - * - :exc:`asyncio.BrokenBarrierError` - - Raised when a Barrier is broken. See also :meth:`Barrier.wait`. - - -.. rubric:: Examples - -* :ref:`Handling CancelledError to run code on cancellation request - `. - -* See also the full list of - :ref:`asyncio-specific exceptions `. diff --git a/Doc/library/asyncio-dev.rst b/Doc/library/asyncio-dev.rst deleted file mode 100644 index a9c3a0183bb72d..00000000000000 --- a/Doc/library/asyncio-dev.rst +++ /dev/null @@ -1,252 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-dev: - -======================= -Developing with asyncio -======================= - -Asynchronous programming is different from classic "sequential" -programming. - -This page lists common mistakes and traps and explains how -to avoid them. - - -.. _asyncio-debug-mode: - -Debug Mode -========== - -By default asyncio runs in production mode. In order to ease -the development asyncio has a *debug mode*. - -There are several ways to enable asyncio debug mode: - -* Setting the :envvar:`PYTHONASYNCIODEBUG` environment variable to ``1``. - -* Using the :ref:`Python Development Mode `. - -* Passing ``debug=True`` to :func:`asyncio.run`. - -* Calling :meth:`loop.set_debug`. - -In addition to enabling the debug mode, consider also: - -* setting the log level of the :ref:`asyncio logger ` to - :py:const:`logging.DEBUG`, for example the following snippet of code - can be run at startup of the application:: - - logging.basicConfig(level=logging.DEBUG) - -* configuring the :mod:`warnings` module to display - :exc:`ResourceWarning` warnings. One way of doing that is by - using the :option:`-W` ``default`` command line option. - - -When the debug mode is enabled: - -* asyncio checks for :ref:`coroutines that were not awaited - ` and logs them; this mitigates - the "forgotten await" pitfall. - -* Many non-threadsafe asyncio APIs (such as :meth:`loop.call_soon` and - :meth:`loop.call_at` methods) raise an exception if they are called - from a wrong thread. - -* The execution time of the I/O selector is logged if it takes too long to - perform an I/O operation. - -* Callbacks taking longer than 100 milliseconds are logged. The - :attr:`loop.slow_callback_duration` attribute can be used to set the - minimum execution duration in seconds that is considered "slow". - - -.. _asyncio-multithreading: - -Concurrency and Multithreading -============================== - -An event loop runs in a thread (typically the main thread) and executes -all callbacks and Tasks in its thread. While a Task is running in the -event loop, no other Tasks can run in the same thread. When a Task -executes an ``await`` expression, the running Task gets suspended, and -the event loop executes the next Task. - -To schedule a :term:`callback` from another OS thread, the -:meth:`loop.call_soon_threadsafe` method should be used. Example:: - - loop.call_soon_threadsafe(callback, *args) - -Almost all asyncio objects are not thread safe, which is typically -not a problem unless there is code that works with them from outside -of a Task or a callback. If there's a need for such code to call a -low-level asyncio API, the :meth:`loop.call_soon_threadsafe` method -should be used, e.g.:: - - loop.call_soon_threadsafe(fut.cancel) - -To schedule a coroutine object from a different OS thread, the -:func:`run_coroutine_threadsafe` function should be used. It returns a -:class:`concurrent.futures.Future` to access the result:: - - async def coro_func(): - return await asyncio.sleep(1, 42) - - # Later in another OS thread: - - future = asyncio.run_coroutine_threadsafe(coro_func(), loop) - # Wait for the result: - result = future.result() - -To handle signals the event loop must be -run in the main thread. - -The :meth:`loop.run_in_executor` method can be used with a -:class:`concurrent.futures.ThreadPoolExecutor` to execute -blocking code in a different OS thread without blocking the OS thread -that the event loop runs in. - -There is currently no way to schedule coroutines or callbacks directly -from a different process (such as one started with -:mod:`multiprocessing`). The :ref:`asyncio-event-loop-methods` -section lists APIs that can read from pipes and watch file descriptors -without blocking the event loop. In addition, asyncio's -:ref:`Subprocess ` APIs provide a way to start a -process and communicate with it from the event loop. Lastly, the -aforementioned :meth:`loop.run_in_executor` method can also be used -with a :class:`concurrent.futures.ProcessPoolExecutor` to execute -code in a different process. - -.. _asyncio-handle-blocking: - -Running Blocking Code -===================== - -Blocking (CPU-bound) code should not be called directly. For example, -if a function performs a CPU-intensive calculation for 1 second, -all concurrent asyncio Tasks and IO operations would be delayed -by 1 second. - -An executor can be used to run a task in a different thread or even in -a different process to avoid blocking the OS thread with the -event loop. See the :meth:`loop.run_in_executor` method for more -details. - - -.. _asyncio-logger: - -Logging -======= - -asyncio uses the :mod:`logging` module and all logging is performed -via the ``"asyncio"`` logger. - -The default log level is :py:const:`logging.INFO`, which can be easily -adjusted:: - - logging.getLogger("asyncio").setLevel(logging.WARNING) - - -Network logging can block the event loop. It is recommended to use -a separate thread for handling logs or use non-blocking IO. For example, -see :ref:`blocking-handlers`. - - -.. _asyncio-coroutine-not-scheduled: - -Detect never-awaited coroutines -=============================== - -When a coroutine function is called, but not awaited -(e.g. ``coro()`` instead of ``await coro()``) -or the coroutine is not scheduled with :meth:`asyncio.create_task`, asyncio -will emit a :exc:`RuntimeWarning`:: - - import asyncio - - async def test(): - print("never scheduled") - - async def main(): - test() - - asyncio.run(main()) - -Output:: - - test.py:7: RuntimeWarning: coroutine 'test' was never awaited - test() - -Output in debug mode:: - - test.py:7: RuntimeWarning: coroutine 'test' was never awaited - Coroutine created at (most recent call last) - File "../t.py", line 9, in - asyncio.run(main(), debug=True) - - < .. > - - File "../t.py", line 7, in main - test() - test() - -The usual fix is to either await the coroutine or call the -:meth:`asyncio.create_task` function:: - - async def main(): - await test() - - -Detect never-retrieved exceptions -================================= - -If a :meth:`Future.set_exception` is called but the Future object is -never awaited on, the exception would never be propagated to the -user code. In this case, asyncio would emit a log message when the -Future object is garbage collected. - -Example of an unhandled exception:: - - import asyncio - - async def bug(): - raise Exception("not consumed") - - async def main(): - asyncio.create_task(bug()) - - asyncio.run(main()) - -Output:: - - Task exception was never retrieved - future: - exception=Exception('not consumed')> - - Traceback (most recent call last): - File "test.py", line 4, in bug - raise Exception("not consumed") - Exception: not consumed - -:ref:`Enable the debug mode ` to get the -traceback where the task was created:: - - asyncio.run(main(), debug=True) - -Output in debug mode:: - - Task exception was never retrieved - future: - exception=Exception('not consumed') created at asyncio/tasks.py:321> - - source_traceback: Object created at (most recent call last): - File "../t.py", line 9, in - asyncio.run(main(), debug=True) - - < .. > - - Traceback (most recent call last): - File "../t.py", line 4, in bug - raise Exception("not consumed") - Exception: not consumed diff --git a/Doc/library/asyncio-eventloop.rst b/Doc/library/asyncio-eventloop.rst deleted file mode 100644 index 32daaa7bca2123..00000000000000 --- a/Doc/library/asyncio-eventloop.rst +++ /dev/null @@ -1,1883 +0,0 @@ -.. currentmodule:: asyncio - - -.. _asyncio-event-loop: - -========== -Event Loop -========== - -**Source code:** :source:`Lib/asyncio/events.py`, -:source:`Lib/asyncio/base_events.py` - ------------------------------------- - -.. rubric:: Preface - -The event loop is the core of every asyncio application. -Event loops run asynchronous tasks and callbacks, perform network -IO operations, and run subprocesses. - -Application developers should typically use the high-level asyncio functions, -such as :func:`asyncio.run`, and should rarely need to reference the loop -object or call its methods. This section is intended mostly for authors -of lower-level code, libraries, and frameworks, who need finer control over -the event loop behavior. - -.. rubric:: Obtaining the Event Loop - -The following low-level functions can be used to get, set, or create -an event loop: - -.. function:: get_running_loop() - - Return the running event loop in the current OS thread. - - Raise a :exc:`RuntimeError` if there is no running event loop. - - This function can only be called from a coroutine or a callback. - - .. versionadded:: 3.7 - -.. function:: get_event_loop() - - Get the current event loop. - - When called from a coroutine or a callback (e.g. scheduled with - call_soon or similar API), this function will always return the - running event loop. - - If there is no running event loop set, the function will return - the result of the ``get_event_loop_policy().get_event_loop()`` call. - - Because this function has rather complex behavior (especially - when custom event loop policies are in use), using the - :func:`get_running_loop` function is preferred to :func:`get_event_loop` - in coroutines and callbacks. - - As noted above, consider using the higher-level :func:`asyncio.run` function, - instead of using these lower level functions to manually create and close an - event loop. - - .. note:: - In Python versions 3.10.0--3.10.8 and 3.11.0 this function - (and other functions which use it implicitly) emitted a - :exc:`DeprecationWarning` if there was no running event loop, even if - the current loop was set on the policy. - In Python versions 3.10.9, 3.11.1 and 3.12 they emit a - :exc:`DeprecationWarning` if there is no running event loop and no - current loop is set. - In some future Python release this will become an error. - -.. function:: set_event_loop(loop) - - Set *loop* as the current event loop for the current OS thread. - -.. function:: new_event_loop() - - Create and return a new event loop object. - -Note that the behaviour of :func:`get_event_loop`, :func:`set_event_loop`, -and :func:`new_event_loop` functions can be altered by -:ref:`setting a custom event loop policy `. - - -.. rubric:: Contents - -This documentation page contains the following sections: - -* The `Event Loop Methods`_ section is the reference documentation of - the event loop APIs; - -* The `Callback Handles`_ section documents the :class:`Handle` and - :class:`TimerHandle` instances which are returned from scheduling - methods such as :meth:`loop.call_soon` and :meth:`loop.call_later`; - -* The `Server Objects`_ section documents types returned from - event loop methods like :meth:`loop.create_server`; - -* The `Event Loop Implementations`_ section documents the - :class:`SelectorEventLoop` and :class:`ProactorEventLoop` classes; - -* The `Examples`_ section showcases how to work with some event - loop APIs. - - -.. _asyncio-event-loop-methods: - -Event Loop Methods -================== - -Event loops have **low-level** APIs for the following: - -.. contents:: - :depth: 1 - :local: - - -Running and stopping the loop -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. method:: loop.run_until_complete(future) - - Run until the *future* (an instance of :class:`Future`) has - completed. - - If the argument is a :ref:`coroutine object ` it - is implicitly scheduled to run as a :class:`asyncio.Task`. - - Return the Future's result or raise its exception. - -.. method:: loop.run_forever() - - Run the event loop until :meth:`stop` is called. - - If :meth:`stop` is called before :meth:`run_forever()` is called, - the loop will poll the I/O selector once with a timeout of zero, - run all callbacks scheduled in response to I/O events (and - those that were already scheduled), and then exit. - - If :meth:`stop` is called while :meth:`run_forever` is running, - the loop will run the current batch of callbacks and then exit. - Note that new callbacks scheduled by callbacks will not run in this - case; instead, they will run the next time :meth:`run_forever` or - :meth:`run_until_complete` is called. - -.. method:: loop.stop() - - Stop the event loop. - -.. method:: loop.is_running() - - Return ``True`` if the event loop is currently running. - -.. method:: loop.is_closed() - - Return ``True`` if the event loop was closed. - -.. method:: loop.close() - - Close the event loop. - - The loop must not be running when this function is called. - Any pending callbacks will be discarded. - - This method clears all queues and shuts down the executor, but does - not wait for the executor to finish. - - This method is idempotent and irreversible. No other methods - should be called after the event loop is closed. - -.. coroutinemethod:: loop.shutdown_asyncgens() - - Schedule all currently open :term:`asynchronous generator` objects to - close with an :meth:`~agen.aclose()` call. After calling this method, - the event loop will issue a warning if a new asynchronous generator - is iterated. This should be used to reliably finalize all scheduled - asynchronous generators. - - Note that there is no need to call this function when - :func:`asyncio.run` is used. - - Example:: - - try: - loop.run_forever() - finally: - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.close() - - .. versionadded:: 3.6 - -.. coroutinemethod:: loop.shutdown_default_executor() - - Schedule the closure of the default executor and wait for it to join all of - the threads in the :class:`~concurrent.futures.ThreadPoolExecutor`. - Once this method has been called, - using the default executor with :meth:`loop.run_in_executor` - will raise a :exc:`RuntimeError`. - - .. note:: - - Do not call this method when using :func:`asyncio.run`, - as the latter handles default executor shutdown automatically. - - .. versionadded:: 3.9 - - -Scheduling callbacks -^^^^^^^^^^^^^^^^^^^^ - -.. method:: loop.call_soon(callback, *args, context=None) - - Schedule the *callback* :term:`callback` to be called with - *args* arguments at the next iteration of the event loop. - - Return an instance of :class:`asyncio.Handle`, - which can be used later to cancel the callback. - - Callbacks are called in the order in which they are registered. - Each callback will be called exactly once. - - The optional keyword-only *context* argument specifies a - custom :class:`contextvars.Context` for the *callback* to run in. - Callbacks use the current context when no *context* is provided. - - Unlike :meth:`call_soon_threadsafe`, this method is not thread-safe. - -.. method:: loop.call_soon_threadsafe(callback, *args, context=None) - - A thread-safe variant of :meth:`call_soon`. When scheduling callbacks from - another thread, this function *must* be used, since :meth:`call_soon` is not - thread-safe. - - Raises :exc:`RuntimeError` if called on a loop that's been closed. - This can happen on a secondary thread when the main application is - shutting down. - - See the :ref:`concurrency and multithreading ` - section of the documentation. - - .. versionchanged:: 3.7 - The *context* keyword-only parameter was added. See :pep:`567` - for more details. - -.. _asyncio-pass-keywords: - -.. note:: - - Most :mod:`asyncio` scheduling functions don't allow passing - keyword arguments. To do that, use :func:`functools.partial`:: - - # will schedule "print("Hello", flush=True)" - loop.call_soon( - functools.partial(print, "Hello", flush=True)) - - Using partial objects is usually more convenient than using lambdas, - as asyncio can render partial objects better in debug and error - messages. - - -.. _asyncio-delayed-calls: - -Scheduling delayed callbacks -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Event loop provides mechanisms to schedule callback functions -to be called at some point in the future. Event loop uses monotonic -clocks to track time. - - -.. method:: loop.call_later(delay, callback, *args, context=None) - - Schedule *callback* to be called after the given *delay* - number of seconds (can be either an int or a float). - - An instance of :class:`asyncio.TimerHandle` is returned which can - be used to cancel the callback. - - *callback* will be called exactly once. If two callbacks are - scheduled for exactly the same time, the order in which they - are called is undefined. - - The optional positional *args* will be passed to the callback when - it is called. If you want the callback to be called with keyword - arguments use :func:`functools.partial`. - - An optional keyword-only *context* argument allows specifying a - custom :class:`contextvars.Context` for the *callback* to run in. - The current context is used when no *context* is provided. - - .. versionchanged:: 3.7 - The *context* keyword-only parameter was added. See :pep:`567` - for more details. - - .. versionchanged:: 3.8 - In Python 3.7 and earlier with the default event loop implementation, - the *delay* could not exceed one day. - This has been fixed in Python 3.8. - -.. method:: loop.call_at(when, callback, *args, context=None) - - Schedule *callback* to be called at the given absolute timestamp - *when* (an int or a float), using the same time reference as - :meth:`loop.time`. - - This method's behavior is the same as :meth:`call_later`. - - An instance of :class:`asyncio.TimerHandle` is returned which can - be used to cancel the callback. - - .. versionchanged:: 3.7 - The *context* keyword-only parameter was added. See :pep:`567` - for more details. - - .. versionchanged:: 3.8 - In Python 3.7 and earlier with the default event loop implementation, - the difference between *when* and the current time could not exceed - one day. This has been fixed in Python 3.8. - -.. method:: loop.time() - - Return the current time, as a :class:`float` value, according to - the event loop's internal monotonic clock. - -.. note:: - .. versionchanged:: 3.8 - In Python 3.7 and earlier timeouts (relative *delay* or absolute *when*) - should not exceed one day. This has been fixed in Python 3.8. - -.. seealso:: - - The :func:`asyncio.sleep` function. - - -Creating Futures and Tasks -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. method:: loop.create_future() - - Create an :class:`asyncio.Future` object attached to the event loop. - - This is the preferred way to create Futures in asyncio. This lets - third-party event loops provide alternative implementations of - the Future object (with better performance or instrumentation). - - .. versionadded:: 3.5.2 - -.. method:: loop.create_task(coro, *, name=None, context=None) - - Schedule the execution of :ref:`coroutine ` *coro*. - Return a :class:`Task` object. - - Third-party event loops can use their own subclass of :class:`Task` - for interoperability. In this case, the result type is a subclass - of :class:`Task`. - - If the *name* argument is provided and not ``None``, it is set as - the name of the task using :meth:`Task.set_name`. - - An optional keyword-only *context* argument allows specifying a - custom :class:`contextvars.Context` for the *coro* to run in. - The current context copy is created when no *context* is provided. - - .. versionchanged:: 3.8 - Added the *name* parameter. - - .. versionchanged:: 3.11 - Added the *context* parameter. - -.. method:: loop.set_task_factory(factory) - - Set a task factory that will be used by - :meth:`loop.create_task`. - - If *factory* is ``None`` the default task factory will be set. - Otherwise, *factory* must be a *callable* with the signature matching - ``(loop, coro, context=None)``, where *loop* is a reference to the active - event loop, and *coro* is a coroutine object. The callable - must return a :class:`asyncio.Future`-compatible object. - -.. method:: loop.get_task_factory() - - Return a task factory or ``None`` if the default one is in use. - - -Opening network connections -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. coroutinemethod:: loop.create_connection(protocol_factory, \ - host=None, port=None, *, ssl=None, \ - family=0, proto=0, flags=0, sock=None, \ - local_addr=None, server_hostname=None, \ - ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, \ - happy_eyeballs_delay=None, interleave=None) - - Open a streaming transport connection to a given - address specified by *host* and *port*. - - The socket family can be either :py:const:`~socket.AF_INET` or - :py:const:`~socket.AF_INET6` depending on *host* (or the *family* - argument, if provided). - - The socket type will be :py:const:`~socket.SOCK_STREAM`. - - *protocol_factory* must be a callable returning an - :ref:`asyncio protocol ` implementation. - - This method will try to establish the connection in the background. - When successful, it returns a ``(transport, protocol)`` pair. - - The chronological synopsis of the underlying operation is as follows: - - #. The connection is established and a :ref:`transport ` - is created for it. - - #. *protocol_factory* is called without arguments and is expected to - return a :ref:`protocol ` instance. - - #. The protocol instance is coupled with the transport by calling its - :meth:`~BaseProtocol.connection_made` method. - - #. A ``(transport, protocol)`` tuple is returned on success. - - The created transport is an implementation-dependent bidirectional - stream. - - Other arguments: - - * *ssl*: if given and not false, a SSL/TLS transport is created - (by default a plain TCP transport is created). If *ssl* is - a :class:`ssl.SSLContext` object, this context is used to create - the transport; if *ssl* is :const:`True`, a default context returned - from :func:`ssl.create_default_context` is used. - - .. seealso:: :ref:`SSL/TLS security considerations ` - - * *server_hostname* sets or overrides the hostname that the target - server's certificate will be matched against. Should only be passed - if *ssl* is not ``None``. By default the value of the *host* argument - is used. If *host* is empty, there is no default and you must pass a - value for *server_hostname*. If *server_hostname* is an empty - string, hostname matching is disabled (which is a serious security - risk, allowing for potential man-in-the-middle attacks). - - * *family*, *proto*, *flags* are the optional address family, protocol - and flags to be passed through to getaddrinfo() for *host* resolution. - If given, these should all be integers from the corresponding - :mod:`socket` module constants. - - * *happy_eyeballs_delay*, if given, enables Happy Eyeballs for this - connection. It should - be a floating-point number representing the amount of time in seconds - to wait for a connection attempt to complete, before starting the next - attempt in parallel. This is the "Connection Attempt Delay" as defined - in :rfc:`8305`. A sensible default value recommended by the RFC is ``0.25`` - (250 milliseconds). - - * *interleave* controls address reordering when a host name resolves to - multiple IP addresses. - If ``0`` or unspecified, no reordering is done, and addresses are - tried in the order returned by :meth:`getaddrinfo`. If a positive integer - is specified, the addresses are interleaved by address family, and the - given integer is interpreted as "First Address Family Count" as defined - in :rfc:`8305`. The default is ``0`` if *happy_eyeballs_delay* is not - specified, and ``1`` if it is. - - * *sock*, if given, should be an existing, already connected - :class:`socket.socket` object to be used by the transport. - If *sock* is given, none of *host*, *port*, *family*, *proto*, *flags*, - *happy_eyeballs_delay*, *interleave* - and *local_addr* should be specified. - - .. note:: - - The *sock* argument transfers ownership of the socket to the - transport created. To close the socket, call the transport's - :meth:`~asyncio.BaseTransport.close` method. - - * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used - to bind the socket locally. The *local_host* and *local_port* - are looked up using ``getaddrinfo()``, similarly to *host* and *port*. - - * *ssl_handshake_timeout* is (for a TLS connection) the time in seconds - to wait for the TLS handshake to complete before aborting the connection. - ``60.0`` seconds if ``None`` (default). - - * *ssl_shutdown_timeout* is the time in seconds to wait for the SSL shutdown - to complete before aborting the connection. ``30.0`` seconds if ``None`` - (default). - - .. versionchanged:: 3.5 - - Added support for SSL/TLS in :class:`ProactorEventLoop`. - - .. versionchanged:: 3.6 - - The socket option :ref:`socket.TCP_NODELAY ` is set by default - for all TCP connections. - - .. versionchanged:: 3.7 - - Added the *ssl_handshake_timeout* parameter. - - .. versionchanged:: 3.8 - - Added the *happy_eyeballs_delay* and *interleave* parameters. - - Happy Eyeballs Algorithm: Success with Dual-Stack Hosts. - When a server's IPv4 path and protocol are working, but the server's - IPv6 path and protocol are not working, a dual-stack client - application experiences significant connection delay compared to an - IPv4-only client. This is undesirable because it causes the - dual-stack client to have a worse user experience. This document - specifies requirements for algorithms that reduce this user-visible - delay and provides an algorithm. - - For more information: https://datatracker.ietf.org/doc/html/rfc6555 - - .. versionchanged:: 3.11 - - Added the *ssl_shutdown_timeout* parameter. - - .. seealso:: - - The :func:`open_connection` function is a high-level alternative - API. It returns a pair of (:class:`StreamReader`, :class:`StreamWriter`) - that can be used directly in async/await code. - -.. coroutinemethod:: loop.create_datagram_endpoint(protocol_factory, \ - local_addr=None, remote_addr=None, *, \ - family=0, proto=0, flags=0, \ - reuse_port=None, \ - allow_broadcast=None, sock=None) - - Create a datagram connection. - - The socket family can be either :py:const:`~socket.AF_INET`, - :py:const:`~socket.AF_INET6`, or :py:const:`~socket.AF_UNIX`, - depending on *host* (or the *family* argument, if provided). - - The socket type will be :py:const:`~socket.SOCK_DGRAM`. - - *protocol_factory* must be a callable returning a - :ref:`protocol ` implementation. - - A tuple of ``(transport, protocol)`` is returned on success. - - Other arguments: - - * *local_addr*, if given, is a ``(local_host, local_port)`` tuple used - to bind the socket locally. The *local_host* and *local_port* - are looked up using :meth:`getaddrinfo`. - - * *remote_addr*, if given, is a ``(remote_host, remote_port)`` tuple used - to connect the socket to a remote address. The *remote_host* and - *remote_port* are looked up using :meth:`getaddrinfo`. - - * *family*, *proto*, *flags* are the optional address family, protocol - and flags to be passed through to :meth:`getaddrinfo` for *host* - resolution. If given, these should all be integers from the - corresponding :mod:`socket` module constants. - - * *reuse_port* tells the kernel to allow this endpoint to be bound to the - same port as other existing endpoints are bound to, so long as they all - set this flag when being created. This option is not supported on Windows - and some Unixes. If the :ref:`socket.SO_REUSEPORT ` constant is not - defined then this capability is unsupported. - - * *allow_broadcast* tells the kernel to allow this endpoint to send - messages to the broadcast address. - - * *sock* can optionally be specified in order to use a preexisting, - already connected, :class:`socket.socket` object to be used by the - transport. If specified, *local_addr* and *remote_addr* should be omitted - (must be :const:`None`). - - .. note:: - - The *sock* argument transfers ownership of the socket to the - transport created. To close the socket, call the transport's - :meth:`~asyncio.BaseTransport.close` method. - - See :ref:`UDP echo client protocol ` and - :ref:`UDP echo server protocol ` examples. - - .. versionchanged:: 3.4.4 - The *family*, *proto*, *flags*, *reuse_address*, *reuse_port*, - *allow_broadcast*, and *sock* parameters were added. - - .. versionchanged:: 3.8.1 - The *reuse_address* parameter is no longer supported, as using - :ref:`socket.SO_REUSEADDR ` - poses a significant security concern for - UDP. Explicitly passing ``reuse_address=True`` will raise an exception. - - When multiple processes with differing UIDs assign sockets to an - identical UDP socket address with ``SO_REUSEADDR``, incoming packets can - become randomly distributed among the sockets. - - For supported platforms, *reuse_port* can be used as a replacement for - similar functionality. With *reuse_port*, - :ref:`socket.SO_REUSEPORT ` - is used instead, which specifically - prevents processes with differing UIDs from assigning sockets to the same - socket address. - - .. versionchanged:: 3.8 - Added support for Windows. - - .. versionchanged:: 3.11 - The *reuse_address* parameter, disabled since Python 3.9.0, 3.8.1, - 3.7.6 and 3.6.10, has been entirely removed. - -.. coroutinemethod:: loop.create_unix_connection(protocol_factory, \ - path=None, *, ssl=None, sock=None, \ - server_hostname=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None) - - Create a Unix connection. - - The socket family will be :py:const:`~socket.AF_UNIX`; socket - type will be :py:const:`~socket.SOCK_STREAM`. - - A tuple of ``(transport, protocol)`` is returned on success. - - *path* is the name of a Unix domain socket and is required, - unless a *sock* parameter is specified. Abstract Unix sockets, - :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths are - supported. - - See the documentation of the :meth:`loop.create_connection` method - for information about arguments to this method. - - .. availability:: Unix. - - .. versionchanged:: 3.7 - Added the *ssl_handshake_timeout* parameter. - The *path* parameter can now be a :term:`path-like object`. - - .. versionchanged:: 3.11 - - Added the *ssl_shutdown_timeout* parameter. - - -Creating network servers -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. _loop_create_server: - -.. coroutinemethod:: loop.create_server(protocol_factory, \ - host=None, port=None, *, \ - family=socket.AF_UNSPEC, \ - flags=socket.AI_PASSIVE, \ - sock=None, backlog=100, ssl=None, \ - reuse_address=None, reuse_port=None, \ - ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, \ - start_serving=True) - - Create a TCP server (socket type :const:`~socket.SOCK_STREAM`) listening - on *port* of the *host* address. - - Returns a :class:`Server` object. - - Arguments: - - * *protocol_factory* must be a callable returning a - :ref:`protocol ` implementation. - - * The *host* parameter can be set to several types which determine where - the server would be listening: - - - If *host* is a string, the TCP server is bound to a single network - interface specified by *host*. - - - If *host* is a sequence of strings, the TCP server is bound to all - network interfaces specified by the sequence. - - - If *host* is an empty string or ``None``, all interfaces are - assumed and a list of multiple sockets will be returned (most likely - one for IPv4 and another one for IPv6). - - * The *port* parameter can be set to specify which port the server should - listen on. If ``0`` or ``None`` (the default), a random unused port will - be selected (note that if *host* resolves to multiple network interfaces, - a different random port will be selected for each interface). - - * *family* can be set to either :const:`socket.AF_INET` or - :const:`~socket.AF_INET6` to force the socket to use IPv4 or IPv6. - If not set, the *family* will be determined from host name - (defaults to :const:`~socket.AF_UNSPEC`). - - * *flags* is a bitmask for :meth:`getaddrinfo`. - - * *sock* can optionally be specified in order to use a preexisting - socket object. If specified, *host* and *port* must not be specified. - - .. note:: - - The *sock* argument transfers ownership of the socket to the - server created. To close the socket, call the server's - :meth:`~asyncio.Server.close` method. - - * *backlog* is the maximum number of queued connections passed to - :meth:`~socket.socket.listen` (defaults to 100). - - * *ssl* can be set to an :class:`~ssl.SSLContext` instance to enable - TLS over the accepted connections. - - * *reuse_address* tells the kernel to reuse a local socket in - ``TIME_WAIT`` state, without waiting for its natural timeout to - expire. If not specified will automatically be set to ``True`` on - Unix. - - * *reuse_port* tells the kernel to allow this endpoint to be bound to the - same port as other existing endpoints are bound to, so long as they all - set this flag when being created. This option is not supported on - Windows. - - * *ssl_handshake_timeout* is (for a TLS server) the time in seconds to wait - for the TLS handshake to complete before aborting the connection. - ``60.0`` seconds if ``None`` (default). - - * *ssl_shutdown_timeout* is the time in seconds to wait for the SSL shutdown - to complete before aborting the connection. ``30.0`` seconds if ``None`` - (default). - - * *start_serving* set to ``True`` (the default) causes the created server - to start accepting connections immediately. When set to ``False``, - the user should await on :meth:`Server.start_serving` or - :meth:`Server.serve_forever` to make the server to start accepting - connections. - - .. versionchanged:: 3.5 - - Added support for SSL/TLS in :class:`ProactorEventLoop`. - - .. versionchanged:: 3.5.1 - - The *host* parameter can be a sequence of strings. - - .. versionchanged:: 3.6 - - Added *ssl_handshake_timeout* and *start_serving* parameters. - The socket option :ref:`socket.TCP_NODELAY ` is set by default - for all TCP connections. - - .. versionchanged:: 3.11 - - Added the *ssl_shutdown_timeout* parameter. - - .. seealso:: - - The :func:`start_server` function is a higher-level alternative API - that returns a pair of :class:`StreamReader` and :class:`StreamWriter` - that can be used in an async/await code. - - -.. coroutinemethod:: loop.create_unix_server(protocol_factory, path=None, \ - *, sock=None, backlog=100, ssl=None, \ - ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, \ - start_serving=True) - - Similar to :meth:`loop.create_server` but works with the - :py:const:`~socket.AF_UNIX` socket family. - - *path* is the name of a Unix domain socket, and is required, - unless a *sock* argument is provided. Abstract Unix sockets, - :class:`str`, :class:`bytes`, and :class:`~pathlib.Path` paths - are supported. - - See the documentation of the :meth:`loop.create_server` method - for information about arguments to this method. - - .. availability:: Unix. - - .. versionchanged:: 3.7 - - Added the *ssl_handshake_timeout* and *start_serving* parameters. - The *path* parameter can now be a :class:`~pathlib.Path` object. - - .. versionchanged:: 3.11 - - Added the *ssl_shutdown_timeout* parameter. - - -.. coroutinemethod:: loop.connect_accepted_socket(protocol_factory, \ - sock, *, ssl=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None) - - Wrap an already accepted connection into a transport/protocol pair. - - This method can be used by servers that accept connections outside - of asyncio but that use asyncio to handle them. - - Parameters: - - * *protocol_factory* must be a callable returning a - :ref:`protocol ` implementation. - - * *sock* is a preexisting socket object returned from - :meth:`socket.accept `. - - .. note:: - - The *sock* argument transfers ownership of the socket to the - transport created. To close the socket, call the transport's - :meth:`~asyncio.BaseTransport.close` method. - - * *ssl* can be set to an :class:`~ssl.SSLContext` to enable SSL over - the accepted connections. - - * *ssl_handshake_timeout* is (for an SSL connection) the time in seconds to - wait for the SSL handshake to complete before aborting the connection. - ``60.0`` seconds if ``None`` (default). - - * *ssl_shutdown_timeout* is the time in seconds to wait for the SSL shutdown - to complete before aborting the connection. ``30.0`` seconds if ``None`` - (default). - - Returns a ``(transport, protocol)`` pair. - - .. versionadded:: 3.5.3 - - .. versionchanged:: 3.7 - - Added the *ssl_handshake_timeout* parameter. - - .. versionchanged:: 3.11 - - Added the *ssl_shutdown_timeout* parameter. - - -Transferring files -^^^^^^^^^^^^^^^^^^ - -.. coroutinemethod:: loop.sendfile(transport, file, \ - offset=0, count=None, *, fallback=True) - - Send a *file* over a *transport*. Return the total number of bytes - sent. - - The method uses high-performance :meth:`os.sendfile` if available. - - *file* must be a regular file object opened in binary mode. - - *offset* tells from where to start reading the file. If specified, - *count* is the total number of bytes to transmit as opposed to - sending the file until EOF is reached. File position is always updated, - even when this method raises an error, and - :meth:`file.tell() ` can be used to obtain the actual - number of bytes sent. - - *fallback* set to ``True`` makes asyncio to manually read and send - the file when the platform does not support the sendfile system call - (e.g. Windows or SSL socket on Unix). - - Raise :exc:`SendfileNotAvailableError` if the system does not support - the *sendfile* syscall and *fallback* is ``False``. - - .. versionadded:: 3.7 - - -TLS Upgrade -^^^^^^^^^^^ - -.. coroutinemethod:: loop.start_tls(transport, protocol, \ - sslcontext, *, server_side=False, \ - server_hostname=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None) - - Upgrade an existing transport-based connection to TLS. - - Create a TLS coder/decoder instance and insert it between the *transport* - and the *protocol*. The coder/decoder implements both *transport*-facing - protocol and *protocol*-facing transport. - - Return the created two-interface instance. After *await*, the *protocol* - must stop using the original *transport* and communicate with the returned - object only because the coder caches *protocol*-side data and sporadically - exchanges extra TLS session packets with *transport*. - - In some situations (e.g. when the passed transport is already closing) this - may return ``None``. - - Parameters: - - * *transport* and *protocol* instances that methods like - :meth:`~loop.create_server` and - :meth:`~loop.create_connection` return. - - * *sslcontext*: a configured instance of :class:`~ssl.SSLContext`. - - * *server_side* pass ``True`` when a server-side connection is being - upgraded (like the one created by :meth:`~loop.create_server`). - - * *server_hostname*: sets or overrides the host name that the target - server's certificate will be matched against. - - * *ssl_handshake_timeout* is (for a TLS connection) the time in seconds to - wait for the TLS handshake to complete before aborting the connection. - ``60.0`` seconds if ``None`` (default). - - * *ssl_shutdown_timeout* is the time in seconds to wait for the SSL shutdown - to complete before aborting the connection. ``30.0`` seconds if ``None`` - (default). - - .. versionadded:: 3.7 - - .. versionchanged:: 3.11 - - Added the *ssl_shutdown_timeout* parameter. - - - -Watching file descriptors -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. method:: loop.add_reader(fd, callback, *args) - - Start monitoring the *fd* file descriptor for read availability and - invoke *callback* with the specified arguments once *fd* is available for - reading. - -.. method:: loop.remove_reader(fd) - - Stop monitoring the *fd* file descriptor for read availability. Returns - ``True`` if *fd* was previously being monitored for reads. - -.. method:: loop.add_writer(fd, callback, *args) - - Start monitoring the *fd* file descriptor for write availability and - invoke *callback* with the specified arguments once *fd* is available for - writing. - - Use :func:`functools.partial` :ref:`to pass keyword arguments - ` to *callback*. - -.. method:: loop.remove_writer(fd) - - Stop monitoring the *fd* file descriptor for write availability. Returns - ``True`` if *fd* was previously being monitored for writes. - -See also :ref:`Platform Support ` section -for some limitations of these methods. - - -Working with socket objects directly -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In general, protocol implementations that use transport-based APIs -such as :meth:`loop.create_connection` and :meth:`loop.create_server` -are faster than implementations that work with sockets directly. -However, there are some use cases when performance is not critical, and -working with :class:`~socket.socket` objects directly is more -convenient. - -.. coroutinemethod:: loop.sock_recv(sock, nbytes) - - Receive up to *nbytes* from *sock*. Asynchronous version of - :meth:`socket.recv() `. - - Return the received data as a bytes object. - - *sock* must be a non-blocking socket. - - .. versionchanged:: 3.7 - Even though this method was always documented as a coroutine - method, releases before Python 3.7 returned a :class:`Future`. - Since Python 3.7 this is an ``async def`` method. - -.. coroutinemethod:: loop.sock_recv_into(sock, buf) - - Receive data from *sock* into the *buf* buffer. Modeled after the blocking - :meth:`socket.recv_into() ` method. - - Return the number of bytes written to the buffer. - - *sock* must be a non-blocking socket. - - .. versionadded:: 3.7 - -.. coroutinemethod:: loop.sock_recvfrom(sock, bufsize) - - Receive a datagram of up to *bufsize* from *sock*. Asynchronous version of - :meth:`socket.recvfrom() `. - - Return a tuple of (received data, remote address). - - *sock* must be a non-blocking socket. - - .. versionadded:: 3.11 - -.. coroutinemethod:: loop.sock_recvfrom_into(sock, buf, nbytes=0) - - Receive a datagram of up to *nbytes* from *sock* into *buf*. - Asynchronous version of - :meth:`socket.recvfrom_into() `. - - Return a tuple of (number of bytes received, remote address). - - *sock* must be a non-blocking socket. - - .. versionadded:: 3.11 - -.. coroutinemethod:: loop.sock_sendall(sock, data) - - Send *data* to the *sock* socket. Asynchronous version of - :meth:`socket.sendall() `. - - This method continues to send to the socket until either all data - in *data* has been sent or an error occurs. ``None`` is returned - on success. On error, an exception is raised. Additionally, there is no way - to determine how much data, if any, was successfully processed by the - receiving end of the connection. - - *sock* must be a non-blocking socket. - - .. versionchanged:: 3.7 - Even though the method was always documented as a coroutine - method, before Python 3.7 it returned a :class:`Future`. - Since Python 3.7, this is an ``async def`` method. - -.. coroutinemethod:: loop.sock_sendto(sock, data, address) - - Send a datagram from *sock* to *address*. - Asynchronous version of - :meth:`socket.sendto() `. - - Return the number of bytes sent. - - *sock* must be a non-blocking socket. - - .. versionadded:: 3.11 - -.. coroutinemethod:: loop.sock_connect(sock, address) - - Connect *sock* to a remote socket at *address*. - - Asynchronous version of :meth:`socket.connect() `. - - *sock* must be a non-blocking socket. - - .. versionchanged:: 3.5.2 - ``address`` no longer needs to be resolved. ``sock_connect`` - will try to check if the *address* is already resolved by calling - :func:`socket.inet_pton`. If not, - :meth:`loop.getaddrinfo` will be used to resolve the - *address*. - - .. seealso:: - - :meth:`loop.create_connection` - and :func:`asyncio.open_connection() `. - - -.. coroutinemethod:: loop.sock_accept(sock) - - Accept a connection. Modeled after the blocking - :meth:`socket.accept() ` method. - - The socket must be bound to an address and listening - for connections. The return value is a pair ``(conn, address)`` where *conn* - is a *new* socket object usable to send and receive data on the connection, - and *address* is the address bound to the socket on the other end of the - connection. - - *sock* must be a non-blocking socket. - - .. versionchanged:: 3.7 - Even though the method was always documented as a coroutine - method, before Python 3.7 it returned a :class:`Future`. - Since Python 3.7, this is an ``async def`` method. - - .. seealso:: - - :meth:`loop.create_server` and :func:`start_server`. - -.. coroutinemethod:: loop.sock_sendfile(sock, file, offset=0, count=None, \ - *, fallback=True) - - Send a file using high-performance :mod:`os.sendfile` if possible. - Return the total number of bytes sent. - - Asynchronous version of :meth:`socket.sendfile() `. - - *sock* must be a non-blocking :const:`socket.SOCK_STREAM` - :class:`~socket.socket`. - - *file* must be a regular file object open in binary mode. - - *offset* tells from where to start reading the file. If specified, - *count* is the total number of bytes to transmit as opposed to - sending the file until EOF is reached. File position is always updated, - even when this method raises an error, and - :meth:`file.tell() ` can be used to obtain the actual - number of bytes sent. - - *fallback*, when set to ``True``, makes asyncio manually read and send - the file when the platform does not support the sendfile syscall - (e.g. Windows or SSL socket on Unix). - - Raise :exc:`SendfileNotAvailableError` if the system does not support - *sendfile* syscall and *fallback* is ``False``. - - *sock* must be a non-blocking socket. - - .. versionadded:: 3.7 - - -DNS -^^^ - -.. coroutinemethod:: loop.getaddrinfo(host, port, *, family=0, \ - type=0, proto=0, flags=0) - - Asynchronous version of :meth:`socket.getaddrinfo`. - -.. coroutinemethod:: loop.getnameinfo(sockaddr, flags=0) - - Asynchronous version of :meth:`socket.getnameinfo`. - -.. versionchanged:: 3.7 - Both *getaddrinfo* and *getnameinfo* methods were always documented - to return a coroutine, but prior to Python 3.7 they were, in fact, - returning :class:`asyncio.Future` objects. Starting with Python 3.7 - both methods are coroutines. - - -Working with pipes -^^^^^^^^^^^^^^^^^^ - -.. coroutinemethod:: loop.connect_read_pipe(protocol_factory, pipe) - - Register the read end of *pipe* in the event loop. - - *protocol_factory* must be a callable returning an - :ref:`asyncio protocol ` implementation. - - *pipe* is a :term:`file-like object `. - - Return pair ``(transport, protocol)``, where *transport* supports - the :class:`ReadTransport` interface and *protocol* is an object - instantiated by the *protocol_factory*. - - With :class:`SelectorEventLoop` event loop, the *pipe* is set to - non-blocking mode. - -.. coroutinemethod:: loop.connect_write_pipe(protocol_factory, pipe) - - Register the write end of *pipe* in the event loop. - - *protocol_factory* must be a callable returning an - :ref:`asyncio protocol ` implementation. - - *pipe* is :term:`file-like object `. - - Return pair ``(transport, protocol)``, where *transport* supports - :class:`WriteTransport` interface and *protocol* is an object - instantiated by the *protocol_factory*. - - With :class:`SelectorEventLoop` event loop, the *pipe* is set to - non-blocking mode. - -.. note:: - - :class:`SelectorEventLoop` does not support the above methods on - Windows. Use :class:`ProactorEventLoop` instead for Windows. - -.. seealso:: - - The :meth:`loop.subprocess_exec` and - :meth:`loop.subprocess_shell` methods. - - -Unix signals -^^^^^^^^^^^^ - -.. _loop_add_signal_handler: - -.. method:: loop.add_signal_handler(signum, callback, *args) - - Set *callback* as the handler for the *signum* signal. - - The callback will be invoked by *loop*, along with other queued callbacks - and runnable coroutines of that event loop. Unlike signal handlers - registered using :func:`signal.signal`, a callback registered with this - function is allowed to interact with the event loop. - - Raise :exc:`ValueError` if the signal number is invalid or uncatchable. - Raise :exc:`RuntimeError` if there is a problem setting up the handler. - - Use :func:`functools.partial` :ref:`to pass keyword arguments - ` to *callback*. - - Like :func:`signal.signal`, this function must be invoked in the main - thread. - -.. method:: loop.remove_signal_handler(sig) - - Remove the handler for the *sig* signal. - - Return ``True`` if the signal handler was removed, or ``False`` if - no handler was set for the given signal. - - .. availability:: Unix. - -.. seealso:: - - The :mod:`signal` module. - - -Executing code in thread or process pools -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. awaitablemethod:: loop.run_in_executor(executor, func, *args) - - Arrange for *func* to be called in the specified executor. - - The *executor* argument should be an :class:`concurrent.futures.Executor` - instance. The default executor is used if *executor* is ``None``. - - Example:: - - import asyncio - import concurrent.futures - - def blocking_io(): - # File operations (such as logging) can block the - # event loop: run them in a thread pool. - with open('/dev/urandom', 'rb') as f: - return f.read(100) - - def cpu_bound(): - # CPU-bound operations will block the event loop: - # in general it is preferable to run them in a - # process pool. - return sum(i * i for i in range(10 ** 7)) - - async def main(): - loop = asyncio.get_running_loop() - - ## Options: - - # 1. Run in the default loop's executor: - result = await loop.run_in_executor( - None, blocking_io) - print('default thread pool', result) - - # 2. Run in a custom thread pool: - with concurrent.futures.ThreadPoolExecutor() as pool: - result = await loop.run_in_executor( - pool, blocking_io) - print('custom thread pool', result) - - # 3. Run in a custom process pool: - with concurrent.futures.ProcessPoolExecutor() as pool: - result = await loop.run_in_executor( - pool, cpu_bound) - print('custom process pool', result) - - if __name__ == '__main__': - asyncio.run(main()) - - Note that the entry point guard (``if __name__ == '__main__'``) - is required for option 3 due to the peculiarities of :mod:`multiprocessing`, - which is used by :class:`~concurrent.futures.ProcessPoolExecutor`. - See :ref:`Safe importing of main module `. - - This method returns a :class:`asyncio.Future` object. - - Use :func:`functools.partial` :ref:`to pass keyword arguments - ` to *func*. - - .. versionchanged:: 3.5.3 - :meth:`loop.run_in_executor` no longer configures the - ``max_workers`` of the thread pool executor it creates, instead - leaving it up to the thread pool executor - (:class:`~concurrent.futures.ThreadPoolExecutor`) to set the - default. - -.. method:: loop.set_default_executor(executor) - - Set *executor* as the default executor used by :meth:`run_in_executor`. - *executor* must be an instance of - :class:`~concurrent.futures.ThreadPoolExecutor`. - - .. versionchanged:: 3.11 - *executor* must be an instance of - :class:`~concurrent.futures.ThreadPoolExecutor`. - - -Error Handling API -^^^^^^^^^^^^^^^^^^ - -Allows customizing how exceptions are handled in the event loop. - -.. method:: loop.set_exception_handler(handler) - - Set *handler* as the new event loop exception handler. - - If *handler* is ``None``, the default exception handler will - be set. Otherwise, *handler* must be a callable with the signature - matching ``(loop, context)``, where ``loop`` - is a reference to the active event loop, and ``context`` - is a ``dict`` object containing the details of the exception - (see :meth:`call_exception_handler` documentation for details - about context). - -.. method:: loop.get_exception_handler() - - Return the current exception handler, or ``None`` if no custom - exception handler was set. - - .. versionadded:: 3.5.2 - -.. method:: loop.default_exception_handler(context) - - Default exception handler. - - This is called when an exception occurs and no exception - handler is set. This can be called by a custom exception - handler that wants to defer to the default handler behavior. - - *context* parameter has the same meaning as in - :meth:`call_exception_handler`. - -.. method:: loop.call_exception_handler(context) - - Call the current event loop exception handler. - - *context* is a ``dict`` object containing the following keys - (new keys may be introduced in future Python versions): - - * 'message': Error message; - * 'exception' (optional): Exception object; - * 'future' (optional): :class:`asyncio.Future` instance; - * 'task' (optional): :class:`asyncio.Task` instance; - * 'handle' (optional): :class:`asyncio.Handle` instance; - * 'protocol' (optional): :ref:`Protocol ` instance; - * 'transport' (optional): :ref:`Transport ` instance; - * 'socket' (optional): :class:`socket.socket` instance; - * 'asyncgen' (optional): Asynchronous generator that caused - the exception. - - .. note:: - - This method should not be overloaded in subclassed - event loops. For custom exception handling, use - the :meth:`set_exception_handler()` method. - -Enabling debug mode -^^^^^^^^^^^^^^^^^^^ - -.. method:: loop.get_debug() - - Get the debug mode (:class:`bool`) of the event loop. - - The default value is ``True`` if the environment variable - :envvar:`PYTHONASYNCIODEBUG` is set to a non-empty string, ``False`` - otherwise. - -.. method:: loop.set_debug(enabled: bool) - - Set the debug mode of the event loop. - - .. versionchanged:: 3.7 - - The new :ref:`Python Development Mode ` can now also be used - to enable the debug mode. - -.. attribute:: loop.slow_callback_duration - - This attribute can be used to set the - minimum execution duration in seconds that is considered "slow". - When debug mode is enabled, "slow" callbacks are logged. - - Default value is 100 milliseconds. - -.. seealso:: - - The :ref:`debug mode of asyncio `. - - -Running Subprocesses -^^^^^^^^^^^^^^^^^^^^ - -Methods described in this subsections are low-level. In regular -async/await code consider using the high-level -:func:`asyncio.create_subprocess_shell` and -:func:`asyncio.create_subprocess_exec` convenience functions instead. - -.. note:: - - On Windows, the default event loop :class:`ProactorEventLoop` supports - subprocesses, whereas :class:`SelectorEventLoop` does not. See - :ref:`Subprocess Support on Windows ` for - details. - -.. _loop_subprocess_exec: - -.. coroutinemethod:: loop.subprocess_exec(protocol_factory, *args, \ - stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, **kwargs) - - Create a subprocess from one or more string arguments specified by - *args*. - - *args* must be a list of strings represented by: - - * :class:`str`; - * or :class:`bytes`, encoded to the - :ref:`filesystem encoding `. - - The first string specifies the program executable, - and the remaining strings specify the arguments. Together, string - arguments form the ``argv`` of the program. - - This is similar to the standard library :class:`subprocess.Popen` - class called with ``shell=False`` and the list of strings passed as - the first argument; however, where :class:`~subprocess.Popen` takes - a single argument which is list of strings, *subprocess_exec* - takes multiple string arguments. - - The *protocol_factory* must be a callable returning a subclass of the - :class:`asyncio.SubprocessProtocol` class. - - Other parameters: - - * *stdin* can be any of these: - - * a file-like object representing a pipe to be connected to the - subprocess's standard input stream using - :meth:`~loop.connect_write_pipe` - * the :const:`subprocess.PIPE` constant (default) which will create a new - pipe and connect it, - * the value ``None`` which will make the subprocess inherit the file - descriptor from this process - * the :const:`subprocess.DEVNULL` constant which indicates that the - special :data:`os.devnull` file will be used - - * *stdout* can be any of these: - - * a file-like object representing a pipe to be connected to the - subprocess's standard output stream using - :meth:`~loop.connect_write_pipe` - * the :const:`subprocess.PIPE` constant (default) which will create a new - pipe and connect it, - * the value ``None`` which will make the subprocess inherit the file - descriptor from this process - * the :const:`subprocess.DEVNULL` constant which indicates that the - special :data:`os.devnull` file will be used - - * *stderr* can be any of these: - - * a file-like object representing a pipe to be connected to the - subprocess's standard error stream using - :meth:`~loop.connect_write_pipe` - * the :const:`subprocess.PIPE` constant (default) which will create a new - pipe and connect it, - * the value ``None`` which will make the subprocess inherit the file - descriptor from this process - * the :const:`subprocess.DEVNULL` constant which indicates that the - special :data:`os.devnull` file will be used - * the :const:`subprocess.STDOUT` constant which will connect the standard - error stream to the process' standard output stream - - * All other keyword arguments are passed to :class:`subprocess.Popen` - without interpretation, except for *bufsize*, *universal_newlines*, - *shell*, *text*, *encoding* and *errors*, which should not be specified - at all. - - The ``asyncio`` subprocess API does not support decoding the streams - as text. :func:`bytes.decode` can be used to convert the bytes returned - from the stream to text. - - See the constructor of the :class:`subprocess.Popen` class - for documentation on other arguments. - - Returns a pair of ``(transport, protocol)``, where *transport* - conforms to the :class:`asyncio.SubprocessTransport` base class and - *protocol* is an object instantiated by the *protocol_factory*. - -.. coroutinemethod:: loop.subprocess_shell(protocol_factory, cmd, *, \ - stdin=subprocess.PIPE, stdout=subprocess.PIPE, \ - stderr=subprocess.PIPE, **kwargs) - - Create a subprocess from *cmd*, which can be a :class:`str` or a - :class:`bytes` string encoded to the - :ref:`filesystem encoding `, - using the platform's "shell" syntax. - - This is similar to the standard library :class:`subprocess.Popen` - class called with ``shell=True``. - - The *protocol_factory* must be a callable returning a subclass of the - :class:`SubprocessProtocol` class. - - See :meth:`~loop.subprocess_exec` for more details about - the remaining arguments. - - Returns a pair of ``(transport, protocol)``, where *transport* - conforms to the :class:`SubprocessTransport` base class and - *protocol* is an object instantiated by the *protocol_factory*. - -.. note:: - It is the application's responsibility to ensure that all whitespace - and special characters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to - properly escape whitespace and special characters in strings that - are going to be used to construct shell commands. - - -Callback Handles -================ - -.. class:: Handle - - A callback wrapper object returned by :meth:`loop.call_soon`, - :meth:`loop.call_soon_threadsafe`. - - .. method:: cancel() - - Cancel the callback. If the callback has already been canceled - or executed, this method has no effect. - - .. method:: cancelled() - - Return ``True`` if the callback was cancelled. - - .. versionadded:: 3.7 - -.. class:: TimerHandle - - A callback wrapper object returned by :meth:`loop.call_later`, - and :meth:`loop.call_at`. - - This class is a subclass of :class:`Handle`. - - .. method:: when() - - Return a scheduled callback time as :class:`float` seconds. - - The time is an absolute timestamp, using the same time - reference as :meth:`loop.time`. - - .. versionadded:: 3.7 - - -Server Objects -============== - -Server objects are created by :meth:`loop.create_server`, -:meth:`loop.create_unix_server`, :func:`start_server`, -and :func:`start_unix_server` functions. - -Do not instantiate the :class:`Server` class directly. - -.. class:: Server - - *Server* objects are asynchronous context managers. When used in an - ``async with`` statement, it's guaranteed that the Server object is - closed and not accepting new connections when the ``async with`` - statement is completed:: - - srv = await loop.create_server(...) - - async with srv: - # some code - - # At this point, srv is closed and no longer accepts new connections. - - - .. versionchanged:: 3.7 - Server object is an asynchronous context manager since Python 3.7. - - .. versionchanged:: 3.11 - This class was exposed publicly as ``asyncio.Server`` in Python 3.9.11, 3.10.3 and 3.11. - - .. method:: close() - - Stop serving: close listening sockets and set the :attr:`sockets` - attribute to ``None``. - - The sockets that represent existing incoming client connections - are left open. - - The server is closed asynchronously, use the :meth:`wait_closed` - coroutine to wait until the server is closed. - - .. method:: get_loop() - - Return the event loop associated with the server object. - - .. versionadded:: 3.7 - - .. coroutinemethod:: start_serving() - - Start accepting connections. - - This method is idempotent, so it can be called when - the server is already serving. - - The *start_serving* keyword-only parameter to - :meth:`loop.create_server` and - :meth:`asyncio.start_server` allows creating a Server object - that is not accepting connections initially. In this case - ``Server.start_serving()``, or :meth:`Server.serve_forever` can be used - to make the Server start accepting connections. - - .. versionadded:: 3.7 - - .. coroutinemethod:: serve_forever() - - Start accepting connections until the coroutine is cancelled. - Cancellation of ``serve_forever`` task causes the server - to be closed. - - This method can be called if the server is already accepting - connections. Only one ``serve_forever`` task can exist per - one *Server* object. - - Example:: - - async def client_connected(reader, writer): - # Communicate with the client with - # reader/writer streams. For example: - await reader.readline() - - async def main(host, port): - srv = await asyncio.start_server( - client_connected, host, port) - await srv.serve_forever() - - asyncio.run(main('127.0.0.1', 0)) - - .. versionadded:: 3.7 - - .. method:: is_serving() - - Return ``True`` if the server is accepting new connections. - - .. versionadded:: 3.7 - - .. coroutinemethod:: wait_closed() - - Wait until the :meth:`close` method completes. - - .. attribute:: sockets - - List of socket-like objects, ``asyncio.trsock.TransportSocket``, which - the server is listening on. - - .. versionchanged:: 3.7 - Prior to Python 3.7 ``Server.sockets`` used to return an - internal list of server sockets directly. In 3.7 a copy - of that list is returned. - - -.. _asyncio-event-loops: -.. _asyncio-event-loop-implementations: - -Event Loop Implementations -========================== - -asyncio ships with two different event loop implementations: -:class:`SelectorEventLoop` and :class:`ProactorEventLoop`. - -By default asyncio is configured to use :class:`SelectorEventLoop` -on Unix and :class:`ProactorEventLoop` on Windows. - - -.. class:: SelectorEventLoop - - An event loop based on the :mod:`selectors` module. - - Uses the most efficient *selector* available for the given - platform. It is also possible to manually configure the - exact selector implementation to be used:: - - import asyncio - import selectors - - class MyPolicy(asyncio.DefaultEventLoopPolicy): - def new_event_loop(self): - selector = selectors.SelectSelector() - return asyncio.SelectorEventLoop(selector) - - asyncio.set_event_loop_policy(MyPolicy()) - - - .. availability:: Unix, Windows. - - -.. class:: ProactorEventLoop - - An event loop for Windows that uses "I/O Completion Ports" (IOCP). - - .. availability:: Windows. - - .. seealso:: - - `MSDN documentation on I/O Completion Ports - `_. - - -.. class:: AbstractEventLoop - - Abstract base class for asyncio-compliant event loops. - - The :ref:`asyncio-event-loop-methods` section lists all - methods that an alternative implementation of ``AbstractEventLoop`` - should have defined. - - -Examples -======== - -Note that all examples in this section **purposefully** show how -to use the low-level event loop APIs, such as :meth:`loop.run_forever` -and :meth:`loop.call_soon`. Modern asyncio applications rarely -need to be written this way; consider using the high-level functions -like :func:`asyncio.run`. - - -.. _asyncio_example_lowlevel_helloworld: - -Hello World with call_soon() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -An example using the :meth:`loop.call_soon` method to schedule a -callback. The callback displays ``"Hello World"`` and then stops the -event loop:: - - import asyncio - - def hello_world(loop): - """A callback to print 'Hello World' and stop the event loop""" - print('Hello World') - loop.stop() - - loop = asyncio.new_event_loop() - - # Schedule a call to hello_world() - loop.call_soon(hello_world, loop) - - # Blocking call interrupted by loop.stop() - try: - loop.run_forever() - finally: - loop.close() - -.. seealso:: - - A similar :ref:`Hello World ` - example created with a coroutine and the :func:`run` function. - - -.. _asyncio_example_call_later: - -Display the current date with call_later() -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -An example of a callback displaying the current date every second. The -callback uses the :meth:`loop.call_later` method to reschedule itself -after 5 seconds, and then stops the event loop:: - - import asyncio - import datetime - - def display_date(end_time, loop): - print(datetime.datetime.now()) - if (loop.time() + 1.0) < end_time: - loop.call_later(1, display_date, end_time, loop) - else: - loop.stop() - - loop = asyncio.new_event_loop() - - # Schedule the first call to display_date() - end_time = loop.time() + 5.0 - loop.call_soon(display_date, end_time, loop) - - # Blocking call interrupted by loop.stop() - try: - loop.run_forever() - finally: - loop.close() - -.. seealso:: - - A similar :ref:`current date ` example - created with a coroutine and the :func:`run` function. - - -.. _asyncio_example_watch_fd: - -Watch a file descriptor for read events -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Wait until a file descriptor received some data using the -:meth:`loop.add_reader` method and then close the event loop:: - - import asyncio - from socket import socketpair - - # Create a pair of connected file descriptors - rsock, wsock = socketpair() - - loop = asyncio.new_event_loop() - - def reader(): - data = rsock.recv(100) - print("Received:", data.decode()) - - # We are done: unregister the file descriptor - loop.remove_reader(rsock) - - # Stop the event loop - loop.stop() - - # Register the file descriptor for read event - loop.add_reader(rsock, reader) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) - - try: - # Run the event loop - loop.run_forever() - finally: - # We are done. Close sockets and the event loop. - rsock.close() - wsock.close() - loop.close() - -.. seealso:: - - * A similar :ref:`example ` - using transports, protocols, and the - :meth:`loop.create_connection` method. - - * Another similar :ref:`example ` - using the high-level :func:`asyncio.open_connection` function - and streams. - - -.. _asyncio_example_unix_signals: - -Set signal handlers for SIGINT and SIGTERM -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -(This ``signals`` example only works on Unix.) - -Register handlers for signals :const:`~signal.SIGINT` and :const:`~signal.SIGTERM` -using the :meth:`loop.add_signal_handler` method:: - - import asyncio - import functools - import os - import signal - - def ask_exit(signame, loop): - print("got signal %s: exit" % signame) - loop.stop() - - async def main(): - loop = asyncio.get_running_loop() - - for signame in {'SIGINT', 'SIGTERM'}: - loop.add_signal_handler( - getattr(signal, signame), - functools.partial(ask_exit, signame, loop)) - - await asyncio.sleep(3600) - - print("Event loop running for 1 hour, press Ctrl+C to interrupt.") - print(f"pid {os.getpid()}: send SIGINT or SIGTERM to exit.") - - asyncio.run(main()) diff --git a/Doc/library/asyncio-exceptions.rst b/Doc/library/asyncio-exceptions.rst deleted file mode 100644 index 7ad9103ca3fdfc..00000000000000 --- a/Doc/library/asyncio-exceptions.rst +++ /dev/null @@ -1,78 +0,0 @@ -.. currentmodule:: asyncio - - -.. _asyncio-exceptions: - -========== -Exceptions -========== - -**Source code:** :source:`Lib/asyncio/exceptions.py` - ----------------------------------------------------- - -.. exception:: TimeoutError - - A deprecated alias of :exc:`TimeoutError`, - raised when the operation has exceeded the given deadline. - - .. versionchanged:: 3.11 - - This class was made an alias of :exc:`TimeoutError`. - - -.. exception:: CancelledError - - The operation has been cancelled. - - This exception can be caught to perform custom operations - when asyncio Tasks are cancelled. In almost all situations the - exception must be re-raised. - - .. versionchanged:: 3.8 - - :exc:`CancelledError` is now a subclass of :class:`BaseException` rather than :class:`Exception`. - - -.. exception:: InvalidStateError - - Invalid internal state of :class:`Task` or :class:`Future`. - - Can be raised in situations like setting a result value for a - *Future* object that already has a result value set. - - -.. exception:: SendfileNotAvailableError - - The "sendfile" syscall is not available for the given - socket or file type. - - A subclass of :exc:`RuntimeError`. - - -.. exception:: IncompleteReadError - - The requested read operation did not complete fully. - - Raised by the :ref:`asyncio stream APIs`. - - This exception is a subclass of :exc:`EOFError`. - - .. attribute:: expected - - The total number (:class:`int`) of expected bytes. - - .. attribute:: partial - - A string of :class:`bytes` read before the end of stream was reached. - - -.. exception:: LimitOverrunError - - Reached the buffer size limit while looking for a separator. - - Raised by the :ref:`asyncio stream APIs `. - - .. attribute:: consumed - - The total number of to be consumed bytes. diff --git a/Doc/library/asyncio-future.rst b/Doc/library/asyncio-future.rst deleted file mode 100644 index 893ae5518f757d..00000000000000 --- a/Doc/library/asyncio-future.rst +++ /dev/null @@ -1,279 +0,0 @@ -.. currentmodule:: asyncio - - -.. _asyncio-futures: - -======= -Futures -======= - -**Source code:** :source:`Lib/asyncio/futures.py`, -:source:`Lib/asyncio/base_futures.py` - -------------------------------------- - -*Future* objects are used to bridge **low-level callback-based code** -with high-level async/await code. - - -Future Functions -================ - -.. function:: isfuture(obj) - - Return ``True`` if *obj* is either of: - - * an instance of :class:`asyncio.Future`, - * an instance of :class:`asyncio.Task`, - * a Future-like object with a ``_asyncio_future_blocking`` - attribute. - - .. versionadded:: 3.5 - - -.. function:: ensure_future(obj, *, loop=None) - - Return: - - * *obj* argument as is, if *obj* is a :class:`Future`, - a :class:`Task`, or a Future-like object (:func:`isfuture` - is used for the test.) - - * a :class:`Task` object wrapping *obj*, if *obj* is a - coroutine (:func:`iscoroutine` is used for the test); - in this case the coroutine will be scheduled by - ``ensure_future()``. - - * a :class:`Task` object that would await on *obj*, if *obj* is an - awaitable (:func:`inspect.isawaitable` is used for the test.) - - If *obj* is neither of the above a :exc:`TypeError` is raised. - - .. important:: - - See also the :func:`create_task` function which is the - preferred way for creating new Tasks. - - Save a reference to the result of this function, to avoid - a task disappearing mid-execution. - - .. versionchanged:: 3.5.1 - The function accepts any :term:`awaitable` object. - - .. deprecated:: 3.10 - Deprecation warning is emitted if *obj* is not a Future-like object - and *loop* is not specified and there is no running event loop. - - -.. function:: wrap_future(future, *, loop=None) - - Wrap a :class:`concurrent.futures.Future` object in a - :class:`asyncio.Future` object. - - .. deprecated:: 3.10 - Deprecation warning is emitted if *future* is not a Future-like object - and *loop* is not specified and there is no running event loop. - - -Future Object -============= - -.. class:: Future(*, loop=None) - - A Future represents an eventual result of an asynchronous - operation. Not thread-safe. - - Future is an :term:`awaitable` object. Coroutines can await on - Future objects until they either have a result or an exception - set, or until they are cancelled. A Future can be awaited multiple - times and the result is same. - - Typically Futures are used to enable low-level - callback-based code (e.g. in protocols implemented using asyncio - :ref:`transports `) - to interoperate with high-level async/await code. - - The rule of thumb is to never expose Future objects in user-facing - APIs, and the recommended way to create a Future object is to call - :meth:`loop.create_future`. This way alternative event loop - implementations can inject their own optimized implementations - of a Future object. - - .. versionchanged:: 3.7 - Added support for the :mod:`contextvars` module. - - .. deprecated:: 3.10 - Deprecation warning is emitted if *loop* is not specified - and there is no running event loop. - - .. method:: result() - - Return the result of the Future. - - If the Future is *done* and has a result set by the - :meth:`set_result` method, the result value is returned. - - If the Future is *done* and has an exception set by the - :meth:`set_exception` method, this method raises the exception. - - If the Future has been *cancelled*, this method raises - a :exc:`CancelledError` exception. - - If the Future's result isn't yet available, this method raises - a :exc:`InvalidStateError` exception. - - .. method:: set_result(result) - - Mark the Future as *done* and set its result. - - Raises a :exc:`InvalidStateError` error if the Future is - already *done*. - - .. method:: set_exception(exception) - - Mark the Future as *done* and set an exception. - - Raises a :exc:`InvalidStateError` error if the Future is - already *done*. - - .. method:: done() - - Return ``True`` if the Future is *done*. - - A Future is *done* if it was *cancelled* or if it has a result - or an exception set with :meth:`set_result` or - :meth:`set_exception` calls. - - .. method:: cancelled() - - Return ``True`` if the Future was *cancelled*. - - The method is usually used to check if a Future is not - *cancelled* before setting a result or an exception for it:: - - if not fut.cancelled(): - fut.set_result(42) - - .. method:: add_done_callback(callback, *, context=None) - - Add a callback to be run when the Future is *done*. - - The *callback* is called with the Future object as its only - argument. - - If the Future is already *done* when this method is called, - the callback is scheduled with :meth:`loop.call_soon`. - - An optional keyword-only *context* argument allows specifying a - custom :class:`contextvars.Context` for the *callback* to run in. - The current context is used when no *context* is provided. - - :func:`functools.partial` can be used to pass parameters - to the callback, e.g.:: - - # Call 'print("Future:", fut)' when "fut" is done. - fut.add_done_callback( - functools.partial(print, "Future:")) - - .. versionchanged:: 3.7 - The *context* keyword-only parameter was added. - See :pep:`567` for more details. - - .. method:: remove_done_callback(callback) - - Remove *callback* from the callbacks list. - - Returns the number of callbacks removed, which is typically 1, - unless a callback was added more than once. - - .. method:: cancel(msg=None) - - Cancel the Future and schedule callbacks. - - If the Future is already *done* or *cancelled*, return ``False``. - Otherwise, change the Future's state to *cancelled*, - schedule the callbacks, and return ``True``. - - .. versionchanged:: 3.9 - Added the *msg* parameter. - - .. method:: exception() - - Return the exception that was set on this Future. - - The exception (or ``None`` if no exception was set) is - returned only if the Future is *done*. - - If the Future has been *cancelled*, this method raises a - :exc:`CancelledError` exception. - - If the Future isn't *done* yet, this method raises an - :exc:`InvalidStateError` exception. - - .. method:: get_loop() - - Return the event loop the Future object is bound to. - - .. versionadded:: 3.7 - - -.. _asyncio_example_future: - -This example creates a Future object, creates and schedules an -asynchronous Task to set result for the Future, and waits until -the Future has a result:: - - async def set_after(fut, delay, value): - # Sleep for *delay* seconds. - await asyncio.sleep(delay) - - # Set *value* as a result of *fut* Future. - fut.set_result(value) - - async def main(): - # Get the current event loop. - loop = asyncio.get_running_loop() - - # Create a new Future object. - fut = loop.create_future() - - # Run "set_after()" coroutine in a parallel Task. - # We are using the low-level "loop.create_task()" API here because - # we already have a reference to the event loop at hand. - # Otherwise we could have just used "asyncio.create_task()". - loop.create_task( - set_after(fut, 1, '... world')) - - print('hello ...') - - # Wait until *fut* has a result (1 second) and print it. - print(await fut) - - asyncio.run(main()) - - -.. important:: - - The Future object was designed to mimic - :class:`concurrent.futures.Future`. Key differences include: - - - unlike asyncio Futures, :class:`concurrent.futures.Future` - instances cannot be awaited. - - - :meth:`asyncio.Future.result` and :meth:`asyncio.Future.exception` - do not accept the *timeout* argument. - - - :meth:`asyncio.Future.result` and :meth:`asyncio.Future.exception` - raise an :exc:`InvalidStateError` exception when the Future is not - *done*. - - - Callbacks registered with :meth:`asyncio.Future.add_done_callback` - are not called immediately. They are scheduled with - :meth:`loop.call_soon` instead. - - - asyncio Future is not compatible with the - :func:`concurrent.futures.wait` and - :func:`concurrent.futures.as_completed` functions. - - - :meth:`asyncio.Future.cancel` accepts an optional ``msg`` argument, - but :meth:`concurrent.futures.Future.cancel` does not. diff --git a/Doc/library/asyncio-llapi-index.rst b/Doc/library/asyncio-llapi-index.rst deleted file mode 100644 index 67136ba69ec875..00000000000000 --- a/Doc/library/asyncio-llapi-index.rst +++ /dev/null @@ -1,523 +0,0 @@ -.. currentmodule:: asyncio - - -=================== -Low-level API Index -=================== - -This page lists all low-level asyncio APIs. - - -Obtaining the Event Loop -======================== - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :func:`asyncio.get_running_loop` - - The **preferred** function to get the running event loop. - - * - :func:`asyncio.get_event_loop` - - Get an event loop instance (running or current via the current policy). - - * - :func:`asyncio.set_event_loop` - - Set the event loop as current via the current policy. - - * - :func:`asyncio.new_event_loop` - - Create a new event loop. - - -.. rubric:: Examples - -* :ref:`Using asyncio.get_running_loop() `. - - -Event Loop Methods -================== - -See also the main documentation section about the -:ref:`asyncio-event-loop-methods`. - -.. rubric:: Lifecycle -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`loop.run_until_complete` - - Run a Future/Task/awaitable until complete. - - * - :meth:`loop.run_forever` - - Run the event loop forever. - - * - :meth:`loop.stop` - - Stop the event loop. - - * - :meth:`loop.close` - - Close the event loop. - - * - :meth:`loop.is_running()` - - Return ``True`` if the event loop is running. - - * - :meth:`loop.is_closed()` - - Return ``True`` if the event loop is closed. - - * - ``await`` :meth:`loop.shutdown_asyncgens` - - Close asynchronous generators. - - -.. rubric:: Debugging -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`loop.set_debug` - - Enable or disable the debug mode. - - * - :meth:`loop.get_debug` - - Get the current debug mode. - - -.. rubric:: Scheduling Callbacks -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`loop.call_soon` - - Invoke a callback soon. - - * - :meth:`loop.call_soon_threadsafe` - - A thread-safe variant of :meth:`loop.call_soon`. - - * - :meth:`loop.call_later` - - Invoke a callback *after* the given time. - - * - :meth:`loop.call_at` - - Invoke a callback *at* the given time. - - -.. rubric:: Thread/Process Pool -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :meth:`loop.run_in_executor` - - Run a CPU-bound or other blocking function in - a :mod:`concurrent.futures` executor. - - * - :meth:`loop.set_default_executor` - - Set the default executor for :meth:`loop.run_in_executor`. - - -.. rubric:: Tasks and Futures -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`loop.create_future` - - Create a :class:`Future` object. - - * - :meth:`loop.create_task` - - Schedule coroutine as a :class:`Task`. - - * - :meth:`loop.set_task_factory` - - Set a factory used by :meth:`loop.create_task` to - create :class:`Tasks `. - - * - :meth:`loop.get_task_factory` - - Get the factory :meth:`loop.create_task` uses - to create :class:`Tasks `. - - -.. rubric:: DNS -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :meth:`loop.getaddrinfo` - - Asynchronous version of :meth:`socket.getaddrinfo`. - - * - ``await`` :meth:`loop.getnameinfo` - - Asynchronous version of :meth:`socket.getnameinfo`. - - -.. rubric:: Networking and IPC -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :meth:`loop.create_connection` - - Open a TCP connection. - - * - ``await`` :meth:`loop.create_server` - - Create a TCP server. - - * - ``await`` :meth:`loop.create_unix_connection` - - Open a Unix socket connection. - - * - ``await`` :meth:`loop.create_unix_server` - - Create a Unix socket server. - - * - ``await`` :meth:`loop.connect_accepted_socket` - - Wrap a :class:`~socket.socket` into a ``(transport, protocol)`` - pair. - - * - ``await`` :meth:`loop.create_datagram_endpoint` - - Open a datagram (UDP) connection. - - * - ``await`` :meth:`loop.sendfile` - - Send a file over a transport. - - * - ``await`` :meth:`loop.start_tls` - - Upgrade an existing connection to TLS. - - * - ``await`` :meth:`loop.connect_read_pipe` - - Wrap a read end of a pipe into a ``(transport, protocol)`` pair. - - * - ``await`` :meth:`loop.connect_write_pipe` - - Wrap a write end of a pipe into a ``(transport, protocol)`` pair. - - -.. rubric:: Sockets -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``await`` :meth:`loop.sock_recv` - - Receive data from the :class:`~socket.socket`. - - * - ``await`` :meth:`loop.sock_recv_into` - - Receive data from the :class:`~socket.socket` into a buffer. - - * - ``await`` :meth:`loop.sock_recvfrom` - - Receive a datagram from the :class:`~socket.socket`. - - * - ``await`` :meth:`loop.sock_recvfrom_into` - - Receive a datagram from the :class:`~socket.socket` into a buffer. - - * - ``await`` :meth:`loop.sock_sendall` - - Send data to the :class:`~socket.socket`. - - * - ``await`` :meth:`loop.sock_sendto` - - Send a datagram via the :class:`~socket.socket` to the given address. - - * - ``await`` :meth:`loop.sock_connect` - - Connect the :class:`~socket.socket`. - - * - ``await`` :meth:`loop.sock_accept` - - Accept a :class:`~socket.socket` connection. - - * - ``await`` :meth:`loop.sock_sendfile` - - Send a file over the :class:`~socket.socket`. - - * - :meth:`loop.add_reader` - - Start watching a file descriptor for read availability. - - * - :meth:`loop.remove_reader` - - Stop watching a file descriptor for read availability. - - * - :meth:`loop.add_writer` - - Start watching a file descriptor for write availability. - - * - :meth:`loop.remove_writer` - - Stop watching a file descriptor for write availability. - - -.. rubric:: Unix Signals -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`loop.add_signal_handler` - - Add a handler for a :mod:`signal`. - - * - :meth:`loop.remove_signal_handler` - - Remove a handler for a :mod:`signal`. - - -.. rubric:: Subprocesses -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`loop.subprocess_exec` - - Spawn a subprocess. - - * - :meth:`loop.subprocess_shell` - - Spawn a subprocess from a shell command. - - -.. rubric:: Error Handling -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`loop.call_exception_handler` - - Call the exception handler. - - * - :meth:`loop.set_exception_handler` - - Set a new exception handler. - - * - :meth:`loop.get_exception_handler` - - Get the current exception handler. - - * - :meth:`loop.default_exception_handler` - - The default exception handler implementation. - - -.. rubric:: Examples - -* :ref:`Using asyncio.new_event_loop() and loop.run_forever() - `. - -* :ref:`Using loop.call_later() `. - -* Using ``loop.create_connection()`` to implement - :ref:`an echo-client `. - -* Using ``loop.create_connection()`` to - :ref:`connect a socket `. - -* :ref:`Using add_reader() to watch an FD for read events - `. - -* :ref:`Using loop.add_signal_handler() `. - -* :ref:`Using loop.subprocess_exec() `. - - -Transports -========== - -All transports implement the following methods: - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`transport.close() ` - - Close the transport. - - * - :meth:`transport.is_closing() ` - - Return ``True`` if the transport is closing or is closed. - - * - :meth:`transport.get_extra_info() ` - - Request for information about the transport. - - * - :meth:`transport.set_protocol() ` - - Set a new protocol. - - * - :meth:`transport.get_protocol() ` - - Return the current protocol. - - -Transports that can receive data (TCP and Unix connections, -pipes, etc). Returned from methods like -:meth:`loop.create_connection`, :meth:`loop.create_unix_connection`, -:meth:`loop.connect_read_pipe`, etc: - -.. rubric:: Read Transports -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`transport.is_reading() ` - - Return ``True`` if the transport is receiving. - - * - :meth:`transport.pause_reading() ` - - Pause receiving. - - * - :meth:`transport.resume_reading() ` - - Resume receiving. - - -Transports that can Send data (TCP and Unix connections, -pipes, etc). Returned from methods like -:meth:`loop.create_connection`, :meth:`loop.create_unix_connection`, -:meth:`loop.connect_write_pipe`, etc: - -.. rubric:: Write Transports -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`transport.write() ` - - Write data to the transport. - - * - :meth:`transport.writelines() ` - - Write buffers to the transport. - - * - :meth:`transport.can_write_eof() ` - - Return :const:`True` if the transport supports sending EOF. - - * - :meth:`transport.write_eof() ` - - Close and send EOF after flushing buffered data. - - * - :meth:`transport.abort() ` - - Close the transport immediately. - - * - :meth:`transport.get_write_buffer_size() - ` - - Return the current size of the output buffer. - - * - :meth:`transport.get_write_buffer_limits() - ` - - Return high and low water marks for write flow control. - - * - :meth:`transport.set_write_buffer_limits() - ` - - Set new high and low water marks for write flow control. - - -Transports returned by :meth:`loop.create_datagram_endpoint`: - -.. rubric:: Datagram Transports -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`transport.sendto() ` - - Send data to the remote peer. - - * - :meth:`transport.abort() ` - - Close the transport immediately. - - -Low-level transport abstraction over subprocesses. -Returned by :meth:`loop.subprocess_exec` and -:meth:`loop.subprocess_shell`: - -.. rubric:: Subprocess Transports -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`transport.get_pid() ` - - Return the subprocess process id. - - * - :meth:`transport.get_pipe_transport() - ` - - Return the transport for the requested communication pipe - (*stdin*, *stdout*, or *stderr*). - - * - :meth:`transport.get_returncode() ` - - Return the subprocess return code. - - * - :meth:`transport.kill() ` - - Kill the subprocess. - - * - :meth:`transport.send_signal() ` - - Send a signal to the subprocess. - - * - :meth:`transport.terminate() ` - - Stop the subprocess. - - * - :meth:`transport.close() ` - - Kill the subprocess and close all pipes. - - -Protocols -========= - -Protocol classes can implement the following **callback methods**: - -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``callback`` :meth:`connection_made() ` - - Called when a connection is made. - - * - ``callback`` :meth:`connection_lost() ` - - Called when the connection is lost or closed. - - * - ``callback`` :meth:`pause_writing() ` - - Called when the transport's buffer goes over the high water mark. - - * - ``callback`` :meth:`resume_writing() ` - - Called when the transport's buffer drains below the low water mark. - - -.. rubric:: Streaming Protocols (TCP, Unix Sockets, Pipes) -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``callback`` :meth:`data_received() ` - - Called when some data is received. - - * - ``callback`` :meth:`eof_received() ` - - Called when an EOF is received. - - -.. rubric:: Buffered Streaming Protocols -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``callback`` :meth:`get_buffer() ` - - Called to allocate a new receive buffer. - - * - ``callback`` :meth:`buffer_updated() ` - - Called when the buffer was updated with the received data. - - * - ``callback`` :meth:`eof_received() ` - - Called when an EOF is received. - - -.. rubric:: Datagram Protocols -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``callback`` :meth:`datagram_received() - ` - - Called when a datagram is received. - - * - ``callback`` :meth:`error_received() ` - - Called when a previous send or receive operation raises an - :class:`OSError`. - - -.. rubric:: Subprocess Protocols -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - ``callback`` :meth:`~SubprocessProtocol.pipe_data_received` - - Called when the child process writes data into its - *stdout* or *stderr* pipe. - - * - ``callback`` :meth:`~SubprocessProtocol.pipe_connection_lost` - - Called when one of the pipes communicating with - the child process is closed. - - * - ``callback`` :meth:`process_exited() - ` - - Called when the child process has exited. It can be called before - :meth:`~SubprocessProtocol.pipe_data_received` and - :meth:`~SubprocessProtocol.pipe_connection_lost` methods. - - -Event Loop Policies -=================== - -Policies is a low-level mechanism to alter the behavior of -functions like :func:`asyncio.get_event_loop`. See also -the main :ref:`policies section ` for more -details. - - -.. rubric:: Accessing Policies -.. list-table:: - :widths: 50 50 - :class: full-width-table - - * - :meth:`asyncio.get_event_loop_policy` - - Return the current process-wide policy. - - * - :meth:`asyncio.set_event_loop_policy` - - Set a new process-wide policy. - - * - :class:`AbstractEventLoopPolicy` - - Base class for policy objects. diff --git a/Doc/library/asyncio-platforms.rst b/Doc/library/asyncio-platforms.rst deleted file mode 100644 index 19ec726c1be060..00000000000000 --- a/Doc/library/asyncio-platforms.rst +++ /dev/null @@ -1,105 +0,0 @@ -.. currentmodule:: asyncio - - -.. _asyncio-platform-support: - - -================ -Platform Support -================ - -The :mod:`asyncio` module is designed to be portable, -but some platforms have subtle differences and limitations -due to the platforms' underlying architecture and capabilities. - - -All Platforms -============= - -* :meth:`loop.add_reader` and :meth:`loop.add_writer` - cannot be used to monitor file I/O. - - -Windows -======= - -**Source code:** :source:`Lib/asyncio/proactor_events.py`, -:source:`Lib/asyncio/windows_events.py`, -:source:`Lib/asyncio/windows_utils.py` - --------------------------------------- - -.. versionchanged:: 3.8 - - On Windows, :class:`ProactorEventLoop` is now the default event loop. - -All event loops on Windows do not support the following methods: - -* :meth:`loop.create_unix_connection` and - :meth:`loop.create_unix_server` are not supported. - The :const:`socket.AF_UNIX` socket family is specific to Unix. - -* :meth:`loop.add_signal_handler` and - :meth:`loop.remove_signal_handler` are not supported. - -:class:`SelectorEventLoop` has the following limitations: - -* :class:`~selectors.SelectSelector` is used to wait on socket events: - it supports sockets and is limited to 512 sockets. - -* :meth:`loop.add_reader` and :meth:`loop.add_writer` only accept - socket handles (e.g. pipe file descriptors are not supported). - -* Pipes are not supported, so the :meth:`loop.connect_read_pipe` - and :meth:`loop.connect_write_pipe` methods are not implemented. - -* :ref:`Subprocesses ` are not supported, i.e. - :meth:`loop.subprocess_exec` and :meth:`loop.subprocess_shell` - methods are not implemented. - -:class:`ProactorEventLoop` has the following limitations: - -* The :meth:`loop.add_reader` and :meth:`loop.add_writer` - methods are not supported. - -The resolution of the monotonic clock on Windows is usually around 15.6 -milliseconds. The best resolution is 0.5 milliseconds. The resolution depends on the -hardware (availability of `HPET -`_) and on the -Windows configuration. - - -.. _asyncio-windows-subprocess: - -Subprocess Support on Windows ------------------------------ - -On Windows, the default event loop :class:`ProactorEventLoop` supports -subprocesses, whereas :class:`SelectorEventLoop` does not. - -The :meth:`policy.set_child_watcher() -` function is also -not supported, as :class:`ProactorEventLoop` has a different mechanism -to watch child processes. - - -macOS -===== - -Modern macOS versions are fully supported. - -.. rubric:: macOS <= 10.8 - -On macOS 10.6, 10.7 and 10.8, the default event loop -uses :class:`selectors.KqueueSelector`, which does not support -character devices on these versions. The :class:`SelectorEventLoop` -can be manually configured to use :class:`~selectors.SelectSelector` -or :class:`~selectors.PollSelector` to support character devices on -these older versions of macOS. Example:: - - import asyncio - import selectors - - selector = selectors.SelectSelector() - loop = asyncio.SelectorEventLoop(selector) - asyncio.set_event_loop(loop) diff --git a/Doc/library/asyncio-policy.rst b/Doc/library/asyncio-policy.rst deleted file mode 100644 index f846f76ca095f4..00000000000000 --- a/Doc/library/asyncio-policy.rst +++ /dev/null @@ -1,311 +0,0 @@ -.. currentmodule:: asyncio - - -.. _asyncio-policies: - -======== -Policies -======== - -An event loop policy is a global object -used to get and set the current :ref:`event loop `, -as well as create new event loops. -The default policy can be :ref:`replaced ` with -:ref:`built-in alternatives ` -to use different event loop implementations, -or substituted by a :ref:`custom policy ` -that can override these behaviors. - -The :ref:`policy object ` -gets and sets a separate event loop per *context*. -This is per-thread by default, -though custom policies could define *context* differently. - -Custom event loop policies can control the behavior of -:func:`get_event_loop`, :func:`set_event_loop`, and :func:`new_event_loop`. - -Policy objects should implement the APIs defined -in the :class:`AbstractEventLoopPolicy` abstract base class. - - -.. _asyncio-policy-get-set: - -Getting and Setting the Policy -============================== - -The following functions can be used to get and set the policy -for the current process: - -.. function:: get_event_loop_policy() - - Return the current process-wide policy. - -.. function:: set_event_loop_policy(policy) - - Set the current process-wide policy to *policy*. - - If *policy* is set to ``None``, the default policy is restored. - - -.. _asyncio-policy-objects: - -Policy Objects -============== - -The abstract event loop policy base class is defined as follows: - -.. class:: AbstractEventLoopPolicy - - An abstract base class for asyncio policies. - - .. method:: get_event_loop() - - Get the event loop for the current context. - - Return an event loop object implementing the - :class:`AbstractEventLoop` interface. - - This method should never return ``None``. - - .. versionchanged:: 3.6 - - .. method:: set_event_loop(loop) - - Set the event loop for the current context to *loop*. - - .. method:: new_event_loop() - - Create and return a new event loop object. - - This method should never return ``None``. - - .. method:: get_child_watcher() - - Get a child process watcher object. - - Return a watcher object implementing the - :class:`AbstractChildWatcher` interface. - - This function is Unix specific. - - .. method:: set_child_watcher(watcher) - - Set the current child process watcher to *watcher*. - - This function is Unix specific. - - -.. _asyncio-policy-builtin: - -asyncio ships with the following built-in policies: - - -.. class:: DefaultEventLoopPolicy - - The default asyncio policy. Uses :class:`SelectorEventLoop` - on Unix and :class:`ProactorEventLoop` on Windows. - - There is no need to install the default policy manually. asyncio - is configured to use the default policy automatically. - - .. versionchanged:: 3.8 - - On Windows, :class:`ProactorEventLoop` is now used by default. - - .. note:: - In Python versions 3.10.9, 3.11.1 and 3.12 the :meth:`get_event_loop` - method of the default asyncio policy emits a :exc:`DeprecationWarning` - if there is no running event loop and no current loop is set. - In some future Python release this will become an error. - - -.. class:: WindowsSelectorEventLoopPolicy - - An alternative event loop policy that uses the - :class:`SelectorEventLoop` event loop implementation. - - .. availability:: Windows. - - -.. class:: WindowsProactorEventLoopPolicy - - An alternative event loop policy that uses the - :class:`ProactorEventLoop` event loop implementation. - - .. availability:: Windows. - - -.. _asyncio-watchers: - -Process Watchers -================ - -A process watcher allows customization of how an event loop monitors -child processes on Unix. Specifically, the event loop needs to know -when a child process has exited. - -In asyncio, child processes are created with -:func:`create_subprocess_exec` and :meth:`loop.subprocess_exec` -functions. - -asyncio defines the :class:`AbstractChildWatcher` abstract base class, which child -watchers should implement, and has four different implementations: -:class:`ThreadedChildWatcher` (configured to be used by default), -:class:`MultiLoopChildWatcher`, :class:`SafeChildWatcher`, and -:class:`FastChildWatcher`. - -See also the :ref:`Subprocess and Threads ` -section. - -The following two functions can be used to customize the child process watcher -implementation used by the asyncio event loop: - -.. function:: get_child_watcher() - - Return the current child watcher for the current policy. - -.. function:: set_child_watcher(watcher) - - Set the current child watcher to *watcher* for the current - policy. *watcher* must implement methods defined in the - :class:`AbstractChildWatcher` base class. - -.. note:: - Third-party event loops implementations might not support - custom child watchers. For such event loops, using - :func:`set_child_watcher` might be prohibited or have no effect. - -.. class:: AbstractChildWatcher - - .. method:: add_child_handler(pid, callback, *args) - - Register a new child handler. - - Arrange for ``callback(pid, returncode, *args)`` to be called - when a process with PID equal to *pid* terminates. Specifying - another callback for the same process replaces the previous - handler. - - The *callback* callable must be thread-safe. - - .. method:: remove_child_handler(pid) - - Removes the handler for process with PID equal to *pid*. - - The function returns ``True`` if the handler was successfully - removed, ``False`` if there was nothing to remove. - - .. method:: attach_loop(loop) - - Attach the watcher to an event loop. - - If the watcher was previously attached to an event loop, then - it is first detached before attaching to the new loop. - - Note: loop may be ``None``. - - .. method:: is_active() - - Return ``True`` if the watcher is ready to use. - - Spawning a subprocess with *inactive* current child watcher raises - :exc:`RuntimeError`. - - .. versionadded:: 3.8 - - .. method:: close() - - Close the watcher. - - This method has to be called to ensure that underlying - resources are cleaned-up. - -.. class:: ThreadedChildWatcher - - This implementation starts a new waiting thread for every subprocess spawn. - - It works reliably even when the asyncio event loop is run in a non-main OS thread. - - There is no noticeable overhead when handling a big number of children (*O(1)* each - time a child terminates), but starting a thread per process requires extra memory. - - This watcher is used by default. - - .. versionadded:: 3.8 - -.. class:: MultiLoopChildWatcher - - This implementation registers a :py:data:`SIGCHLD` signal handler on - instantiation. That can break third-party code that installs a custom handler for - :py:data:`SIGCHLD` signal. - - The watcher avoids disrupting other code spawning processes - by polling every process explicitly on a :py:data:`SIGCHLD` signal. - - There is no limitation for running subprocesses from different threads once the - watcher is installed. - - The solution is safe but it has a significant overhead when - handling a big number of processes (*O(n)* each time a - :py:data:`SIGCHLD` is received). - - .. versionadded:: 3.8 - -.. class:: SafeChildWatcher - - This implementation uses active event loop from the main thread to handle - :py:data:`SIGCHLD` signal. If the main thread has no running event loop another - thread cannot spawn a subprocess (:exc:`RuntimeError` is raised). - - The watcher avoids disrupting other code spawning processes - by polling every process explicitly on a :py:data:`SIGCHLD` signal. - - This solution is as safe as :class:`MultiLoopChildWatcher` and has the same *O(N)* - complexity but requires a running event loop in the main thread to work. - -.. class:: FastChildWatcher - - This implementation reaps every terminated processes by calling - ``os.waitpid(-1)`` directly, possibly breaking other code spawning - processes and waiting for their termination. - - There is no noticeable overhead when handling a big number of - children (*O(1)* each time a child terminates). - - This solution requires a running event loop in the main thread to work, as - :class:`SafeChildWatcher`. - -.. class:: PidfdChildWatcher - - This implementation polls process file descriptors (pidfds) to await child - process termination. In some respects, :class:`PidfdChildWatcher` is a - "Goldilocks" child watcher implementation. It doesn't require signals or - threads, doesn't interfere with any processes launched outside the event - loop, and scales linearly with the number of subprocesses launched by the - event loop. The main disadvantage is that pidfds are specific to Linux, and - only work on recent (5.3+) kernels. - - .. versionadded:: 3.9 - - -.. _asyncio-custom-policies: - -Custom Policies -=============== - -To implement a new event loop policy, it is recommended to subclass -:class:`DefaultEventLoopPolicy` and override the methods for which -custom behavior is wanted, e.g.:: - - class MyEventLoopPolicy(asyncio.DefaultEventLoopPolicy): - - def get_event_loop(self): - """Get the event loop. - - This may be None or an instance of EventLoop. - """ - loop = super().get_event_loop() - # Do something with loop ... - return loop - - asyncio.set_event_loop_policy(MyEventLoopPolicy()) diff --git a/Doc/library/asyncio-protocol.rst b/Doc/library/asyncio-protocol.rst deleted file mode 100644 index 48fa02937b5237..00000000000000 --- a/Doc/library/asyncio-protocol.rst +++ /dev/null @@ -1,1061 +0,0 @@ -.. currentmodule:: asyncio - - -.. _asyncio-transports-protocols: - - -======================== -Transports and Protocols -======================== - -.. rubric:: Preface - -Transports and Protocols are used by the **low-level** event loop -APIs such as :meth:`loop.create_connection`. They use -callback-based programming style and enable high-performance -implementations of network or IPC protocols (e.g. HTTP). - -Essentially, transports and protocols should only be used in -libraries and frameworks and never in high-level asyncio -applications. - -This documentation page covers both `Transports`_ and `Protocols`_. - -.. rubric:: Introduction - -At the highest level, the transport is concerned with *how* bytes -are transmitted, while the protocol determines *which* bytes to -transmit (and to some extent when). - -A different way of saying the same thing: a transport is an -abstraction for a socket (or similar I/O endpoint) while a protocol -is an abstraction for an application, from the transport's point -of view. - -Yet another view is the transport and protocol interfaces -together define an abstract interface for using network I/O and -interprocess I/O. - -There is always a 1:1 relationship between transport and protocol -objects: the protocol calls transport methods to send data, -while the transport calls protocol methods to pass it data that -has been received. - -Most of connection oriented event loop methods -(such as :meth:`loop.create_connection`) usually accept a -*protocol_factory* argument used to create a *Protocol* object -for an accepted connection, represented by a *Transport* object. -Such methods usually return a tuple of ``(transport, protocol)``. - -.. rubric:: Contents - -This documentation page contains the following sections: - -* The `Transports`_ section documents asyncio :class:`BaseTransport`, - :class:`ReadTransport`, :class:`WriteTransport`, :class:`Transport`, - :class:`DatagramTransport`, and :class:`SubprocessTransport` - classes. - -* The `Protocols`_ section documents asyncio :class:`BaseProtocol`, - :class:`Protocol`, :class:`BufferedProtocol`, - :class:`DatagramProtocol`, and :class:`SubprocessProtocol` classes. - -* The `Examples`_ section showcases how to work with transports, - protocols, and low-level event loop APIs. - - -.. _asyncio-transport: - -Transports -========== - -**Source code:** :source:`Lib/asyncio/transports.py` - ----------------------------------------------------- - -Transports are classes provided by :mod:`asyncio` in order to abstract -various kinds of communication channels. - -Transport objects are always instantiated by an -:ref:`asyncio event loop `. - -asyncio implements transports for TCP, UDP, SSL, and subprocess pipes. -The methods available on a transport depend on the transport's kind. - -The transport classes are :ref:`not thread safe `. - - -Transports Hierarchy --------------------- - -.. class:: BaseTransport - - Base class for all transports. Contains methods that all - asyncio transports share. - -.. class:: WriteTransport(BaseTransport) - - A base transport for write-only connections. - - Instances of the *WriteTransport* class are returned from - the :meth:`loop.connect_write_pipe` event loop method and - are also used by subprocess-related methods like - :meth:`loop.subprocess_exec`. - -.. class:: ReadTransport(BaseTransport) - - A base transport for read-only connections. - - Instances of the *ReadTransport* class are returned from - the :meth:`loop.connect_read_pipe` event loop method and - are also used by subprocess-related methods like - :meth:`loop.subprocess_exec`. - -.. class:: Transport(WriteTransport, ReadTransport) - - Interface representing a bidirectional transport, such as a - TCP connection. - - The user does not instantiate a transport directly; they call a - utility function, passing it a protocol factory and other - information necessary to create the transport and protocol. - - Instances of the *Transport* class are returned from or used by - event loop methods like :meth:`loop.create_connection`, - :meth:`loop.create_unix_connection`, - :meth:`loop.create_server`, :meth:`loop.sendfile`, etc. - - -.. class:: DatagramTransport(BaseTransport) - - A transport for datagram (UDP) connections. - - Instances of the *DatagramTransport* class are returned from - the :meth:`loop.create_datagram_endpoint` event loop method. - - -.. class:: SubprocessTransport(BaseTransport) - - An abstraction to represent a connection between a parent and its - child OS process. - - Instances of the *SubprocessTransport* class are returned from - event loop methods :meth:`loop.subprocess_shell` and - :meth:`loop.subprocess_exec`. - - -Base Transport --------------- - -.. method:: BaseTransport.close() - - Close the transport. - - If the transport has a buffer for outgoing - data, buffered data will be flushed asynchronously. No more data - will be received. After all buffered data is flushed, the - protocol's :meth:`protocol.connection_lost() - ` method will be called with - :const:`None` as its argument. The transport should not be - used once it is closed. - -.. method:: BaseTransport.is_closing() - - Return ``True`` if the transport is closing or is closed. - -.. method:: BaseTransport.get_extra_info(name, default=None) - - Return information about the transport or underlying resources - it uses. - - *name* is a string representing the piece of transport-specific - information to get. - - *default* is the value to return if the information is not - available, or if the transport does not support querying it - with the given third-party event loop implementation or on the - current platform. - - For example, the following code attempts to get the underlying - socket object of the transport:: - - sock = transport.get_extra_info('socket') - if sock is not None: - print(sock.getsockopt(...)) - - Categories of information that can be queried on some transports: - - * socket: - - - ``'peername'``: the remote address to which the socket is - connected, result of :meth:`socket.socket.getpeername` - (``None`` on error) - - - ``'socket'``: :class:`socket.socket` instance - - - ``'sockname'``: the socket's own address, - result of :meth:`socket.socket.getsockname` - - * SSL socket: - - - ``'compression'``: the compression algorithm being used as a - string, or ``None`` if the connection isn't compressed; result - of :meth:`ssl.SSLSocket.compression` - - - ``'cipher'``: a three-value tuple containing the name of the - cipher being used, the version of the SSL protocol that defines - its use, and the number of secret bits being used; result of - :meth:`ssl.SSLSocket.cipher` - - - ``'peercert'``: peer certificate; result of - :meth:`ssl.SSLSocket.getpeercert` - - - ``'sslcontext'``: :class:`ssl.SSLContext` instance - - - ``'ssl_object'``: :class:`ssl.SSLObject` or - :class:`ssl.SSLSocket` instance - - * pipe: - - - ``'pipe'``: pipe object - - * subprocess: - - - ``'subprocess'``: :class:`subprocess.Popen` instance - -.. method:: BaseTransport.set_protocol(protocol) - - Set a new protocol. - - Switching protocol should only be done when both - protocols are documented to support the switch. - -.. method:: BaseTransport.get_protocol() - - Return the current protocol. - - -Read-only Transports --------------------- - -.. method:: ReadTransport.is_reading() - - Return ``True`` if the transport is receiving new data. - - .. versionadded:: 3.7 - -.. method:: ReadTransport.pause_reading() - - Pause the receiving end of the transport. No data will be passed to - the protocol's :meth:`protocol.data_received() ` - method until :meth:`resume_reading` is called. - - .. versionchanged:: 3.7 - The method is idempotent, i.e. it can be called when the - transport is already paused or closed. - -.. method:: ReadTransport.resume_reading() - - Resume the receiving end. The protocol's - :meth:`protocol.data_received() ` method - will be called once again if some data is available for reading. - - .. versionchanged:: 3.7 - The method is idempotent, i.e. it can be called when the - transport is already reading. - - -Write-only Transports ---------------------- - -.. method:: WriteTransport.abort() - - Close the transport immediately, without waiting for pending operations - to complete. Buffered data will be lost. No more data will be received. - The protocol's :meth:`protocol.connection_lost() - ` method will eventually be - called with :const:`None` as its argument. - -.. method:: WriteTransport.can_write_eof() - - Return :const:`True` if the transport supports - :meth:`~WriteTransport.write_eof`, :const:`False` if not. - -.. method:: WriteTransport.get_write_buffer_size() - - Return the current size of the output buffer used by the transport. - -.. method:: WriteTransport.get_write_buffer_limits() - - Get the *high* and *low* watermarks for write flow control. Return a - tuple ``(low, high)`` where *low* and *high* are positive number of - bytes. - - Use :meth:`set_write_buffer_limits` to set the limits. - - .. versionadded:: 3.4.2 - -.. method:: WriteTransport.set_write_buffer_limits(high=None, low=None) - - Set the *high* and *low* watermarks for write flow control. - - These two values (measured in number of - bytes) control when the protocol's - :meth:`protocol.pause_writing() ` - and :meth:`protocol.resume_writing() ` - methods are called. If specified, the low watermark must be less - than or equal to the high watermark. Neither *high* nor *low* - can be negative. - - :meth:`~BaseProtocol.pause_writing` is called when the buffer size - becomes greater than or equal to the *high* value. If writing has - been paused, :meth:`~BaseProtocol.resume_writing` is called when - the buffer size becomes less than or equal to the *low* value. - - The defaults are implementation-specific. If only the - high watermark is given, the low watermark defaults to an - implementation-specific value less than or equal to the - high watermark. Setting *high* to zero forces *low* to zero as - well, and causes :meth:`~BaseProtocol.pause_writing` to be called - whenever the buffer becomes non-empty. Setting *low* to zero causes - :meth:`~BaseProtocol.resume_writing` to be called only once the - buffer is empty. Use of zero for either limit is generally - sub-optimal as it reduces opportunities for doing I/O and - computation concurrently. - - Use :meth:`~WriteTransport.get_write_buffer_limits` - to get the limits. - -.. method:: WriteTransport.write(data) - - Write some *data* bytes to the transport. - - This method does not block; it buffers the data and arranges for it - to be sent out asynchronously. - -.. method:: WriteTransport.writelines(list_of_data) - - Write a list (or any iterable) of data bytes to the transport. - This is functionally equivalent to calling :meth:`write` on each - element yielded by the iterable, but may be implemented more - efficiently. - -.. method:: WriteTransport.write_eof() - - Close the write end of the transport after flushing all buffered data. - Data may still be received. - - This method can raise :exc:`NotImplementedError` if the transport - (e.g. SSL) doesn't support half-closed connections. - - -Datagram Transports -------------------- - -.. method:: DatagramTransport.sendto(data, addr=None) - - Send the *data* bytes to the remote peer given by *addr* (a - transport-dependent target address). If *addr* is :const:`None`, - the data is sent to the target address given on transport - creation. - - This method does not block; it buffers the data and arranges - for it to be sent out asynchronously. - -.. method:: DatagramTransport.abort() - - Close the transport immediately, without waiting for pending - operations to complete. Buffered data will be lost. - No more data will be received. The protocol's - :meth:`protocol.connection_lost() ` - method will eventually be called with :const:`None` as its argument. - - -.. _asyncio-subprocess-transports: - -Subprocess Transports ---------------------- - -.. method:: SubprocessTransport.get_pid() - - Return the subprocess process id as an integer. - -.. method:: SubprocessTransport.get_pipe_transport(fd) - - Return the transport for the communication pipe corresponding to the - integer file descriptor *fd*: - - * ``0``: readable streaming transport of the standard input (*stdin*), - or :const:`None` if the subprocess was not created with ``stdin=PIPE`` - * ``1``: writable streaming transport of the standard output (*stdout*), - or :const:`None` if the subprocess was not created with ``stdout=PIPE`` - * ``2``: writable streaming transport of the standard error (*stderr*), - or :const:`None` if the subprocess was not created with ``stderr=PIPE`` - * other *fd*: :const:`None` - -.. method:: SubprocessTransport.get_returncode() - - Return the subprocess return code as an integer or :const:`None` - if it hasn't returned, which is similar to the - :attr:`subprocess.Popen.returncode` attribute. - -.. method:: SubprocessTransport.kill() - - Kill the subprocess. - - On POSIX systems, the function sends SIGKILL to the subprocess. - On Windows, this method is an alias for :meth:`terminate`. - - See also :meth:`subprocess.Popen.kill`. - -.. method:: SubprocessTransport.send_signal(signal) - - Send the *signal* number to the subprocess, as in - :meth:`subprocess.Popen.send_signal`. - -.. method:: SubprocessTransport.terminate() - - Stop the subprocess. - - On POSIX systems, this method sends SIGTERM to the subprocess. - On Windows, the Windows API function TerminateProcess() is called to - stop the subprocess. - - See also :meth:`subprocess.Popen.terminate`. - -.. method:: SubprocessTransport.close() - - Kill the subprocess by calling the :meth:`kill` method. - - If the subprocess hasn't returned yet, and close transports of - *stdin*, *stdout*, and *stderr* pipes. - - -.. _asyncio-protocol: - -Protocols -========= - -**Source code:** :source:`Lib/asyncio/protocols.py` - ---------------------------------------------------- - -asyncio provides a set of abstract base classes that should be used -to implement network protocols. Those classes are meant to be used -together with :ref:`transports `. - -Subclasses of abstract base protocol classes may implement some or -all methods. All these methods are callbacks: they are called by -transports on certain events, for example when some data is received. -A base protocol method should be called by the corresponding transport. - - -Base Protocols --------------- - -.. class:: BaseProtocol - - Base protocol with methods that all protocols share. - -.. class:: Protocol(BaseProtocol) - - The base class for implementing streaming protocols - (TCP, Unix sockets, etc). - -.. class:: BufferedProtocol(BaseProtocol) - - A base class for implementing streaming protocols with manual - control of the receive buffer. - -.. class:: DatagramProtocol(BaseProtocol) - - The base class for implementing datagram (UDP) protocols. - -.. class:: SubprocessProtocol(BaseProtocol) - - The base class for implementing protocols communicating with child - processes (unidirectional pipes). - - -Base Protocol -------------- - -All asyncio protocols can implement Base Protocol callbacks. - -.. rubric:: Connection Callbacks - -Connection callbacks are called on all protocols, exactly once per -a successful connection. All other protocol callbacks can only be -called between those two methods. - -.. method:: BaseProtocol.connection_made(transport) - - Called when a connection is made. - - The *transport* argument is the transport representing the - connection. The protocol is responsible for storing the reference - to its transport. - -.. method:: BaseProtocol.connection_lost(exc) - - Called when the connection is lost or closed. - - The argument is either an exception object or :const:`None`. - The latter means a regular EOF is received, or the connection was - aborted or closed by this side of the connection. - - -.. rubric:: Flow Control Callbacks - -Flow control callbacks can be called by transports to pause or -resume writing performed by the protocol. - -See the documentation of the :meth:`~WriteTransport.set_write_buffer_limits` -method for more details. - -.. method:: BaseProtocol.pause_writing() - - Called when the transport's buffer goes over the high watermark. - -.. method:: BaseProtocol.resume_writing() - - Called when the transport's buffer drains below the low watermark. - -If the buffer size equals the high watermark, -:meth:`~BaseProtocol.pause_writing` is not called: the buffer size must -go strictly over. - -Conversely, :meth:`~BaseProtocol.resume_writing` is called when the -buffer size is equal or lower than the low watermark. These end -conditions are important to ensure that things go as expected when -either mark is zero. - - -Streaming Protocols -------------------- - -Event methods, such as :meth:`loop.create_server`, -:meth:`loop.create_unix_server`, :meth:`loop.create_connection`, -:meth:`loop.create_unix_connection`, :meth:`loop.connect_accepted_socket`, -:meth:`loop.connect_read_pipe`, and :meth:`loop.connect_write_pipe` -accept factories that return streaming protocols. - -.. method:: Protocol.data_received(data) - - Called when some data is received. *data* is a non-empty bytes - object containing the incoming data. - - Whether the data is buffered, chunked or reassembled depends on - the transport. In general, you shouldn't rely on specific semantics - and instead make your parsing generic and flexible. However, - data is always received in the correct order. - - The method can be called an arbitrary number of times while - a connection is open. - - However, :meth:`protocol.eof_received() ` - is called at most once. Once ``eof_received()`` is called, - ``data_received()`` is not called anymore. - -.. method:: Protocol.eof_received() - - Called when the other end signals it won't send any more data - (for example by calling :meth:`transport.write_eof() - `, if the other end also uses - asyncio). - - This method may return a false value (including ``None``), in which case - the transport will close itself. Conversely, if this method returns a - true value, the protocol used determines whether to close the transport. - Since the default implementation returns ``None``, it implicitly closes the - connection. - - Some transports, including SSL, don't support half-closed connections, - in which case returning true from this method will result in the connection - being closed. - - -State machine: - -.. code-block:: none - - start -> connection_made - [-> data_received]* - [-> eof_received]? - -> connection_lost -> end - - -Buffered Streaming Protocols ----------------------------- - -.. versionadded:: 3.7 - -Buffered Protocols can be used with any event loop method -that supports `Streaming Protocols`_. - -``BufferedProtocol`` implementations allow explicit manual allocation -and control of the receive buffer. Event loops can then use the buffer -provided by the protocol to avoid unnecessary data copies. This -can result in noticeable performance improvement for protocols that -receive big amounts of data. Sophisticated protocol implementations -can significantly reduce the number of buffer allocations. - -The following callbacks are called on :class:`BufferedProtocol` -instances: - -.. method:: BufferedProtocol.get_buffer(sizehint) - - Called to allocate a new receive buffer. - - *sizehint* is the recommended minimum size for the returned - buffer. It is acceptable to return smaller or larger buffers - than what *sizehint* suggests. When set to -1, the buffer size - can be arbitrary. It is an error to return a buffer with a zero size. - - ``get_buffer()`` must return an object implementing the - :ref:`buffer protocol `. - -.. method:: BufferedProtocol.buffer_updated(nbytes) - - Called when the buffer was updated with the received data. - - *nbytes* is the total number of bytes that were written to the buffer. - -.. method:: BufferedProtocol.eof_received() - - See the documentation of the :meth:`protocol.eof_received() - ` method. - - -:meth:`~BufferedProtocol.get_buffer` can be called an arbitrary number -of times during a connection. However, :meth:`protocol.eof_received() -` is called at most once -and, if called, :meth:`~BufferedProtocol.get_buffer` and -:meth:`~BufferedProtocol.buffer_updated` won't be called after it. - -State machine: - -.. code-block:: none - - start -> connection_made - [-> get_buffer - [-> buffer_updated]? - ]* - [-> eof_received]? - -> connection_lost -> end - - -Datagram Protocols ------------------- - -Datagram Protocol instances should be constructed by protocol -factories passed to the :meth:`loop.create_datagram_endpoint` method. - -.. method:: DatagramProtocol.datagram_received(data, addr) - - Called when a datagram is received. *data* is a bytes object containing - the incoming data. *addr* is the address of the peer sending the data; - the exact format depends on the transport. - -.. method:: DatagramProtocol.error_received(exc) - - Called when a previous send or receive operation raises an - :class:`OSError`. *exc* is the :class:`OSError` instance. - - This method is called in rare conditions, when the transport (e.g. UDP) - detects that a datagram could not be delivered to its recipient. - In many conditions though, undeliverable datagrams will be silently - dropped. - -.. note:: - - On BSD systems (macOS, FreeBSD, etc.) flow control is not supported - for datagram protocols, because there is no reliable way to detect send - failures caused by writing too many packets. - - The socket always appears 'ready' and excess packets are dropped. An - :class:`OSError` with ``errno`` set to :const:`errno.ENOBUFS` may - or may not be raised; if it is raised, it will be reported to - :meth:`DatagramProtocol.error_received` but otherwise ignored. - - -.. _asyncio-subprocess-protocols: - -Subprocess Protocols --------------------- - -Subprocess Protocol instances should be constructed by protocol -factories passed to the :meth:`loop.subprocess_exec` and -:meth:`loop.subprocess_shell` methods. - -.. method:: SubprocessProtocol.pipe_data_received(fd, data) - - Called when the child process writes data into its stdout or stderr - pipe. - - *fd* is the integer file descriptor of the pipe. - - *data* is a non-empty bytes object containing the received data. - -.. method:: SubprocessProtocol.pipe_connection_lost(fd, exc) - - Called when one of the pipes communicating with the child process - is closed. - - *fd* is the integer file descriptor that was closed. - -.. method:: SubprocessProtocol.process_exited() - - Called when the child process has exited. - - It can be called before :meth:`~SubprocessProtocol.pipe_data_received` and - :meth:`~SubprocessProtocol.pipe_connection_lost` methods. - - -Examples -======== - -.. _asyncio_example_tcp_echo_server_protocol: - -TCP Echo Server ---------------- - -Create a TCP echo server using the :meth:`loop.create_server` method, send back -received data, and close the connection:: - - import asyncio - - - class EchoServerProtocol(asyncio.Protocol): - def connection_made(self, transport): - peername = transport.get_extra_info('peername') - print('Connection from {}'.format(peername)) - self.transport = transport - - def data_received(self, data): - message = data.decode() - print('Data received: {!r}'.format(message)) - - print('Send: {!r}'.format(message)) - self.transport.write(data) - - print('Close the client socket') - self.transport.close() - - - async def main(): - # Get a reference to the event loop as we plan to use - # low-level APIs. - loop = asyncio.get_running_loop() - - server = await loop.create_server( - lambda: EchoServerProtocol(), - '127.0.0.1', 8888) - - async with server: - await server.serve_forever() - - - asyncio.run(main()) - - -.. seealso:: - - The :ref:`TCP echo server using streams ` - example uses the high-level :func:`asyncio.start_server` function. - -.. _asyncio_example_tcp_echo_client_protocol: - -TCP Echo Client ---------------- - -A TCP echo client using the :meth:`loop.create_connection` method, sends -data, and waits until the connection is closed:: - - import asyncio - - - class EchoClientProtocol(asyncio.Protocol): - def __init__(self, message, on_con_lost): - self.message = message - self.on_con_lost = on_con_lost - - def connection_made(self, transport): - transport.write(self.message.encode()) - print('Data sent: {!r}'.format(self.message)) - - def data_received(self, data): - print('Data received: {!r}'.format(data.decode())) - - def connection_lost(self, exc): - print('The server closed the connection') - self.on_con_lost.set_result(True) - - - async def main(): - # Get a reference to the event loop as we plan to use - # low-level APIs. - loop = asyncio.get_running_loop() - - on_con_lost = loop.create_future() - message = 'Hello World!' - - transport, protocol = await loop.create_connection( - lambda: EchoClientProtocol(message, on_con_lost), - '127.0.0.1', 8888) - - # Wait until the protocol signals that the connection - # is lost and close the transport. - try: - await on_con_lost - finally: - transport.close() - - - asyncio.run(main()) - - -.. seealso:: - - The :ref:`TCP echo client using streams ` - example uses the high-level :func:`asyncio.open_connection` function. - - -.. _asyncio-udp-echo-server-protocol: - -UDP Echo Server ---------------- - -A UDP echo server, using the :meth:`loop.create_datagram_endpoint` -method, sends back received data:: - - import asyncio - - - class EchoServerProtocol: - def connection_made(self, transport): - self.transport = transport - - def datagram_received(self, data, addr): - message = data.decode() - print('Received %r from %s' % (message, addr)) - print('Send %r to %s' % (message, addr)) - self.transport.sendto(data, addr) - - - async def main(): - print("Starting UDP server") - - # Get a reference to the event loop as we plan to use - # low-level APIs. - loop = asyncio.get_running_loop() - - # One protocol instance will be created to serve all - # client requests. - transport, protocol = await loop.create_datagram_endpoint( - lambda: EchoServerProtocol(), - local_addr=('127.0.0.1', 9999)) - - try: - await asyncio.sleep(3600) # Serve for 1 hour. - finally: - transport.close() - - - asyncio.run(main()) - - -.. _asyncio-udp-echo-client-protocol: - -UDP Echo Client ---------------- - -A UDP echo client, using the :meth:`loop.create_datagram_endpoint` -method, sends data and closes the transport when it receives the answer:: - - import asyncio - - - class EchoClientProtocol: - def __init__(self, message, on_con_lost): - self.message = message - self.on_con_lost = on_con_lost - self.transport = None - - def connection_made(self, transport): - self.transport = transport - print('Send:', self.message) - self.transport.sendto(self.message.encode()) - - def datagram_received(self, data, addr): - print("Received:", data.decode()) - - print("Close the socket") - self.transport.close() - - def error_received(self, exc): - print('Error received:', exc) - - def connection_lost(self, exc): - print("Connection closed") - self.on_con_lost.set_result(True) - - - async def main(): - # Get a reference to the event loop as we plan to use - # low-level APIs. - loop = asyncio.get_running_loop() - - on_con_lost = loop.create_future() - message = "Hello World!" - - transport, protocol = await loop.create_datagram_endpoint( - lambda: EchoClientProtocol(message, on_con_lost), - remote_addr=('127.0.0.1', 9999)) - - try: - await on_con_lost - finally: - transport.close() - - - asyncio.run(main()) - - -.. _asyncio_example_create_connection: - -Connecting Existing Sockets ---------------------------- - -Wait until a socket receives data using the -:meth:`loop.create_connection` method with a protocol:: - - import asyncio - import socket - - - class MyProtocol(asyncio.Protocol): - - def __init__(self, on_con_lost): - self.transport = None - self.on_con_lost = on_con_lost - - def connection_made(self, transport): - self.transport = transport - - def data_received(self, data): - print("Received:", data.decode()) - - # We are done: close the transport; - # connection_lost() will be called automatically. - self.transport.close() - - def connection_lost(self, exc): - # The socket has been closed - self.on_con_lost.set_result(True) - - - async def main(): - # Get a reference to the event loop as we plan to use - # low-level APIs. - loop = asyncio.get_running_loop() - on_con_lost = loop.create_future() - - # Create a pair of connected sockets - rsock, wsock = socket.socketpair() - - # Register the socket to wait for data. - transport, protocol = await loop.create_connection( - lambda: MyProtocol(on_con_lost), sock=rsock) - - # Simulate the reception of data from the network. - loop.call_soon(wsock.send, 'abc'.encode()) - - try: - await protocol.on_con_lost - finally: - transport.close() - wsock.close() - - asyncio.run(main()) - -.. seealso:: - - The :ref:`watch a file descriptor for read events - ` example uses the low-level - :meth:`loop.add_reader` method to register an FD. - - The :ref:`register an open socket to wait for data using streams - ` example uses high-level streams - created by the :func:`open_connection` function in a coroutine. - -.. _asyncio_example_subprocess_proto: - -loop.subprocess_exec() and SubprocessProtocol ---------------------------------------------- - -An example of a subprocess protocol used to get the output of a -subprocess and to wait for the subprocess exit. - -The subprocess is created by the :meth:`loop.subprocess_exec` method:: - - import asyncio - import sys - - class DateProtocol(asyncio.SubprocessProtocol): - def __init__(self, exit_future): - self.exit_future = exit_future - self.output = bytearray() - self.pipe_closed = False - self.exited = False - - def pipe_connection_lost(self, fd, exc): - self.pipe_closed = True - self.check_for_exit() - - def pipe_data_received(self, fd, data): - self.output.extend(data) - - def process_exited(self): - self.exited = True - # process_exited() method can be called before - # pipe_connection_lost() method: wait until both methods are - # called. - self.check_for_exit() - - def check_for_exit(self): - if self.pipe_closed and self.exited: - self.exit_future.set_result(True) - - async def get_date(): - # Get a reference to the event loop as we plan to use - # low-level APIs. - loop = asyncio.get_running_loop() - - code = 'import datetime; print(datetime.datetime.now())' - exit_future = asyncio.Future(loop=loop) - - # Create the subprocess controlled by DateProtocol; - # redirect the standard output into a pipe. - transport, protocol = await loop.subprocess_exec( - lambda: DateProtocol(exit_future), - sys.executable, '-c', code, - stdin=None, stderr=None) - - # Wait for the subprocess exit using the process_exited() - # method of the protocol. - await exit_future - - # Close the stdout pipe. - transport.close() - - # Read the output which was collected by the - # pipe_data_received() method of the protocol. - data = bytes(protocol.output) - return data.decode('ascii').rstrip() - - date = asyncio.run(get_date()) - print(f"Current date: {date}") - -See also the :ref:`same example ` -written using high-level APIs. diff --git a/Doc/library/asyncio-queue.rst b/Doc/library/asyncio-queue.rst deleted file mode 100644 index d86fbc21351e2d..00000000000000 --- a/Doc/library/asyncio-queue.rst +++ /dev/null @@ -1,208 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-queues: - -====== -Queues -====== - -**Source code:** :source:`Lib/asyncio/queues.py` - ------------------------------------------------- - -asyncio queues are designed to be similar to classes of the -:mod:`queue` module. Although asyncio queues are not thread-safe, -they are designed to be used specifically in async/await code. - -Note that methods of asyncio queues don't have a *timeout* parameter; -use :func:`asyncio.wait_for` function to do queue operations with a -timeout. - -See also the `Examples`_ section below. - -Queue -===== - -.. class:: Queue(maxsize=0) - - A first in, first out (FIFO) queue. - - If *maxsize* is less than or equal to zero, the queue size is - infinite. If it is an integer greater than ``0``, then - ``await put()`` blocks when the queue reaches *maxsize* - until an item is removed by :meth:`get`. - - Unlike the standard library threading :mod:`queue`, the size of - the queue is always known and can be returned by calling the - :meth:`qsize` method. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - - This class is :ref:`not thread safe `. - - .. attribute:: maxsize - - Number of items allowed in the queue. - - .. method:: empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. - - .. method:: full() - - Return ``True`` if there are :attr:`maxsize` items in the queue. - - If the queue was initialized with ``maxsize=0`` (the default), - then :meth:`full()` never returns ``True``. - - .. coroutinemethod:: get() - - Remove and return an item from the queue. If queue is empty, - wait until an item is available. - - .. method:: get_nowait() - - Return an item if one is immediately available, else raise - :exc:`QueueEmpty`. - - .. coroutinemethod:: join() - - Block until all items in the queue have been received and processed. - - The count of unfinished tasks goes up whenever an item is added - to the queue. The count goes down whenever a consumer coroutine calls - :meth:`task_done` to indicate that the item was retrieved and all - work on it is complete. When the count of unfinished tasks drops - to zero, :meth:`join` unblocks. - - .. coroutinemethod:: put(item) - - Put an item into the queue. If the queue is full, wait until a - free slot is available before adding the item. - - .. method:: put_nowait(item) - - Put an item into the queue without blocking. - - If no free slot is immediately available, raise :exc:`QueueFull`. - - .. method:: qsize() - - Return the number of items in the queue. - - .. method:: task_done() - - Indicate that a formerly enqueued task is complete. - - Used by queue consumers. For each :meth:`~Queue.get` used to - fetch a task, a subsequent call to :meth:`task_done` tells the - queue that the processing on the task is complete. - - If a :meth:`join` is currently blocking, it will resume when all - items have been processed (meaning that a :meth:`task_done` - call was received for every item that had been :meth:`~Queue.put` - into the queue). - - Raises :exc:`ValueError` if called more times than there were - items placed in the queue. - - -Priority Queue -============== - -.. class:: PriorityQueue - - A variant of :class:`Queue`; retrieves entries in priority order - (lowest first). - - Entries are typically tuples of the form - ``(priority_number, data)``. - - -LIFO Queue -========== - -.. class:: LifoQueue - - A variant of :class:`Queue` that retrieves most recently added - entries first (last in, first out). - - -Exceptions -========== - -.. exception:: QueueEmpty - - This exception is raised when the :meth:`~Queue.get_nowait` method - is called on an empty queue. - - -.. exception:: QueueFull - - Exception raised when the :meth:`~Queue.put_nowait` method is called - on a queue that has reached its *maxsize*. - - -Examples -======== - -.. _asyncio_example_queue_dist: - -Queues can be used to distribute workload between several -concurrent tasks:: - - import asyncio - import random - import time - - - async def worker(name, queue): - while True: - # Get a "work item" out of the queue. - sleep_for = await queue.get() - - # Sleep for the "sleep_for" seconds. - await asyncio.sleep(sleep_for) - - # Notify the queue that the "work item" has been processed. - queue.task_done() - - print(f'{name} has slept for {sleep_for:.2f} seconds') - - - async def main(): - # Create a queue that we will use to store our "workload". - queue = asyncio.Queue() - - # Generate random timings and put them into the queue. - total_sleep_time = 0 - for _ in range(20): - sleep_for = random.uniform(0.05, 1.0) - total_sleep_time += sleep_for - queue.put_nowait(sleep_for) - - # Create three worker tasks to process the queue concurrently. - tasks = [] - for i in range(3): - task = asyncio.create_task(worker(f'worker-{i}', queue)) - tasks.append(task) - - # Wait until the queue is fully processed. - started_at = time.monotonic() - await queue.join() - total_slept_for = time.monotonic() - started_at - - # Cancel our worker tasks. - for task in tasks: - task.cancel() - # Wait until all worker tasks are cancelled. - await asyncio.gather(*tasks, return_exceptions=True) - - print('====') - print(f'3 workers slept in parallel for {total_slept_for:.2f} seconds') - print(f'total expected sleep time: {total_sleep_time:.2f} seconds') - - - asyncio.run(main()) diff --git a/Doc/library/asyncio-stream.rst b/Doc/library/asyncio-stream.rst deleted file mode 100644 index e9d466d95e547e..00000000000000 --- a/Doc/library/asyncio-stream.rst +++ /dev/null @@ -1,558 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-streams: - -======= -Streams -======= - -**Source code:** :source:`Lib/asyncio/streams.py` - -------------------------------------------------- - -Streams are high-level async/await-ready primitives to work with -network connections. Streams allow sending and receiving data without -using callbacks or low-level protocols and transports. - -.. _asyncio_example_stream: - -Here is an example of a TCP echo client written using asyncio -streams:: - - import asyncio - - async def tcp_echo_client(message): - reader, writer = await asyncio.open_connection( - '127.0.0.1', 8888) - - print(f'Send: {message!r}') - writer.write(message.encode()) - await writer.drain() - - data = await reader.read(100) - print(f'Received: {data.decode()!r}') - - print('Close the connection') - writer.close() - await writer.wait_closed() - - asyncio.run(tcp_echo_client('Hello World!')) - - -See also the `Examples`_ section below. - - -.. rubric:: Stream Functions - -The following top-level asyncio functions can be used to create -and work with streams: - - -.. coroutinefunction:: open_connection(host=None, port=None, *, \ - limit=None, ssl=None, family=0, proto=0, \ - flags=0, sock=None, local_addr=None, \ - server_hostname=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, \ - happy_eyeballs_delay=None, interleave=None) - - Establish a network connection and return a pair of - ``(reader, writer)`` objects. - - The returned *reader* and *writer* objects are instances of - :class:`StreamReader` and :class:`StreamWriter` classes. - - *limit* determines the buffer size limit used by the - returned :class:`StreamReader` instance. By default the *limit* - is set to 64 KiB. - - The rest of the arguments are passed directly to - :meth:`loop.create_connection`. - - .. note:: - - The *sock* argument transfers ownership of the socket to the - :class:`StreamWriter` created. To close the socket, call its - :meth:`~asyncio.StreamWriter.close` method. - - .. versionchanged:: 3.7 - Added the *ssl_handshake_timeout* parameter. - - .. versionadded:: 3.8 - Added *happy_eyeballs_delay* and *interleave* parameters. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. versionchanged:: 3.11 - Added the *ssl_shutdown_timeout* parameter. - - -.. coroutinefunction:: start_server(client_connected_cb, host=None, \ - port=None, *, limit=None, \ - family=socket.AF_UNSPEC, \ - flags=socket.AI_PASSIVE, sock=None, \ - backlog=100, ssl=None, reuse_address=None, \ - reuse_port=None, ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, start_serving=True) - - Start a socket server. - - The *client_connected_cb* callback is called whenever a new client - connection is established. It receives a ``(reader, writer)`` pair - as two arguments, instances of the :class:`StreamReader` and - :class:`StreamWriter` classes. - - *client_connected_cb* can be a plain callable or a - :ref:`coroutine function `; if it is a coroutine function, - it will be automatically scheduled as a :class:`Task`. - - *limit* determines the buffer size limit used by the - returned :class:`StreamReader` instance. By default the *limit* - is set to 64 KiB. - - The rest of the arguments are passed directly to - :meth:`loop.create_server`. - - .. note:: - - The *sock* argument transfers ownership of the socket to the - server created. To close the socket, call the server's - :meth:`~asyncio.Server.close` method. - - .. versionchanged:: 3.7 - Added the *ssl_handshake_timeout* and *start_serving* parameters. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. versionchanged:: 3.11 - Added the *ssl_shutdown_timeout* parameter. - - -.. rubric:: Unix Sockets - -.. coroutinefunction:: open_unix_connection(path=None, *, limit=None, \ - ssl=None, sock=None, server_hostname=None, \ - ssl_handshake_timeout=None, ssl_shutdown_timeout=None) - - Establish a Unix socket connection and return a pair of - ``(reader, writer)``. - - Similar to :func:`open_connection` but operates on Unix sockets. - - See also the documentation of :meth:`loop.create_unix_connection`. - - .. note:: - - The *sock* argument transfers ownership of the socket to the - :class:`StreamWriter` created. To close the socket, call its - :meth:`~asyncio.StreamWriter.close` method. - - .. availability:: Unix. - - .. versionchanged:: 3.7 - Added the *ssl_handshake_timeout* parameter. - The *path* parameter can now be a :term:`path-like object` - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. versionchanged:: 3.11 - Added the *ssl_shutdown_timeout* parameter. - - -.. coroutinefunction:: start_unix_server(client_connected_cb, path=None, \ - *, limit=None, sock=None, backlog=100, ssl=None, \ - ssl_handshake_timeout=None, \ - ssl_shutdown_timeout=None, start_serving=True) - - Start a Unix socket server. - - Similar to :func:`start_server` but works with Unix sockets. - - See also the documentation of :meth:`loop.create_unix_server`. - - .. note:: - - The *sock* argument transfers ownership of the socket to the - server created. To close the socket, call the server's - :meth:`~asyncio.Server.close` method. - - .. availability:: Unix. - - .. versionchanged:: 3.7 - Added the *ssl_handshake_timeout* and *start_serving* parameters. - The *path* parameter can now be a :term:`path-like object`. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. versionchanged:: 3.11 - Added the *ssl_shutdown_timeout* parameter. - - -StreamReader -============ - -.. class:: StreamReader - - Represents a reader object that provides APIs to read data - from the IO stream. As an :term:`asynchronous iterable`, the - object supports the :keyword:`async for` statement. - - It is not recommended to instantiate *StreamReader* objects - directly; use :func:`open_connection` and :func:`start_server` - instead. - - .. coroutinemethod:: read(n=-1) - - Read up to *n* bytes from the stream. - - If *n* is not provided or set to ``-1``, - read until EOF, then return all read :class:`bytes`. - If EOF was received and the internal buffer is empty, - return an empty ``bytes`` object. - - If *n* is ``0``, return an empty ``bytes`` object immediately. - - If *n* is positive, return at most *n* available ``bytes`` - as soon as at least 1 byte is available in the internal buffer. - If EOF is received before any byte is read, return an empty - ``bytes`` object. - - .. coroutinemethod:: readline() - - Read one line, where "line" is a sequence of bytes - ending with ``\n``. - - If EOF is received and ``\n`` was not found, the method - returns partially read data. - - If EOF is received and the internal buffer is empty, - return an empty ``bytes`` object. - - .. coroutinemethod:: readexactly(n) - - Read exactly *n* bytes. - - Raise an :exc:`IncompleteReadError` if EOF is reached before *n* - can be read. Use the :attr:`IncompleteReadError.partial` - attribute to get the partially read data. - - .. coroutinemethod:: readuntil(separator=b'\n') - - Read data from the stream until *separator* is found. - - On success, the data and separator will be removed from the - internal buffer (consumed). Returned data will include the - separator at the end. - - If the amount of data read exceeds the configured stream limit, a - :exc:`LimitOverrunError` exception is raised, and the data - is left in the internal buffer and can be read again. - - If EOF is reached before the complete separator is found, - an :exc:`IncompleteReadError` exception is raised, and the internal - buffer is reset. The :attr:`IncompleteReadError.partial` attribute - may contain a portion of the separator. - - .. versionadded:: 3.5.2 - - .. method:: at_eof() - - Return ``True`` if the buffer is empty and :meth:`feed_eof` - was called. - - -StreamWriter -============ - -.. class:: StreamWriter - - Represents a writer object that provides APIs to write data - to the IO stream. - - It is not recommended to instantiate *StreamWriter* objects - directly; use :func:`open_connection` and :func:`start_server` - instead. - - .. method:: write(data) - - The method attempts to write the *data* to the underlying socket immediately. - If that fails, the data is queued in an internal write buffer until it can be - sent. - - The method should be used along with the ``drain()`` method:: - - stream.write(data) - await stream.drain() - - .. method:: writelines(data) - - The method writes a list (or any iterable) of bytes to the underlying socket - immediately. - If that fails, the data is queued in an internal write buffer until it can be - sent. - - The method should be used along with the ``drain()`` method:: - - stream.writelines(lines) - await stream.drain() - - .. method:: close() - - The method closes the stream and the underlying socket. - - The method should be used, though not mandatory, - along with the ``wait_closed()`` method:: - - stream.close() - await stream.wait_closed() - - .. method:: can_write_eof() - - Return ``True`` if the underlying transport supports - the :meth:`write_eof` method, ``False`` otherwise. - - .. method:: write_eof() - - Close the write end of the stream after the buffered write - data is flushed. - - .. attribute:: transport - - Return the underlying asyncio transport. - - .. method:: get_extra_info(name, default=None) - - Access optional transport information; see - :meth:`BaseTransport.get_extra_info` for details. - - .. coroutinemethod:: drain() - - Wait until it is appropriate to resume writing to the stream. - Example:: - - writer.write(data) - await writer.drain() - - This is a flow control method that interacts with the underlying - IO write buffer. When the size of the buffer reaches - the high watermark, *drain()* blocks until the size of the - buffer is drained down to the low watermark and writing can - be resumed. When there is nothing to wait for, the :meth:`drain` - returns immediately. - - .. coroutinemethod:: start_tls(sslcontext, \*, server_hostname=None, \ - ssl_handshake_timeout=None) - - Upgrade an existing stream-based connection to TLS. - - Parameters: - - * *sslcontext*: a configured instance of :class:`~ssl.SSLContext`. - - * *server_hostname*: sets or overrides the host name that the target - server's certificate will be matched against. - - * *ssl_handshake_timeout* is the time in seconds to wait for the TLS - handshake to complete before aborting the connection. ``60.0`` seconds - if ``None`` (default). - - .. versionadded:: 3.11 - - .. method:: is_closing() - - Return ``True`` if the stream is closed or in the process of - being closed. - - .. versionadded:: 3.7 - - .. coroutinemethod:: wait_closed() - - Wait until the stream is closed. - - Should be called after :meth:`close` to wait until the underlying - connection is closed, ensuring that all data has been flushed - before e.g. exiting the program. - - .. versionadded:: 3.7 - - -Examples -======== - -.. _asyncio-tcp-echo-client-streams: - -TCP echo client using streams ------------------------------ - -TCP echo client using the :func:`asyncio.open_connection` function:: - - import asyncio - - async def tcp_echo_client(message): - reader, writer = await asyncio.open_connection( - '127.0.0.1', 8888) - - print(f'Send: {message!r}') - writer.write(message.encode()) - await writer.drain() - - data = await reader.read(100) - print(f'Received: {data.decode()!r}') - - print('Close the connection') - writer.close() - await writer.wait_closed() - - asyncio.run(tcp_echo_client('Hello World!')) - - -.. seealso:: - - The :ref:`TCP echo client protocol ` - example uses the low-level :meth:`loop.create_connection` method. - - -.. _asyncio-tcp-echo-server-streams: - -TCP echo server using streams ------------------------------ - -TCP echo server using the :func:`asyncio.start_server` function:: - - import asyncio - - async def handle_echo(reader, writer): - data = await reader.read(100) - message = data.decode() - addr = writer.get_extra_info('peername') - - print(f"Received {message!r} from {addr!r}") - - print(f"Send: {message!r}") - writer.write(data) - await writer.drain() - - print("Close the connection") - writer.close() - await writer.wait_closed() - - async def main(): - server = await asyncio.start_server( - handle_echo, '127.0.0.1', 8888) - - addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets) - print(f'Serving on {addrs}') - - async with server: - await server.serve_forever() - - asyncio.run(main()) - - -.. seealso:: - - The :ref:`TCP echo server protocol ` - example uses the :meth:`loop.create_server` method. - - -Get HTTP headers ----------------- - -Simple example querying HTTP headers of the URL passed on the command line:: - - import asyncio - import urllib.parse - import sys - - async def print_http_headers(url): - url = urllib.parse.urlsplit(url) - if url.scheme == 'https': - reader, writer = await asyncio.open_connection( - url.hostname, 443, ssl=True) - else: - reader, writer = await asyncio.open_connection( - url.hostname, 80) - - query = ( - f"HEAD {url.path or '/'} HTTP/1.0\r\n" - f"Host: {url.hostname}\r\n" - f"\r\n" - ) - - writer.write(query.encode('latin-1')) - while True: - line = await reader.readline() - if not line: - break - - line = line.decode('latin1').rstrip() - if line: - print(f'HTTP header> {line}') - - # Ignore the body, close the socket - writer.close() - await writer.wait_closed() - - url = sys.argv[1] - asyncio.run(print_http_headers(url)) - - -Usage:: - - python example.py http://example.com/path/page.html - -or with HTTPS:: - - python example.py https://example.com/path/page.html - - -.. _asyncio_example_create_connection-streams: - -Register an open socket to wait for data using streams ------------------------------------------------------- - -Coroutine waiting until a socket receives data using the -:func:`open_connection` function:: - - import asyncio - import socket - - async def wait_for_data(): - # Get a reference to the current event loop because - # we want to access low-level APIs. - loop = asyncio.get_running_loop() - - # Create a pair of connected sockets. - rsock, wsock = socket.socketpair() - - # Register the open socket to wait for data. - reader, writer = await asyncio.open_connection(sock=rsock) - - # Simulate the reception of data from the network - loop.call_soon(wsock.send, 'abc'.encode()) - - # Wait for data - data = await reader.read(100) - - # Got data, we are done: close the socket - print("Received:", data.decode()) - writer.close() - await writer.wait_closed() - - # Close the second socket - wsock.close() - - asyncio.run(wait_for_data()) - -.. seealso:: - - The :ref:`register an open socket to wait for data using a protocol - ` example uses a low-level protocol and - the :meth:`loop.create_connection` method. - - The :ref:`watch a file descriptor for read events - ` example uses the low-level - :meth:`loop.add_reader` method to watch a file descriptor. diff --git a/Doc/library/asyncio-subprocess.rst b/Doc/library/asyncio-subprocess.rst deleted file mode 100644 index 77c3aae43099c3..00000000000000 --- a/Doc/library/asyncio-subprocess.rst +++ /dev/null @@ -1,372 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-subprocess: - -============ -Subprocesses -============ - -**Source code:** :source:`Lib/asyncio/subprocess.py`, -:source:`Lib/asyncio/base_subprocess.py` - ----------------------------------------- - -This section describes high-level async/await asyncio APIs to -create and manage subprocesses. - -.. _asyncio_example_subprocess_shell: - -Here's an example of how asyncio can run a shell command and -obtain its result:: - - import asyncio - - async def run(cmd): - proc = await asyncio.create_subprocess_shell( - cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE) - - stdout, stderr = await proc.communicate() - - print(f'[{cmd!r} exited with {proc.returncode}]') - if stdout: - print(f'[stdout]\n{stdout.decode()}') - if stderr: - print(f'[stderr]\n{stderr.decode()}') - - asyncio.run(run('ls /zzz')) - -will print:: - - ['ls /zzz' exited with 1] - [stderr] - ls: /zzz: No such file or directory - -Because all asyncio subprocess functions are asynchronous and asyncio -provides many tools to work with such functions, it is easy to execute -and monitor multiple subprocesses in parallel. It is indeed trivial -to modify the above example to run several commands simultaneously:: - - async def main(): - await asyncio.gather( - run('ls /zzz'), - run('sleep 1; echo "hello"')) - - asyncio.run(main()) - -See also the `Examples`_ subsection. - - -Creating Subprocesses -===================== - -.. coroutinefunction:: create_subprocess_exec(program, *args, stdin=None, \ - stdout=None, stderr=None, limit=None, **kwds) - - Create a subprocess. - - The *limit* argument sets the buffer limit for :class:`StreamReader` - wrappers for :attr:`Process.stdout` and :attr:`Process.stderr` - (if :const:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). - - Return a :class:`~asyncio.subprocess.Process` instance. - - See the documentation of :meth:`loop.subprocess_exec` for other - parameters. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - -.. coroutinefunction:: create_subprocess_shell(cmd, stdin=None, \ - stdout=None, stderr=None, limit=None, **kwds) - - Run the *cmd* shell command. - - The *limit* argument sets the buffer limit for :class:`StreamReader` - wrappers for :attr:`Process.stdout` and :attr:`Process.stderr` - (if :const:`subprocess.PIPE` is passed to *stdout* and *stderr* arguments). - - Return a :class:`~asyncio.subprocess.Process` instance. - - See the documentation of :meth:`loop.subprocess_shell` for other - parameters. - - .. important:: - - It is the application's responsibility to ensure that all whitespace and - special characters are quoted appropriately to avoid `shell injection - `_ - vulnerabilities. The :func:`shlex.quote` function can be used to properly - escape whitespace and special shell characters in strings that are going - to be used to construct shell commands. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - -.. note:: - - Subprocesses are available for Windows if a :class:`ProactorEventLoop` is - used. See :ref:`Subprocess Support on Windows ` - for details. - -.. seealso:: - - asyncio also has the following *low-level* APIs to work with subprocesses: - :meth:`loop.subprocess_exec`, :meth:`loop.subprocess_shell`, - :meth:`loop.connect_read_pipe`, :meth:`loop.connect_write_pipe`, - as well as the :ref:`Subprocess Transports ` - and :ref:`Subprocess Protocols `. - - -Constants -========= - -.. data:: asyncio.subprocess.PIPE - :module: - - Can be passed to the *stdin*, *stdout* or *stderr* parameters. - - If *PIPE* is passed to *stdin* argument, the - :attr:`Process.stdin ` attribute - will point to a :class:`StreamWriter` instance. - - If *PIPE* is passed to *stdout* or *stderr* arguments, the - :attr:`Process.stdout ` and - :attr:`Process.stderr ` - attributes will point to :class:`StreamReader` instances. - -.. data:: asyncio.subprocess.STDOUT - :module: - - Special value that can be used as the *stderr* argument and indicates - that standard error should be redirected into standard output. - -.. data:: asyncio.subprocess.DEVNULL - :module: - - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to process creation functions. It indicates that the special file - :data:`os.devnull` will be used for the corresponding subprocess stream. - - -Interacting with Subprocesses -============================= - -Both :func:`create_subprocess_exec` and :func:`create_subprocess_shell` -functions return instances of the *Process* class. *Process* is a high-level -wrapper that allows communicating with subprocesses and watching for -their completion. - -.. class:: asyncio.subprocess.Process - :module: - - An object that wraps OS processes created by the - :func:`create_subprocess_exec` and :func:`create_subprocess_shell` - functions. - - This class is designed to have a similar API to the - :class:`subprocess.Popen` class, but there are some - notable differences: - - * unlike Popen, Process instances do not have an equivalent to - the :meth:`~subprocess.Popen.poll` method; - - * the :meth:`~asyncio.subprocess.Process.communicate` and - :meth:`~asyncio.subprocess.Process.wait` methods don't have a - *timeout* parameter: use the :func:`~asyncio.wait_for` function; - - * the :meth:`Process.wait() ` method - is asynchronous, whereas :meth:`subprocess.Popen.wait` method - is implemented as a blocking busy loop; - - * the *universal_newlines* parameter is not supported. - - This class is :ref:`not thread safe `. - - See also the :ref:`Subprocess and Threads ` - section. - - .. coroutinemethod:: wait() - - Wait for the child process to terminate. - - Set and return the :attr:`returncode` attribute. - - .. note:: - - This method can deadlock when using ``stdout=PIPE`` or - ``stderr=PIPE`` and the child process generates so much output - that it blocks waiting for the OS pipe buffer to accept - more data. Use the :meth:`communicate` method when using pipes - to avoid this condition. - - .. coroutinemethod:: communicate(input=None) - - Interact with process: - - 1. send data to *stdin* (if *input* is not ``None``); - 2. read data from *stdout* and *stderr*, until EOF is reached; - 3. wait for process to terminate. - - The optional *input* argument is the data (:class:`bytes` object) - that will be sent to the child process. - - Return a tuple ``(stdout_data, stderr_data)``. - - If either :exc:`BrokenPipeError` or :exc:`ConnectionResetError` - exception is raised when writing *input* into *stdin*, the - exception is ignored. This condition occurs when the process - exits before all data are written into *stdin*. - - If it is desired to send data to the process' *stdin*, - the process needs to be created with ``stdin=PIPE``. Similarly, - to get anything other than ``None`` in the result tuple, the - process has to be created with ``stdout=PIPE`` and/or - ``stderr=PIPE`` arguments. - - Note, that the data read is buffered in memory, so do not use - this method if the data size is large or unlimited. - - .. method:: send_signal(signal) - - Sends the signal *signal* to the child process. - - .. note:: - - On Windows, :py:data:`SIGTERM` is an alias for :meth:`terminate`. - ``CTRL_C_EVENT`` and ``CTRL_BREAK_EVENT`` can be sent to processes - started with a *creationflags* parameter which includes - ``CREATE_NEW_PROCESS_GROUP``. - - .. method:: terminate() - - Stop the child process. - - On POSIX systems this method sends :py:const:`signal.SIGTERM` to the - child process. - - On Windows the Win32 API function :c:func:`TerminateProcess` is - called to stop the child process. - - .. method:: kill() - - Kill the child process. - - On POSIX systems this method sends :py:data:`SIGKILL` to the child - process. - - On Windows this method is an alias for :meth:`terminate`. - - .. attribute:: stdin - - Standard input stream (:class:`StreamWriter`) or ``None`` - if the process was created with ``stdin=None``. - - .. attribute:: stdout - - Standard output stream (:class:`StreamReader`) or ``None`` - if the process was created with ``stdout=None``. - - .. attribute:: stderr - - Standard error stream (:class:`StreamReader`) or ``None`` - if the process was created with ``stderr=None``. - - .. warning:: - - Use the :meth:`communicate` method rather than - :attr:`process.stdin.write() `, - :attr:`await process.stdout.read() ` or - :attr:`await process.stderr.read() `. - This avoids deadlocks due to streams pausing reading or writing - and blocking the child process. - - .. attribute:: pid - - Process identification number (PID). - - Note that for processes created by the :func:`create_subprocess_shell` - function, this attribute is the PID of the spawned shell. - - .. attribute:: returncode - - Return code of the process when it exits. - - A ``None`` value indicates that the process has not terminated yet. - - A negative value ``-N`` indicates that the child was terminated - by signal ``N`` (POSIX only). - - -.. _asyncio-subprocess-threads: - -Subprocess and Threads ----------------------- - -Standard asyncio event loop supports running subprocesses from different threads by -default. - -On Windows subprocesses are provided by :class:`ProactorEventLoop` only (default), -:class:`SelectorEventLoop` has no subprocess support. - -On UNIX *child watchers* are used for subprocess finish waiting, see -:ref:`asyncio-watchers` for more info. - - -.. versionchanged:: 3.8 - - UNIX switched to use :class:`ThreadedChildWatcher` for spawning subprocesses from - different threads without any limitation. - - Spawning a subprocess with *inactive* current child watcher raises - :exc:`RuntimeError`. - -Note that alternative event loop implementations might have own limitations; -please refer to their documentation. - -.. seealso:: - - The :ref:`Concurrency and multithreading in asyncio - ` section. - - -Examples --------- - -An example using the :class:`~asyncio.subprocess.Process` class to -control a subprocess and the :class:`StreamReader` class to read from -its standard output. - -.. _asyncio_example_create_subprocess_exec: - -The subprocess is created by the :func:`create_subprocess_exec` -function:: - - import asyncio - import sys - - async def get_date(): - code = 'import datetime; print(datetime.datetime.now())' - - # Create the subprocess; redirect the standard output - # into a pipe. - proc = await asyncio.create_subprocess_exec( - sys.executable, '-c', code, - stdout=asyncio.subprocess.PIPE) - - # Read one line of output. - data = await proc.stdout.readline() - line = data.decode('ascii').rstrip() - - # Wait for the subprocess exit. - await proc.wait() - return line - - date = asyncio.run(get_date()) - print(f"Current date: {date}") - - -See also the :ref:`same example ` -written using low-level APIs. diff --git a/Doc/library/asyncio-sync.rst b/Doc/library/asyncio-sync.rst deleted file mode 100644 index 05bdf5488af143..00000000000000 --- a/Doc/library/asyncio-sync.rst +++ /dev/null @@ -1,460 +0,0 @@ -.. currentmodule:: asyncio - -.. _asyncio-sync: - -========================== -Synchronization Primitives -========================== - -**Source code:** :source:`Lib/asyncio/locks.py` - ------------------------------------------------ - -asyncio synchronization primitives are designed to be similar to -those of the :mod:`threading` module with two important caveats: - -* asyncio primitives are not thread-safe, therefore they should not - be used for OS thread synchronization (use :mod:`threading` for - that); - -* methods of these synchronization primitives do not accept the *timeout* - argument; use the :func:`asyncio.wait_for` function to perform - operations with timeouts. - -asyncio has the following basic synchronization primitives: - -* :class:`Lock` -* :class:`Event` -* :class:`Condition` -* :class:`Semaphore` -* :class:`BoundedSemaphore` -* :class:`Barrier` - - ---------- - - -Lock -==== - -.. class:: Lock() - - Implements a mutex lock for asyncio tasks. Not thread-safe. - - An asyncio lock can be used to guarantee exclusive access to a - shared resource. - - The preferred way to use a Lock is an :keyword:`async with` - statement:: - - lock = asyncio.Lock() - - # ... later - async with lock: - # access shared state - - which is equivalent to:: - - lock = asyncio.Lock() - - # ... later - await lock.acquire() - try: - # access shared state - finally: - lock.release() - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. coroutinemethod:: acquire() - - Acquire the lock. - - This method waits until the lock is *unlocked*, sets it to - *locked* and returns ``True``. - - When more than one coroutine is blocked in :meth:`acquire` - waiting for the lock to be unlocked, only one coroutine - eventually proceeds. - - Acquiring a lock is *fair*: the coroutine that proceeds will be - the first coroutine that started waiting on the lock. - - .. method:: release() - - Release the lock. - - When the lock is *locked*, reset it to *unlocked* and return. - - If the lock is *unlocked*, a :exc:`RuntimeError` is raised. - - .. method:: locked() - - Return ``True`` if the lock is *locked*. - - -Event -===== - -.. class:: Event() - - An event object. Not thread-safe. - - An asyncio event can be used to notify multiple asyncio tasks - that some event has happened. - - An Event object manages an internal flag that can be set to *true* - with the :meth:`~Event.set` method and reset to *false* with the - :meth:`clear` method. The :meth:`~Event.wait` method blocks until the - flag is set to *true*. The flag is set to *false* initially. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. _asyncio_example_sync_event: - - Example:: - - async def waiter(event): - print('waiting for it ...') - await event.wait() - print('... got it!') - - async def main(): - # Create an Event object. - event = asyncio.Event() - - # Spawn a Task to wait until 'event' is set. - waiter_task = asyncio.create_task(waiter(event)) - - # Sleep for 1 second and set the event. - await asyncio.sleep(1) - event.set() - - # Wait until the waiter task is finished. - await waiter_task - - asyncio.run(main()) - - .. coroutinemethod:: wait() - - Wait until the event is set. - - If the event is set, return ``True`` immediately. - Otherwise block until another task calls :meth:`~Event.set`. - - .. method:: set() - - Set the event. - - All tasks waiting for event to be set will be immediately - awakened. - - .. method:: clear() - - Clear (unset) the event. - - Tasks awaiting on :meth:`~Event.wait` will now block until the - :meth:`~Event.set` method is called again. - - .. method:: is_set() - - Return ``True`` if the event is set. - - -Condition -========= - -.. class:: Condition(lock=None) - - A Condition object. Not thread-safe. - - An asyncio condition primitive can be used by a task to wait for - some event to happen and then get exclusive access to a shared - resource. - - In essence, a Condition object combines the functionality - of an :class:`Event` and a :class:`Lock`. It is possible to have - multiple Condition objects share one Lock, which allows coordinating - exclusive access to a shared resource between different tasks - interested in particular states of that shared resource. - - The optional *lock* argument must be a :class:`Lock` object or - ``None``. In the latter case a new Lock object is created - automatically. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - The preferred way to use a Condition is an :keyword:`async with` - statement:: - - cond = asyncio.Condition() - - # ... later - async with cond: - await cond.wait() - - which is equivalent to:: - - cond = asyncio.Condition() - - # ... later - await cond.acquire() - try: - await cond.wait() - finally: - cond.release() - - .. coroutinemethod:: acquire() - - Acquire the underlying lock. - - This method waits until the underlying lock is *unlocked*, - sets it to *locked* and returns ``True``. - - .. method:: notify(n=1) - - Wake up at most *n* tasks (1 by default) waiting on this - condition. The method is no-op if no tasks are waiting. - - The lock must be acquired before this method is called and - released shortly after. If called with an *unlocked* lock - a :exc:`RuntimeError` error is raised. - - .. method:: locked() - - Return ``True`` if the underlying lock is acquired. - - .. method:: notify_all() - - Wake up all tasks waiting on this condition. - - This method acts like :meth:`notify`, but wakes up all waiting - tasks. - - The lock must be acquired before this method is called and - released shortly after. If called with an *unlocked* lock - a :exc:`RuntimeError` error is raised. - - .. method:: release() - - Release the underlying lock. - - When invoked on an unlocked lock, a :exc:`RuntimeError` is - raised. - - .. coroutinemethod:: wait() - - Wait until notified. - - If the calling task has not acquired the lock when this method is - called, a :exc:`RuntimeError` is raised. - - This method releases the underlying lock, and then blocks until - it is awakened by a :meth:`notify` or :meth:`notify_all` call. - Once awakened, the Condition re-acquires its lock and this method - returns ``True``. - - .. coroutinemethod:: wait_for(predicate) - - Wait until a predicate becomes *true*. - - The predicate must be a callable which result will be - interpreted as a boolean value. The final value is the - return value. - - -Semaphore -========= - -.. class:: Semaphore(value=1) - - A Semaphore object. Not thread-safe. - - A semaphore manages an internal counter which is decremented by each - :meth:`acquire` call and incremented by each :meth:`release` call. - The counter can never go below zero; when :meth:`acquire` finds - that it is zero, it blocks, waiting until some task calls - :meth:`release`. - - The optional *value* argument gives the initial value for the - internal counter (``1`` by default). If the given value is - less than ``0`` a :exc:`ValueError` is raised. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - The preferred way to use a Semaphore is an :keyword:`async with` - statement:: - - sem = asyncio.Semaphore(10) - - # ... later - async with sem: - # work with shared resource - - which is equivalent to:: - - sem = asyncio.Semaphore(10) - - # ... later - await sem.acquire() - try: - # work with shared resource - finally: - sem.release() - - .. coroutinemethod:: acquire() - - Acquire a semaphore. - - If the internal counter is greater than zero, decrement - it by one and return ``True`` immediately. If it is zero, wait - until a :meth:`release` is called and return ``True``. - - .. method:: locked() - - Returns ``True`` if semaphore can not be acquired immediately. - - .. method:: release() - - Release a semaphore, incrementing the internal counter by one. - Can wake up a task waiting to acquire the semaphore. - - Unlike :class:`BoundedSemaphore`, :class:`Semaphore` allows - making more ``release()`` calls than ``acquire()`` calls. - - -BoundedSemaphore -================ - -.. class:: BoundedSemaphore(value=1) - - A bounded semaphore object. Not thread-safe. - - Bounded Semaphore is a version of :class:`Semaphore` that raises - a :exc:`ValueError` in :meth:`~Semaphore.release` if it - increases the internal counter above the initial *value*. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - -Barrier -======= - -.. class:: Barrier(parties) - - A barrier object. Not thread-safe. - - A barrier is a simple synchronization primitive that allows to block until - *parties* number of tasks are waiting on it. - Tasks can wait on the :meth:`~Barrier.wait` method and would be blocked until - the specified number of tasks end up waiting on :meth:`~Barrier.wait`. - At that point all of the waiting tasks would unblock simultaneously. - - :keyword:`async with` can be used as an alternative to awaiting on - :meth:`~Barrier.wait`. - - The barrier can be reused any number of times. - - .. _asyncio_example_barrier: - - Example:: - - async def example_barrier(): - # barrier with 3 parties - b = asyncio.Barrier(3) - - # create 2 new waiting tasks - asyncio.create_task(b.wait()) - asyncio.create_task(b.wait()) - - await asyncio.sleep(0) - print(b) - - # The third .wait() call passes the barrier - await b.wait() - print(b) - print("barrier passed") - - await asyncio.sleep(0) - print(b) - - asyncio.run(example_barrier()) - - Result of this example is:: - - - - barrier passed - - - .. versionadded:: 3.11 - - .. coroutinemethod:: wait() - - Pass the barrier. When all the tasks party to the barrier have called - this function, they are all unblocked simultaneously. - - When a waiting or blocked task in the barrier is cancelled, - this task exits the barrier which stays in the same state. - If the state of the barrier is "filling", the number of waiting task - decreases by 1. - - The return value is an integer in the range of 0 to ``parties-1``, different - for each task. This can be used to select a task to do some special - housekeeping, e.g.:: - - ... - async with barrier as position: - if position == 0: - # Only one task prints this - print('End of *draining phase*') - - This method may raise a :class:`BrokenBarrierError` exception if the - barrier is broken or reset while a task is waiting. - It could raise a :exc:`CancelledError` if a task is cancelled. - - .. coroutinemethod:: reset() - - Return the barrier to the default, empty state. Any tasks waiting on it - will receive the :class:`BrokenBarrierError` exception. - - If a barrier is broken it may be better to just leave it and create a new one. - - .. coroutinemethod:: abort() - - Put the barrier into a broken state. This causes any active or future - calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. - Use this for example if one of the tasks needs to abort, to avoid infinite - waiting tasks. - - .. attribute:: parties - - The number of tasks required to pass the barrier. - - .. attribute:: n_waiting - - The number of tasks currently waiting in the barrier while filling. - - .. attribute:: broken - - A boolean that is ``True`` if the barrier is in the broken state. - - -.. exception:: BrokenBarrierError - - This exception, a subclass of :exc:`RuntimeError`, is raised when the - :class:`Barrier` object is reset or broken. - ---------- - - -.. versionchanged:: 3.9 - - Acquiring a lock using ``await lock`` or ``yield from lock`` and/or - :keyword:`with` statement (``with await lock``, ``with (yield from - lock)``) was removed. Use ``async with lock`` instead. diff --git a/Doc/library/asyncio-task.rst b/Doc/library/asyncio-task.rst deleted file mode 100644 index 0944aef0ab619d..00000000000000 --- a/Doc/library/asyncio-task.rst +++ /dev/null @@ -1,1261 +0,0 @@ -.. currentmodule:: asyncio - - -==================== -Coroutines and Tasks -==================== - -This section outlines high-level asyncio APIs to work with coroutines -and Tasks. - -.. contents:: - :depth: 1 - :local: - - -.. _coroutine: - -Coroutines -========== - -**Source code:** :source:`Lib/asyncio/coroutines.py` - ----------------------------------------------------- - -:term:`Coroutines ` declared with the async/await syntax is the -preferred way of writing asyncio applications. For example, the following -snippet of code prints "hello", waits 1 second, -and then prints "world":: - - >>> import asyncio - - >>> async def main(): - ... print('hello') - ... await asyncio.sleep(1) - ... print('world') - - >>> asyncio.run(main()) - hello - world - -Note that simply calling a coroutine will not schedule it to -be executed:: - - >>> main() - - -To actually run a coroutine, asyncio provides the following mechanisms: - -* The :func:`asyncio.run` function to run the top-level - entry point "main()" function (see the above example.) - -* Awaiting on a coroutine. The following snippet of code will - print "hello" after waiting for 1 second, and then print "world" - after waiting for *another* 2 seconds:: - - import asyncio - import time - - async def say_after(delay, what): - await asyncio.sleep(delay) - print(what) - - async def main(): - print(f"started at {time.strftime('%X')}") - - await say_after(1, 'hello') - await say_after(2, 'world') - - print(f"finished at {time.strftime('%X')}") - - asyncio.run(main()) - - Expected output:: - - started at 17:13:52 - hello - world - finished at 17:13:55 - -* The :func:`asyncio.create_task` function to run coroutines - concurrently as asyncio :class:`Tasks `. - - Let's modify the above example and run two ``say_after`` coroutines - *concurrently*:: - - async def main(): - task1 = asyncio.create_task( - say_after(1, 'hello')) - - task2 = asyncio.create_task( - say_after(2, 'world')) - - print(f"started at {time.strftime('%X')}") - - # Wait until both tasks are completed (should take - # around 2 seconds.) - await task1 - await task2 - - print(f"finished at {time.strftime('%X')}") - - Note that expected output now shows that the snippet runs - 1 second faster than before:: - - started at 17:14:32 - hello - world - finished at 17:14:34 - -* The :class:`asyncio.TaskGroup` class provides a more modern - alternative to :func:`create_task`. - Using this API, the last example becomes:: - - async def main(): - async with asyncio.TaskGroup() as tg: - task1 = tg.create_task( - say_after(1, 'hello')) - - task2 = tg.create_task( - say_after(2, 'world')) - - print(f"started at {time.strftime('%X')}") - - # The await is implicit when the context manager exits. - - print(f"finished at {time.strftime('%X')}") - - The timing and output should be the same as for the previous version. - - .. versionadded:: 3.11 - :class:`asyncio.TaskGroup`. - - -.. _asyncio-awaitables: - -Awaitables -========== - -We say that an object is an **awaitable** object if it can be used -in an :keyword:`await` expression. Many asyncio APIs are designed to -accept awaitables. - -There are three main types of *awaitable* objects: -**coroutines**, **Tasks**, and **Futures**. - - -.. rubric:: Coroutines - -Python coroutines are *awaitables* and therefore can be awaited from -other coroutines:: - - import asyncio - - async def nested(): - return 42 - - async def main(): - # Nothing happens if we just call "nested()". - # A coroutine object is created but not awaited, - # so it *won't run at all*. - nested() - - # Let's do it differently now and await it: - print(await nested()) # will print "42". - - asyncio.run(main()) - -.. important:: - - In this documentation the term "coroutine" can be used for - two closely related concepts: - - * a *coroutine function*: an :keyword:`async def` function; - - * a *coroutine object*: an object returned by calling a - *coroutine function*. - - -.. rubric:: Tasks - -*Tasks* are used to schedule coroutines *concurrently*. - -When a coroutine is wrapped into a *Task* with functions like -:func:`asyncio.create_task` the coroutine is automatically -scheduled to run soon:: - - import asyncio - - async def nested(): - return 42 - - async def main(): - # Schedule nested() to run soon concurrently - # with "main()". - task = asyncio.create_task(nested()) - - # "task" can now be used to cancel "nested()", or - # can simply be awaited to wait until it is complete: - await task - - asyncio.run(main()) - - -.. rubric:: Futures - -A :class:`Future` is a special **low-level** awaitable object that -represents an **eventual result** of an asynchronous operation. - -When a Future object is *awaited* it means that the coroutine will -wait until the Future is resolved in some other place. - -Future objects in asyncio are needed to allow callback-based code -to be used with async/await. - -Normally **there is no need** to create Future objects at the -application level code. - -Future objects, sometimes exposed by libraries and some asyncio -APIs, can be awaited:: - - async def main(): - await function_that_returns_a_future_object() - - # this is also valid: - await asyncio.gather( - function_that_returns_a_future_object(), - some_python_coroutine() - ) - -A good example of a low-level function that returns a Future object -is :meth:`loop.run_in_executor`. - - -Creating Tasks -============== - -**Source code:** :source:`Lib/asyncio/tasks.py` - ------------------------------------------------ - -.. function:: create_task(coro, *, name=None, context=None) - - Wrap the *coro* :ref:`coroutine ` into a :class:`Task` - and schedule its execution. Return the Task object. - - If *name* is not ``None``, it is set as the name of the task using - :meth:`Task.set_name`. - - An optional keyword-only *context* argument allows specifying a - custom :class:`contextvars.Context` for the *coro* to run in. - The current context copy is created when no *context* is provided. - - The task is executed in the loop returned by :func:`get_running_loop`, - :exc:`RuntimeError` is raised if there is no running loop in - current thread. - - .. note:: - - :meth:`asyncio.TaskGroup.create_task` is a newer alternative - that allows for convenient waiting for a group of related tasks. - - .. important:: - - Save a reference to the result of this function, to avoid - a task disappearing mid-execution. The event loop only keeps - weak references to tasks. A task that isn't referenced elsewhere - may get garbage collected at any time, even before it's done. - For reliable "fire-and-forget" background tasks, gather them in - a collection:: - - background_tasks = set() - - for i in range(10): - task = asyncio.create_task(some_coro(param=i)) - - # Add task to the set. This creates a strong reference. - background_tasks.add(task) - - # To prevent keeping references to finished tasks forever, - # make each task remove its own reference from the set after - # completion: - task.add_done_callback(background_tasks.discard) - - .. versionadded:: 3.7 - - .. versionchanged:: 3.8 - Added the *name* parameter. - - .. versionchanged:: 3.11 - Added the *context* parameter. - - -Task Cancellation -================= - -Tasks can easily and safely be cancelled. -When a task is cancelled, :exc:`asyncio.CancelledError` will be raised -in the task at the next opportunity. - -It is recommended that coroutines use ``try/finally`` blocks to robustly -perform clean-up logic. In case :exc:`asyncio.CancelledError` -is explicitly caught, it should generally be propagated when -clean-up is complete. :exc:`asyncio.CancelledError` directly subclasses -:exc:`BaseException` so most code will not need to be aware of it. - -The asyncio components that enable structured concurrency, like -:class:`asyncio.TaskGroup` and :func:`asyncio.timeout`, -are implemented using cancellation internally and might misbehave if -a coroutine swallows :exc:`asyncio.CancelledError`. Similarly, user code -should not generally call :meth:`uncancel `. -However, in cases when suppressing :exc:`asyncio.CancelledError` is -truly desired, it is necessary to also call ``uncancel()`` to completely -remove the cancellation state. - -.. _taskgroups: - -Task Groups -=========== - -Task groups combine a task creation API with a convenient -and reliable way to wait for all tasks in the group to finish. - -.. class:: TaskGroup() - - An :ref:`asynchronous context manager ` - holding a group of tasks. - Tasks can be added to the group using :meth:`create_task`. - All tasks are awaited when the context manager exits. - - .. versionadded:: 3.11 - - .. method:: create_task(coro, *, name=None, context=None) - - Create a task in this task group. - The signature matches that of :func:`asyncio.create_task`. - -Example:: - - async def main(): - async with asyncio.TaskGroup() as tg: - task1 = tg.create_task(some_coro(...)) - task2 = tg.create_task(another_coro(...)) - print("Both tasks have completed now.") - -The ``async with`` statement will wait for all tasks in the group to finish. -While waiting, new tasks may still be added to the group -(for example, by passing ``tg`` into one of the coroutines -and calling ``tg.create_task()`` in that coroutine). -Once the last task has finished and the ``async with`` block is exited, -no new tasks may be added to the group. - -The first time any of the tasks belonging to the group fails -with an exception other than :exc:`asyncio.CancelledError`, -the remaining tasks in the group are cancelled. -No further tasks can then be added to the group. -At this point, if the body of the ``async with`` statement is still active -(i.e., :meth:`~object.__aexit__` hasn't been called yet), -the task directly containing the ``async with`` statement is also cancelled. -The resulting :exc:`asyncio.CancelledError` will interrupt an ``await``, -but it will not bubble out of the containing ``async with`` statement. - -Once all tasks have finished, if any tasks have failed -with an exception other than :exc:`asyncio.CancelledError`, -those exceptions are combined in an -:exc:`ExceptionGroup` or :exc:`BaseExceptionGroup` -(as appropriate; see their documentation) -which is then raised. - -Two base exceptions are treated specially: -If any task fails with :exc:`KeyboardInterrupt` or :exc:`SystemExit`, -the task group still cancels the remaining tasks and waits for them, -but then the initial :exc:`KeyboardInterrupt` or :exc:`SystemExit` -is re-raised instead of :exc:`ExceptionGroup` or :exc:`BaseExceptionGroup`. - -If the body of the ``async with`` statement exits with an exception -(so :meth:`~object.__aexit__` is called with an exception set), -this is treated the same as if one of the tasks failed: -the remaining tasks are cancelled and then waited for, -and non-cancellation exceptions are grouped into an -exception group and raised. -The exception passed into :meth:`~object.__aexit__`, -unless it is :exc:`asyncio.CancelledError`, -is also included in the exception group. -The same special case is made for -:exc:`KeyboardInterrupt` and :exc:`SystemExit` as in the previous paragraph. - - -Sleeping -======== - -.. coroutinefunction:: sleep(delay, result=None) - - Block for *delay* seconds. - - If *result* is provided, it is returned to the caller - when the coroutine completes. - - ``sleep()`` always suspends the current task, allowing other tasks - to run. - - Setting the delay to 0 provides an optimized path to allow other - tasks to run. This can be used by long-running functions to avoid - blocking the event loop for the full duration of the function call. - - .. _asyncio_example_sleep: - - Example of coroutine displaying the current date every second - for 5 seconds:: - - import asyncio - import datetime - - async def display_date(): - loop = asyncio.get_running_loop() - end_time = loop.time() + 5.0 - while True: - print(datetime.datetime.now()) - if (loop.time() + 1.0) >= end_time: - break - await asyncio.sleep(1) - - asyncio.run(display_date()) - - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - -Running Tasks Concurrently -========================== - -.. awaitablefunction:: gather(*aws, return_exceptions=False) - - Run :ref:`awaitable objects ` in the *aws* - sequence *concurrently*. - - If any awaitable in *aws* is a coroutine, it is automatically - scheduled as a Task. - - If all awaitables are completed successfully, the result is an - aggregate list of returned values. The order of result values - corresponds to the order of awaitables in *aws*. - - If *return_exceptions* is ``False`` (default), the first - raised exception is immediately propagated to the task that - awaits on ``gather()``. Other awaitables in the *aws* sequence - **won't be cancelled** and will continue to run. - - If *return_exceptions* is ``True``, exceptions are treated the - same as successful results, and aggregated in the result list. - - If ``gather()`` is *cancelled*, all submitted awaitables - (that have not completed yet) are also *cancelled*. - - If any Task or Future from the *aws* sequence is *cancelled*, it is - treated as if it raised :exc:`CancelledError` -- the ``gather()`` - call is **not** cancelled in this case. This is to prevent the - cancellation of one submitted Task/Future to cause other - Tasks/Futures to be cancelled. - - .. note:: - A more modern way to create and run tasks concurrently and - wait for their completion is :class:`asyncio.TaskGroup`. - - .. _asyncio_example_gather: - - Example:: - - import asyncio - - async def factorial(name, number): - f = 1 - for i in range(2, number + 1): - print(f"Task {name}: Compute factorial({number}), currently i={i}...") - await asyncio.sleep(1) - f *= i - print(f"Task {name}: factorial({number}) = {f}") - return f - - async def main(): - # Schedule three calls *concurrently*: - L = await asyncio.gather( - factorial("A", 2), - factorial("B", 3), - factorial("C", 4), - ) - print(L) - - asyncio.run(main()) - - # Expected output: - # - # Task A: Compute factorial(2), currently i=2... - # Task B: Compute factorial(3), currently i=2... - # Task C: Compute factorial(4), currently i=2... - # Task A: factorial(2) = 2 - # Task B: Compute factorial(3), currently i=3... - # Task C: Compute factorial(4), currently i=3... - # Task B: factorial(3) = 6 - # Task C: Compute factorial(4), currently i=4... - # Task C: factorial(4) = 24 - # [2, 6, 24] - - .. note:: - If *return_exceptions* is False, cancelling gather() after it - has been marked done won't cancel any submitted awaitables. - For instance, gather can be marked done after propagating an - exception to the caller, therefore, calling ``gather.cancel()`` - after catching an exception (raised by one of the awaitables) from - gather won't cancel any other awaitables. - - .. versionchanged:: 3.7 - If the *gather* itself is cancelled, the cancellation is - propagated regardless of *return_exceptions*. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. deprecated:: 3.10 - Deprecation warning is emitted if no positional arguments are provided - or not all positional arguments are Future-like objects - and there is no running event loop. - - -Shielding From Cancellation -=========================== - -.. awaitablefunction:: shield(aw) - - Protect an :ref:`awaitable object ` - from being :meth:`cancelled `. - - If *aw* is a coroutine it is automatically scheduled as a Task. - - The statement:: - - task = asyncio.create_task(something()) - res = await shield(task) - - is equivalent to:: - - res = await something() - - *except* that if the coroutine containing it is cancelled, the - Task running in ``something()`` is not cancelled. From the point - of view of ``something()``, the cancellation did not happen. - Although its caller is still cancelled, so the "await" expression - still raises a :exc:`CancelledError`. - - If ``something()`` is cancelled by other means (i.e. from within - itself) that would also cancel ``shield()``. - - If it is desired to completely ignore cancellation (not recommended) - the ``shield()`` function should be combined with a try/except - clause, as follows:: - - task = asyncio.create_task(something()) - try: - res = await shield(task) - except CancelledError: - res = None - - .. important:: - - Save a reference to tasks passed to this function, to avoid - a task disappearing mid-execution. The event loop only keeps - weak references to tasks. A task that isn't referenced elsewhere - may get garbage collected at any time, even before it's done. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. deprecated:: 3.10 - Deprecation warning is emitted if *aw* is not Future-like object - and there is no running event loop. - - -Timeouts -======== - -.. function:: timeout(delay) - - Return an :ref:`asynchronous context manager ` - that can be used to limit the amount of time spent waiting on - something. - - *delay* can either be ``None``, or a float/int number of - seconds to wait. If *delay* is ``None``, no time limit will - be applied; this can be useful if the delay is unknown when - the context manager is created. - - In either case, the context manager can be rescheduled after - creation using :meth:`Timeout.reschedule`. - - Example:: - - async def main(): - async with asyncio.timeout(10): - await long_running_task() - - If ``long_running_task`` takes more than 10 seconds to complete, - the context manager will cancel the current task and handle - the resulting :exc:`asyncio.CancelledError` internally, transforming it - into a :exc:`TimeoutError` which can be caught and handled. - - .. note:: - - The :func:`asyncio.timeout` context manager is what transforms - the :exc:`asyncio.CancelledError` into a :exc:`TimeoutError`, - which means the :exc:`TimeoutError` can only be caught - *outside* of the context manager. - - Example of catching :exc:`TimeoutError`:: - - async def main(): - try: - async with asyncio.timeout(10): - await long_running_task() - except TimeoutError: - print("The long operation timed out, but we've handled it.") - - print("This statement will run regardless.") - - The context manager produced by :func:`asyncio.timeout` can be - rescheduled to a different deadline and inspected. - - .. class:: Timeout(when) - - An :ref:`asynchronous context manager ` - for cancelling overdue coroutines. - - ``when`` should be an absolute time at which the context should time out, - as measured by the event loop's clock: - - - If ``when`` is ``None``, the timeout will never trigger. - - If ``when < loop.time()``, the timeout will trigger on the next - iteration of the event loop. - - .. method:: when() -> float | None - - Return the current deadline, or ``None`` if the current - deadline is not set. - - .. method:: reschedule(when: float | None) - - Reschedule the timeout. - - .. method:: expired() -> bool - - Return whether the context manager has exceeded its deadline - (expired). - - Example:: - - async def main(): - try: - # We do not know the timeout when starting, so we pass ``None``. - async with asyncio.timeout(None) as cm: - # We know the timeout now, so we reschedule it. - new_deadline = get_running_loop().time() + 10 - cm.reschedule(new_deadline) - - await long_running_task() - except TimeoutError: - pass - - if cm.expired(): - print("Looks like we haven't finished on time.") - - Timeout context managers can be safely nested. - - .. versionadded:: 3.11 - -.. function:: timeout_at(when) - - Similar to :func:`asyncio.timeout`, except *when* is the absolute time - to stop waiting, or ``None``. - - Example:: - - async def main(): - loop = get_running_loop() - deadline = loop.time() + 20 - try: - async with asyncio.timeout_at(deadline): - await long_running_task() - except TimeoutError: - print("The long operation timed out, but we've handled it.") - - print("This statement will run regardless.") - - .. versionadded:: 3.11 - -.. coroutinefunction:: wait_for(aw, timeout) - - Wait for the *aw* :ref:`awaitable ` - to complete with a timeout. - - If *aw* is a coroutine it is automatically scheduled as a Task. - - *timeout* can either be ``None`` or a float or int number of seconds - to wait for. If *timeout* is ``None``, block until the future - completes. - - If a timeout occurs, it cancels the task and raises - :exc:`TimeoutError`. - - To avoid the task :meth:`cancellation `, - wrap it in :func:`shield`. - - The function will wait until the future is actually cancelled, - so the total wait time may exceed the *timeout*. If an exception - happens during cancellation, it is propagated. - - If the wait is cancelled, the future *aw* is also cancelled. - - .. _asyncio_example_waitfor: - - Example:: - - async def eternity(): - # Sleep for one hour - await asyncio.sleep(3600) - print('yay!') - - async def main(): - # Wait for at most 1 second - try: - await asyncio.wait_for(eternity(), timeout=1.0) - except TimeoutError: - print('timeout!') - - asyncio.run(main()) - - # Expected output: - # - # timeout! - - .. versionchanged:: 3.7 - When *aw* is cancelled due to a timeout, ``wait_for`` waits - for *aw* to be cancelled. Previously, it raised - :exc:`TimeoutError` immediately. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. versionchanged:: 3.11 - Raises :exc:`TimeoutError` instead of :exc:`asyncio.TimeoutError`. - - -Waiting Primitives -================== - -.. coroutinefunction:: wait(aws, *, timeout=None, return_when=ALL_COMPLETED) - - Run :class:`~asyncio.Future` and :class:`~asyncio.Task` instances in the *aws* - iterable concurrently and block until the condition specified - by *return_when*. - - The *aws* iterable must not be empty and generators yielding tasks are not accepted. - - Returns two sets of Tasks/Futures: ``(done, pending)``. - - Usage:: - - done, pending = await asyncio.wait(aws) - - *timeout* (a float or int), if specified, can be used to control - the maximum number of seconds to wait before returning. - - Note that this function does not raise :exc:`TimeoutError`. - Futures or Tasks that aren't done when the timeout occurs are simply - returned in the second set. - - *return_when* indicates when this function should return. It must - be one of the following constants: - - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ - - Unlike :func:`~asyncio.wait_for`, ``wait()`` does not cancel the - futures when a timeout occurs. - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. versionchanged:: 3.11 - Passing coroutine objects to ``wait()`` directly is forbidden. - -.. function:: as_completed(aws, *, timeout=None) - - Run :ref:`awaitable objects ` in the *aws* - iterable concurrently. Generators yielding tasks are not accepted - as *aws* iterable. Return an iterator of coroutines. - Each coroutine returned can be awaited to get the earliest next - result from the iterable of the remaining awaitables. - - Raises :exc:`TimeoutError` if the timeout occurs before - all Futures are done. - - Example:: - - for coro in as_completed(aws): - earliest_result = await coro - # ... - - .. versionchanged:: 3.10 - Removed the *loop* parameter. - - .. deprecated:: 3.10 - Deprecation warning is emitted if not all awaitable objects in the *aws* - iterable are Future-like objects and there is no running event loop. - - -Running in Threads -================== - -.. coroutinefunction:: to_thread(func, /, *args, **kwargs) - - Asynchronously run function *func* in a separate thread. - - Any \*args and \*\*kwargs supplied for this function are directly passed - to *func*. Also, the current :class:`contextvars.Context` is propagated, - allowing context variables from the event loop thread to be accessed in the - separate thread. - - Return a coroutine that can be awaited to get the eventual result of *func*. - - This coroutine function is primarily intended to be used for executing - IO-bound functions/methods that would otherwise block the event loop if - they were run in the main thread. For example:: - - def blocking_io(): - print(f"start blocking_io at {time.strftime('%X')}") - # Note that time.sleep() can be replaced with any blocking - # IO-bound operation, such as file operations. - time.sleep(1) - print(f"blocking_io complete at {time.strftime('%X')}") - - async def main(): - print(f"started main at {time.strftime('%X')}") - - await asyncio.gather( - asyncio.to_thread(blocking_io), - asyncio.sleep(1)) - - print(f"finished main at {time.strftime('%X')}") - - - asyncio.run(main()) - - # Expected output: - # - # started main at 19:50:53 - # start blocking_io at 19:50:53 - # blocking_io complete at 19:50:54 - # finished main at 19:50:54 - - Directly calling ``blocking_io()`` in any coroutine would block the event loop - for its duration, resulting in an additional 1 second of run time. Instead, - by using ``asyncio.to_thread()``, we can run it in a separate thread without - blocking the event loop. - - .. note:: - - Due to the :term:`GIL`, ``asyncio.to_thread()`` can typically only be used - to make IO-bound functions non-blocking. However, for extension modules - that release the GIL or alternative Python implementations that don't - have one, ``asyncio.to_thread()`` can also be used for CPU-bound functions. - - .. versionadded:: 3.9 - - -Scheduling From Other Threads -============================= - -.. function:: run_coroutine_threadsafe(coro, loop) - - Submit a coroutine to the given event loop. Thread-safe. - - Return a :class:`concurrent.futures.Future` to wait for the result - from another OS thread. - - This function is meant to be called from a different OS thread - than the one where the event loop is running. Example:: - - # Create a coroutine - coro = asyncio.sleep(1, result=3) - - # Submit the coroutine to a given loop - future = asyncio.run_coroutine_threadsafe(coro, loop) - - # Wait for the result with an optional timeout argument - assert future.result(timeout) == 3 - - If an exception is raised in the coroutine, the returned Future - will be notified. It can also be used to cancel the task in - the event loop:: - - try: - result = future.result(timeout) - except TimeoutError: - print('The coroutine took too long, cancelling the task...') - future.cancel() - except Exception as exc: - print(f'The coroutine raised an exception: {exc!r}') - else: - print(f'The coroutine returned: {result!r}') - - See the :ref:`concurrency and multithreading ` - section of the documentation. - - Unlike other asyncio functions this function requires the *loop* - argument to be passed explicitly. - - .. versionadded:: 3.5.1 - - -Introspection -============= - - -.. function:: current_task(loop=None) - - Return the currently running :class:`Task` instance, or ``None`` if - no task is running. - - If *loop* is ``None`` :func:`get_running_loop` is used to get - the current loop. - - .. versionadded:: 3.7 - - -.. function:: all_tasks(loop=None) - - Return a set of not yet finished :class:`Task` objects run by - the loop. - - If *loop* is ``None``, :func:`get_running_loop` is used for getting - current loop. - - .. versionadded:: 3.7 - - -.. function:: iscoroutine(obj) - - Return ``True`` if *obj* is a coroutine object. - - .. versionadded:: 3.4 - - -Task Object -=========== - -.. class:: Task(coro, *, loop=None, name=None, context=None) - - A :class:`Future-like ` object that runs a Python - :ref:`coroutine `. Not thread-safe. - - Tasks are used to run coroutines in event loops. - If a coroutine awaits on a Future, the Task suspends - the execution of the coroutine and waits for the completion - of the Future. When the Future is *done*, the execution of - the wrapped coroutine resumes. - - Event loops use cooperative scheduling: an event loop runs - one Task at a time. While a Task awaits for the completion of a - Future, the event loop runs other Tasks, callbacks, or performs - IO operations. - - Use the high-level :func:`asyncio.create_task` function to create - Tasks, or the low-level :meth:`loop.create_task` or - :func:`ensure_future` functions. Manual instantiation of Tasks - is discouraged. - - To cancel a running Task use the :meth:`cancel` method. Calling it - will cause the Task to throw a :exc:`CancelledError` exception into - the wrapped coroutine. If a coroutine is awaiting on a Future - object during cancellation, the Future object will be cancelled. - - :meth:`cancelled` can be used to check if the Task was cancelled. - The method returns ``True`` if the wrapped coroutine did not - suppress the :exc:`CancelledError` exception and was actually - cancelled. - - :class:`asyncio.Task` inherits from :class:`Future` all of its - APIs except :meth:`Future.set_result` and - :meth:`Future.set_exception`. - - An optional keyword-only *context* argument allows specifying a - custom :class:`contextvars.Context` for the *coro* to run in. - If no *context* is provided, the Task copies the current context - and later runs its coroutine in the copied context. - - .. versionchanged:: 3.7 - Added support for the :mod:`contextvars` module. - - .. versionchanged:: 3.8 - Added the *name* parameter. - - .. deprecated:: 3.10 - Deprecation warning is emitted if *loop* is not specified - and there is no running event loop. - - .. versionchanged:: 3.11 - Added the *context* parameter. - - .. method:: done() - - Return ``True`` if the Task is *done*. - - A Task is *done* when the wrapped coroutine either returned - a value, raised an exception, or the Task was cancelled. - - .. method:: result() - - Return the result of the Task. - - If the Task is *done*, the result of the wrapped coroutine - is returned (or if the coroutine raised an exception, that - exception is re-raised.) - - If the Task has been *cancelled*, this method raises - a :exc:`CancelledError` exception. - - If the Task's result isn't yet available, this method raises - a :exc:`InvalidStateError` exception. - - .. method:: exception() - - Return the exception of the Task. - - If the wrapped coroutine raised an exception that exception - is returned. If the wrapped coroutine returned normally - this method returns ``None``. - - If the Task has been *cancelled*, this method raises a - :exc:`CancelledError` exception. - - If the Task isn't *done* yet, this method raises an - :exc:`InvalidStateError` exception. - - .. method:: add_done_callback(callback, *, context=None) - - Add a callback to be run when the Task is *done*. - - This method should only be used in low-level callback-based code. - - See the documentation of :meth:`Future.add_done_callback` - for more details. - - .. method:: remove_done_callback(callback) - - Remove *callback* from the callbacks list. - - This method should only be used in low-level callback-based code. - - See the documentation of :meth:`Future.remove_done_callback` - for more details. - - .. method:: get_stack(*, limit=None) - - Return the list of stack frames for this Task. - - If the wrapped coroutine is not done, this returns the stack - where it is suspended. If the coroutine has completed - successfully or was cancelled, this returns an empty list. - If the coroutine was terminated by an exception, this returns - the list of traceback frames. - - The frames are always ordered from oldest to newest. - - Only one stack frame is returned for a suspended coroutine. - - The optional *limit* argument sets the maximum number of frames - to return; by default all available frames are returned. - The ordering of the returned list differs depending on whether - a stack or a traceback is returned: the newest frames of a - stack are returned, but the oldest frames of a traceback are - returned. (This matches the behavior of the traceback module.) - - .. method:: print_stack(*, limit=None, file=None) - - Print the stack or traceback for this Task. - - This produces output similar to that of the traceback module - for the frames retrieved by :meth:`get_stack`. - - The *limit* argument is passed to :meth:`get_stack` directly. - - The *file* argument is an I/O stream to which the output - is written; by default output is written to :data:`sys.stdout`. - - .. method:: get_coro() - - Return the coroutine object wrapped by the :class:`Task`. - - .. versionadded:: 3.8 - - .. method:: get_name() - - Return the name of the Task. - - If no name has been explicitly assigned to the Task, the default - asyncio Task implementation generates a default name during - instantiation. - - .. versionadded:: 3.8 - - .. method:: set_name(value) - - Set the name of the Task. - - The *value* argument can be any object, which is then - converted to a string. - - In the default Task implementation, the name will be visible - in the :func:`repr` output of a task object. - - .. versionadded:: 3.8 - - .. method:: cancel(msg=None) - - Request the Task to be cancelled. - - This arranges for a :exc:`CancelledError` exception to be thrown - into the wrapped coroutine on the next cycle of the event loop. - - The coroutine then has a chance to clean up or even deny the - request by suppressing the exception with a :keyword:`try` ... - ... ``except CancelledError`` ... :keyword:`finally` block. - Therefore, unlike :meth:`Future.cancel`, :meth:`Task.cancel` does - not guarantee that the Task will be cancelled, although - suppressing cancellation completely is not common and is actively - discouraged. Should the coroutine nevertheless decide to suppress - the cancellation, it needs to call :meth:`Task.uncancel` in addition - to catching the exception. - - .. versionchanged:: 3.9 - Added the *msg* parameter. - - .. versionchanged:: 3.11 - The ``msg`` parameter is propagated from cancelled task to its awaiter. - - .. _asyncio_example_task_cancel: - - The following example illustrates how coroutines can intercept - the cancellation request:: - - async def cancel_me(): - print('cancel_me(): before sleep') - - try: - # Wait for 1 hour - await asyncio.sleep(3600) - except asyncio.CancelledError: - print('cancel_me(): cancel sleep') - raise - finally: - print('cancel_me(): after sleep') - - async def main(): - # Create a "cancel_me" Task - task = asyncio.create_task(cancel_me()) - - # Wait for 1 second - await asyncio.sleep(1) - - task.cancel() - try: - await task - except asyncio.CancelledError: - print("main(): cancel_me is cancelled now") - - asyncio.run(main()) - - # Expected output: - # - # cancel_me(): before sleep - # cancel_me(): cancel sleep - # cancel_me(): after sleep - # main(): cancel_me is cancelled now - - .. method:: cancelled() - - Return ``True`` if the Task is *cancelled*. - - The Task is *cancelled* when the cancellation was requested with - :meth:`cancel` and the wrapped coroutine propagated the - :exc:`CancelledError` exception thrown into it. - - .. method:: uncancel() - - Decrement the count of cancellation requests to this Task. - - Returns the remaining number of cancellation requests. - - Note that once execution of a cancelled task completed, further - calls to :meth:`uncancel` are ineffective. - - .. versionadded:: 3.11 - - This method is used by asyncio's internals and isn't expected to be - used by end-user code. In particular, if a Task gets successfully - uncancelled, this allows for elements of structured concurrency like - :ref:`taskgroups` and :func:`asyncio.timeout` to continue running, - isolating cancellation to the respective structured block. - For example:: - - async def make_request_with_timeout(): - try: - async with asyncio.timeout(1): - # Structured block affected by the timeout: - await make_request() - await make_another_request() - except TimeoutError: - log("There was a timeout") - # Outer code not affected by the timeout: - await unrelated_code() - - While the block with ``make_request()`` and ``make_another_request()`` - might get cancelled due to the timeout, ``unrelated_code()`` should - continue running even in case of the timeout. This is implemented - with :meth:`uncancel`. :class:`TaskGroup` context managers use - :func:`uncancel` in a similar fashion. - - If end-user code is, for some reason, suppresing cancellation by - catching :exc:`CancelledError`, it needs to call this method to remove - the cancellation state. - - .. method:: cancelling() - - Return the number of pending cancellation requests to this Task, i.e., - the number of calls to :meth:`cancel` less the number of - :meth:`uncancel` calls. - - Note that if this number is greater than zero but the Task is - still executing, :meth:`cancelled` will still return ``False``. - This is because this number can be lowered by calling :meth:`uncancel`, - which can lead to the task not being cancelled after all if the - cancellation requests go down to zero. - - This method is used by asyncio's internals and isn't expected to be - used by end-user code. See :meth:`uncancel` for more details. - - .. versionadded:: 3.11 diff --git a/Doc/library/asyncio.rst b/Doc/library/asyncio.rst deleted file mode 100644 index 5f33c6813e74c0..00000000000000 --- a/Doc/library/asyncio.rst +++ /dev/null @@ -1,112 +0,0 @@ -:mod:`asyncio` --- Asynchronous I/O -=================================== - -.. module:: asyncio - :synopsis: Asynchronous I/O. - -------------------------------- - -.. sidebar:: Hello World! - - :: - - import asyncio - - async def main(): - print('Hello ...') - await asyncio.sleep(1) - print('... World!') - - asyncio.run(main()) - -asyncio is a library to write **concurrent** code using -the **async/await** syntax. - -asyncio is used as a foundation for multiple Python asynchronous -frameworks that provide high-performance network and web-servers, -database connection libraries, distributed task queues, etc. - -asyncio is often a perfect fit for IO-bound and high-level -**structured** network code. - -asyncio provides a set of **high-level** APIs to: - -* :ref:`run Python coroutines ` concurrently and - have full control over their execution; - -* perform :ref:`network IO and IPC `; - -* control :ref:`subprocesses `; - -* distribute tasks via :ref:`queues `; - -* :ref:`synchronize ` concurrent code; - -Additionally, there are **low-level** APIs for -*library and framework developers* to: - -* create and manage :ref:`event loops `, which - provide asynchronous APIs for :ref:`networking `, - running :ref:`subprocesses `, - handling :ref:`OS signals `, etc; - -* implement efficient protocols using - :ref:`transports `; - -* :ref:`bridge ` callback-based libraries and code - with async/await syntax. - -.. _asyncio-cli: - -You can experiment with an ``asyncio`` concurrent context in the REPL: - -.. code-block:: pycon - - $ python -m asyncio - asyncio REPL ... - Use "await" directly instead of "asyncio.run()". - Type "help", "copyright", "credits" or "license" for more information. - >>> import asyncio - >>> await asyncio.sleep(10, result='hello') - 'hello' - -.. include:: ../includes/wasm-notavail.rst - -.. We use the "rubric" directive here to avoid creating - the "Reference" subsection in the TOC. - -.. rubric:: Reference - -.. toctree:: - :caption: High-level APIs - :maxdepth: 1 - - asyncio-runner.rst - asyncio-task.rst - asyncio-stream.rst - asyncio-sync.rst - asyncio-subprocess.rst - asyncio-queue.rst - asyncio-exceptions.rst - -.. toctree:: - :caption: Low-level APIs - :maxdepth: 1 - - asyncio-eventloop.rst - asyncio-future.rst - asyncio-protocol.rst - asyncio-policy.rst - asyncio-platforms.rst - asyncio-extending.rst - -.. toctree:: - :caption: Guides and Tutorials - :maxdepth: 1 - - asyncio-api-index.rst - asyncio-llapi-index.rst - asyncio-dev.rst - -.. note:: - The source code for asyncio can be found in :source:`Lib/asyncio/`. diff --git a/Doc/library/asyncore.rst b/Doc/library/asyncore.rst deleted file mode 100644 index a3a4e90d05277e..00000000000000 --- a/Doc/library/asyncore.rst +++ /dev/null @@ -1,365 +0,0 @@ -:mod:`asyncore` --- Asynchronous socket handler -=============================================== - -.. module:: asyncore - :synopsis: A base class for developing asynchronous socket handling - services. - :deprecated: - -.. moduleauthor:: Sam Rushing -.. sectionauthor:: Christopher Petrilli -.. sectionauthor:: Steve Holden -.. heavily adapted from original documentation by Sam Rushing - -**Source code:** :source:`Lib/asyncore.py` - -.. deprecated-removed:: 3.6 3.12 - The :mod:`asyncore` module is deprecated - (see :pep:`PEP 594 <594#asyncore>` for details). - Please use :mod:`asyncio` instead. - --------------- - -.. note:: - - This module exists for backwards compatibility only. For new code we - recommend using :mod:`asyncio`. - -This module provides the basic infrastructure for writing asynchronous socket -service clients and servers. - -.. include:: ../includes/wasm-notavail.rst - -There are only two ways to have a program on a single processor do "more than -one thing at a time." Multi-threaded programming is the simplest and most -popular way to do it, but there is another very different technique, that lets -you have nearly all the advantages of multi-threading, without actually using -multiple threads. It's really only practical if your program is largely I/O -bound. If your program is processor bound, then pre-emptive scheduled threads -are probably what you really need. Network servers are rarely processor -bound, however. - -If your operating system supports the :c:func:`select` system call in its I/O -library (and nearly all do), then you can use it to juggle multiple -communication channels at once; doing other work while your I/O is taking -place in the "background." Although this strategy can seem strange and -complex, especially at first, it is in many ways easier to understand and -control than multi-threaded programming. The :mod:`asyncore` module solves -many of the difficult problems for you, making the task of building -sophisticated high-performance network servers and clients a snap. For -"conversational" applications and protocols the companion :mod:`asynchat` -module is invaluable. - -The basic idea behind both modules is to create one or more network -*channels*, instances of class :class:`asyncore.dispatcher` and -:class:`asynchat.async_chat`. Creating the channels adds them to a global -map, used by the :func:`loop` function if you do not provide it with your own -*map*. - -Once the initial channel(s) is(are) created, calling the :func:`loop` function -activates channel service, which continues until the last channel (including -any that have been added to the map during asynchronous service) is closed. - - -.. function:: loop([timeout[, use_poll[, map[,count]]]]) - - Enter a polling loop that terminates after count passes or all open - channels have been closed. All arguments are optional. The *count* - parameter defaults to ``None``, resulting in the loop terminating only when all - channels have been closed. The *timeout* argument sets the timeout - parameter for the appropriate :func:`~select.select` or :func:`~select.poll` - call, measured in seconds; the default is 30 seconds. The *use_poll* - parameter, if true, indicates that :func:`~select.poll` should be used in - preference to :func:`~select.select` (the default is ``False``). - - The *map* parameter is a dictionary whose items are the channels to watch. - As channels are closed they are deleted from their map. If *map* is - omitted, a global map is used. Channels (instances of - :class:`asyncore.dispatcher`, :class:`asynchat.async_chat` and subclasses - thereof) can freely be mixed in the map. - - -.. class:: dispatcher() - - The :class:`dispatcher` class is a thin wrapper around a low-level socket - object. To make it more useful, it has a few methods for event-handling - which are called from the asynchronous loop. Otherwise, it can be treated - as a normal non-blocking socket object. - - The firing of low-level events at certain times or in certain connection - states tells the asynchronous loop that certain higher-level events have - taken place. For example, if we have asked for a socket to connect to - another host, we know that the connection has been made when the socket - becomes writable for the first time (at this point you know that you may - write to it with the expectation of success). The implied higher-level - events are: - - +----------------------+----------------------------------------+ - | Event | Description | - +======================+========================================+ - | ``handle_connect()`` | Implied by the first read or write | - | | event | - +----------------------+----------------------------------------+ - | ``handle_close()`` | Implied by a read event with no data | - | | available | - +----------------------+----------------------------------------+ - | ``handle_accepted()``| Implied by a read event on a listening | - | | socket | - +----------------------+----------------------------------------+ - - During asynchronous processing, each mapped channel's :meth:`readable` and - :meth:`writable` methods are used to determine whether the channel's socket - should be added to the list of channels :c:func:`select`\ ed or - :c:func:`poll`\ ed for read and write events. - - Thus, the set of channel events is larger than the basic socket events. The - full set of methods that can be overridden in your subclass follows: - - - .. method:: handle_read() - - Called when the asynchronous loop detects that a :meth:`read` call on the - channel's socket will succeed. - - - .. method:: handle_write() - - Called when the asynchronous loop detects that a writable socket can be - written. Often this method will implement the necessary buffering for - performance. For example:: - - def handle_write(self): - sent = self.send(self.buffer) - self.buffer = self.buffer[sent:] - - - .. method:: handle_expt() - - Called when there is out of band (OOB) data for a socket connection. This - will almost never happen, as OOB is tenuously supported and rarely used. - - - .. method:: handle_connect() - - Called when the active opener's socket actually makes a connection. Might - send a "welcome" banner, or initiate a protocol negotiation with the - remote endpoint, for example. - - - .. method:: handle_close() - - Called when the socket is closed. - - - .. method:: handle_error() - - Called when an exception is raised and not otherwise handled. The default - version prints a condensed traceback. - - - .. method:: handle_accept() - - Called on listening channels (passive openers) when a connection can be - established with a new remote endpoint that has issued a :meth:`connect` - call for the local endpoint. Deprecated in version 3.2; use - :meth:`handle_accepted` instead. - - .. deprecated:: 3.2 - - - .. method:: handle_accepted(sock, addr) - - Called on listening channels (passive openers) when a connection has been - established with a new remote endpoint that has issued a :meth:`connect` - call for the local endpoint. *sock* is a *new* socket object usable to - send and receive data on the connection, and *addr* is the address - bound to the socket on the other end of the connection. - - .. versionadded:: 3.2 - - - .. method:: readable() - - Called each time around the asynchronous loop to determine whether a - channel's socket should be added to the list on which read events can - occur. The default method simply returns ``True``, indicating that by - default, all channels will be interested in read events. - - - .. method:: writable() - - Called each time around the asynchronous loop to determine whether a - channel's socket should be added to the list on which write events can - occur. The default method simply returns ``True``, indicating that by - default, all channels will be interested in write events. - - - In addition, each channel delegates or extends many of the socket methods. - Most of these are nearly identical to their socket partners. - - - .. method:: create_socket(family=socket.AF_INET, type=socket.SOCK_STREAM) - - This is identical to the creation of a normal socket, and will use the - same options for creation. Refer to the :mod:`socket` documentation for - information on creating sockets. - - .. versionchanged:: 3.3 - *family* and *type* arguments can be omitted. - - - .. method:: connect(address) - - As with the normal socket object, *address* is a tuple with the first - element the host to connect to, and the second the port number. - - - .. method:: send(data) - - Send *data* to the remote end-point of the socket. - - - .. method:: recv(buffer_size) - - Read at most *buffer_size* bytes from the socket's remote end-point. An - empty bytes object implies that the channel has been closed from the - other end. - - Note that :meth:`recv` may raise :exc:`BlockingIOError` , even though - :func:`select.select` or :func:`select.poll` has reported the socket - ready for reading. - - - .. method:: listen(backlog) - - Listen for connections made to the socket. The *backlog* argument - specifies the maximum number of queued connections and should be at least - 1; the maximum value is system-dependent (usually 5). - - - .. method:: bind(address) - - Bind the socket to *address*. The socket must not already be bound. (The - format of *address* depends on the address family --- refer to the - :mod:`socket` documentation for more information.) To mark - the socket as re-usable (setting the :const:`SO_REUSEADDR` option), call - the :class:`dispatcher` object's :meth:`set_reuse_addr` method. - - - .. method:: accept() - - Accept a connection. The socket must be bound to an address and listening - for connections. The return value can be either ``None`` or a pair - ``(conn, address)`` where *conn* is a *new* socket object usable to send - and receive data on the connection, and *address* is the address bound to - the socket on the other end of the connection. - When ``None`` is returned it means the connection didn't take place, in - which case the server should just ignore this event and keep listening - for further incoming connections. - - - .. method:: close() - - Close the socket. All future operations on the socket object will fail. - The remote end-point will receive no more data (after queued data is - flushed). Sockets are automatically closed when they are - garbage-collected. - - -.. class:: dispatcher_with_send() - - A :class:`dispatcher` subclass which adds simple buffered output capability, - useful for simple clients. For more sophisticated usage use - :class:`asynchat.async_chat`. - -.. class:: file_dispatcher() - - A file_dispatcher takes a file descriptor or :term:`file object` along - with an optional map argument and wraps it for use with the :c:func:`poll` - or :c:func:`loop` functions. If provided a file object or anything with a - :c:func:`fileno` method, that method will be called and passed to the - :class:`file_wrapper` constructor. - - .. availability:: Unix. - -.. class:: file_wrapper() - - A file_wrapper takes an integer file descriptor and calls :func:`os.dup` to - duplicate the handle so that the original handle may be closed independently - of the file_wrapper. This class implements sufficient methods to emulate a - socket for use by the :class:`file_dispatcher` class. - - .. availability:: Unix. - - -.. _asyncore-example-1: - -asyncore Example basic HTTP client ----------------------------------- - -Here is a very basic HTTP client that uses the :class:`dispatcher` class to -implement its socket handling:: - - import asyncore - - class HTTPClient(asyncore.dispatcher): - - def __init__(self, host, path): - asyncore.dispatcher.__init__(self) - self.create_socket() - self.connect( (host, 80) ) - self.buffer = bytes('GET %s HTTP/1.0\r\nHost: %s\r\n\r\n' % - (path, host), 'ascii') - - def handle_connect(self): - pass - - def handle_close(self): - self.close() - - def handle_read(self): - print(self.recv(8192)) - - def writable(self): - return (len(self.buffer) > 0) - - def handle_write(self): - sent = self.send(self.buffer) - self.buffer = self.buffer[sent:] - - - client = HTTPClient('www.python.org', '/') - asyncore.loop() - -.. _asyncore-example-2: - -asyncore Example basic echo server ----------------------------------- - -Here is a basic echo server that uses the :class:`dispatcher` class to accept -connections and dispatches the incoming connections to a handler:: - - import asyncore - - class EchoHandler(asyncore.dispatcher_with_send): - - def handle_read(self): - data = self.recv(8192) - if data: - self.send(data) - - class EchoServer(asyncore.dispatcher): - - def __init__(self, host, port): - asyncore.dispatcher.__init__(self) - self.create_socket() - self.set_reuse_addr() - self.bind((host, port)) - self.listen(5) - - def handle_accepted(self, sock, addr): - print('Incoming connection from %s' % repr(addr)) - handler = EchoHandler(sock) - - server = EchoServer('localhost', 8080) - asyncore.loop() diff --git a/Doc/library/atexit.rst b/Doc/library/atexit.rst deleted file mode 100644 index a2bd85b31c9a8d..00000000000000 --- a/Doc/library/atexit.rst +++ /dev/null @@ -1,117 +0,0 @@ -:mod:`atexit` --- Exit handlers -=============================== - -.. module:: atexit - :synopsis: Register and execute cleanup functions. - -.. moduleauthor:: Skip Montanaro -.. sectionauthor:: Skip Montanaro - --------------- - -The :mod:`atexit` module defines functions to register and unregister cleanup -functions. Functions thus registered are automatically executed upon normal -interpreter termination. :mod:`atexit` runs these functions in the *reverse* -order in which they were registered; if you register ``A``, ``B``, and ``C``, -at interpreter termination time they will be run in the order ``C``, ``B``, -``A``. - -**Note:** The functions registered via this module are not called when the -program is killed by a signal not handled by Python, when a Python fatal -internal error is detected, or when :func:`os._exit` is called. - -**Note:** The effect of registering or unregistering functions from within -a cleanup function is undefined. - -.. versionchanged:: 3.7 - When used with C-API subinterpreters, registered functions - are local to the interpreter they were registered in. - -.. function:: register(func, *args, **kwargs) - - Register *func* as a function to be executed at termination. Any optional - arguments that are to be passed to *func* must be passed as arguments to - :func:`register`. It is possible to register the same function and arguments - more than once. - - At normal program termination (for instance, if :func:`sys.exit` is called or - the main module's execution completes), all functions registered are called in - last in, first out order. The assumption is that lower level modules will - normally be imported before higher level modules and thus must be cleaned up - later. - - If an exception is raised during execution of the exit handlers, a traceback is - printed (unless :exc:`SystemExit` is raised) and the exception information is - saved. After all exit handlers have had a chance to run, the last exception to - be raised is re-raised. - - This function returns *func*, which makes it possible to use it as a - decorator. - - -.. function:: unregister(func) - - Remove *func* from the list of functions to be run at interpreter shutdown. - :func:`unregister` silently does nothing if *func* was not previously - registered. If *func* has been registered more than once, every occurrence - of that function in the :mod:`atexit` call stack will be removed. Equality - comparisons (``==``) are used internally during unregistration, so function - references do not need to have matching identities. - - -.. seealso:: - - Module :mod:`readline` - Useful example of :mod:`atexit` to read and write :mod:`readline` history - files. - - -.. _atexit-example: - -:mod:`atexit` Example ---------------------- - -The following simple example demonstrates how a module can initialize a counter -from a file when it is imported and save the counter's updated value -automatically when the program terminates without relying on the application -making an explicit call into this module at termination. :: - - try: - with open('counterfile') as infile: - _count = int(infile.read()) - except FileNotFoundError: - _count = 0 - - def incrcounter(n): - global _count - _count = _count + n - - def savecounter(): - with open('counterfile', 'w') as outfile: - outfile.write('%d' % _count) - - import atexit - - atexit.register(savecounter) - -Positional and keyword arguments may also be passed to :func:`register` to be -passed along to the registered function when it is called:: - - def goodbye(name, adjective): - print('Goodbye %s, it was %s to meet you.' % (name, adjective)) - - import atexit - - atexit.register(goodbye, 'Donny', 'nice') - # or: - atexit.register(goodbye, adjective='nice', name='Donny') - -Usage as a :term:`decorator`:: - - import atexit - - @atexit.register - def goodbye(): - print('You are now leaving the Python sector.') - -This only works with functions that can be called without arguments. diff --git a/Doc/library/audioop.rst b/Doc/library/audioop.rst deleted file mode 100644 index 1f96575d08f5b1..00000000000000 --- a/Doc/library/audioop.rst +++ /dev/null @@ -1,287 +0,0 @@ -:mod:`audioop` --- Manipulate raw audio data -============================================ - -.. module:: audioop - :synopsis: Manipulate raw audio data. - :deprecated: - -.. deprecated-removed:: 3.11 3.13 - The :mod:`audioop` module is deprecated - (see :pep:`PEP 594 <594#audioop>` for details). - --------------- - -The :mod:`audioop` module contains some useful operations on sound fragments. -It operates on sound fragments consisting of signed integer samples 8, 16, 24 -or 32 bits wide, stored in :term:`bytes-like objects `. All scalar items are -integers, unless specified otherwise. - -.. versionchanged:: 3.4 - Support for 24-bit samples was added. - All functions now accept any :term:`bytes-like object`. - String input now results in an immediate error. - -.. index:: - single: Intel/DVI ADPCM - single: ADPCM, Intel/DVI - single: a-LAW - single: u-LAW - -This module provides support for a-LAW, u-LAW and Intel/DVI ADPCM encodings. - -.. This para is mostly here to provide an excuse for the index entries... - -A few of the more complicated operations only take 16-bit samples, otherwise the -sample size (in bytes) is always a parameter of the operation. - -The module defines the following variables and functions: - - -.. exception:: error - - This exception is raised on all errors, such as unknown number of bytes per - sample, etc. - - -.. function:: add(fragment1, fragment2, width) - - Return a fragment which is the addition of the two samples passed as parameters. - *width* is the sample width in bytes, either ``1``, ``2``, ``3`` or ``4``. Both - fragments should have the same length. Samples are truncated in case of overflow. - - -.. function:: adpcm2lin(adpcmfragment, width, state) - - Decode an Intel/DVI ADPCM coded fragment to a linear fragment. See the - description of :func:`lin2adpcm` for details on ADPCM coding. Return a tuple - ``(sample, newstate)`` where the sample has the width specified in *width*. - - -.. function:: alaw2lin(fragment, width) - - Convert sound fragments in a-LAW encoding to linearly encoded sound fragments. - a-LAW encoding always uses 8 bits samples, so *width* refers only to the sample - width of the output fragment here. - - -.. function:: avg(fragment, width) - - Return the average over all samples in the fragment. - - -.. function:: avgpp(fragment, width) - - Return the average peak-peak value over all samples in the fragment. No - filtering is done, so the usefulness of this routine is questionable. - - -.. function:: bias(fragment, width, bias) - - Return a fragment that is the original fragment with a bias added to each - sample. Samples wrap around in case of overflow. - - -.. function:: byteswap(fragment, width) - - "Byteswap" all samples in a fragment and returns the modified fragment. - Converts big-endian samples to little-endian and vice versa. - - .. versionadded:: 3.4 - - -.. function:: cross(fragment, width) - - Return the number of zero crossings in the fragment passed as an argument. - - -.. function:: findfactor(fragment, reference) - - Return a factor *F* such that ``rms(add(fragment, mul(reference, -F)))`` is - minimal, i.e., return the factor with which you should multiply *reference* to - make it match as well as possible to *fragment*. The fragments should both - contain 2-byte samples. - - The time taken by this routine is proportional to ``len(fragment)``. - - -.. function:: findfit(fragment, reference) - - Try to match *reference* as well as possible to a portion of *fragment* (which - should be the longer fragment). This is (conceptually) done by taking slices - out of *fragment*, using :func:`findfactor` to compute the best match, and - minimizing the result. The fragments should both contain 2-byte samples. - Return a tuple ``(offset, factor)`` where *offset* is the (integer) offset into - *fragment* where the optimal match started and *factor* is the (floating-point) - factor as per :func:`findfactor`. - - -.. function:: findmax(fragment, length) - - Search *fragment* for a slice of length *length* samples (not bytes!) with - maximum energy, i.e., return *i* for which ``rms(fragment[i*2:(i+length)*2])`` - is maximal. The fragments should both contain 2-byte samples. - - The routine takes time proportional to ``len(fragment)``. - - -.. function:: getsample(fragment, width, index) - - Return the value of sample *index* from the fragment. - - -.. function:: lin2adpcm(fragment, width, state) - - Convert samples to 4 bit Intel/DVI ADPCM encoding. ADPCM coding is an adaptive - coding scheme, whereby each 4 bit number is the difference between one sample - and the next, divided by a (varying) step. The Intel/DVI ADPCM algorithm has - been selected for use by the IMA, so it may well become a standard. - - *state* is a tuple containing the state of the coder. The coder returns a tuple - ``(adpcmfrag, newstate)``, and the *newstate* should be passed to the next call - of :func:`lin2adpcm`. In the initial call, ``None`` can be passed as the state. - *adpcmfrag* is the ADPCM coded fragment packed 2 4-bit values per byte. - - -.. function:: lin2alaw(fragment, width) - - Convert samples in the audio fragment to a-LAW encoding and return this as a - bytes object. a-LAW is an audio encoding format whereby you get a dynamic - range of about 13 bits using only 8 bit samples. It is used by the Sun audio - hardware, among others. - - -.. function:: lin2lin(fragment, width, newwidth) - - Convert samples between 1-, 2-, 3- and 4-byte formats. - - .. note:: - - In some audio formats, such as .WAV files, 16, 24 and 32 bit samples are - signed, but 8 bit samples are unsigned. So when converting to 8 bit wide - samples for these formats, you need to also add 128 to the result:: - - new_frames = audioop.lin2lin(frames, old_width, 1) - new_frames = audioop.bias(new_frames, 1, 128) - - The same, in reverse, has to be applied when converting from 8 to 16, 24 - or 32 bit width samples. - - -.. function:: lin2ulaw(fragment, width) - - Convert samples in the audio fragment to u-LAW encoding and return this as a - bytes object. u-LAW is an audio encoding format whereby you get a dynamic - range of about 14 bits using only 8 bit samples. It is used by the Sun audio - hardware, among others. - - -.. function:: max(fragment, width) - - Return the maximum of the *absolute value* of all samples in a fragment. - - -.. function:: maxpp(fragment, width) - - Return the maximum peak-peak value in the sound fragment. - - -.. function:: minmax(fragment, width) - - Return a tuple consisting of the minimum and maximum values of all samples in - the sound fragment. - - -.. function:: mul(fragment, width, factor) - - Return a fragment that has all samples in the original fragment multiplied by - the floating-point value *factor*. Samples are truncated in case of overflow. - - -.. function:: ratecv(fragment, width, nchannels, inrate, outrate, state[, weightA[, weightB]]) - - Convert the frame rate of the input fragment. - - *state* is a tuple containing the state of the converter. The converter returns - a tuple ``(newfragment, newstate)``, and *newstate* should be passed to the next - call of :func:`ratecv`. The initial call should pass ``None`` as the state. - - The *weightA* and *weightB* arguments are parameters for a simple digital filter - and default to ``1`` and ``0`` respectively. - - -.. function:: reverse(fragment, width) - - Reverse the samples in a fragment and returns the modified fragment. - - -.. function:: rms(fragment, width) - - Return the root-mean-square of the fragment, i.e. ``sqrt(sum(S_i^2)/n)``. - - This is a measure of the power in an audio signal. - - -.. function:: tomono(fragment, width, lfactor, rfactor) - - Convert a stereo fragment to a mono fragment. The left channel is multiplied by - *lfactor* and the right channel by *rfactor* before adding the two channels to - give a mono signal. - - -.. function:: tostereo(fragment, width, lfactor, rfactor) - - Generate a stereo fragment from a mono fragment. Each pair of samples in the - stereo fragment are computed from the mono sample, whereby left channel samples - are multiplied by *lfactor* and right channel samples by *rfactor*. - - -.. function:: ulaw2lin(fragment, width) - - Convert sound fragments in u-LAW encoding to linearly encoded sound fragments. - u-LAW encoding always uses 8 bits samples, so *width* refers only to the sample - width of the output fragment here. - -Note that operations such as :func:`.mul` or :func:`.max` make no distinction -between mono and stereo fragments, i.e. all samples are treated equal. If this -is a problem the stereo fragment should be split into two mono fragments first -and recombined later. Here is an example of how to do that:: - - def mul_stereo(sample, width, lfactor, rfactor): - lsample = audioop.tomono(sample, width, 1, 0) - rsample = audioop.tomono(sample, width, 0, 1) - lsample = audioop.mul(lsample, width, lfactor) - rsample = audioop.mul(rsample, width, rfactor) - lsample = audioop.tostereo(lsample, width, 1, 0) - rsample = audioop.tostereo(rsample, width, 0, 1) - return audioop.add(lsample, rsample, width) - -If you use the ADPCM coder to build network packets and you want your protocol -to be stateless (i.e. to be able to tolerate packet loss) you should not only -transmit the data but also the state. Note that you should send the *initial* -state (the one you passed to :func:`lin2adpcm`) along to the decoder, not the -final state (as returned by the coder). If you want to use -:class:`struct.Struct` to store the state in binary you can code the first -element (the predicted value) in 16 bits and the second (the delta index) in 8. - -The ADPCM coders have never been tried against other ADPCM coders, only against -themselves. It could well be that I misinterpreted the standards in which case -they will not be interoperable with the respective standards. - -The :func:`find\*` routines might look a bit funny at first sight. They are -primarily meant to do echo cancellation. A reasonably fast way to do this is to -pick the most energetic piece of the output sample, locate that in the input -sample and subtract the whole output sample from the input sample:: - - def echocancel(outputdata, inputdata): - pos = audioop.findmax(outputdata, 800) # one tenth second - out_test = outputdata[pos*2:] - in_test = inputdata[pos*2:] - ipos, factor = audioop.findfit(in_test, out_test) - # Optional (for better cancellation): - # factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)], - # out_test) - prefill = '\0'*(pos+ipos)*2 - postfill = '\0'*(len(inputdata)-len(prefill)-len(outputdata)) - outputdata = prefill + audioop.mul(outputdata, 2, -factor) + postfill - return audioop.add(inputdata, outputdata, 2) - diff --git a/Doc/library/audit_events.rst b/Doc/library/audit_events.rst deleted file mode 100644 index 8227a7955bef81..00000000000000 --- a/Doc/library/audit_events.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. _audit-events: - -.. index:: single: audit events - -Audit events table -================== - -This table contains all events raised by :func:`sys.audit` or -:c:func:`PySys_Audit` calls throughout the CPython runtime and the -standard library. These calls were added in 3.8.0 or later (see :pep:`578`). - -See :func:`sys.addaudithook` and :c:func:`PySys_AddAuditHook` for -information on handling these events. - -.. impl-detail:: - - This table is generated from the CPython documentation, and may not - represent events raised by other implementations. See your runtime - specific documentation for actual events raised. - -.. audit-event-table:: - -The following events are raised internally and do not correspond to any -public API of CPython: - -+--------------------------+-------------------------------------------+ -| Audit event | Arguments | -+==========================+===========================================+ -| _winapi.CreateFile | ``file_name``, ``desired_access``, | -| | ``share_mode``, ``creation_disposition``, | -| | ``flags_and_attributes`` | -+--------------------------+-------------------------------------------+ -| _winapi.CreateJunction | ``src_path``, ``dst_path`` | -+--------------------------+-------------------------------------------+ -| _winapi.CreateNamedPipe | ``name``, ``open_mode``, ``pipe_mode`` | -+--------------------------+-------------------------------------------+ -| _winapi.CreatePipe | | -+--------------------------+-------------------------------------------+ -| _winapi.CreateProcess | ``application_name``, ``command_line``, | -| | ``current_directory`` | -+--------------------------+-------------------------------------------+ -| _winapi.OpenProcess | ``process_id``, ``desired_access`` | -+--------------------------+-------------------------------------------+ -| _winapi.TerminateProcess | ``handle``, ``exit_code`` | -+--------------------------+-------------------------------------------+ -| ctypes.PyObj_FromPtr | ``obj`` | -+--------------------------+-------------------------------------------+ diff --git a/Doc/library/base64.rst b/Doc/library/base64.rst deleted file mode 100644 index d5b6af8c1928ef..00000000000000 --- a/Doc/library/base64.rst +++ /dev/null @@ -1,311 +0,0 @@ -:mod:`base64` --- Base16, Base32, Base64, Base85 Data Encodings -=============================================================== - -.. module:: base64 - :synopsis: RFC 4648: Base16, Base32, Base64 Data Encodings; - Base85 and Ascii85 - -**Source code:** :source:`Lib/base64.py` - -.. index:: - pair: base64; encoding - single: MIME; base64 encoding - --------------- - -This module provides functions for encoding binary data to printable -ASCII characters and decoding such encodings back to binary data. -It provides encoding and decoding functions for the encodings specified in -:rfc:`4648`, which defines the Base16, Base32, and Base64 algorithms, -and for the de-facto standard Ascii85 and Base85 encodings. - -The :rfc:`4648` encodings are suitable for encoding binary data so that it can be -safely sent by email, used as parts of URLs, or included as part of an HTTP -POST request. The encoding algorithm is not the same as the -:program:`uuencode` program. - -There are two interfaces provided by this module. The modern interface -supports encoding :term:`bytes-like objects ` to ASCII -:class:`bytes`, and decoding :term:`bytes-like objects ` or -strings containing ASCII to :class:`bytes`. Both base-64 alphabets -defined in :rfc:`4648` (normal, and URL- and filesystem-safe) are supported. - -The legacy interface does not support decoding from strings, but it does -provide functions for encoding and decoding to and from :term:`file objects -`. It only supports the Base64 standard alphabet, and it adds -newlines every 76 characters as per :rfc:`2045`. Note that if you are looking -for :rfc:`2045` support you probably want to be looking at the :mod:`email` -package instead. - - -.. versionchanged:: 3.3 - ASCII-only Unicode strings are now accepted by the decoding functions of - the modern interface. - -.. versionchanged:: 3.4 - Any :term:`bytes-like objects ` are now accepted by all - encoding and decoding functions in this module. Ascii85/Base85 support added. - -The modern interface provides: - -.. function:: b64encode(s, altchars=None) - - Encode the :term:`bytes-like object` *s* using Base64 and return the encoded - :class:`bytes`. - - Optional *altchars* must be a :term:`bytes-like object` of length 2 which - specifies an alternative alphabet for the ``+`` and ``/`` characters. - This allows an application to e.g. generate URL or filesystem safe Base64 - strings. The default is ``None``, for which the standard Base64 alphabet is used. - - May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. Raises a - :exc:`TypeError` if *altchars* is not a :term:`bytes-like object`. - - -.. function:: b64decode(s, altchars=None, validate=False) - - Decode the Base64 encoded :term:`bytes-like object` or ASCII string - *s* and return the decoded :class:`bytes`. - - Optional *altchars* must be a :term:`bytes-like object` or ASCII string - of length 2 which specifies the alternative alphabet used instead of the - ``+`` and ``/`` characters. - - A :exc:`binascii.Error` exception is raised - if *s* is incorrectly padded. - - If *validate* is ``False`` (the default), characters that are neither - in the normal base-64 alphabet nor the alternative alphabet are - discarded prior to the padding check. If *validate* is ``True``, - these non-alphabet characters in the input result in a - :exc:`binascii.Error`. - - For more information about the strict base64 check, see :func:`binascii.a2b_base64` - - May assert or raise a :exc:`ValueError` if the length of *altchars* is not 2. - -.. function:: standard_b64encode(s) - - Encode :term:`bytes-like object` *s* using the standard Base64 alphabet - and return the encoded :class:`bytes`. - - -.. function:: standard_b64decode(s) - - Decode :term:`bytes-like object` or ASCII string *s* using the standard - Base64 alphabet and return the decoded :class:`bytes`. - - -.. function:: urlsafe_b64encode(s) - - Encode :term:`bytes-like object` *s* using the - URL- and filesystem-safe alphabet, which - substitutes ``-`` instead of ``+`` and ``_`` instead of ``/`` in the - standard Base64 alphabet, and return the encoded :class:`bytes`. The result - can still contain ``=``. - - -.. function:: urlsafe_b64decode(s) - - Decode :term:`bytes-like object` or ASCII string *s* - using the URL- and filesystem-safe - alphabet, which substitutes ``-`` instead of ``+`` and ``_`` instead of - ``/`` in the standard Base64 alphabet, and return the decoded - :class:`bytes`. - - -.. function:: b32encode(s) - - Encode the :term:`bytes-like object` *s* using Base32 and return the - encoded :class:`bytes`. - - -.. function:: b32decode(s, casefold=False, map01=None) - - Decode the Base32 encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. - - Optional *casefold* is a flag specifying - whether a lowercase alphabet is acceptable as input. For security purposes, - the default is ``False``. - - :rfc:`4648` allows for optional mapping of the digit 0 (zero) to the letter O - (oh), and for optional mapping of the digit 1 (one) to either the letter I (eye) - or letter L (el). The optional argument *map01* when not ``None``, specifies - which letter the digit 1 should be mapped to (when *map01* is not ``None``, the - digit 0 is always mapped to the letter O). For security purposes the default is - ``None``, so that 0 and 1 are not allowed in the input. - - A :exc:`binascii.Error` is raised if *s* is - incorrectly padded or if there are non-alphabet characters present in the - input. - - -.. function:: b32hexencode(s) - - Similar to :func:`b32encode` but uses the Extended Hex Alphabet, as defined in - :rfc:`4648`. - - .. versionadded:: 3.10 - - -.. function:: b32hexdecode(s, casefold=False) - - Similar to :func:`b32decode` but uses the Extended Hex Alphabet, as defined in - :rfc:`4648`. - - This version does not allow the digit 0 (zero) to the letter O (oh) and digit - 1 (one) to either the letter I (eye) or letter L (el) mappings, all these - characters are included in the Extended Hex Alphabet and are not - interchangeable. - - .. versionadded:: 3.10 - - -.. function:: b16encode(s) - - Encode the :term:`bytes-like object` *s* using Base16 and return the - encoded :class:`bytes`. - - -.. function:: b16decode(s, casefold=False) - - Decode the Base16 encoded :term:`bytes-like object` or ASCII string *s* and - return the decoded :class:`bytes`. - - Optional *casefold* is a flag specifying whether a - lowercase alphabet is acceptable as input. For security purposes, the default - is ``False``. - - A :exc:`binascii.Error` is raised if *s* is - incorrectly padded or if there are non-alphabet characters present in the - input. - - -.. function:: a85encode(b, *, foldspaces=False, wrapcol=0, pad=False, adobe=False) - - Encode the :term:`bytes-like object` *b* using Ascii85 and return the - encoded :class:`bytes`. - - *foldspaces* is an optional flag that uses the special short sequence 'y' - instead of 4 consecutive spaces (ASCII 0x20) as supported by 'btoa'. This - feature is not supported by the "standard" Ascii85 encoding. - - *wrapcol* controls whether the output should have newline (``b'\n'``) - characters added to it. If this is non-zero, each output line will be - at most this many characters long. - - *pad* controls whether the input is padded to a multiple of 4 - before encoding. Note that the ``btoa`` implementation always pads. - - *adobe* controls whether the encoded byte sequence is framed with ``<~`` - and ``~>``, which is used by the Adobe implementation. - - .. versionadded:: 3.4 - - -.. function:: a85decode(b, *, foldspaces=False, adobe=False, ignorechars=b' \t\n\r\v') - - Decode the Ascii85 encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. - - *foldspaces* is a flag that specifies whether the 'y' short sequence - should be accepted as shorthand for 4 consecutive spaces (ASCII 0x20). - This feature is not supported by the "standard" Ascii85 encoding. - - *adobe* controls whether the input sequence is in Adobe Ascii85 format - (i.e. is framed with <~ and ~>). - - *ignorechars* should be a :term:`bytes-like object` or ASCII string - containing characters to ignore - from the input. This should only contain whitespace characters, and by - default contains all whitespace characters in ASCII. - - .. versionadded:: 3.4 - - -.. function:: b85encode(b, pad=False) - - Encode the :term:`bytes-like object` *b* using base85 (as used in e.g. - git-style binary diffs) and return the encoded :class:`bytes`. - - If *pad* is true, the input is padded with ``b'\0'`` so its length is a - multiple of 4 bytes before encoding. - - .. versionadded:: 3.4 - - -.. function:: b85decode(b) - - Decode the base85-encoded :term:`bytes-like object` or ASCII string *b* and - return the decoded :class:`bytes`. Padding is implicitly removed, if - necessary. - - .. versionadded:: 3.4 - - -The legacy interface: - -.. function:: decode(input, output) - - Decode the contents of the binary *input* file and write the resulting binary - data to the *output* file. *input* and *output* must be :term:`file objects - `. *input* will be read until ``input.readline()`` returns an - empty bytes object. - - -.. function:: decodebytes(s) - - Decode the :term:`bytes-like object` *s*, which must contain one or more - lines of base64 encoded data, and return the decoded :class:`bytes`. - - .. versionadded:: 3.1 - - -.. function:: encode(input, output) - - Encode the contents of the binary *input* file and write the resulting base64 - encoded data to the *output* file. *input* and *output* must be :term:`file - objects `. *input* will be read until ``input.read()`` returns - an empty bytes object. :func:`encode` inserts a newline character (``b'\n'``) - after every 76 bytes of the output, as well as ensuring that the output - always ends with a newline, as per :rfc:`2045` (MIME). - - -.. function:: encodebytes(s) - - Encode the :term:`bytes-like object` *s*, which can contain arbitrary binary - data, and return :class:`bytes` containing the base64-encoded data, with newlines - (``b'\n'``) inserted after every 76 bytes of output, and ensuring that - there is a trailing newline, as per :rfc:`2045` (MIME). - - .. versionadded:: 3.1 - - -An example usage of the module: - - >>> import base64 - >>> encoded = base64.b64encode(b'data to be encoded') - >>> encoded - b'ZGF0YSB0byBiZSBlbmNvZGVk' - >>> data = base64.b64decode(encoded) - >>> data - b'data to be encoded' - -.. _base64-security: - -Security Considerations ------------------------ - -A new security considerations section was added to :rfc:`4648` (section 12); it's -recommended to review the security section for any code deployed to production. - -.. seealso:: - - Module :mod:`binascii` - Support module containing ASCII-to-binary and binary-to-ASCII conversions. - - :rfc:`1521` - MIME (Multipurpose Internet Mail Extensions) Part One: Mechanisms for Specifying and Describing the Format of Internet Message Bodies - Section 5.2, "Base64 Content-Transfer-Encoding," provides the definition of the - base64 encoding. - diff --git a/Doc/library/bdb.rst b/Doc/library/bdb.rst deleted file mode 100644 index d201dc963b5995..00000000000000 --- a/Doc/library/bdb.rst +++ /dev/null @@ -1,437 +0,0 @@ -:mod:`bdb` --- Debugger framework -================================= - -.. module:: bdb - :synopsis: Debugger framework. - -**Source code:** :source:`Lib/bdb.py` - --------------- - -The :mod:`bdb` module handles basic debugger functions, like setting breakpoints -or managing execution via the debugger. - -The following exception is defined: - -.. exception:: BdbQuit - - Exception raised by the :class:`Bdb` class for quitting the debugger. - - -The :mod:`bdb` module also defines two classes: - -.. class:: Breakpoint(self, file, line, temporary=False, cond=None, funcname=None) - - This class implements temporary breakpoints, ignore counts, disabling and - (re-)enabling, and conditionals. - - Breakpoints are indexed by number through a list called :attr:`bpbynumber` - and by ``(file, line)`` pairs through :attr:`bplist`. The former points to - a single instance of class :class:`Breakpoint`. The latter points to a list - of such instances since there may be more than one breakpoint per line. - - When creating a breakpoint, its associated :attr:`file name ` should - be in canonical form. If a :attr:`funcname` is defined, a breakpoint - :attr:`hit ` will be counted when the first line of that function is - executed. A :attr:`conditional ` breakpoint always counts a - :attr:`hit `. - - :class:`Breakpoint` instances have the following methods: - - .. method:: deleteMe() - - Delete the breakpoint from the list associated to a file/line. If it is - the last breakpoint in that position, it also deletes the entry for the - file/line. - - - .. method:: enable() - - Mark the breakpoint as enabled. - - - .. method:: disable() - - Mark the breakpoint as disabled. - - - .. method:: bpformat() - - Return a string with all the information about the breakpoint, nicely - formatted: - - * Breakpoint number. - * Temporary status (del or keep). - * File/line position. - * Break condition. - * Number of times to ignore. - * Number of times hit. - - .. versionadded:: 3.2 - - .. method:: bpprint(out=None) - - Print the output of :meth:`bpformat` to the file *out*, or if it is - ``None``, to standard output. - - :class:`Breakpoint` instances have the following attributes: - - .. attribute:: file - - File name of the :class:`Breakpoint`. - - .. attribute:: line - - Line number of the :class:`Breakpoint` within :attr:`file`. - - .. attribute:: temporary - - True if a :class:`Breakpoint` at (file, line) is temporary. - - .. attribute:: cond - - Condition for evaluating a :class:`Breakpoint` at (file, line). - - .. attribute:: funcname - - Function name that defines whether a :class:`Breakpoint` is hit upon - entering the function. - - .. attribute:: enabled - - True if :class:`Breakpoint` is enabled. - - .. attribute:: bpbynumber - - Numeric index for a single instance of a :class:`Breakpoint`. - - .. attribute:: bplist - - Dictionary of :class:`Breakpoint` instances indexed by - (:attr:`file`, :attr:`line`) tuples. - - .. attribute:: ignore - - Number of times to ignore a :class:`Breakpoint`. - - .. attribute:: hits - - Count of the number of times a :class:`Breakpoint` has been hit. - -.. class:: Bdb(skip=None) - - The :class:`Bdb` class acts as a generic Python debugger base class. - - This class takes care of the details of the trace facility; a derived class - should implement user interaction. The standard debugger class - (:class:`pdb.Pdb`) is an example. - - The *skip* argument, if given, must be an iterable of glob-style - module name patterns. The debugger will not step into frames that - originate in a module that matches one of these patterns. Whether a - frame is considered to originate in a certain module is determined - by the ``__name__`` in the frame globals. - - .. versionadded:: 3.1 - The *skip* argument. - - The following methods of :class:`Bdb` normally don't need to be overridden. - - .. method:: canonic(filename) - - Return canonical form of *filename*. - - For real file names, the canonical form is an operating-system-dependent, - :func:`case-normalized ` :func:`absolute path - `. A *filename* with angle brackets, such as ``""`` - generated in interactive mode, is returned unchanged. - - .. method:: reset() - - Set the :attr:`botframe`, :attr:`stopframe`, :attr:`returnframe` and - :attr:`quitting` attributes with values ready to start debugging. - - .. method:: trace_dispatch(frame, event, arg) - - This function is installed as the trace function of debugged frames. Its - return value is the new trace function (in most cases, that is, itself). - - The default implementation decides how to dispatch a frame, depending on - the type of event (passed as a string) that is about to be executed. - *event* can be one of the following: - - * ``"line"``: A new line of code is going to be executed. - * ``"call"``: A function is about to be called, or another code block - entered. - * ``"return"``: A function or other code block is about to return. - * ``"exception"``: An exception has occurred. - * ``"c_call"``: A C function is about to be called. - * ``"c_return"``: A C function has returned. - * ``"c_exception"``: A C function has raised an exception. - - For the Python events, specialized functions (see below) are called. For - the C events, no action is taken. - - The *arg* parameter depends on the previous event. - - See the documentation for :func:`sys.settrace` for more information on the - trace function. For more information on code and frame objects, refer to - :ref:`types`. - - .. method:: dispatch_line(frame) - - If the debugger should stop on the current line, invoke the - :meth:`user_line` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set - (which can be set from :meth:`user_line`). Return a reference to the - :meth:`trace_dispatch` method for further tracing in that scope. - - .. method:: dispatch_call(frame, arg) - - If the debugger should stop on this function call, invoke the - :meth:`user_call` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set - (which can be set from :meth:`user_call`). Return a reference to the - :meth:`trace_dispatch` method for further tracing in that scope. - - .. method:: dispatch_return(frame, arg) - - If the debugger should stop on this function return, invoke the - :meth:`user_return` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set - (which can be set from :meth:`user_return`). Return a reference to the - :meth:`trace_dispatch` method for further tracing in that scope. - - .. method:: dispatch_exception(frame, arg) - - If the debugger should stop at this exception, invokes the - :meth:`user_exception` method (which should be overridden in subclasses). - Raise a :exc:`BdbQuit` exception if the :attr:`Bdb.quitting` flag is set - (which can be set from :meth:`user_exception`). Return a reference to the - :meth:`trace_dispatch` method for further tracing in that scope. - - Normally derived classes don't override the following methods, but they may - if they want to redefine the definition of stopping and breakpoints. - - .. method:: is_skipped_line(module_name) - - Return True if *module_name* matches any skip pattern. - - .. method:: stop_here(frame) - - Return True if *frame* is below the starting frame in the stack. - - .. method:: break_here(frame) - - Return True if there is an effective breakpoint for this line. - - Check whether a line or function breakpoint exists and is in effect. Delete temporary - breakpoints based on information from :func:`effective`. - - .. method:: break_anywhere(frame) - - Return True if any breakpoint exists for *frame*'s filename. - - Derived classes should override these methods to gain control over debugger - operation. - - .. method:: user_call(frame, argument_list) - - Called from :meth:`dispatch_call` if a break might stop inside the - called function. - - .. method:: user_line(frame) - - Called from :meth:`dispatch_line` when either :meth:`stop_here` or - :meth:`break_here` returns ``True``. - - .. method:: user_return(frame, return_value) - - Called from :meth:`dispatch_return` when :meth:`stop_here` returns ``True``. - - .. method:: user_exception(frame, exc_info) - - Called from :meth:`dispatch_exception` when :meth:`stop_here` - returns ``True``. - - .. method:: do_clear(arg) - - Handle how a breakpoint must be removed when it is a temporary one. - - This method must be implemented by derived classes. - - - Derived classes and clients can call the following methods to affect the - stepping state. - - .. method:: set_step() - - Stop after one line of code. - - .. method:: set_next(frame) - - Stop on the next line in or below the given frame. - - .. method:: set_return(frame) - - Stop when returning from the given frame. - - .. method:: set_until(frame, lineno=None) - - Stop when the line with the *lineno* greater than the current one is - reached or when returning from current frame. - - .. method:: set_trace([frame]) - - Start debugging from *frame*. If *frame* is not specified, debugging - starts from caller's frame. - - .. method:: set_continue() - - Stop only at breakpoints or when finished. If there are no breakpoints, - set the system trace function to ``None``. - - .. method:: set_quit() - - Set the :attr:`quitting` attribute to ``True``. This raises :exc:`BdbQuit` in - the next call to one of the :meth:`dispatch_\*` methods. - - - Derived classes and clients can call the following methods to manipulate - breakpoints. These methods return a string containing an error message if - something went wrong, or ``None`` if all is well. - - .. method:: set_break(filename, lineno, temporary=False, cond=None, funcname=None) - - Set a new breakpoint. If the *lineno* line doesn't exist for the - *filename* passed as argument, return an error message. The *filename* - should be in canonical form, as described in the :meth:`canonic` method. - - .. method:: clear_break(filename, lineno) - - Delete the breakpoints in *filename* and *lineno*. If none were set, - return an error message. - - .. method:: clear_bpbynumber(arg) - - Delete the breakpoint which has the index *arg* in the - :attr:`Breakpoint.bpbynumber`. If *arg* is not numeric or out of range, - return an error message. - - .. method:: clear_all_file_breaks(filename) - - Delete all breakpoints in *filename*. If none were set, return an error - message. - - .. method:: clear_all_breaks() - - Delete all existing breakpoints. If none were set, return an error - message. - - .. method:: get_bpbynumber(arg) - - Return a breakpoint specified by the given number. If *arg* is a string, - it will be converted to a number. If *arg* is a non-numeric string, if - the given breakpoint never existed or has been deleted, a - :exc:`ValueError` is raised. - - .. versionadded:: 3.2 - - .. method:: get_break(filename, lineno) - - Return True if there is a breakpoint for *lineno* in *filename*. - - .. method:: get_breaks(filename, lineno) - - Return all breakpoints for *lineno* in *filename*, or an empty list if - none are set. - - .. method:: get_file_breaks(filename) - - Return all breakpoints in *filename*, or an empty list if none are set. - - .. method:: get_all_breaks() - - Return all breakpoints that are set. - - - Derived classes and clients can call the following methods to get a data - structure representing a stack trace. - - .. method:: get_stack(f, t) - - Return a list of (frame, lineno) tuples in a stack trace, and a size. - - The most recently called frame is last in the list. The size is the number - of frames below the frame where the debugger was invoked. - - .. method:: format_stack_entry(frame_lineno, lprefix=': ') - - Return a string with information about a stack entry, which is a - ``(frame, lineno)`` tuple. The return string contains: - - * The canonical filename which contains the frame. - * The function name or ``""``. - * The input arguments. - * The return value. - * The line of code (if it exists). - - - The following two methods can be called by clients to use a debugger to debug - a :term:`statement`, given as a string. - - .. method:: run(cmd, globals=None, locals=None) - - Debug a statement executed via the :func:`exec` function. *globals* - defaults to :attr:`__main__.__dict__`, *locals* defaults to *globals*. - - .. method:: runeval(expr, globals=None, locals=None) - - Debug an expression executed via the :func:`eval` function. *globals* and - *locals* have the same meaning as in :meth:`run`. - - .. method:: runctx(cmd, globals, locals) - - For backwards compatibility. Calls the :meth:`run` method. - - .. method:: runcall(func, /, *args, **kwds) - - Debug a single function call, and return its result. - - -Finally, the module defines the following functions: - -.. function:: checkfuncname(b, frame) - - Return True if we should break here, depending on the way the - :class:`Breakpoint` *b* was set. - - If it was set via line number, it checks if - :attr:`b.line ` is the same as the one in *frame*. - If the breakpoint was set via - :attr:`function name `, we have to check we are in - the right *frame* (the right function) and if we are on its first executable - line. - -.. function:: effective(file, line, frame) - - Return ``(active breakpoint, delete temporary flag)`` or ``(None, None)`` as the - breakpoint to act upon. - - The *active breakpoint* is the first entry in - :attr:`bplist ` for the - (:attr:`file `, :attr:`line `) - (which must exist) that is :attr:`enabled `, for - which :func:`checkfuncname` is True, and that has neither a False - :attr:`condition ` nor positive - :attr:`ignore ` count. The *flag*, meaning that a - temporary breakpoint should be deleted, is False only when the - :attr:`cond ` cannot be evaluated (in which case, - :attr:`ignore ` count is ignored). - - If no such entry exists, then (None, None) is returned. - - -.. function:: set_trace() - - Start debugging with a :class:`Bdb` instance from caller's frame. diff --git a/Doc/library/binary.rst b/Doc/library/binary.rst deleted file mode 100644 index 51fbdc1a44b15d..00000000000000 --- a/Doc/library/binary.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. _binaryservices: - -******************** -Binary Data Services -******************** - -The modules described in this chapter provide some basic services operations -for manipulation of binary data. Other operations on binary data, specifically -in relation to file formats and network protocols, are described in the -relevant sections. - -Some libraries described under :ref:`textservices` also work with either -ASCII-compatible binary formats (for example, :mod:`re`) or all binary data -(for example, :mod:`difflib`). - -In addition, see the documentation for Python's built-in binary data types in -:ref:`binaryseq`. - -.. toctree:: - - struct.rst - codecs.rst - diff --git a/Doc/library/binascii.rst b/Doc/library/binascii.rst deleted file mode 100644 index d065fa3506da11..00000000000000 --- a/Doc/library/binascii.rst +++ /dev/null @@ -1,187 +0,0 @@ -:mod:`binascii` --- Convert between binary and ASCII -==================================================== - -.. module:: binascii - :synopsis: Tools for converting between binary and various ASCII-encoded binary - representations. - -.. index:: - pair: module; uu - pair: module; base64 - --------------- - -The :mod:`binascii` module contains a number of methods to convert between -binary and various ASCII-encoded binary representations. Normally, you will not -use these functions directly but use wrapper modules like :mod:`uu` or -:mod:`base64` instead. The :mod:`binascii` module contains -low-level functions written in C for greater speed that are used by the -higher-level modules. - -.. note:: - - ``a2b_*`` functions accept Unicode strings containing only ASCII characters. - Other functions only accept :term:`bytes-like objects ` (such as - :class:`bytes`, :class:`bytearray` and other objects that support the buffer - protocol). - - .. versionchanged:: 3.3 - ASCII-only unicode strings are now accepted by the ``a2b_*`` functions. - - -The :mod:`binascii` module defines the following functions: - - -.. function:: a2b_uu(string) - - Convert a single line of uuencoded data back to binary and return the binary - data. Lines normally contain 45 (binary) bytes, except for the last line. Line - data may be followed by whitespace. - - -.. function:: b2a_uu(data, *, backtick=False) - - Convert binary data to a line of ASCII characters, the return value is the - converted line, including a newline char. The length of *data* should be at most - 45. If *backtick* is true, zeros are represented by ``'`'`` instead of spaces. - - .. versionchanged:: 3.7 - Added the *backtick* parameter. - - -.. function:: a2b_base64(string, /, *, strict_mode=False) - - Convert a block of base64 data back to binary and return the binary data. More - than one line may be passed at a time. - - If *strict_mode* is true, only valid base64 data will be converted. Invalid base64 - data will raise :exc:`binascii.Error`. - - Valid base64: - - * Conforms to :rfc:`3548`. - * Contains only characters from the base64 alphabet. - * Contains no excess data after padding (including excess padding, newlines, etc.). - * Does not start with a padding. - - .. versionchanged:: 3.11 - Added the *strict_mode* parameter. - - -.. function:: b2a_base64(data, *, newline=True) - - Convert binary data to a line of ASCII characters in base64 coding. The return - value is the converted line, including a newline char if *newline* is - true. The output of this function conforms to :rfc:`3548`. - - .. versionchanged:: 3.6 - Added the *newline* parameter. - - -.. function:: a2b_qp(data, header=False) - - Convert a block of quoted-printable data back to binary and return the binary - data. More than one line may be passed at a time. If the optional argument - *header* is present and true, underscores will be decoded as spaces. - - -.. function:: b2a_qp(data, quotetabs=False, istext=True, header=False) - - Convert binary data to a line(s) of ASCII characters in quoted-printable - encoding. The return value is the converted line(s). If the optional argument - *quotetabs* is present and true, all tabs and spaces will be encoded. If the - optional argument *istext* is present and true, newlines are not encoded but - trailing whitespace will be encoded. If the optional argument *header* is - present and true, spaces will be encoded as underscores per :rfc:`1522`. If the - optional argument *header* is present and false, newline characters will be - encoded as well; otherwise linefeed conversion might corrupt the binary data - stream. - - -.. function:: crc_hqx(data, value) - - Compute a 16-bit CRC value of *data*, starting with *value* as the - initial CRC, and return the result. This uses the CRC-CCITT polynomial - *x*:sup:`16` + *x*:sup:`12` + *x*:sup:`5` + 1, often represented as - 0x1021. This CRC is used in the binhex4 format. - - -.. function:: crc32(data[, value]) - - Compute CRC-32, the unsigned 32-bit checksum of *data*, starting with an - initial CRC of *value*. The default initial CRC is zero. The algorithm - is consistent with the ZIP file checksum. Since the algorithm is designed for - use as a checksum algorithm, it is not suitable for use as a general hash - algorithm. Use as follows:: - - print(binascii.crc32(b"hello world")) - # Or, in two pieces: - crc = binascii.crc32(b"hello") - crc = binascii.crc32(b" world", crc) - print('crc32 = {:#010x}'.format(crc)) - - .. versionchanged:: 3.0 - The result is always unsigned. - -.. function:: b2a_hex(data[, sep[, bytes_per_sep=1]]) - hexlify(data[, sep[, bytes_per_sep=1]]) - - Return the hexadecimal representation of the binary *data*. Every byte of - *data* is converted into the corresponding 2-digit hex representation. The - returned bytes object is therefore twice as long as the length of *data*. - - Similar functionality (but returning a text string) is also conveniently - accessible using the :meth:`bytes.hex` method. - - If *sep* is specified, it must be a single character str or bytes object. - It will be inserted in the output after every *bytes_per_sep* input bytes. - Separator placement is counted from the right end of the output by default, - if you wish to count from the left, supply a negative *bytes_per_sep* value. - - >>> import binascii - >>> binascii.b2a_hex(b'\xb9\x01\xef') - b'b901ef' - >>> binascii.hexlify(b'\xb9\x01\xef', '-') - b'b9-01-ef' - >>> binascii.b2a_hex(b'\xb9\x01\xef', b'_', 2) - b'b9_01ef' - >>> binascii.b2a_hex(b'\xb9\x01\xef', b' ', -2) - b'b901 ef' - - .. versionchanged:: 3.8 - The *sep* and *bytes_per_sep* parameters were added. - -.. function:: a2b_hex(hexstr) - unhexlify(hexstr) - - Return the binary data represented by the hexadecimal string *hexstr*. This - function is the inverse of :func:`b2a_hex`. *hexstr* must contain an even number - of hexadecimal digits (which can be upper or lower case), otherwise an - :exc:`Error` exception is raised. - - Similar functionality (accepting only text string arguments, but more - liberal towards whitespace) is also accessible using the - :meth:`bytes.fromhex` class method. - -.. exception:: Error - - Exception raised on errors. These are usually programming errors. - - -.. exception:: Incomplete - - Exception raised on incomplete data. These are usually not programming errors, - but may be handled by reading a little more data and trying again. - - -.. seealso:: - - Module :mod:`base64` - Support for RFC compliant base64-style encoding in base 16, 32, 64, - and 85. - - Module :mod:`uu` - Support for UU encoding used on Unix. - - Module :mod:`quopri` - Support for quoted-printable encoding used in MIME email messages. diff --git a/Doc/library/bisect.rst b/Doc/library/bisect.rst deleted file mode 100644 index d4abd7a6313dfd..00000000000000 --- a/Doc/library/bisect.rst +++ /dev/null @@ -1,250 +0,0 @@ -:mod:`bisect` --- Array bisection algorithm -=========================================== - -.. module:: bisect - :synopsis: Array bisection algorithms for binary searching. -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Raymond Hettinger -.. example based on the PyModules FAQ entry by Aaron Watters - -**Source code:** :source:`Lib/bisect.py` - --------------- - -This module provides support for maintaining a list in sorted order without -having to sort the list after each insertion. For long lists of items with -expensive comparison operations, this can be an improvement over the more common -approach. The module is called :mod:`bisect` because it uses a basic bisection -algorithm to do its work. The source code may be most useful as a working -example of the algorithm (the boundary conditions are already right!). - -.. _bisect functions: - -The following functions are provided: - - -.. function:: bisect_left(a, x, lo=0, hi=len(a), *, key=None) - - Locate the insertion point for *x* in *a* to maintain sorted order. - The parameters *lo* and *hi* may be used to specify a subset of the list - which should be considered; by default the entire list is used. If *x* is - already present in *a*, the insertion point will be before (to the left of) - any existing entries. The return value is suitable for use as the first - parameter to ``list.insert()`` assuming that *a* is already sorted. - - The returned insertion point *i* partitions the array *a* into two halves so - that ``all(val < x for val in a[lo : i])`` for the left side and - ``all(val >= x for val in a[i : hi])`` for the right side. - - *key* specifies a :term:`key function` of one argument that is used to - extract a comparison key from each element in the array. To support - searching complex records, the key function is not applied to the *x* value. - - If *key* is ``None``, the elements are compared directly with no - intervening function call. - - .. versionchanged:: 3.10 - Added the *key* parameter. - - -.. function:: bisect_right(a, x, lo=0, hi=len(a), *, key=None) - bisect(a, x, lo=0, hi=len(a), *, key=None) - - Similar to :py:func:`~bisect.bisect_left`, but returns an insertion point which comes - after (to the right of) any existing entries of *x* in *a*. - - The returned insertion point *i* partitions the array *a* into two halves so - that ``all(val <= x for val in a[lo : i])`` for the left side and - ``all(val > x for val in a[i : hi])`` for the right side. - - *key* specifies a :term:`key function` of one argument that is used to - extract a comparison key from each element in the array. To support - searching complex records, the key function is not applied to the *x* value. - - If *key* is ``None``, the elements are compared directly with no - intervening function call. - - .. versionchanged:: 3.10 - Added the *key* parameter. - - -.. function:: insort_left(a, x, lo=0, hi=len(a), *, key=None) - - Insert *x* in *a* in sorted order. - - This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point. - Next, it runs the :meth:`insert` method on *a* to insert *x* at the - appropriate position to maintain sort order. - - To support inserting records in a table, the *key* function (if any) is - applied to *x* for the search step but not for the insertion step. - - Keep in mind that the ``O(log n)`` search is dominated by the slow O(n) - insertion step. - - .. versionchanged:: 3.10 - Added the *key* parameter. - - -.. function:: insort_right(a, x, lo=0, hi=len(a), *, key=None) - insort(a, x, lo=0, hi=len(a), *, key=None) - - Similar to :py:func:`~bisect.insort_left`, but inserting *x* in *a* after any existing - entries of *x*. - - This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point. - Next, it runs the :meth:`insert` method on *a* to insert *x* at the - appropriate position to maintain sort order. - - To support inserting records in a table, the *key* function (if any) is - applied to *x* for the search step but not for the insertion step. - - Keep in mind that the ``O(log n)`` search is dominated by the slow O(n) - insertion step. - - .. versionchanged:: 3.10 - Added the *key* parameter. - - -Performance Notes ------------------ - -When writing time sensitive code using *bisect()* and *insort()*, keep these -thoughts in mind: - -* Bisection is effective for searching ranges of values. - For locating specific values, dictionaries are more performant. - -* The *insort()* functions are ``O(n)`` because the logarithmic search step - is dominated by the linear time insertion step. - -* The search functions are stateless and discard key function results after - they are used. Consequently, if the search functions are used in a loop, - the key function may be called again and again on the same array elements. - If the key function isn't fast, consider wrapping it with - :py:func:`functools.cache` to avoid duplicate computations. Alternatively, - consider searching an array of precomputed keys to locate the insertion - point (as shown in the examples section below). - -.. seealso:: - - * `Sorted Collections - `_ is a high performance - module that uses *bisect* to managed sorted collections of data. - - * The `SortedCollection recipe - `_ uses - bisect to build a full-featured collection class with straight-forward search - methods and support for a key-function. The keys are precomputed to save - unnecessary calls to the key function during searches. - - -Searching Sorted Lists ----------------------- - -The above `bisect functions`_ are useful for finding insertion points but -can be tricky or awkward to use for common searching tasks. The following five -functions show how to transform them into the standard lookups for sorted -lists:: - - def index(a, x): - 'Locate the leftmost value exactly equal to x' - i = bisect_left(a, x) - if i != len(a) and a[i] == x: - return i - raise ValueError - - def find_lt(a, x): - 'Find rightmost value less than x' - i = bisect_left(a, x) - if i: - return a[i-1] - raise ValueError - - def find_le(a, x): - 'Find rightmost value less than or equal to x' - i = bisect_right(a, x) - if i: - return a[i-1] - raise ValueError - - def find_gt(a, x): - 'Find leftmost value greater than x' - i = bisect_right(a, x) - if i != len(a): - return a[i] - raise ValueError - - def find_ge(a, x): - 'Find leftmost item greater than or equal to x' - i = bisect_left(a, x) - if i != len(a): - return a[i] - raise ValueError - - -Examples --------- - -.. _bisect-example: - -The :py:func:`~bisect.bisect` function can be useful for numeric table lookups. This -example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam score (say) -based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is -a 'B', and so on:: - - >>> def grade(score, breakpoints=[60, 70, 80, 90], grades='FDCBA'): - ... i = bisect(breakpoints, score) - ... return grades[i] - ... - >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]] - ['F', 'A', 'C', 'C', 'B', 'A', 'A'] - -The :py:func:`~bisect.bisect` and :py:func:`~bisect.insort` functions also work with -lists of tuples. The *key* argument can serve to extract the field used for ordering -records in a table:: - - >>> from collections import namedtuple - >>> from operator import attrgetter - >>> from bisect import bisect, insort - >>> from pprint import pprint - - >>> Movie = namedtuple('Movie', ('name', 'released', 'director')) - - >>> movies = [ - ... Movie('Jaws', 1975, 'Spielberg'), - ... Movie('Titanic', 1997, 'Cameron'), - ... Movie('The Birds', 1963, 'Hitchcock'), - ... Movie('Aliens', 1986, 'Cameron') - ... ] - - >>> # Find the first movie released after 1960 - >>> by_year = attrgetter('released') - >>> movies.sort(key=by_year) - >>> movies[bisect(movies, 1960, key=by_year)] - Movie(name='The Birds', released=1963, director='Hitchcock') - - >>> # Insert a movie while maintaining sort order - >>> romance = Movie('Love Story', 1970, 'Hiller') - >>> insort(movies, romance, key=by_year) - >>> pprint(movies) - [Movie(name='The Birds', released=1963, director='Hitchcock'), - Movie(name='Love Story', released=1970, director='Hiller'), - Movie(name='Jaws', released=1975, director='Spielberg'), - Movie(name='Aliens', released=1986, director='Cameron'), - Movie(name='Titanic', released=1997, director='Cameron')] - -If the key function is expensive, it is possible to avoid repeated function -calls by searching a list of precomputed keys to find the index of a record:: - - >>> data = [('red', 5), ('blue', 1), ('yellow', 8), ('black', 0)] - >>> data.sort(key=lambda r: r[1]) # Or use operator.itemgetter(1). - >>> keys = [r[1] for r in data] # Precompute a list of keys. - >>> data[bisect_left(keys, 0)] - ('black', 0) - >>> data[bisect_left(keys, 1)] - ('blue', 1) - >>> data[bisect_left(keys, 5)] - ('red', 5) - >>> data[bisect_left(keys, 8)] - ('yellow', 8) diff --git a/Doc/library/builtins.rst b/Doc/library/builtins.rst deleted file mode 100644 index 7e4f8fe0531567..00000000000000 --- a/Doc/library/builtins.rst +++ /dev/null @@ -1,42 +0,0 @@ -:mod:`builtins` --- Built-in objects -==================================== - -.. module:: builtins - :synopsis: The module that provides the built-in namespace. - --------------- - -This module provides direct access to all 'built-in' identifiers of Python; for -example, ``builtins.open`` is the full name for the built-in function -:func:`open`. See :ref:`built-in-funcs` and :ref:`built-in-consts` for -documentation. - - -This module is not normally accessed explicitly by most applications, but can be -useful in modules that provide objects with the same name as a built-in value, -but in which the built-in of that name is also needed. For example, in a module -that wants to implement an :func:`open` function that wraps the built-in -:func:`open`, this module can be used directly:: - - import builtins - - def open(path): - f = builtins.open(path, 'r') - return UpperCaser(f) - - class UpperCaser: - '''Wrapper around a file that converts output to uppercase.''' - - def __init__(self, f): - self._f = f - - def read(self, count=-1): - return self._f.read(count).upper() - - # ... - -As an implementation detail, most modules have the name ``__builtins__`` made -available as part of their globals. The value of ``__builtins__`` is normally -either this module or the value of this module's :attr:`~object.__dict__` attribute. -Since this is an implementation detail, it may not be used by alternate -implementations of Python. diff --git a/Doc/library/bz2.rst b/Doc/library/bz2.rst deleted file mode 100644 index 51b1da89683ead..00000000000000 --- a/Doc/library/bz2.rst +++ /dev/null @@ -1,371 +0,0 @@ -:mod:`bz2` --- Support for :program:`bzip2` compression -======================================================= - -.. module:: bz2 - :synopsis: Interfaces for bzip2 compression and decompression. - -.. moduleauthor:: Gustavo Niemeyer -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Gustavo Niemeyer -.. sectionauthor:: Nadeem Vawda - -**Source code:** :source:`Lib/bz2.py` - --------------- - -This module provides a comprehensive interface for compressing and -decompressing data using the bzip2 compression algorithm. - -The :mod:`bz2` module contains: - -* The :func:`.open` function and :class:`BZ2File` class for reading and - writing compressed files. -* The :class:`BZ2Compressor` and :class:`BZ2Decompressor` classes for - incremental (de)compression. -* The :func:`compress` and :func:`decompress` functions for one-shot - (de)compression. - - -(De)compression of files ------------------------- - -.. function:: open(filename, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None) - - Open a bzip2-compressed file in binary or text mode, returning a :term:`file - object`. - - As with the constructor for :class:`BZ2File`, the *filename* argument can be - an actual filename (a :class:`str` or :class:`bytes` object), or an existing - file object to read from or write to. - - The *mode* argument can be any of ``'r'``, ``'rb'``, ``'w'``, ``'wb'``, - ``'x'``, ``'xb'``, ``'a'`` or ``'ab'`` for binary mode, or ``'rt'``, - ``'wt'``, ``'xt'``, or ``'at'`` for text mode. The default is ``'rb'``. - - The *compresslevel* argument is an integer from 1 to 9, as for the - :class:`BZ2File` constructor. - - For binary mode, this function is equivalent to the :class:`BZ2File` - constructor: ``BZ2File(filename, mode, compresslevel=compresslevel)``. In - this case, the *encoding*, *errors* and *newline* arguments must not be - provided. - - For text mode, a :class:`BZ2File` object is created, and wrapped in an - :class:`io.TextIOWrapper` instance with the specified encoding, error - handling behavior, and line ending(s). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The ``'x'`` (exclusive creation) mode was added. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. class:: BZ2File(filename, mode='r', *, compresslevel=9) - - Open a bzip2-compressed file in binary mode. - - If *filename* is a :class:`str` or :class:`bytes` object, open the named file - directly. Otherwise, *filename* should be a :term:`file object`, which will - be used to read or write the compressed data. - - The *mode* argument can be either ``'r'`` for reading (default), ``'w'`` for - overwriting, ``'x'`` for exclusive creation, or ``'a'`` for appending. These - can equivalently be given as ``'rb'``, ``'wb'``, ``'xb'`` and ``'ab'`` - respectively. - - If *filename* is a file object (rather than an actual file name), a mode of - ``'w'`` does not truncate the file, and is instead equivalent to ``'a'``. - - If *mode* is ``'w'`` or ``'a'``, *compresslevel* can be an integer between - ``1`` and ``9`` specifying the level of compression: ``1`` produces the - least compression, and ``9`` (default) produces the most compression. - - If *mode* is ``'r'``, the input file may be the concatenation of multiple - compressed streams. - - :class:`BZ2File` provides all of the members specified by the - :class:`io.BufferedIOBase`, except for :meth:`~io.BufferedIOBase.detach` - and :meth:`~io.IOBase.truncate`. - Iteration and the :keyword:`with` statement are supported. - - :class:`BZ2File` also provides the following methods: - - .. method:: peek([n]) - - Return buffered data without advancing the file position. At least one - byte of data will be returned (unless at EOF). The exact number of bytes - returned is unspecified. - - .. note:: While calling :meth:`peek` does not change the file position of - the :class:`BZ2File`, it may change the position of the underlying file - object (e.g. if the :class:`BZ2File` was constructed by passing a file - object for *filename*). - - .. versionadded:: 3.3 - - .. method:: fileno() - - Return the file descriptor for the underlying file. - - .. versionadded:: 3.3 - - .. method:: readable() - - Return whether the file was opened for reading. - - .. versionadded:: 3.3 - - .. method:: seekable() - - Return whether the file supports seeking. - - .. versionadded:: 3.3 - - .. method:: writable() - - Return whether the file was opened for writing. - - .. versionadded:: 3.3 - - .. method:: read1(size=-1) - - Read up to *size* uncompressed bytes, while trying to avoid - making multiple reads from the underlying stream. Reads up to a - buffer's worth of data if size is negative. - - Returns ``b''`` if the file is at EOF. - - .. versionadded:: 3.3 - - .. method:: readinto(b) - - Read bytes into *b*. - - Returns the number of bytes read (0 for EOF). - - .. versionadded:: 3.3 - - - .. versionchanged:: 3.1 - Support for the :keyword:`with` statement was added. - - .. versionchanged:: 3.3 - Support was added for *filename* being a :term:`file object` instead of an - actual filename. - - .. versionchanged:: 3.3 - The ``'a'`` (append) mode was added, along with support for reading - multi-stream files. - - .. versionchanged:: 3.4 - The ``'x'`` (exclusive creation) mode was added. - - .. versionchanged:: 3.5 - The :meth:`~io.BufferedIOBase.read` method now accepts an argument of - ``None``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. versionchanged:: 3.9 - The *buffering* parameter has been removed. It was ignored and deprecated - since Python 3.0. Pass an open file object to control how the file is - opened. - - The *compresslevel* parameter became keyword-only. - - .. versionchanged:: 3.10 - This class is thread unsafe in the face of multiple simultaneous - readers or writers, just like its equivalent classes in :mod:`gzip` and - :mod:`lzma` have always been. - - -Incremental (de)compression ---------------------------- - -.. class:: BZ2Compressor(compresslevel=9) - - Create a new compressor object. This object may be used to compress data - incrementally. For one-shot compression, use the :func:`compress` function - instead. - - *compresslevel*, if given, must be an integer between ``1`` and ``9``. The - default is ``9``. - - .. method:: compress(data) - - Provide data to the compressor object. Returns a chunk of compressed data - if possible, or an empty byte string otherwise. - - When you have finished providing data to the compressor, call the - :meth:`flush` method to finish the compression process. - - - .. method:: flush() - - Finish the compression process. Returns the compressed data left in - internal buffers. - - The compressor object may not be used after this method has been called. - - -.. class:: BZ2Decompressor() - - Create a new decompressor object. This object may be used to decompress data - incrementally. For one-shot compression, use the :func:`decompress` function - instead. - - .. note:: - This class does not transparently handle inputs containing multiple - compressed streams, unlike :func:`decompress` and :class:`BZ2File`. If - you need to decompress a multi-stream input with :class:`BZ2Decompressor`, - you must use a new decompressor for each stream. - - .. method:: decompress(data, max_length=-1) - - Decompress *data* (a :term:`bytes-like object`), returning - uncompressed data as bytes. Some of *data* may be buffered - internally, for use in later calls to :meth:`decompress`. The - returned data should be concatenated with the output of any - previous calls to :meth:`decompress`. - - If *max_length* is nonnegative, returns at most *max_length* - bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to - :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. - - If all of the input data was decompressed and returned (either - because this was less than *max_length* bytes, or because - *max_length* was negative), the :attr:`~.needs_input` attribute - will be set to ``True``. - - Attempting to decompress data after the end of stream is reached - raises an :exc:`EOFError`. Any data found after the end of the - stream is ignored and saved in the :attr:`~.unused_data` attribute. - - .. versionchanged:: 3.5 - Added the *max_length* parameter. - - .. attribute:: eof - - ``True`` if the end-of-stream marker has been reached. - - .. versionadded:: 3.3 - - - .. attribute:: unused_data - - Data found after the end of the compressed stream. - - If this attribute is accessed before the end of the stream has been - reached, its value will be ``b''``. - - .. attribute:: needs_input - - ``False`` if the :meth:`.decompress` method can provide more - decompressed data before requiring new uncompressed input. - - .. versionadded:: 3.5 - - -One-shot (de)compression ------------------------- - -.. function:: compress(data, compresslevel=9) - - Compress *data*, a :term:`bytes-like object `. - - *compresslevel*, if given, must be an integer between ``1`` and ``9``. The - default is ``9``. - - For incremental compression, use a :class:`BZ2Compressor` instead. - - -.. function:: decompress(data) - - Decompress *data*, a :term:`bytes-like object `. - - If *data* is the concatenation of multiple compressed streams, decompress - all of the streams. - - For incremental decompression, use a :class:`BZ2Decompressor` instead. - - .. versionchanged:: 3.3 - Support for multi-stream inputs was added. - -.. _bz2-usage-examples: - -Examples of usage ------------------ - -Below are some examples of typical usage of the :mod:`bz2` module. - -Using :func:`compress` and :func:`decompress` to demonstrate round-trip compression: - - >>> import bz2 - >>> data = b"""\ - ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue - ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, - ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus - ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat. - ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo - ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum - ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" - >>> c = bz2.compress(data) - >>> len(data) / len(c) # Data compression ratio - 1.513595166163142 - >>> d = bz2.decompress(c) - >>> data == d # Check equality to original object after round-trip - True - -Using :class:`BZ2Compressor` for incremental compression: - - >>> import bz2 - >>> def gen_data(chunks=10, chunksize=1000): - ... """Yield incremental blocks of chunksize bytes.""" - ... for _ in range(chunks): - ... yield b"z" * chunksize - ... - >>> comp = bz2.BZ2Compressor() - >>> out = b"" - >>> for chunk in gen_data(): - ... # Provide data to the compressor object - ... out = out + comp.compress(chunk) - ... - >>> # Finish the compression process. Call this once you have - >>> # finished providing data to the compressor. - >>> out = out + comp.flush() - -The example above uses a very "nonrandom" stream of data -(a stream of ``b"z"`` chunks). Random data tends to compress poorly, -while ordered, repetitive data usually yields a high compression ratio. - -Writing and reading a bzip2-compressed file in binary mode: - - >>> import bz2 - >>> data = b"""\ - ... Donec rhoncus quis sapien sit amet molestie. Fusce scelerisque vel augue - ... nec ullamcorper. Nam rutrum pretium placerat. Aliquam vel tristique lorem, - ... sit amet cursus ante. In interdum laoreet mi, sit amet ultrices purus - ... pulvinar a. Nam gravida euismod magna, non varius justo tincidunt feugiat. - ... Aliquam pharetra lacus non risus vehicula rutrum. Maecenas aliquam leo - ... felis. Pellentesque semper nunc sit amet nibh ullamcorper, ac elementum - ... dolor luctus. Curabitur lacinia mi ornare consectetur vestibulum.""" - >>> with bz2.open("myfile.bz2", "wb") as f: - ... # Write compressed data to file - ... unused = f.write(data) - >>> with bz2.open("myfile.bz2", "rb") as f: - ... # Decompress data from file - ... content = f.read() - >>> content == data # Check equality to original object after round-trip - True - -.. testcleanup:: - - import os - os.remove("myfile.bz2") diff --git a/Doc/library/calendar.rst b/Doc/library/calendar.rst deleted file mode 100644 index dbef171c487dba..00000000000000 --- a/Doc/library/calendar.rst +++ /dev/null @@ -1,593 +0,0 @@ -:mod:`calendar` --- General calendar-related functions -====================================================== - -.. module:: calendar - :synopsis: Functions for working with calendars, including some emulation - of the Unix cal program. - -.. sectionauthor:: Drew Csillag - -**Source code:** :source:`Lib/calendar.py` - --------------- - -This module allows you to output calendars like the Unix :program:`cal` program, -and provides additional useful functions related to the calendar. By default, -these calendars have Monday as the first day of the week, and Sunday as the last -(the European convention). Use :func:`setfirstweekday` to set the first day of -the week to Sunday (6) or to any other weekday. Parameters that specify dates -are given as integers. For related -functionality, see also the :mod:`datetime` and :mod:`time` modules. - -The functions and classes defined in this module -use an idealized calendar, the current Gregorian calendar extended indefinitely -in both directions. This matches the definition of the "proleptic Gregorian" -calendar in Dershowitz and Reingold's book "Calendrical Calculations", where -it's the base calendar for all computations. Zero and negative years are -interpreted as prescribed by the ISO 8601 standard. Year 0 is 1 BC, year -1 is -2 BC, and so on. - - -.. class:: Calendar(firstweekday=0) - - Creates a :class:`Calendar` object. *firstweekday* is an integer specifying the - first day of the week. :const:`MONDAY` is ``0`` (the default), :const:`SUNDAY` is ``6``. - - A :class:`Calendar` object provides several methods that can be used for - preparing the calendar data for formatting. This class doesn't do any formatting - itself. This is the job of subclasses. - - - :class:`Calendar` instances have the following methods: - - .. method:: iterweekdays() - - Return an iterator for the week day numbers that will be used for one - week. The first value from the iterator will be the same as the value of - the :attr:`firstweekday` property. - - - .. method:: itermonthdates(year, month) - - Return an iterator for the month *month* (1--12) in the year *year*. This - iterator will return all days (as :class:`datetime.date` objects) for the - month and all days before the start of the month or after the end of the - month that are required to get a complete week. - - - .. method:: itermonthdays(year, month) - - Return an iterator for the month *month* in the year *year* similar to - :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` - range. Days returned will simply be day of the month numbers. For the - days outside of the specified month, the day number is ``0``. - - - .. method:: itermonthdays2(year, month) - - Return an iterator for the month *month* in the year *year* similar to - :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` - range. Days returned will be tuples consisting of a day of the month - number and a week day number. - - - .. method:: itermonthdays3(year, month) - - Return an iterator for the month *month* in the year *year* similar to - :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` - range. Days returned will be tuples consisting of a year, a month and a day - of the month numbers. - - .. versionadded:: 3.7 - - - .. method:: itermonthdays4(year, month) - - Return an iterator for the month *month* in the year *year* similar to - :meth:`itermonthdates`, but not restricted by the :class:`datetime.date` - range. Days returned will be tuples consisting of a year, a month, a day - of the month, and a day of the week numbers. - - .. versionadded:: 3.7 - - - .. method:: monthdatescalendar(year, month) - - Return a list of the weeks in the month *month* of the *year* as full - weeks. Weeks are lists of seven :class:`datetime.date` objects. - - - .. method:: monthdays2calendar(year, month) - - Return a list of the weeks in the month *month* of the *year* as full - weeks. Weeks are lists of seven tuples of day numbers and weekday - numbers. - - - .. method:: monthdayscalendar(year, month) - - Return a list of the weeks in the month *month* of the *year* as full - weeks. Weeks are lists of seven day numbers. - - - .. method:: yeardatescalendar(year, width=3) - - Return the data for the specified year ready for formatting. The return - value is a list of month rows. Each month row contains up to *width* - months (defaulting to 3). Each month contains between 4 and 6 weeks and - each week contains 1--7 days. Days are :class:`datetime.date` objects. - - - .. method:: yeardays2calendar(year, width=3) - - Return the data for the specified year ready for formatting (similar to - :meth:`yeardatescalendar`). Entries in the week lists are tuples of day - numbers and weekday numbers. Day numbers outside this month are zero. - - - .. method:: yeardayscalendar(year, width=3) - - Return the data for the specified year ready for formatting (similar to - :meth:`yeardatescalendar`). Entries in the week lists are day numbers. Day - numbers outside this month are zero. - - -.. class:: TextCalendar(firstweekday=0) - - This class can be used to generate plain text calendars. - - :class:`TextCalendar` instances have the following methods: - - .. method:: formatmonth(theyear, themonth, w=0, l=0) - - Return a month's calendar in a multi-line string. If *w* is provided, it - specifies the width of the date columns, which are centered. If *l* is - given, it specifies the number of lines that each week will use. Depends - on the first weekday as specified in the constructor or set by the - :meth:`setfirstweekday` method. - - - .. method:: prmonth(theyear, themonth, w=0, l=0) - - Print a month's calendar as returned by :meth:`formatmonth`. - - - .. method:: formatyear(theyear, w=2, l=1, c=6, m=3) - - Return a *m*-column calendar for an entire year as a multi-line string. - Optional parameters *w*, *l*, and *c* are for date column width, lines per - week, and number of spaces between month columns, respectively. Depends on - the first weekday as specified in the constructor or set by the - :meth:`setfirstweekday` method. The earliest year for which a calendar - can be generated is platform-dependent. - - - .. method:: pryear(theyear, w=2, l=1, c=6, m=3) - - Print the calendar for an entire year as returned by :meth:`formatyear`. - - -.. class:: HTMLCalendar(firstweekday=0) - - This class can be used to generate HTML calendars. - - - :class:`!HTMLCalendar` instances have the following methods: - - .. method:: formatmonth(theyear, themonth, withyear=True) - - Return a month's calendar as an HTML table. If *withyear* is true the year - will be included in the header, otherwise just the month name will be - used. - - - .. method:: formatyear(theyear, width=3) - - Return a year's calendar as an HTML table. *width* (defaulting to 3) - specifies the number of months per row. - - - .. method:: formatyearpage(theyear, width=3, css='calendar.css', encoding=None) - - Return a year's calendar as a complete HTML page. *width* (defaulting to - 3) specifies the number of months per row. *css* is the name for the - cascading style sheet to be used. :const:`None` can be passed if no style - sheet should be used. *encoding* specifies the encoding to be used for the - output (defaulting to the system default encoding). - - - :class:`!HTMLCalendar` has the following attributes you can override to - customize the CSS classes used by the calendar: - - .. attribute:: cssclasses - - A list of CSS classes used for each weekday. The default class list is:: - - cssclasses = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"] - - more styles can be added for each day:: - - cssclasses = ["mon text-bold", "tue", "wed", "thu", "fri", "sat", "sun red"] - - Note that the length of this list must be seven items. - - - .. attribute:: cssclass_noday - - The CSS class for a weekday occurring in the previous or coming month. - - .. versionadded:: 3.7 - - - .. attribute:: cssclasses_weekday_head - - A list of CSS classes used for weekday names in the header row. - The default is the same as :attr:`cssclasses`. - - .. versionadded:: 3.7 - - - .. attribute:: cssclass_month_head - - The month's head CSS class (used by :meth:`formatmonthname`). - The default value is ``"month"``. - - .. versionadded:: 3.7 - - - .. attribute:: cssclass_month - - The CSS class for the whole month's table (used by :meth:`formatmonth`). - The default value is ``"month"``. - - .. versionadded:: 3.7 - - - .. attribute:: cssclass_year - - The CSS class for the whole year's table of tables (used by - :meth:`formatyear`). The default value is ``"year"``. - - .. versionadded:: 3.7 - - - .. attribute:: cssclass_year_head - - The CSS class for the table head for the whole year (used by - :meth:`formatyear`). The default value is ``"year"``. - - .. versionadded:: 3.7 - - - Note that although the naming for the above described class attributes is - singular (e.g. ``cssclass_month`` ``cssclass_noday``), one can replace the - single CSS class with a space separated list of CSS classes, for example:: - - "text-bold text-red" - - Here is an example how :class:`!HTMLCalendar` can be customized:: - - class CustomHTMLCal(calendar.HTMLCalendar): - cssclasses = [style + " text-nowrap" for style in - calendar.HTMLCalendar.cssclasses] - cssclass_month_head = "text-center month-head" - cssclass_month = "text-center month" - cssclass_year = "text-italic lead" - - -.. class:: LocaleTextCalendar(firstweekday=0, locale=None) - - This subclass of :class:`TextCalendar` can be passed a locale name in the - constructor and will return month and weekday names in the specified locale. - - -.. class:: LocaleHTMLCalendar(firstweekday=0, locale=None) - - This subclass of :class:`HTMLCalendar` can be passed a locale name in the - constructor and will return month and weekday names in the specified - locale. - -.. note:: - - The constructor, :meth:`formatweekday` and :meth:`formatmonthname` methods - of these two classes temporarily change the ``LC_TIME`` locale to the given - *locale*. Because the current locale is a process-wide setting, they are - not thread-safe. - - -For simple text calendars this module provides the following functions. - -.. function:: setfirstweekday(weekday) - - Sets the weekday (``0`` is Monday, ``6`` is Sunday) to start each week. The - values :const:`MONDAY`, :const:`TUESDAY`, :const:`WEDNESDAY`, :const:`THURSDAY`, - :const:`FRIDAY`, :const:`SATURDAY`, and :const:`SUNDAY` are provided for - convenience. For example, to set the first weekday to Sunday:: - - import calendar - calendar.setfirstweekday(calendar.SUNDAY) - - -.. function:: firstweekday() - - Returns the current setting for the weekday to start each week. - - -.. function:: isleap(year) - - Returns :const:`True` if *year* is a leap year, otherwise :const:`False`. - - -.. function:: leapdays(y1, y2) - - Returns the number of leap years in the range from *y1* to *y2* (exclusive), - where *y1* and *y2* are years. - - This function works for ranges spanning a century change. - - -.. function:: weekday(year, month, day) - - Returns the day of the week (``0`` is Monday) for *year* (``1970``--...), - *month* (``1``--``12``), *day* (``1``--``31``). - - -.. function:: weekheader(n) - - Return a header containing abbreviated weekday names. *n* specifies the width in - characters for one weekday. - - -.. function:: monthrange(year, month) - - Returns weekday of first day of the month and number of days in month, for the - specified *year* and *month*. - - -.. function:: monthcalendar(year, month) - - Returns a matrix representing a month's calendar. Each row represents a week; - days outside of the month are represented by zeros. Each week begins with Monday - unless set by :func:`setfirstweekday`. - - -.. function:: prmonth(theyear, themonth, w=0, l=0) - - Prints a month's calendar as returned by :func:`month`. - - -.. function:: month(theyear, themonth, w=0, l=0) - - Returns a month's calendar in a multi-line string using the :meth:`formatmonth` - of the :class:`TextCalendar` class. - - -.. function:: prcal(year, w=0, l=0, c=6, m=3) - - Prints the calendar for an entire year as returned by :func:`calendar`. - - -.. function:: calendar(year, w=2, l=1, c=6, m=3) - - Returns a 3-column calendar for an entire year as a multi-line string using - the :meth:`formatyear` of the :class:`TextCalendar` class. - - -.. function:: timegm(tuple) - - An unrelated but handy function that takes a time tuple such as returned by - the :func:`~time.gmtime` function in the :mod:`time` module, and returns the - corresponding Unix timestamp value, assuming an epoch of 1970, and the POSIX - encoding. In fact, :func:`time.gmtime` and :func:`timegm` are each others' - inverse. - - -The :mod:`calendar` module exports the following data attributes: - -.. data:: day_name - - An array that represents the days of the week in the current locale. - - -.. data:: day_abbr - - An array that represents the abbreviated days of the week in the current locale. - - -.. data:: month_name - - An array that represents the months of the year in the current locale. This - follows normal convention of January being month number 1, so it has a length of - 13 and ``month_name[0]`` is the empty string. - - -.. data:: month_abbr - - An array that represents the abbreviated months of the year in the current - locale. This follows normal convention of January being month number 1, so it - has a length of 13 and ``month_abbr[0]`` is the empty string. - -.. data:: MONDAY - TUESDAY - WEDNESDAY - THURSDAY - FRIDAY - SATURDAY - SUNDAY - - Aliases for day numbers, where ``MONDAY`` is ``0`` and ``SUNDAY`` is ``6``. - - -The :mod:`calendar` module defines the following exceptions: - -.. exception:: IllegalMonthError(month) - - A subclass of :exc:`ValueError`, - raised when the given month number is outside of the range 1-12 (inclusive). - - .. attribute:: month - - The invalid month number. - - -.. exception:: IllegalWeekdayError(weekday) - - A subclass of :exc:`ValueError`, - raised when the given weekday number is outside of the range 0-6 (inclusive). - - .. attribute:: weekday - - The invalid weekday number. - - -.. seealso:: - - Module :mod:`datetime` - Object-oriented interface to dates and times with similar functionality to the - :mod:`time` module. - - Module :mod:`time` - Low-level time related functions. - - -.. _calendar-cli: - -Command-Line Usage ------------------- - -.. versionadded:: 2.5 - -The :mod:`calendar` module can be executed as a script from the command line -to interactively print a calendar. - -.. code-block:: shell - - python -m calendar [-h] [-L LOCALE] [-e ENCODING] [-t {text,html}] - [-w WIDTH] [-l LINES] [-s SPACING] [-m MONTHS] [-c CSS] - [year] [month] - - -For example, to print a calendar for the year 2000: - -.. code-block:: console - - $ python -m calendar 2000 - 2000 - - January February March - Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su - 1 2 1 2 3 4 5 6 1 2 3 4 5 - 3 4 5 6 7 8 9 7 8 9 10 11 12 13 6 7 8 9 10 11 12 - 10 11 12 13 14 15 16 14 15 16 17 18 19 20 13 14 15 16 17 18 19 - 17 18 19 20 21 22 23 21 22 23 24 25 26 27 20 21 22 23 24 25 26 - 24 25 26 27 28 29 30 28 29 27 28 29 30 31 - 31 - - April May June - Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su - 1 2 1 2 3 4 5 6 7 1 2 3 4 - 3 4 5 6 7 8 9 8 9 10 11 12 13 14 5 6 7 8 9 10 11 - 10 11 12 13 14 15 16 15 16 17 18 19 20 21 12 13 14 15 16 17 18 - 17 18 19 20 21 22 23 22 23 24 25 26 27 28 19 20 21 22 23 24 25 - 24 25 26 27 28 29 30 29 30 31 26 27 28 29 30 - - July August September - Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su - 1 2 1 2 3 4 5 6 1 2 3 - 3 4 5 6 7 8 9 7 8 9 10 11 12 13 4 5 6 7 8 9 10 - 10 11 12 13 14 15 16 14 15 16 17 18 19 20 11 12 13 14 15 16 17 - 17 18 19 20 21 22 23 21 22 23 24 25 26 27 18 19 20 21 22 23 24 - 24 25 26 27 28 29 30 28 29 30 31 25 26 27 28 29 30 - 31 - - October November December - Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su Mo Tu We Th Fr Sa Su - 1 1 2 3 4 5 1 2 3 - 2 3 4 5 6 7 8 6 7 8 9 10 11 12 4 5 6 7 8 9 10 - 9 10 11 12 13 14 15 13 14 15 16 17 18 19 11 12 13 14 15 16 17 - 16 17 18 19 20 21 22 20 21 22 23 24 25 26 18 19 20 21 22 23 24 - 23 24 25 26 27 28 29 27 28 29 30 25 26 27 28 29 30 31 - 30 31 - - -The following options are accepted: - -.. program:: calendar - - -.. option:: --help, -h - - Show the help message and exit. - - -.. option:: --locale LOCALE, -L LOCALE - - The locale to use for month and weekday names. - Defaults to English. - - -.. option:: --encoding ENCODING, -e ENCODING - - The encoding to use for output. - :option:`--encoding` is required if :option:`--locale` is set. - - -.. option:: --type {text,html}, -t {text,html} - - Print the calendar to the terminal as text, - or as an HTML document. - - -.. option:: year - - The year to print the calendar for. - Must be a number between 1 and 9999. - Defaults to the current year. - - -.. option:: month - - The month of the specified :option:`year` to print the calendar for. - Must be a number between 1 and 12, - and may only be used in text mode. - Defaults to printing a calendar for the full year. - - -*Text-mode options:* - -.. option:: --width WIDTH, -w WIDTH - - The width of the date column in terminal columns. - The date is printed centred in the column. - Any value lower than 2 is ignored. - Defaults to 2. - - -.. option:: --lines LINES, -l LINES - - The number of lines for each week in terminal rows. - The date is printed top-aligned. - Any value lower than 1 is ignored. - Defaults to 1. - - -.. option:: --spacing SPACING, -s SPACING - - The space between months in columns. - Any value lower than 2 is ignored. - Defaults to 6. - - -.. option:: --months MONTHS, -m MONTHS - - The number of months printed per row. - Defaults to 3. - - -*HTML-mode options:* - -.. option:: --css CSS, -c CSS - - The path of a CSS stylesheet to use for the calendar. - This must either be relative to the generated HTML, - or an absolute HTTP or ``file:///`` URL. diff --git a/Doc/library/cgi.rst b/Doc/library/cgi.rst deleted file mode 100644 index 295a601a7bf197..00000000000000 --- a/Doc/library/cgi.rst +++ /dev/null @@ -1,564 +0,0 @@ -:mod:`cgi` --- Common Gateway Interface support -=============================================== - -.. module:: cgi - :synopsis: Helpers for running Python scripts via the Common Gateway Interface. - :deprecated: - -**Source code:** :source:`Lib/cgi.py` - -.. index:: - pair: WWW; server - pair: CGI; protocol - pair: HTTP; protocol - pair: MIME; headers - single: URL - single: Common Gateway Interface - -.. deprecated-removed:: 3.11 3.13 - The :mod:`cgi` module is deprecated - (see :pep:`PEP 594 <594#cgi>` for details and alternatives). - - The :class:`FieldStorage` class can typically be replaced with - :func:`urllib.parse.parse_qsl` for ``GET`` and ``HEAD`` requests, - and the :mod:`email.message` module or - `multipart `_ for ``POST`` and ``PUT``. - Most :ref:`utility functions ` have replacements. - --------------- - -Support module for Common Gateway Interface (CGI) scripts. - -This module defines a number of utilities for use by CGI scripts written in -Python. - -The global variable ``maxlen`` can be set to an integer indicating the maximum -size of a POST request. POST requests larger than this size will result in a -:exc:`ValueError` being raised during parsing. The default value of this -variable is ``0``, meaning the request size is unlimited. - -.. include:: ../includes/wasm-notavail.rst - -Introduction ------------- - -.. _cgi-intro: - -A CGI script is invoked by an HTTP server, usually to process user input -submitted through an HTML ``
    `` or ```` element. - -Most often, CGI scripts live in the server's special :file:`cgi-bin` directory. -The HTTP server places all sorts of information about the request (such as the -client's hostname, the requested URL, the query string, and lots of other -goodies) in the script's shell environment, executes the script, and sends the -script's output back to the client. - -The script's input is connected to the client too, and sometimes the form data -is read this way; at other times the form data is passed via the "query string" -part of the URL. This module is intended to take care of the different cases -and provide a simpler interface to the Python script. It also provides a number -of utilities that help in debugging scripts, and the latest addition is support -for file uploads from a form (if your browser supports it). - -The output of a CGI script should consist of two sections, separated by a blank -line. The first section contains a number of headers, telling the client what -kind of data is following. Python code to generate a minimal header section -looks like this:: - - print("Content-Type: text/html") # HTML is following - print() # blank line, end of headers - -The second section is usually HTML, which allows the client software to display -nicely formatted text with header, in-line images, etc. Here's Python code that -prints a simple piece of HTML:: - - print("CGI script output") - print("

    This is my first CGI script

    ") - print("Hello, world!") - - -.. _using-the-cgi-module: - -Using the cgi module --------------------- - -Begin by writing ``import cgi``. - -When you write a new script, consider adding these lines:: - - import cgitb - cgitb.enable() - -This activates a special exception handler that will display detailed reports in -the web browser if any errors occur. If you'd rather not show the guts of your -program to users of your script, you can have the reports saved to files -instead, with code like this:: - - import cgitb - cgitb.enable(display=0, logdir="/path/to/logdir") - -It's very helpful to use this feature during script development. The reports -produced by :mod:`cgitb` provide information that can save you a lot of time in -tracking down bugs. You can always remove the ``cgitb`` line later when you -have tested your script and are confident that it works correctly. - -To get at submitted form data, use the :class:`FieldStorage` class. If the form -contains non-ASCII characters, use the *encoding* keyword parameter set to the -value of the encoding defined for the document. It is usually contained in the -META tag in the HEAD section of the HTML document or by the -:mailheader:`Content-Type` header. This reads the form contents from the -standard input or the environment (depending on the value of various -environment variables set according to the CGI standard). Since it may consume -standard input, it should be instantiated only once. - -The :class:`FieldStorage` instance can be indexed like a Python dictionary. -It allows membership testing with the :keyword:`in` operator, and also supports -the standard dictionary method :meth:`~dict.keys` and the built-in function -:func:`len`. Form fields containing empty strings are ignored and do not appear -in the dictionary; to keep such values, provide a true value for the optional -*keep_blank_values* keyword parameter when creating the :class:`FieldStorage` -instance. - -For instance, the following code (which assumes that the -:mailheader:`Content-Type` header and blank line have already been printed) -checks that the fields ``name`` and ``addr`` are both set to a non-empty -string:: - - form = cgi.FieldStorage() - if "name" not in form or "addr" not in form: - print("

    Error

    ") - print("Please fill in the name and addr fields.") - return - print("

    name:", form["name"].value) - print("

    addr:", form["addr"].value) - ...further form processing here... - -Here the fields, accessed through ``form[key]``, are themselves instances of -:class:`FieldStorage` (or :class:`MiniFieldStorage`, depending on the form -encoding). The :attr:`~FieldStorage.value` attribute of the instance yields -the string value of the field. The :meth:`~FieldStorage.getvalue` method -returns this string value directly; it also accepts an optional second argument -as a default to return if the requested key is not present. - -If the submitted form data contains more than one field with the same name, the -object retrieved by ``form[key]`` is not a :class:`FieldStorage` or -:class:`MiniFieldStorage` instance but a list of such instances. Similarly, in -this situation, ``form.getvalue(key)`` would return a list of strings. If you -expect this possibility (when your HTML form contains multiple fields with the -same name), use the :meth:`~FieldStorage.getlist` method, which always returns -a list of values (so that you do not need to special-case the single item -case). For example, this code concatenates any number of username fields, -separated by commas:: - - value = form.getlist("username") - usernames = ",".join(value) - -If a field represents an uploaded file, accessing the value via the -:attr:`~FieldStorage.value` attribute or the :meth:`~FieldStorage.getvalue` -method reads the entire file in memory as bytes. This may not be what you -want. You can test for an uploaded file by testing either the -:attr:`~FieldStorage.filename` attribute or the :attr:`~FieldStorage.file` -attribute. You can then read the data from the :attr:`!file` -attribute before it is automatically closed as part of the garbage collection of -the :class:`FieldStorage` instance -(the :func:`~io.RawIOBase.read` and :func:`~io.IOBase.readline` methods will -return bytes):: - - fileitem = form["userfile"] - if fileitem.file: - # It's an uploaded file; count lines - linecount = 0 - while True: - line = fileitem.file.readline() - if not line: break - linecount = linecount + 1 - -:class:`FieldStorage` objects also support being used in a :keyword:`with` -statement, which will automatically close them when done. - -If an error is encountered when obtaining the contents of an uploaded file -(for example, when the user interrupts the form submission by clicking on -a Back or Cancel button) the :attr:`~FieldStorage.done` attribute of the -object for the field will be set to the value -1. - -The file upload draft standard entertains the possibility of uploading multiple -files from one field (using a recursive :mimetype:`multipart/\*` encoding). -When this occurs, the item will be a dictionary-like :class:`FieldStorage` item. -This can be determined by testing its :attr:`!type` attribute, which should be -:mimetype:`multipart/form-data` (or perhaps another MIME type matching -:mimetype:`multipart/\*`). In this case, it can be iterated over recursively -just like the top-level form object. - -When a form is submitted in the "old" format (as the query string or as a single -data part of type :mimetype:`application/x-www-form-urlencoded`), the items will -actually be instances of the class :class:`MiniFieldStorage`. In this case, the -:attr:`!list`, :attr:`!file`, and :attr:`filename` attributes are always ``None``. - -A form submitted via POST that also has a query string will contain both -:class:`FieldStorage` and :class:`MiniFieldStorage` items. - -.. versionchanged:: 3.4 - The :attr:`~FieldStorage.file` attribute is automatically closed upon the - garbage collection of the creating :class:`FieldStorage` instance. - -.. versionchanged:: 3.5 - Added support for the context management protocol to the - :class:`FieldStorage` class. - - -Higher Level Interface ----------------------- - -The previous section explains how to read CGI form data using the -:class:`FieldStorage` class. This section describes a higher level interface -which was added to this class to allow one to do it in a more readable and -intuitive way. The interface doesn't make the techniques described in previous -sections obsolete --- they are still useful to process file uploads efficiently, -for example. - -.. XXX: Is this true ? - -The interface consists of two simple methods. Using the methods you can process -form data in a generic way, without the need to worry whether only one or more -values were posted under one name. - -In the previous section, you learned to write following code anytime you -expected a user to post more than one value under one name:: - - item = form.getvalue("item") - if isinstance(item, list): - # The user is requesting more than one item. - else: - # The user is requesting only one item. - -This situation is common for example when a form contains a group of multiple -checkboxes with the same name:: - - - - -In most situations, however, there's only one form control with a particular -name in a form and then you expect and need only one value associated with this -name. So you write a script containing for example this code:: - - user = form.getvalue("user").upper() - -The problem with the code is that you should never expect that a client will -provide valid input to your scripts. For example, if a curious user appends -another ``user=foo`` pair to the query string, then the script would crash, -because in this situation the ``getvalue("user")`` method call returns a list -instead of a string. Calling the :meth:`~str.upper` method on a list is not valid -(since lists do not have a method of this name) and results in an -:exc:`AttributeError` exception. - -Therefore, the appropriate way to read form data values was to always use the -code which checks whether the obtained value is a single value or a list of -values. That's annoying and leads to less readable scripts. - -A more convenient approach is to use the methods :meth:`~FieldStorage.getfirst` -and :meth:`~FieldStorage.getlist` provided by this higher level interface. - - -.. method:: FieldStorage.getfirst(name, default=None) - - This method always returns only one value associated with form field *name*. - The method returns only the first value in case that more values were posted - under such name. Please note that the order in which the values are received - may vary from browser to browser and should not be counted on. [#]_ If no such - form field or value exists then the method returns the value specified by the - optional parameter *default*. This parameter defaults to ``None`` if not - specified. - - -.. method:: FieldStorage.getlist(name) - - This method always returns a list of values associated with form field *name*. - The method returns an empty list if no such form field or value exists for - *name*. It returns a list consisting of one item if only one such value exists. - -Using these methods you can write nice compact code:: - - import cgi - form = cgi.FieldStorage() - user = form.getfirst("user", "").upper() # This way it's safe. - for item in form.getlist("item"): - do_something(item) - - -.. _functions-in-cgi-module: - -Functions ---------- - -These are useful if you want more control, or if you want to employ some of the -algorithms implemented in this module in other circumstances. - - -.. function:: parse(fp=None, environ=os.environ, keep_blank_values=False, strict_parsing=False, separator="&") - - Parse a query in the environment or from a file (the file defaults to - ``sys.stdin``). The *keep_blank_values*, *strict_parsing* and *separator* parameters are - passed to :func:`urllib.parse.parse_qs` unchanged. - - .. deprecated-removed:: 3.11 3.13 - This function, like the rest of the :mod:`cgi` module, is deprecated. - It can be replaced by calling :func:`urllib.parse.parse_qs` directly - on the desired query string (except for ``multipart/form-data`` input, - which can be handled as described for :func:`parse_multipart`). - - -.. function:: parse_multipart(fp, pdict, encoding="utf-8", errors="replace", separator="&") - - Parse input of type :mimetype:`multipart/form-data` (for file uploads). - Arguments are *fp* for the input file, *pdict* for a dictionary containing - other parameters in the :mailheader:`Content-Type` header, and *encoding*, - the request encoding. - - Returns a dictionary just like :func:`urllib.parse.parse_qs`: keys are the - field names, each value is a list of values for that field. For non-file - fields, the value is a list of strings. - - This is easy to use but not much good if you are expecting megabytes to be - uploaded --- in that case, use the :class:`FieldStorage` class instead - which is much more flexible. - - .. versionchanged:: 3.7 - Added the *encoding* and *errors* parameters. For non-file fields, the - value is now a list of strings, not bytes. - - .. versionchanged:: 3.10 - Added the *separator* parameter. - - .. deprecated-removed:: 3.11 3.13 - This function, like the rest of the :mod:`cgi` module, is deprecated. - It can be replaced with the functionality in the :mod:`email` package - (e.g. :class:`email.message.EmailMessage`/:class:`email.message.Message`) - which implements the same MIME RFCs, or with the - `multipart `__ PyPI project. - - -.. function:: parse_header(string) - - Parse a MIME header (such as :mailheader:`Content-Type`) into a main value and a - dictionary of parameters. - - .. deprecated-removed:: 3.11 3.13 - This function, like the rest of the :mod:`cgi` module, is deprecated. - It can be replaced with the functionality in the :mod:`email` package, - which implements the same MIME RFCs. - - For example, with :class:`email.message.EmailMessage`:: - - from email.message import EmailMessage - msg = EmailMessage() - msg['content-type'] = 'application/json; charset="utf8"' - main, params = msg.get_content_type(), msg['content-type'].params - - -.. function:: test() - - Robust test CGI script, usable as main program. Writes minimal HTTP headers and - formats all information provided to the script in HTML format. - - -.. function:: print_environ() - - Format the shell environment in HTML. - - -.. function:: print_form(form) - - Format a form in HTML. - - -.. function:: print_directory() - - Format the current directory in HTML. - - -.. function:: print_environ_usage() - - Print a list of useful (used by CGI) environment variables in HTML. - - -.. _cgi-security: - -Caring about security ---------------------- - -.. index:: pair: CGI; security - -There's one important rule: if you invoke an external program (via -:func:`os.system`, :func:`os.popen` or other functions with similar -functionality), make very sure you don't pass arbitrary strings received from -the client to the shell. This is a well-known security hole whereby clever -hackers anywhere on the web can exploit a gullible CGI script to invoke -arbitrary shell commands. Even parts of the URL or field names cannot be -trusted, since the request doesn't have to come from your form! - -To be on the safe side, if you must pass a string gotten from a form to a shell -command, you should make sure the string contains only alphanumeric characters, -dashes, underscores, and periods. - - -Installing your CGI script on a Unix system -------------------------------------------- - -Read the documentation for your HTTP server and check with your local system -administrator to find the directory where CGI scripts should be installed; -usually this is in a directory :file:`cgi-bin` in the server tree. - -Make sure that your script is readable and executable by "others"; the Unix file -mode should be ``0o755`` octal (use ``chmod 0755 filename``). Make sure that the -first line of the script contains ``#!`` starting in column 1 followed by the -pathname of the Python interpreter, for instance:: - - #!/usr/local/bin/python - -Make sure the Python interpreter exists and is executable by "others". - -Make sure that any files your script needs to read or write are readable or -writable, respectively, by "others" --- their mode should be ``0o644`` for -readable and ``0o666`` for writable. This is because, for security reasons, the -HTTP server executes your script as user "nobody", without any special -privileges. It can only read (write, execute) files that everybody can read -(write, execute). The current directory at execution time is also different (it -is usually the server's cgi-bin directory) and the set of environment variables -is also different from what you get when you log in. In particular, don't count -on the shell's search path for executables (:envvar:`PATH`) or the Python module -search path (:envvar:`PYTHONPATH`) to be set to anything interesting. - -If you need to load modules from a directory which is not on Python's default -module search path, you can change the path in your script, before importing -other modules. For example:: - - import sys - sys.path.insert(0, "/usr/home/joe/lib/python") - sys.path.insert(0, "/usr/local/lib/python") - -(This way, the directory inserted last will be searched first!) - -Instructions for non-Unix systems will vary; check your HTTP server's -documentation (it will usually have a section on CGI scripts). - - -Testing your CGI script ------------------------ - -Unfortunately, a CGI script will generally not run when you try it from the -command line, and a script that works perfectly from the command line may fail -mysteriously when run from the server. There's one reason why you should still -test your script from the command line: if it contains a syntax error, the -Python interpreter won't execute it at all, and the HTTP server will most likely -send a cryptic error to the client. - -Assuming your script has no syntax errors, yet it does not work, you have no -choice but to read the next section. - - -Debugging CGI scripts ---------------------- - -.. index:: pair: CGI; debugging - -First of all, check for trivial installation errors --- reading the section -above on installing your CGI script carefully can save you a lot of time. If -you wonder whether you have understood the installation procedure correctly, try -installing a copy of this module file (:file:`cgi.py`) as a CGI script. When -invoked as a script, the file will dump its environment and the contents of the -form in HTML format. Give it the right mode etc., and send it a request. If it's -installed in the standard :file:`cgi-bin` directory, it should be possible to -send it a request by entering a URL into your browser of the form: - -.. code-block:: none - - http://yourhostname/cgi-bin/cgi.py?name=Joe+Blow&addr=At+Home - -If this gives an error of type 404, the server cannot find the script -- perhaps -you need to install it in a different directory. If it gives another error, -there's an installation problem that you should fix before trying to go any -further. If you get a nicely formatted listing of the environment and form -content (in this example, the fields should be listed as "addr" with value "At -Home" and "name" with value "Joe Blow"), the :file:`cgi.py` script has been -installed correctly. If you follow the same procedure for your own script, you -should now be able to debug it. - -The next step could be to call the :mod:`cgi` module's :func:`test` function -from your script: replace its main code with the single statement :: - - cgi.test() - -This should produce the same results as those gotten from installing the -:file:`cgi.py` file itself. - -When an ordinary Python script raises an unhandled exception (for whatever -reason: of a typo in a module name, a file that can't be opened, etc.), the -Python interpreter prints a nice traceback and exits. While the Python -interpreter will still do this when your CGI script raises an exception, most -likely the traceback will end up in one of the HTTP server's log files, or be -discarded altogether. - -Fortunately, once you have managed to get your script to execute *some* code, -you can easily send tracebacks to the web browser using the :mod:`cgitb` module. -If you haven't done so already, just add the lines:: - - import cgitb - cgitb.enable() - -to the top of your script. Then try running it again; when a problem occurs, -you should see a detailed report that will likely make apparent the cause of the -crash. - -If you suspect that there may be a problem in importing the :mod:`cgitb` module, -you can use an even more robust approach (which only uses built-in modules):: - - import sys - sys.stderr = sys.stdout - print("Content-Type: text/plain") - print() - ...your code here... - -This relies on the Python interpreter to print the traceback. The content type -of the output is set to plain text, which disables all HTML processing. If your -script works, the raw HTML will be displayed by your client. If it raises an -exception, most likely after the first two lines have been printed, a traceback -will be displayed. Because no HTML interpretation is going on, the traceback -will be readable. - - -Common problems and solutions ------------------------------ - -* Most HTTP servers buffer the output from CGI scripts until the script is - completed. This means that it is not possible to display a progress report on - the client's display while the script is running. - -* Check the installation instructions above. - -* Check the HTTP server's log files. (``tail -f logfile`` in a separate window - may be useful!) - -* Always check a script for syntax errors first, by doing something like - ``python script.py``. - -* If your script does not have any syntax errors, try adding ``import cgitb; - cgitb.enable()`` to the top of the script. - -* When invoking external programs, make sure they can be found. Usually, this - means using absolute path names --- :envvar:`PATH` is usually not set to a very - useful value in a CGI script. - -* When reading or writing external files, make sure they can be read or written - by the userid under which your CGI script will be running: this is typically the - userid under which the web server is running, or some explicitly specified - userid for a web server's ``suexec`` feature. - -* Don't try to give a CGI script a set-uid mode. This doesn't work on most - systems, and is a security liability as well. - -.. rubric:: Footnotes - -.. [#] Note that some recent versions of the HTML specification do state what - order the field values should be supplied in, but knowing whether a request - was received from a conforming browser, or even from a browser at all, is - tedious and error-prone. diff --git a/Doc/library/cgitb.rst b/Doc/library/cgitb.rst deleted file mode 100644 index 7f00bcd55c1e53..00000000000000 --- a/Doc/library/cgitb.rst +++ /dev/null @@ -1,89 +0,0 @@ -:mod:`cgitb` --- Traceback manager for CGI scripts -================================================== - -.. module:: cgitb - :synopsis: Configurable traceback handler for CGI scripts. - :deprecated: - -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/cgitb.py` - -.. index:: - single: CGI; exceptions - single: CGI; tracebacks - single: exceptions; in CGI scripts - single: tracebacks; in CGI scripts - -.. deprecated-removed:: 3.11 3.13 - The :mod:`cgitb` module is deprecated - (see :pep:`PEP 594 <594#cgitb>` for details). - --------------- - -The :mod:`cgitb` module provides a special exception handler for Python scripts. -(Its name is a bit misleading. It was originally designed to display extensive -traceback information in HTML for CGI scripts. It was later generalized to also -display this information in plain text.) After this module is activated, if an -uncaught exception occurs, a detailed, formatted report will be displayed. The -report includes a traceback showing excerpts of the source code for each level, -as well as the values of the arguments and local variables to currently running -functions, to help you debug the problem. Optionally, you can save this -information to a file instead of sending it to the browser. - -To enable this feature, simply add this to the top of your CGI script:: - - import cgitb - cgitb.enable() - -The options to the :func:`enable` function control whether the report is -displayed in the browser and whether the report is logged to a file for later -analysis. - - -.. function:: enable(display=1, logdir=None, context=5, format="html") - - .. index:: single: excepthook() (in module sys) - - This function causes the :mod:`cgitb` module to take over the interpreter's - default handling for exceptions by setting the value of :attr:`sys.excepthook`. - - The optional argument *display* defaults to ``1`` and can be set to ``0`` to - suppress sending the traceback to the browser. If the argument *logdir* is - present, the traceback reports are written to files. The value of *logdir* - should be a directory where these files will be placed. The optional argument - *context* is the number of lines of context to display around the current line - of source code in the traceback; this defaults to ``5``. If the optional - argument *format* is ``"html"``, the output is formatted as HTML. Any other - value forces plain text output. The default value is ``"html"``. - - -.. function:: text(info, context=5) - - This function handles the exception described by *info* (a 3-tuple containing - the result of :func:`sys.exc_info`), formatting its traceback as text and - returning the result as a string. The optional argument *context* is the - number of lines of context to display around the current line of source code - in the traceback; this defaults to ``5``. - - -.. function:: html(info, context=5) - - This function handles the exception described by *info* (a 3-tuple containing - the result of :func:`sys.exc_info`), formatting its traceback as HTML and - returning the result as a string. The optional argument *context* is the - number of lines of context to display around the current line of source code - in the traceback; this defaults to ``5``. - - -.. function:: handler(info=None) - - This function handles an exception using the default settings (that is, show a - report in the browser, but don't log to a file). This can be used when you've - caught an exception and want to report it using :mod:`cgitb`. The optional - *info* argument should be a 3-tuple containing an exception type, exception - value, and traceback object, exactly like the tuple returned by - :func:`sys.exc_info`. If the *info* argument is not supplied, the current - exception is obtained from :func:`sys.exc_info`. - diff --git a/Doc/library/chunk.rst b/Doc/library/chunk.rst deleted file mode 100644 index 3b88e55b147882..00000000000000 --- a/Doc/library/chunk.rst +++ /dev/null @@ -1,142 +0,0 @@ -:mod:`chunk` --- Read IFF chunked data -====================================== - -.. module:: chunk - :synopsis: Module to read IFF chunks. - :deprecated: - -.. moduleauthor:: Sjoerd Mullender -.. sectionauthor:: Sjoerd Mullender - -**Source code:** :source:`Lib/chunk.py` - -.. index:: - single: Audio Interchange File Format - single: AIFF - single: AIFF-C - single: Real Media File Format - single: RMFF - -.. deprecated-removed:: 3.11 3.13 - The :mod:`chunk` module is deprecated - (see :pep:`PEP 594 <594#chunk>` for details). - --------------- - -This module provides an interface for reading files that use EA IFF 85 chunks. -[#]_ This format is used in at least the Audio Interchange File Format -(AIFF/AIFF-C) and the Real Media File Format (RMFF). The WAVE audio file format -is closely related and can also be read using this module. - -A chunk has the following structure: - -+---------+--------+-------------------------------+ -| Offset | Length | Contents | -+=========+========+===============================+ -| 0 | 4 | Chunk ID | -+---------+--------+-------------------------------+ -| 4 | 4 | Size of chunk in big-endian | -| | | byte order, not including the | -| | | header | -+---------+--------+-------------------------------+ -| 8 | *n* | Data bytes, where *n* is the | -| | | size given in the preceding | -| | | field | -+---------+--------+-------------------------------+ -| 8 + *n* | 0 or 1 | Pad byte needed if *n* is odd | -| | | and chunk alignment is used | -+---------+--------+-------------------------------+ - -The ID is a 4-byte string which identifies the type of chunk. - -The size field (a 32-bit value, encoded using big-endian byte order) gives the -size of the chunk data, not including the 8-byte header. - -Usually an IFF-type file consists of one or more chunks. The proposed usage of -the :class:`Chunk` class defined here is to instantiate an instance at the start -of each chunk and read from the instance until it reaches the end, after which a -new instance can be instantiated. At the end of the file, creating a new -instance will fail with an :exc:`EOFError` exception. - - -.. class:: Chunk(file, align=True, bigendian=True, inclheader=False) - - Class which represents a chunk. The *file* argument is expected to be a - file-like object. An instance of this class is specifically allowed. The - only method that is needed is :meth:`~io.IOBase.read`. If the methods - :meth:`~io.IOBase.seek` and :meth:`~io.IOBase.tell` are present and don't - raise an exception, they are also used. - If these methods are present and raise an exception, they are expected to not - have altered the object. If the optional argument *align* is true, chunks - are assumed to be aligned on 2-byte boundaries. If *align* is false, no - alignment is assumed. The default value is true. If the optional argument - *bigendian* is false, the chunk size is assumed to be in little-endian order. - This is needed for WAVE audio files. The default value is true. If the - optional argument *inclheader* is true, the size given in the chunk header - includes the size of the header. The default value is false. - - A :class:`Chunk` object supports the following methods: - - - .. method:: getname() - - Returns the name (ID) of the chunk. This is the first 4 bytes of the - chunk. - - - .. method:: getsize() - - Returns the size of the chunk. - - - .. method:: close() - - Close and skip to the end of the chunk. This does not close the - underlying file. - - The remaining methods will raise :exc:`OSError` if called after the - :meth:`close` method has been called. Before Python 3.3, they used to - raise :exc:`IOError`, now an alias of :exc:`OSError`. - - - .. method:: isatty() - - Returns ``False``. - - - .. method:: seek(pos, whence=0) - - Set the chunk's current position. The *whence* argument is optional and - defaults to ``0`` (absolute file positioning); other values are ``1`` - (seek relative to the current position) and ``2`` (seek relative to the - file's end). There is no return value. If the underlying file does not - allow seek, only forward seeks are allowed. - - - .. method:: tell() - - Return the current position into the chunk. - - - .. method:: read(size=-1) - - Read at most *size* bytes from the chunk (less if the read hits the end of - the chunk before obtaining *size* bytes). If the *size* argument is - negative or omitted, read all data until the end of the chunk. An empty - bytes object is returned when the end of the chunk is encountered - immediately. - - - .. method:: skip() - - Skip to the end of the chunk. All further calls to :meth:`read` for the - chunk will return ``b''``. If you are not interested in the contents of - the chunk, this method should be called so that the file points to the - start of the next chunk. - - -.. rubric:: Footnotes - -.. [#] "EA IFF 85" Standard for Interchange Format Files, Jerry Morrison, Electronic - Arts, January 1985. - diff --git a/Doc/library/cmath.rst b/Doc/library/cmath.rst deleted file mode 100644 index fdac51d9603ceb..00000000000000 --- a/Doc/library/cmath.rst +++ /dev/null @@ -1,326 +0,0 @@ -:mod:`cmath` --- Mathematical functions for complex numbers -=========================================================== - -.. module:: cmath - :synopsis: Mathematical functions for complex numbers. - --------------- - -This module provides access to mathematical functions for complex numbers. The -functions in this module accept integers, floating-point numbers or complex -numbers as arguments. They will also accept any Python object that has either a -:meth:`~object.__complex__` or a :meth:`~object.__float__` method: these methods are used to -convert the object to a complex or floating-point number, respectively, and -the function is then applied to the result of the conversion. - -.. note:: - - For functions involving branch cuts, we have the problem of deciding how to - define those functions on the cut itself. Following Kahan's "Branch cuts for - complex elementary functions" paper, as well as Annex G of C99 and later C - standards, we use the sign of zero to distinguish one side of the branch cut - from the other: for a branch cut along (a portion of) the real axis we look - at the sign of the imaginary part, while for a branch cut along the - imaginary axis we look at the sign of the real part. - - For example, the :func:`cmath.sqrt` function has a branch cut along the - negative real axis. An argument of ``complex(-2.0, -0.0)`` is treated as - though it lies *below* the branch cut, and so gives a result on the negative - imaginary axis:: - - >>> cmath.sqrt(complex(-2.0, -0.0)) - -1.4142135623730951j - - But an argument of ``complex(-2.0, 0.0)`` is treated as though it lies above - the branch cut:: - - >>> cmath.sqrt(complex(-2.0, 0.0)) - 1.4142135623730951j - - -Conversions to and from polar coordinates ------------------------------------------ - -A Python complex number ``z`` is stored internally using *rectangular* -or *Cartesian* coordinates. It is completely determined by its *real -part* ``z.real`` and its *imaginary part* ``z.imag``. In other -words:: - - z == z.real + z.imag*1j - -*Polar coordinates* give an alternative way to represent a complex -number. In polar coordinates, a complex number *z* is defined by the -modulus *r* and the phase angle *phi*. The modulus *r* is the distance -from *z* to the origin, while the phase *phi* is the counterclockwise -angle, measured in radians, from the positive x-axis to the line -segment that joins the origin to *z*. - -The following functions can be used to convert from the native -rectangular coordinates to polar coordinates and back. - -.. function:: phase(x) - - Return the phase of *x* (also known as the *argument* of *x*), as a float. - ``phase(x)`` is equivalent to ``math.atan2(x.imag, x.real)``. The result - lies in the range [-\ *π*, *π*], and the branch cut for this operation lies - along the negative real axis. The sign of the result is the same as the - sign of ``x.imag``, even when ``x.imag`` is zero:: - - >>> phase(complex(-1.0, 0.0)) - 3.141592653589793 - >>> phase(complex(-1.0, -0.0)) - -3.141592653589793 - - -.. note:: - - The modulus (absolute value) of a complex number *x* can be - computed using the built-in :func:`abs` function. There is no - separate :mod:`cmath` module function for this operation. - - -.. function:: polar(x) - - Return the representation of *x* in polar coordinates. Returns a - pair ``(r, phi)`` where *r* is the modulus of *x* and phi is the - phase of *x*. ``polar(x)`` is equivalent to ``(abs(x), - phase(x))``. - - -.. function:: rect(r, phi) - - Return the complex number *x* with polar coordinates *r* and *phi*. - Equivalent to ``r * (math.cos(phi) + math.sin(phi)*1j)``. - - -Power and logarithmic functions -------------------------------- - -.. function:: exp(x) - - Return *e* raised to the power *x*, where *e* is the base of natural - logarithms. - - -.. function:: log(x[, base]) - - Returns the logarithm of *x* to the given *base*. If the *base* is not - specified, returns the natural logarithm of *x*. There is one branch cut, - from 0 along the negative real axis to -∞. - - -.. function:: log10(x) - - Return the base-10 logarithm of *x*. This has the same branch cut as - :func:`log`. - - -.. function:: sqrt(x) - - Return the square root of *x*. This has the same branch cut as :func:`log`. - - -Trigonometric functions ------------------------ - -.. function:: acos(x) - - Return the arc cosine of *x*. There are two branch cuts: One extends right - from 1 along the real axis to ∞. The other extends left from -1 along the - real axis to -∞. - - -.. function:: asin(x) - - Return the arc sine of *x*. This has the same branch cuts as :func:`acos`. - - -.. function:: atan(x) - - Return the arc tangent of *x*. There are two branch cuts: One extends from - ``1j`` along the imaginary axis to ``∞j``. The other extends from ``-1j`` - along the imaginary axis to ``-∞j``. - - -.. function:: cos(x) - - Return the cosine of *x*. - - -.. function:: sin(x) - - Return the sine of *x*. - - -.. function:: tan(x) - - Return the tangent of *x*. - - -Hyperbolic functions --------------------- - -.. function:: acosh(x) - - Return the inverse hyperbolic cosine of *x*. There is one branch cut, - extending left from 1 along the real axis to -∞. - - -.. function:: asinh(x) - - Return the inverse hyperbolic sine of *x*. There are two branch cuts: - One extends from ``1j`` along the imaginary axis to ``∞j``. The other - extends from ``-1j`` along the imaginary axis to ``-∞j``. - - -.. function:: atanh(x) - - Return the inverse hyperbolic tangent of *x*. There are two branch cuts: One - extends from ``1`` along the real axis to ``∞``. The other extends from - ``-1`` along the real axis to ``-∞``. - - -.. function:: cosh(x) - - Return the hyperbolic cosine of *x*. - - -.. function:: sinh(x) - - Return the hyperbolic sine of *x*. - - -.. function:: tanh(x) - - Return the hyperbolic tangent of *x*. - - -Classification functions ------------------------- - -.. function:: isfinite(x) - - Return ``True`` if both the real and imaginary parts of *x* are finite, and - ``False`` otherwise. - - .. versionadded:: 3.2 - - -.. function:: isinf(x) - - Return ``True`` if either the real or the imaginary part of *x* is an - infinity, and ``False`` otherwise. - - -.. function:: isnan(x) - - Return ``True`` if either the real or the imaginary part of *x* is a NaN, - and ``False`` otherwise. - - -.. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) - - Return ``True`` if the values *a* and *b* are close to each other and - ``False`` otherwise. - - Whether or not two values are considered close is determined according to - given absolute and relative tolerances. - - *rel_tol* is the relative tolerance -- it is the maximum allowed difference - between *a* and *b*, relative to the larger absolute value of *a* or *b*. - For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default - tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. - - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. - - The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be - handled according to IEEE rules. Specifically, ``NaN`` is not considered - close to any other value, including ``NaN``. ``inf`` and ``-inf`` are only - considered close to themselves. - - .. versionadded:: 3.5 - - .. seealso:: - - :pep:`485` -- A function for testing approximate equality - - -Constants ---------- - -.. data:: pi - - The mathematical constant *π*, as a float. - - -.. data:: e - - The mathematical constant *e*, as a float. - - -.. data:: tau - - The mathematical constant *τ*, as a float. - - .. versionadded:: 3.6 - - -.. data:: inf - - Floating-point positive infinity. Equivalent to ``float('inf')``. - - .. versionadded:: 3.6 - - -.. data:: infj - - Complex number with zero real part and positive infinity imaginary - part. Equivalent to ``complex(0.0, float('inf'))``. - - .. versionadded:: 3.6 - - -.. data:: nan - - A floating-point "not a number" (NaN) value. Equivalent to - ``float('nan')``. - - .. versionadded:: 3.6 - - -.. data:: nanj - - Complex number with zero real part and NaN imaginary part. Equivalent to - ``complex(0.0, float('nan'))``. - - .. versionadded:: 3.6 - - -.. index:: pair: module; math - -Note that the selection of functions is similar, but not identical, to that in -module :mod:`math`. The reason for having two modules is that some users aren't -interested in complex numbers, and perhaps don't even know what they are. They -would rather have ``math.sqrt(-1)`` raise an exception than return a complex -number. Also note that the functions defined in :mod:`cmath` always return a -complex number, even if the answer can be expressed as a real number (in which -case the complex number has an imaginary part of zero). - -A note on branch cuts: They are curves along which the given function fails to -be continuous. They are a necessary feature of many complex functions. It is -assumed that if you need to compute with complex functions, you will understand -about branch cuts. Consult almost any (not too elementary) book on complex -variables for enlightenment. For information of the proper choice of branch -cuts for numerical purposes, a good reference should be the following: - - -.. seealso:: - - Kahan, W: Branch cuts for complex elementary functions; or, Much ado about - nothing's sign bit. In Iserles, A., and Powell, M. (eds.), The state of the art - in numerical analysis. Clarendon Press (1987) pp165--211. diff --git a/Doc/library/cmd.rst b/Doc/library/cmd.rst deleted file mode 100644 index fd5df96dfd0b3d..00000000000000 --- a/Doc/library/cmd.rst +++ /dev/null @@ -1,388 +0,0 @@ -:mod:`cmd` --- Support for line-oriented command interpreters -============================================================= - -.. module:: cmd - :synopsis: Build line-oriented command interpreters. - -.. sectionauthor:: Eric S. Raymond - -**Source code:** :source:`Lib/cmd.py` - --------------- - -The :class:`Cmd` class provides a simple framework for writing line-oriented -command interpreters. These are often useful for test harnesses, administrative -tools, and prototypes that will later be wrapped in a more sophisticated -interface. - -.. class:: Cmd(completekey='tab', stdin=None, stdout=None) - - A :class:`Cmd` instance or subclass instance is a line-oriented interpreter - framework. There is no good reason to instantiate :class:`Cmd` itself; rather, - it's useful as a superclass of an interpreter class you define yourself in order - to inherit :class:`Cmd`'s methods and encapsulate action methods. - - The optional argument *completekey* is the :mod:`readline` name of a completion - key; it defaults to :kbd:`Tab`. If *completekey* is not :const:`None` and - :mod:`readline` is available, command completion is done automatically. - - The optional arguments *stdin* and *stdout* specify the input and output file - objects that the Cmd instance or subclass instance will use for input and - output. If not specified, they will default to :data:`sys.stdin` and - :data:`sys.stdout`. - - If you want a given *stdin* to be used, make sure to set the instance's - :attr:`use_rawinput` attribute to ``False``, otherwise *stdin* will be - ignored. - - -.. _cmd-objects: - -Cmd Objects ------------ - -A :class:`Cmd` instance has the following methods: - - -.. method:: Cmd.cmdloop(intro=None) - - Repeatedly issue a prompt, accept input, parse an initial prefix off the - received input, and dispatch to action methods, passing them the remainder of - the line as argument. - - The optional argument is a banner or intro string to be issued before the first - prompt (this overrides the :attr:`intro` class attribute). - - If the :mod:`readline` module is loaded, input will automatically inherit - :program:`bash`\ -like history-list editing (e.g. :kbd:`Control-P` scrolls back - to the last command, :kbd:`Control-N` forward to the next one, :kbd:`Control-F` - moves the cursor to the right non-destructively, :kbd:`Control-B` moves the - cursor to the left non-destructively, etc.). - - An end-of-file on input is passed back as the string ``'EOF'``. - - .. index:: - single: ? (question mark); in a command interpreter - single: ! (exclamation); in a command interpreter - - An interpreter instance will recognize a command name ``foo`` if and only if it - has a method :meth:`do_foo`. As a special case, a line beginning with the - character ``'?'`` is dispatched to the method :meth:`do_help`. As another - special case, a line beginning with the character ``'!'`` is dispatched to the - method :meth:`do_shell` (if such a method is defined). - - This method will return when the :meth:`postcmd` method returns a true value. - The *stop* argument to :meth:`postcmd` is the return value from the command's - corresponding :meth:`do_\*` method. - - If completion is enabled, completing commands will be done automatically, and - completing of commands args is done by calling :meth:`complete_foo` with - arguments *text*, *line*, *begidx*, and *endidx*. *text* is the string prefix - we are attempting to match: all returned matches must begin with it. *line* is - the current input line with leading whitespace removed, *begidx* and *endidx* - are the beginning and ending indexes of the prefix text, which could be used to - provide different completion depending upon which position the argument is in. - - All subclasses of :class:`Cmd` inherit a predefined :meth:`do_help`. This - method, called with an argument ``'bar'``, invokes the corresponding method - :meth:`help_bar`, and if that is not present, prints the docstring of - :meth:`do_bar`, if available. With no argument, :meth:`do_help` lists all - available help topics (that is, all commands with corresponding - :meth:`help_\*` methods or commands that have docstrings), and also lists any - undocumented commands. - - -.. method:: Cmd.onecmd(str) - - Interpret the argument as though it had been typed in response to the prompt. - This may be overridden, but should not normally need to be; see the - :meth:`precmd` and :meth:`postcmd` methods for useful execution hooks. The - return value is a flag indicating whether interpretation of commands by the - interpreter should stop. If there is a :meth:`do_\*` method for the command - *str*, the return value of that method is returned, otherwise the return value - from the :meth:`default` method is returned. - - -.. method:: Cmd.emptyline() - - Method called when an empty line is entered in response to the prompt. If this - method is not overridden, it repeats the last nonempty command entered. - - -.. method:: Cmd.default(line) - - Method called on an input line when the command prefix is not recognized. If - this method is not overridden, it prints an error message and returns. - - -.. method:: Cmd.completedefault(text, line, begidx, endidx) - - Method called to complete an input line when no command-specific - :meth:`complete_\*` method is available. By default, it returns an empty list. - - -.. method:: Cmd.columnize(list, displaywidth=80) - - Method called to display a list of strings as a compact set of columns. - Each column is only as wide as necessary. - Columns are separated by two spaces for readability. - - -.. method:: Cmd.precmd(line) - - Hook method executed just before the command line *line* is interpreted, but - after the input prompt is generated and issued. This method is a stub in - :class:`Cmd`; it exists to be overridden by subclasses. The return value is - used as the command which will be executed by the :meth:`onecmd` method; the - :meth:`precmd` implementation may re-write the command or simply return *line* - unchanged. - - -.. method:: Cmd.postcmd(stop, line) - - Hook method executed just after a command dispatch is finished. This method is - a stub in :class:`Cmd`; it exists to be overridden by subclasses. *line* is the - command line which was executed, and *stop* is a flag which indicates whether - execution will be terminated after the call to :meth:`postcmd`; this will be the - return value of the :meth:`onecmd` method. The return value of this method will - be used as the new value for the internal flag which corresponds to *stop*; - returning false will cause interpretation to continue. - - -.. method:: Cmd.preloop() - - Hook method executed once when :meth:`cmdloop` is called. This method is a stub - in :class:`Cmd`; it exists to be overridden by subclasses. - - -.. method:: Cmd.postloop() - - Hook method executed once when :meth:`cmdloop` is about to return. This method - is a stub in :class:`Cmd`; it exists to be overridden by subclasses. - - -Instances of :class:`Cmd` subclasses have some public instance variables: - -.. attribute:: Cmd.prompt - - The prompt issued to solicit input. - - -.. attribute:: Cmd.identchars - - The string of characters accepted for the command prefix. - - -.. attribute:: Cmd.lastcmd - - The last nonempty command prefix seen. - - -.. attribute:: Cmd.cmdqueue - - A list of queued input lines. The cmdqueue list is checked in - :meth:`cmdloop` when new input is needed; if it is nonempty, its elements - will be processed in order, as if entered at the prompt. - - -.. attribute:: Cmd.intro - - A string to issue as an intro or banner. May be overridden by giving the - :meth:`cmdloop` method an argument. - - -.. attribute:: Cmd.doc_header - - The header to issue if the help output has a section for documented commands. - - -.. attribute:: Cmd.misc_header - - The header to issue if the help output has a section for miscellaneous help - topics (that is, there are :meth:`help_\*` methods without corresponding - :meth:`do_\*` methods). - - -.. attribute:: Cmd.undoc_header - - The header to issue if the help output has a section for undocumented commands - (that is, there are :meth:`do_\*` methods without corresponding :meth:`help_\*` - methods). - - -.. attribute:: Cmd.ruler - - The character used to draw separator lines under the help-message headers. If - empty, no ruler line is drawn. It defaults to ``'='``. - - -.. attribute:: Cmd.use_rawinput - - A flag, defaulting to true. If true, :meth:`cmdloop` uses :func:`input` to - display a prompt and read the next command; if false, :meth:`sys.stdout.write` - and :meth:`sys.stdin.readline` are used. (This means that by importing - :mod:`readline`, on systems that support it, the interpreter will automatically - support :program:`Emacs`\ -like line editing and command-history keystrokes.) - - -.. _cmd-example: - -Cmd Example ------------ - -.. sectionauthor:: Raymond Hettinger - -The :mod:`cmd` module is mainly useful for building custom shells that let a -user work with a program interactively. - -This section presents a simple example of how to build a shell around a few of -the commands in the :mod:`turtle` module. - -Basic turtle commands such as :meth:`~turtle.forward` are added to a -:class:`Cmd` subclass with method named :meth:`do_forward`. The argument is -converted to a number and dispatched to the turtle module. The docstring is -used in the help utility provided by the shell. - -The example also includes a basic record and playback facility implemented with -the :meth:`~Cmd.precmd` method which is responsible for converting the input to -lowercase and writing the commands to a file. The :meth:`do_playback` method -reads the file and adds the recorded commands to the :attr:`cmdqueue` for -immediate playback:: - - import cmd, sys - from turtle import * - - class TurtleShell(cmd.Cmd): - intro = 'Welcome to the turtle shell. Type help or ? to list commands.\n' - prompt = '(turtle) ' - file = None - - # ----- basic turtle commands ----- - def do_forward(self, arg): - 'Move the turtle forward by the specified distance: FORWARD 10' - forward(*parse(arg)) - def do_right(self, arg): - 'Turn turtle right by given number of degrees: RIGHT 20' - right(*parse(arg)) - def do_left(self, arg): - 'Turn turtle left by given number of degrees: LEFT 90' - left(*parse(arg)) - def do_goto(self, arg): - 'Move turtle to an absolute position with changing orientation. GOTO 100 200' - goto(*parse(arg)) - def do_home(self, arg): - 'Return turtle to the home position: HOME' - home() - def do_circle(self, arg): - 'Draw circle with given radius an options extent and steps: CIRCLE 50' - circle(*parse(arg)) - def do_position(self, arg): - 'Print the current turtle position: POSITION' - print('Current position is %d %d\n' % position()) - def do_heading(self, arg): - 'Print the current turtle heading in degrees: HEADING' - print('Current heading is %d\n' % (heading(),)) - def do_color(self, arg): - 'Set the color: COLOR BLUE' - color(arg.lower()) - def do_undo(self, arg): - 'Undo (repeatedly) the last turtle action(s): UNDO' - def do_reset(self, arg): - 'Clear the screen and return turtle to center: RESET' - reset() - def do_bye(self, arg): - 'Stop recording, close the turtle window, and exit: BYE' - print('Thank you for using Turtle') - self.close() - bye() - return True - - # ----- record and playback ----- - def do_record(self, arg): - 'Save future commands to filename: RECORD rose.cmd' - self.file = open(arg, 'w') - def do_playback(self, arg): - 'Playback commands from a file: PLAYBACK rose.cmd' - self.close() - with open(arg) as f: - self.cmdqueue.extend(f.read().splitlines()) - def precmd(self, line): - line = line.lower() - if self.file and 'playback' not in line: - print(line, file=self.file) - return line - def close(self): - if self.file: - self.file.close() - self.file = None - - def parse(arg): - 'Convert a series of zero or more numbers to an argument tuple' - return tuple(map(int, arg.split())) - - if __name__ == '__main__': - TurtleShell().cmdloop() - - -Here is a sample session with the turtle shell showing the help functions, using -blank lines to repeat commands, and the simple record and playback facility: - -.. code-block:: none - - Welcome to the turtle shell. Type help or ? to list commands. - - (turtle) ? - - Documented commands (type help ): - ======================================== - bye color goto home playback record right - circle forward heading left position reset undo - - (turtle) help forward - Move the turtle forward by the specified distance: FORWARD 10 - (turtle) record spiral.cmd - (turtle) position - Current position is 0 0 - - (turtle) heading - Current heading is 0 - - (turtle) reset - (turtle) circle 20 - (turtle) right 30 - (turtle) circle 40 - (turtle) right 30 - (turtle) circle 60 - (turtle) right 30 - (turtle) circle 80 - (turtle) right 30 - (turtle) circle 100 - (turtle) right 30 - (turtle) circle 120 - (turtle) right 30 - (turtle) circle 120 - (turtle) heading - Current heading is 180 - - (turtle) forward 100 - (turtle) - (turtle) right 90 - (turtle) forward 100 - (turtle) - (turtle) right 90 - (turtle) forward 400 - (turtle) right 90 - (turtle) forward 500 - (turtle) right 90 - (turtle) forward 400 - (turtle) right 90 - (turtle) forward 300 - (turtle) playback spiral.cmd - Current position is 0 0 - - Current heading is 0 - - Current heading is 180 - - (turtle) bye - Thank you for using Turtle diff --git a/Doc/library/code.rst b/Doc/library/code.rst deleted file mode 100644 index 3d7f43c86a0557..00000000000000 --- a/Doc/library/code.rst +++ /dev/null @@ -1,184 +0,0 @@ -:mod:`code` --- Interpreter base classes -======================================== - -.. module:: code - :synopsis: Facilities to implement read-eval-print loops. - -**Source code:** :source:`Lib/code.py` - --------------- - -The ``code`` module provides facilities to implement read-eval-print loops in -Python. Two classes and convenience functions are included which can be used to -build applications which provide an interactive interpreter prompt. - - -.. class:: InteractiveInterpreter(locals=None) - - This class deals with parsing and interpreter state (the user's namespace); it - does not deal with input buffering or prompting or input file naming (the - filename is always passed in explicitly). The optional *locals* argument - specifies the dictionary in which code will be executed; it defaults to a newly - created dictionary with key ``'__name__'`` set to ``'__console__'`` and key - ``'__doc__'`` set to ``None``. - - -.. class:: InteractiveConsole(locals=None, filename="") - - Closely emulate the behavior of the interactive Python interpreter. This class - builds on :class:`InteractiveInterpreter` and adds prompting using the familiar - ``sys.ps1`` and ``sys.ps2``, and input buffering. - - -.. function:: interact(banner=None, readfunc=None, local=None, exitmsg=None) - - Convenience function to run a read-eval-print loop. This creates a new - instance of :class:`InteractiveConsole` and sets *readfunc* to be used as - the :meth:`InteractiveConsole.raw_input` method, if provided. If *local* is - provided, it is passed to the :class:`InteractiveConsole` constructor for - use as the default namespace for the interpreter loop. The :meth:`interact` - method of the instance is then run with *banner* and *exitmsg* passed as the - banner and exit message to use, if provided. The console object is discarded - after use. - - .. versionchanged:: 3.6 - Added *exitmsg* parameter. - - -.. function:: compile_command(source, filename="", symbol="single") - - This function is useful for programs that want to emulate Python's interpreter - main loop (a.k.a. the read-eval-print loop). The tricky part is to determine - when the user has entered an incomplete command that can be completed by - entering more text (as opposed to a complete command or a syntax error). This - function *almost* always makes the same decision as the real interpreter main - loop. - - *source* is the source string; *filename* is the optional filename from which - source was read, defaulting to ``''``; and *symbol* is the optional - grammar start symbol, which should be ``'single'`` (the default), ``'eval'`` - or ``'exec'``. - - Returns a code object (the same as ``compile(source, filename, symbol)``) if the - command is complete and valid; ``None`` if the command is incomplete; raises - :exc:`SyntaxError` if the command is complete and contains a syntax error, or - raises :exc:`OverflowError` or :exc:`ValueError` if the command contains an - invalid literal. - - -.. _interpreter-objects: - -Interactive Interpreter Objects -------------------------------- - - -.. method:: InteractiveInterpreter.runsource(source, filename="", symbol="single") - - Compile and run some source in the interpreter. Arguments are the same as for - :func:`compile_command`; the default for *filename* is ``''``, and for - *symbol* is ``'single'``. One of several things can happen: - - * The input is incorrect; :func:`compile_command` raised an exception - (:exc:`SyntaxError` or :exc:`OverflowError`). A syntax traceback will be - printed by calling the :meth:`showsyntaxerror` method. :meth:`runsource` - returns ``False``. - - * The input is incomplete, and more input is required; :func:`compile_command` - returned ``None``. :meth:`runsource` returns ``True``. - - * The input is complete; :func:`compile_command` returned a code object. The - code is executed by calling the :meth:`runcode` (which also handles run-time - exceptions, except for :exc:`SystemExit`). :meth:`runsource` returns ``False``. - - The return value can be used to decide whether to use ``sys.ps1`` or ``sys.ps2`` - to prompt the next line. - - -.. method:: InteractiveInterpreter.runcode(code) - - Execute a code object. When an exception occurs, :meth:`showtraceback` is called - to display a traceback. All exceptions are caught except :exc:`SystemExit`, - which is allowed to propagate. - - A note about :exc:`KeyboardInterrupt`: this exception may occur elsewhere in - this code, and may not always be caught. The caller should be prepared to deal - with it. - - -.. method:: InteractiveInterpreter.showsyntaxerror(filename=None) - - Display the syntax error that just occurred. This does not display a stack - trace because there isn't one for syntax errors. If *filename* is given, it is - stuffed into the exception instead of the default filename provided by Python's - parser, because it always uses ``''`` when reading from a string. The - output is written by the :meth:`write` method. - - -.. method:: InteractiveInterpreter.showtraceback() - - Display the exception that just occurred. We remove the first stack item - because it is within the interpreter object implementation. The output is - written by the :meth:`write` method. - - .. versionchanged:: 3.5 The full chained traceback is displayed instead - of just the primary traceback. - - -.. method:: InteractiveInterpreter.write(data) - - Write a string to the standard error stream (``sys.stderr``). Derived classes - should override this to provide the appropriate output handling as needed. - - -.. _console-objects: - -Interactive Console Objects ---------------------------- - -The :class:`InteractiveConsole` class is a subclass of -:class:`InteractiveInterpreter`, and so offers all the methods of the -interpreter objects as well as the following additions. - - -.. method:: InteractiveConsole.interact(banner=None, exitmsg=None) - - Closely emulate the interactive Python console. The optional *banner* argument - specify the banner to print before the first interaction; by default it prints a - banner similar to the one printed by the standard Python interpreter, followed - by the class name of the console object in parentheses (so as not to confuse - this with the real interpreter -- since it's so close!). - - The optional *exitmsg* argument specifies an exit message printed when exiting. - Pass the empty string to suppress the exit message. If *exitmsg* is not given or - ``None``, a default message is printed. - - .. versionchanged:: 3.4 - To suppress printing any banner, pass an empty string. - - .. versionchanged:: 3.6 - Print an exit message when exiting. - - -.. method:: InteractiveConsole.push(line) - - Push a line of source text to the interpreter. The line should not have a - trailing newline; it may have internal newlines. The line is appended to a - buffer and the interpreter's :meth:`~InteractiveInterpreter.runsource` method is called with the - concatenated contents of the buffer as source. If this indicates that the - command was executed or invalid, the buffer is reset; otherwise, the command is - incomplete, and the buffer is left as it was after the line was appended. The - return value is ``True`` if more input is required, ``False`` if the line was - dealt with in some way (this is the same as :meth:`!runsource`). - - -.. method:: InteractiveConsole.resetbuffer() - - Remove any unhandled source text from the input buffer. - - -.. method:: InteractiveConsole.raw_input(prompt="") - - Write a prompt and read a line. The returned line does not include the trailing - newline. When the user enters the EOF key sequence, :exc:`EOFError` is raised. - The base implementation reads from ``sys.stdin``; a subclass may replace this - with a different implementation. diff --git a/Doc/library/codecs.rst b/Doc/library/codecs.rst deleted file mode 100644 index 9ce584874783da..00000000000000 --- a/Doc/library/codecs.rst +++ /dev/null @@ -1,1563 +0,0 @@ -:mod:`codecs` --- Codec registry and base classes -================================================= - -.. module:: codecs - :synopsis: Encode and decode data and streams. - -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/codecs.py` - -.. index:: - single: Unicode - single: Codecs - pair: Codecs; encode - pair: Codecs; decode - single: streams - pair: stackable; streams - --------------- - -This module defines base classes for standard Python codecs (encoders and -decoders) and provides access to the internal Python codec registry, which -manages the codec and error handling lookup process. Most standard codecs -are :term:`text encodings `, which encode text to bytes (and -decode bytes to text), but there are also codecs provided that encode text to -text, and bytes to bytes. Custom codecs may encode and decode between arbitrary -types, but some module features are restricted to be used specifically with -:term:`text encodings ` or with codecs that encode to -:class:`bytes`. - -The module defines the following functions for encoding and decoding with -any codec: - -.. function:: encode(obj, encoding='utf-8', errors='strict') - - Encodes *obj* using the codec registered for *encoding*. - - *Errors* may be given to set the desired error handling scheme. The - default error handler is ``'strict'`` meaning that encoding errors raise - :exc:`ValueError` (or a more codec specific subclass, such as - :exc:`UnicodeEncodeError`). Refer to :ref:`codec-base-classes` for more - information on codec error handling. - -.. function:: decode(obj, encoding='utf-8', errors='strict') - - Decodes *obj* using the codec registered for *encoding*. - - *Errors* may be given to set the desired error handling scheme. The - default error handler is ``'strict'`` meaning that decoding errors raise - :exc:`ValueError` (or a more codec specific subclass, such as - :exc:`UnicodeDecodeError`). Refer to :ref:`codec-base-classes` for more - information on codec error handling. - -The full details for each codec can also be looked up directly: - -.. function:: lookup(encoding) - - Looks up the codec info in the Python codec registry and returns a - :class:`CodecInfo` object as defined below. - - Encodings are first looked up in the registry's cache. If not found, the list of - registered search functions is scanned. If no :class:`CodecInfo` object is - found, a :exc:`LookupError` is raised. Otherwise, the :class:`CodecInfo` object - is stored in the cache and returned to the caller. - -.. class:: CodecInfo(encode, decode, streamreader=None, streamwriter=None, incrementalencoder=None, incrementaldecoder=None, name=None) - - Codec details when looking up the codec registry. The constructor - arguments are stored in attributes of the same name: - - - .. attribute:: name - - The name of the encoding. - - - .. attribute:: encode - decode - - The stateless encoding and decoding functions. These must be - functions or methods which have the same interface as - the :meth:`~Codec.encode` and :meth:`~Codec.decode` methods of Codec - instances (see :ref:`Codec Interface `). - The functions or methods are expected to work in a stateless mode. - - - .. attribute:: incrementalencoder - incrementaldecoder - - Incremental encoder and decoder classes or factory functions. - These have to provide the interface defined by the base classes - :class:`IncrementalEncoder` and :class:`IncrementalDecoder`, - respectively. Incremental codecs can maintain state. - - - .. attribute:: streamwriter - streamreader - - Stream writer and reader classes or factory functions. These have to - provide the interface defined by the base classes - :class:`StreamWriter` and :class:`StreamReader`, respectively. - Stream codecs can maintain state. - -To simplify access to the various codec components, the module provides -these additional functions which use :func:`lookup` for the codec lookup: - -.. function:: getencoder(encoding) - - Look up the codec for the given encoding and return its encoder function. - - Raises a :exc:`LookupError` in case the encoding cannot be found. - - -.. function:: getdecoder(encoding) - - Look up the codec for the given encoding and return its decoder function. - - Raises a :exc:`LookupError` in case the encoding cannot be found. - - -.. function:: getincrementalencoder(encoding) - - Look up the codec for the given encoding and return its incremental encoder - class or factory function. - - Raises a :exc:`LookupError` in case the encoding cannot be found or the codec - doesn't support an incremental encoder. - - -.. function:: getincrementaldecoder(encoding) - - Look up the codec for the given encoding and return its incremental decoder - class or factory function. - - Raises a :exc:`LookupError` in case the encoding cannot be found or the codec - doesn't support an incremental decoder. - - -.. function:: getreader(encoding) - - Look up the codec for the given encoding and return its :class:`StreamReader` - class or factory function. - - Raises a :exc:`LookupError` in case the encoding cannot be found. - - -.. function:: getwriter(encoding) - - Look up the codec for the given encoding and return its :class:`StreamWriter` - class or factory function. - - Raises a :exc:`LookupError` in case the encoding cannot be found. - -Custom codecs are made available by registering a suitable codec search -function: - -.. function:: register(search_function) - - Register a codec search function. Search functions are expected to take one - argument, being the encoding name in all lower case letters with hyphens - and spaces converted to underscores, and return a :class:`CodecInfo` object. - In case a search function cannot find a given encoding, it should return - ``None``. - - .. versionchanged:: 3.9 - Hyphens and spaces are converted to underscore. - - -.. function:: unregister(search_function) - - Unregister a codec search function and clear the registry's cache. - If the search function is not registered, do nothing. - - .. versionadded:: 3.10 - - -While the builtin :func:`open` and the associated :mod:`io` module are the -recommended approach for working with encoded text files, this module -provides additional utility functions and classes that allow the use of a -wider range of codecs when working with binary files: - -.. function:: open(filename, mode='r', encoding=None, errors='strict', buffering=-1) - - Open an encoded file using the given *mode* and return an instance of - :class:`StreamReaderWriter`, providing transparent encoding/decoding. - The default file mode is ``'r'``, meaning to open the file in read mode. - - .. note:: - - If *encoding* is not ``None``, then the - underlying encoded files are always opened in binary mode. - No automatic conversion of ``'\n'`` is done on reading and writing. - The *mode* argument may be any binary mode acceptable to the built-in - :func:`open` function; the ``'b'`` is automatically added. - - *encoding* specifies the encoding which is to be used for the file. - Any encoding that encodes to and decodes from bytes is allowed, and - the data types supported by the file methods depend on the codec used. - - *errors* may be given to define the error handling. It defaults to ``'strict'`` - which causes a :exc:`ValueError` to be raised in case an encoding error occurs. - - *buffering* has the same meaning as for the built-in :func:`open` function. - It defaults to -1 which means that the default buffer size will be used. - - .. versionchanged:: 3.11 - The ``'U'`` mode has been removed. - - -.. function:: EncodedFile(file, data_encoding, file_encoding=None, errors='strict') - - Return a :class:`StreamRecoder` instance, a wrapped version of *file* - which provides transparent transcoding. The original file is closed - when the wrapped version is closed. - - Data written to the wrapped file is decoded according to the given - *data_encoding* and then written to the original file as bytes using - *file_encoding*. Bytes read from the original file are decoded - according to *file_encoding*, and the result is encoded - using *data_encoding*. - - If *file_encoding* is not given, it defaults to *data_encoding*. - - *errors* may be given to define the error handling. It defaults to - ``'strict'``, which causes :exc:`ValueError` to be raised in case an encoding - error occurs. - - -.. function:: iterencode(iterator, encoding, errors='strict', **kwargs) - - Uses an incremental encoder to iteratively encode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any - other keyword argument) is passed through to the incremental encoder. - - This function requires that the codec accept text :class:`str` objects - to encode. Therefore it does not support bytes-to-bytes encoders such as - ``base64_codec``. - - -.. function:: iterdecode(iterator, encoding, errors='strict', **kwargs) - - Uses an incremental decoder to iteratively decode the input provided by - *iterator*. This function is a :term:`generator`. - The *errors* argument (as well as any - other keyword argument) is passed through to the incremental decoder. - - This function requires that the codec accept :class:`bytes` objects - to decode. Therefore it does not support text-to-text encoders such as - ``rot_13``, although ``rot_13`` may be used equivalently with - :func:`iterencode`. - - -The module also provides the following constants which are useful for reading -and writing to platform dependent files: - - -.. data:: BOM - BOM_BE - BOM_LE - BOM_UTF8 - BOM_UTF16 - BOM_UTF16_BE - BOM_UTF16_LE - BOM_UTF32 - BOM_UTF32_BE - BOM_UTF32_LE - - These constants define various byte sequences, - being Unicode byte order marks (BOMs) for several encodings. They are - used in UTF-16 and UTF-32 data streams to indicate the byte order used, - and in UTF-8 as a Unicode signature. :const:`BOM_UTF16` is either - :const:`BOM_UTF16_BE` or :const:`BOM_UTF16_LE` depending on the platform's - native byte order, :const:`BOM` is an alias for :const:`BOM_UTF16`, - :const:`BOM_LE` for :const:`BOM_UTF16_LE` and :const:`BOM_BE` for - :const:`BOM_UTF16_BE`. The others represent the BOM in UTF-8 and UTF-32 - encodings. - - -.. _codec-base-classes: - -Codec Base Classes ------------------- - -The :mod:`codecs` module defines a set of base classes which define the -interfaces for working with codec objects, and can also be used as the basis -for custom codec implementations. - -Each codec has to define four interfaces to make it usable as codec in Python: -stateless encoder, stateless decoder, stream reader and stream writer. The -stream reader and writers typically reuse the stateless encoder/decoder to -implement the file protocols. Codec authors also need to define how the -codec will handle encoding and decoding errors. - - -.. _surrogateescape: -.. _error-handlers: - -Error Handlers -^^^^^^^^^^^^^^ - -To simplify and standardize error handling, codecs may implement different -error handling schemes by accepting the *errors* string argument: - - >>> 'German ß, ♬'.encode(encoding='ascii', errors='backslashreplace') - b'German \\xdf, \\u266c' - >>> 'German ß, ♬'.encode(encoding='ascii', errors='xmlcharrefreplace') - b'German ß, ♬' - -.. index:: - pair: strict; error handler's name - pair: ignore; error handler's name - pair: replace; error handler's name - pair: backslashreplace; error handler's name - pair: surrogateescape; error handler's name - single: ? (question mark); replacement character - single: \ (backslash); escape sequence - single: \x; escape sequence - single: \u; escape sequence - single: \U; escape sequence - -The following error handlers can be used with all Python -:ref:`standard-encodings` codecs: - -.. tabularcolumns:: |l|L| - -+-------------------------+-----------------------------------------------+ -| Value | Meaning | -+=========================+===============================================+ -| ``'strict'`` | Raise :exc:`UnicodeError` (or a subclass), | -| | this is the default. Implemented in | -| | :func:`strict_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'ignore'`` | Ignore the malformed data and continue without| -| | further notice. Implemented in | -| | :func:`ignore_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'replace'`` | Replace with a replacement marker. On | -| | encoding, use ``?`` (ASCII character). On | -| | decoding, use ``�`` (U+FFFD, the official | -| | REPLACEMENT CHARACTER). Implemented in | -| | :func:`replace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'backslashreplace'`` | Replace with backslashed escape sequences. | -| | On encoding, use hexadecimal form of Unicode | -| | code point with formats :samp:`\\x{hh}` | -| | :samp:`\\u{xxxx}` :samp:`\\U{xxxxxxxx}`. | -| | On decoding, use hexadecimal form of byte | -| | value with format :samp:`\\x{hh}`. | -| | Implemented in | -| | :func:`backslashreplace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'surrogateescape'`` | On decoding, replace byte with individual | -| | surrogate code ranging from ``U+DC80`` to | -| | ``U+DCFF``. This code will then be turned | -| | back into the same byte when the | -| | ``'surrogateescape'`` error handler is used | -| | when encoding the data. (See :pep:`383` for | -| | more.) | -+-------------------------+-----------------------------------------------+ - -.. index:: - pair: xmlcharrefreplace; error handler's name - pair: namereplace; error handler's name - single: \N; escape sequence - -The following error handlers are only applicable to encoding (within -:term:`text encodings `): - -+-------------------------+-----------------------------------------------+ -| Value | Meaning | -+=========================+===============================================+ -| ``'xmlcharrefreplace'`` | Replace with XML/HTML numeric character | -| | reference, which is a decimal form of Unicode | -| | code point with format :samp:`&#{num};`. | -| | Implemented in | -| | :func:`xmlcharrefreplace_errors`. | -+-------------------------+-----------------------------------------------+ -| ``'namereplace'`` | Replace with ``\N{...}`` escape sequences, | -| | what appears in the braces is the Name | -| | property from Unicode Character Database. | -| | Implemented in :func:`namereplace_errors`. | -+-------------------------+-----------------------------------------------+ - -.. index:: - pair: surrogatepass; error handler's name - -In addition, the following error handler is specific to the given codecs: - -+-------------------+------------------------+-------------------------------------------+ -| Value | Codecs | Meaning | -+===================+========================+===========================================+ -|``'surrogatepass'``| utf-8, utf-16, utf-32, | Allow encoding and decoding surrogate code| -| | utf-16-be, utf-16-le, | point (``U+D800`` - ``U+DFFF``) as normal | -| | utf-32-be, utf-32-le | code point. Otherwise these codecs treat | -| | | the presence of surrogate code point in | -| | | :class:`str` as an error. | -+-------------------+------------------------+-------------------------------------------+ - -.. versionadded:: 3.1 - The ``'surrogateescape'`` and ``'surrogatepass'`` error handlers. - -.. versionchanged:: 3.4 - The ``'surrogatepass'`` error handler now works with utf-16\* and utf-32\* - codecs. - -.. versionadded:: 3.5 - The ``'namereplace'`` error handler. - -.. versionchanged:: 3.5 - The ``'backslashreplace'`` error handler now works with decoding and - translating. - -The set of allowed values can be extended by registering a new named error -handler: - -.. function:: register_error(name, error_handler) - - Register the error handling function *error_handler* under the name *name*. - The *error_handler* argument will be called during encoding and decoding - in case of an error, when *name* is specified as the errors parameter. - - For encoding, *error_handler* will be called with a :exc:`UnicodeEncodeError` - instance, which contains information about the location of the error. The - error handler must either raise this or a different exception, or return a - tuple with a replacement for the unencodable part of the input and a position - where encoding should continue. The replacement may be either :class:`str` or - :class:`bytes`. If the replacement is bytes, the encoder will simply copy - them into the output buffer. If the replacement is a string, the encoder will - encode the replacement. Encoding continues on original input at the - specified position. Negative position values will be treated as being - relative to the end of the input string. If the resulting position is out of - bound an :exc:`IndexError` will be raised. - - Decoding and translating works similarly, except :exc:`UnicodeDecodeError` or - :exc:`UnicodeTranslateError` will be passed to the handler and that the - replacement from the error handler will be put into the output directly. - - -Previously registered error handlers (including the standard error handlers) -can be looked up by name: - -.. function:: lookup_error(name) - - Return the error handler previously registered under the name *name*. - - Raises a :exc:`LookupError` in case the handler cannot be found. - -The following standard error handlers are also made available as module level -functions: - -.. function:: strict_errors(exception) - - Implements the ``'strict'`` error handling. - - Each encoding or decoding error raises a :exc:`UnicodeError`. - - -.. function:: ignore_errors(exception) - - Implements the ``'ignore'`` error handling. - - Malformed data is ignored; encoding or decoding is continued without - further notice. - - -.. function:: replace_errors(exception) - - Implements the ``'replace'`` error handling. - - Substitutes ``?`` (ASCII character) for encoding errors or ``�`` (U+FFFD, - the official REPLACEMENT CHARACTER) for decoding errors. - - -.. function:: backslashreplace_errors(exception) - - Implements the ``'backslashreplace'`` error handling. - - Malformed data is replaced by a backslashed escape sequence. - On encoding, use the hexadecimal form of Unicode code point with formats - :samp:`\\x{hh}` :samp:`\\u{xxxx}` :samp:`\\U{xxxxxxxx}`. - On decoding, use the hexadecimal form of - byte value with format :samp:`\\x{hh}`. - - .. versionchanged:: 3.5 - Works with decoding and translating. - - -.. function:: xmlcharrefreplace_errors(exception) - - Implements the ``'xmlcharrefreplace'`` error handling (for encoding within - :term:`text encoding` only). - - The unencodable character is replaced by an appropriate XML/HTML numeric - character reference, which is a decimal form of Unicode code point with - format :samp:`&#{num};` . - - -.. function:: namereplace_errors(exception) - - Implements the ``'namereplace'`` error handling (for encoding within - :term:`text encoding` only). - - The unencodable character is replaced by a ``\N{...}`` escape sequence. The - set of characters that appear in the braces is the Name property from - Unicode Character Database. For example, the German lowercase letter ``'ß'`` - will be converted to byte sequence ``\N{LATIN SMALL LETTER SHARP S}`` . - - .. versionadded:: 3.5 - - -.. _codec-objects: - -Stateless Encoding and Decoding -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The base :class:`Codec` class defines these methods which also define the -function interfaces of the stateless encoder and decoder: - - -.. class:: Codec - - .. method:: encode(input, errors='strict') - - Encodes the object *input* and returns a tuple (output object, length consumed). - For instance, :term:`text encoding` converts - a string object to a bytes object using a particular - character set encoding (e.g., ``cp1252`` or ``iso-8859-1``). - - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. - - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamWriter` for codecs which have to keep state in order to make - encoding efficient. - - The encoder must be able to handle zero length input and return an empty object - of the output object type in this situation. - - - .. method:: decode(input, errors='strict') - - Decodes the object *input* and returns a tuple (output object, length - consumed). For instance, for a :term:`text encoding`, decoding converts - a bytes object encoded using a particular - character set encoding to a string object. - - For text encodings and bytes-to-bytes codecs, - *input* must be a bytes object or one which provides the read-only - buffer interface -- for example, buffer objects and memory mapped files. - - The *errors* argument defines the error handling to apply. - It defaults to ``'strict'`` handling. - - The method may not store state in the :class:`Codec` instance. Use - :class:`StreamReader` for codecs which have to keep state in order to make - decoding efficient. - - The decoder must be able to handle zero length input and return an empty object - of the output object type in this situation. - - -Incremental Encoding and Decoding -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :class:`IncrementalEncoder` and :class:`IncrementalDecoder` classes provide -the basic interface for incremental encoding and decoding. Encoding/decoding the -input isn't done with one call to the stateless encoder/decoder function, but -with multiple calls to the -:meth:`~IncrementalEncoder.encode`/:meth:`~IncrementalDecoder.decode` method of -the incremental encoder/decoder. The incremental encoder/decoder keeps track of -the encoding/decoding process during method calls. - -The joined output of calls to the -:meth:`~IncrementalEncoder.encode`/:meth:`~IncrementalDecoder.decode` method is -the same as if all the single inputs were joined into one, and this input was -encoded/decoded with the stateless encoder/decoder. - - -.. _incremental-encoder-objects: - -IncrementalEncoder Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`IncrementalEncoder` class is used for encoding an input in multiple -steps. It defines the following methods which every incremental encoder must -define in order to be compatible with the Python codec registry. - - -.. class:: IncrementalEncoder(errors='strict') - - Constructor for an :class:`IncrementalEncoder` instance. - - All incremental encoders must provide this constructor interface. They are free - to add additional keyword arguments, but only the ones defined here are used by - the Python codec registry. - - The :class:`IncrementalEncoder` may implement different error handling schemes - by providing the *errors* keyword argument. See :ref:`error-handlers` for - possible values. - - The *errors* argument will be assigned to an attribute of the same name. - Assigning to this attribute makes it possible to switch between different error - handling strategies during the lifetime of the :class:`IncrementalEncoder` - object. - - - .. method:: encode(object, final=False) - - Encodes *object* (taking the current state of the encoder into account) - and returns the resulting encoded object. If this is the last call to - :meth:`encode` *final* must be true (the default is false). - - - .. method:: reset() - - Reset the encoder to the initial state. The output is discarded: call - ``.encode(object, final=True)``, passing an empty byte or text string - if necessary, to reset the encoder and to get the output. - - - .. method:: getstate() - - Return the current state of the encoder which must be an integer. The - implementation should make sure that ``0`` is the most common - state. (States that are more complicated than integers can be converted - into an integer by marshaling/pickling the state and encoding the bytes - of the resulting string into an integer.) - - - .. method:: setstate(state) - - Set the state of the encoder to *state*. *state* must be an encoder state - returned by :meth:`getstate`. - - -.. _incremental-decoder-objects: - -IncrementalDecoder Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`IncrementalDecoder` class is used for decoding an input in multiple -steps. It defines the following methods which every incremental decoder must -define in order to be compatible with the Python codec registry. - - -.. class:: IncrementalDecoder(errors='strict') - - Constructor for an :class:`IncrementalDecoder` instance. - - All incremental decoders must provide this constructor interface. They are free - to add additional keyword arguments, but only the ones defined here are used by - the Python codec registry. - - The :class:`IncrementalDecoder` may implement different error handling schemes - by providing the *errors* keyword argument. See :ref:`error-handlers` for - possible values. - - The *errors* argument will be assigned to an attribute of the same name. - Assigning to this attribute makes it possible to switch between different error - handling strategies during the lifetime of the :class:`IncrementalDecoder` - object. - - - .. method:: decode(object, final=False) - - Decodes *object* (taking the current state of the decoder into account) - and returns the resulting decoded object. If this is the last call to - :meth:`decode` *final* must be true (the default is false). If *final* is - true the decoder must decode the input completely and must flush all - buffers. If this isn't possible (e.g. because of incomplete byte sequences - at the end of the input) it must initiate error handling just like in the - stateless case (which might raise an exception). - - - .. method:: reset() - - Reset the decoder to the initial state. - - - .. method:: getstate() - - Return the current state of the decoder. This must be a tuple with two - items, the first must be the buffer containing the still undecoded - input. The second must be an integer and can be additional state - info. (The implementation should make sure that ``0`` is the most common - additional state info.) If this additional state info is ``0`` it must be - possible to set the decoder to the state which has no input buffered and - ``0`` as the additional state info, so that feeding the previously - buffered input to the decoder returns it to the previous state without - producing any output. (Additional state info that is more complicated than - integers can be converted into an integer by marshaling/pickling the info - and encoding the bytes of the resulting string into an integer.) - - - .. method:: setstate(state) - - Set the state of the decoder to *state*. *state* must be a decoder state - returned by :meth:`getstate`. - - -Stream Encoding and Decoding -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - - -The :class:`StreamWriter` and :class:`StreamReader` classes provide generic -working interfaces which can be used to implement new encoding submodules very -easily. See :mod:`!encodings.utf_8` for an example of how this is done. - - -.. _stream-writer-objects: - -StreamWriter Objects -~~~~~~~~~~~~~~~~~~~~ - -The :class:`StreamWriter` class is a subclass of :class:`Codec` and defines the -following methods which every stream writer must define in order to be -compatible with the Python codec registry. - - -.. class:: StreamWriter(stream, errors='strict') - - Constructor for a :class:`StreamWriter` instance. - - All stream writers must provide this constructor interface. They are free to add - additional keyword arguments, but only the ones defined here are used by the - Python codec registry. - - The *stream* argument must be a file-like object open for writing - text or binary data, as appropriate for the specific codec. - - The :class:`StreamWriter` may implement different error handling schemes by - providing the *errors* keyword argument. See :ref:`error-handlers` for - the standard error handlers the underlying stream codec may support. - - The *errors* argument will be assigned to an attribute of the same name. - Assigning to this attribute makes it possible to switch between different error - handling strategies during the lifetime of the :class:`StreamWriter` object. - - .. method:: write(object) - - Writes the object's contents encoded to the stream. - - - .. method:: writelines(list) - - Writes the concatenated iterable of strings to the stream (possibly by reusing - the :meth:`write` method). Infinite or - very large iterables are not supported. The standard bytes-to-bytes codecs - do not support this method. - - - .. method:: reset() - - Resets the codec buffers used for keeping internal state. - - Calling this method should ensure that the data on the output is put into - a clean state that allows appending of new fresh data without having to - rescan the whole stream to recover state. - - -In addition to the above methods, the :class:`StreamWriter` must also inherit -all other methods and attributes from the underlying stream. - - -.. _stream-reader-objects: - -StreamReader Objects -~~~~~~~~~~~~~~~~~~~~ - -The :class:`StreamReader` class is a subclass of :class:`Codec` and defines the -following methods which every stream reader must define in order to be -compatible with the Python codec registry. - - -.. class:: StreamReader(stream, errors='strict') - - Constructor for a :class:`StreamReader` instance. - - All stream readers must provide this constructor interface. They are free to add - additional keyword arguments, but only the ones defined here are used by the - Python codec registry. - - The *stream* argument must be a file-like object open for reading - text or binary data, as appropriate for the specific codec. - - The :class:`StreamReader` may implement different error handling schemes by - providing the *errors* keyword argument. See :ref:`error-handlers` for - the standard error handlers the underlying stream codec may support. - - The *errors* argument will be assigned to an attribute of the same name. - Assigning to this attribute makes it possible to switch between different error - handling strategies during the lifetime of the :class:`StreamReader` object. - - The set of allowed values for the *errors* argument can be extended with - :func:`register_error`. - - - .. method:: read(size=-1, chars=-1, firstline=False) - - Decodes data from the stream and returns the resulting object. - - The *chars* argument indicates the number of decoded - code points or bytes to return. The :func:`read` method will - never return more data than requested, but it might return less, - if there is not enough available. - - The *size* argument indicates the approximate maximum - number of encoded bytes or code points to read - for decoding. The decoder can modify this setting as - appropriate. The default value -1 indicates to read and decode as much as - possible. This parameter is intended to - prevent having to decode huge files in one step. - - The *firstline* flag indicates that - it would be sufficient to only return the first - line, if there are decoding errors on later lines. - - The method should use a greedy read strategy meaning that it should read - as much data as is allowed within the definition of the encoding and the - given size, e.g. if optional encoding endings or state markers are - available on the stream, these should be read too. - - - .. method:: readline(size=None, keepends=True) - - Read one line from the input stream and return the decoded data. - - *size*, if given, is passed as size argument to the stream's - :meth:`read` method. - - If *keepends* is false line-endings will be stripped from the lines - returned. - - - .. method:: readlines(sizehint=None, keepends=True) - - Read all lines available on the input stream and return them as a list of - lines. - - Line-endings are implemented using the codec's :meth:`decode` method and - are included in the list entries if *keepends* is true. - - *sizehint*, if given, is passed as the *size* argument to the stream's - :meth:`read` method. - - - .. method:: reset() - - Resets the codec buffers used for keeping internal state. - - Note that no stream repositioning should take place. This method is - primarily intended to be able to recover from decoding errors. - - -In addition to the above methods, the :class:`StreamReader` must also inherit -all other methods and attributes from the underlying stream. - -.. _stream-reader-writer: - -StreamReaderWriter Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`StreamReaderWriter` is a convenience class that allows wrapping -streams which work in both read and write modes. - -The design is such that one can use the factory functions returned by the -:func:`lookup` function to construct the instance. - - -.. class:: StreamReaderWriter(stream, Reader, Writer, errors='strict') - - Creates a :class:`StreamReaderWriter` instance. *stream* must be a file-like - object. *Reader* and *Writer* must be factory functions or classes providing the - :class:`StreamReader` and :class:`StreamWriter` interface resp. Error handling - is done in the same way as defined for the stream readers and writers. - -:class:`StreamReaderWriter` instances define the combined interfaces of -:class:`StreamReader` and :class:`StreamWriter` classes. They inherit all other -methods and attributes from the underlying stream. - - -.. _stream-recoder-objects: - -StreamRecoder Objects -~~~~~~~~~~~~~~~~~~~~~ - -The :class:`StreamRecoder` translates data from one encoding to another, -which is sometimes useful when dealing with different encoding environments. - -The design is such that one can use the factory functions returned by the -:func:`lookup` function to construct the instance. - - -.. class:: StreamRecoder(stream, encode, decode, Reader, Writer, errors='strict') - - Creates a :class:`StreamRecoder` instance which implements a two-way conversion: - *encode* and *decode* work on the frontend — the data visible to - code calling :meth:`~StreamReader.read` and :meth:`~StreamWriter.write`, - while *Reader* and *Writer* - work on the backend — the data in *stream*. - - You can use these objects to do transparent transcodings, e.g., from Latin-1 - to UTF-8 and back. - - The *stream* argument must be a file-like object. - - The *encode* and *decode* arguments must - adhere to the :class:`Codec` interface. *Reader* and - *Writer* must be factory functions or classes providing objects of the - :class:`StreamReader` and :class:`StreamWriter` interface respectively. - - Error handling is done in the same way as defined for the stream readers and - writers. - - -:class:`StreamRecoder` instances define the combined interfaces of -:class:`StreamReader` and :class:`StreamWriter` classes. They inherit all other -methods and attributes from the underlying stream. - - -.. _encodings-overview: - -Encodings and Unicode ---------------------- - -Strings are stored internally as sequences of code points in -range ``U+0000``--``U+10FFFF``. (See :pep:`393` for -more details about the implementation.) -Once a string object is used outside of CPU and memory, endianness -and how these arrays are stored as bytes become an issue. As with other -codecs, serialising a string into a sequence of bytes is known as *encoding*, -and recreating the string from the sequence of bytes is known as *decoding*. -There are a variety of different text serialisation codecs, which are -collectivity referred to as :term:`text encodings `. - -The simplest text encoding (called ``'latin-1'`` or ``'iso-8859-1'``) maps -the code points 0--255 to the bytes ``0x0``--``0xff``, which means that a string -object that contains code points above ``U+00FF`` can't be encoded with this -codec. Doing so will raise a :exc:`UnicodeEncodeError` that looks -like the following (although the details of the error message may differ): -``UnicodeEncodeError: 'latin-1' codec can't encode character '\u1234' in -position 3: ordinal not in range(256)``. - -There's another group of encodings (the so called charmap encodings) that choose -a different subset of all Unicode code points and how these code points are -mapped to the bytes ``0x0``--``0xff``. To see how this is done simply open -e.g. :file:`encodings/cp1252.py` (which is an encoding that is used primarily on -Windows). There's a string constant with 256 characters that shows you which -character is mapped to which byte value. - -All of these encodings can only encode 256 of the 1114112 code points -defined in Unicode. A simple and straightforward way that can store each Unicode -code point, is to store each code point as four consecutive bytes. There are two -possibilities: store the bytes in big endian or in little endian order. These -two encodings are called ``UTF-32-BE`` and ``UTF-32-LE`` respectively. Their -disadvantage is that if e.g. you use ``UTF-32-BE`` on a little endian machine you -will always have to swap bytes on encoding and decoding. ``UTF-32`` avoids this -problem: bytes will always be in natural endianness. When these bytes are read -by a CPU with a different endianness, then bytes have to be swapped though. To -be able to detect the endianness of a ``UTF-16`` or ``UTF-32`` byte sequence, -there's the so called BOM ("Byte Order Mark"). This is the Unicode character -``U+FEFF``. This character can be prepended to every ``UTF-16`` or ``UTF-32`` -byte sequence. The byte swapped version of this character (``0xFFFE``) is an -illegal character that may not appear in a Unicode text. So when the -first character in a ``UTF-16`` or ``UTF-32`` byte sequence -appears to be a ``U+FFFE`` the bytes have to be swapped on decoding. -Unfortunately the character ``U+FEFF`` had a second purpose as -a ``ZERO WIDTH NO-BREAK SPACE``: a character that has no width and doesn't allow -a word to be split. It can e.g. be used to give hints to a ligature algorithm. -With Unicode 4.0 using ``U+FEFF`` as a ``ZERO WIDTH NO-BREAK SPACE`` has been -deprecated (with ``U+2060`` (``WORD JOINER``) assuming this role). Nevertheless -Unicode software still must be able to handle ``U+FEFF`` in both roles: as a BOM -it's a device to determine the storage layout of the encoded bytes, and vanishes -once the byte sequence has been decoded into a string; as a ``ZERO WIDTH -NO-BREAK SPACE`` it's a normal character that will be decoded like any other. - -There's another encoding that is able to encode the full range of Unicode -characters: UTF-8. UTF-8 is an 8-bit encoding, which means there are no issues -with byte order in UTF-8. Each byte in a UTF-8 byte sequence consists of two -parts: marker bits (the most significant bits) and payload bits. The marker bits -are a sequence of zero to four ``1`` bits followed by a ``0`` bit. Unicode characters are -encoded like this (with x being payload bits, which when concatenated give the -Unicode character): - -+-----------------------------------+----------------------------------------------+ -| Range | Encoding | -+===================================+==============================================+ -| ``U-00000000`` ... ``U-0000007F`` | 0xxxxxxx | -+-----------------------------------+----------------------------------------------+ -| ``U-00000080`` ... ``U-000007FF`` | 110xxxxx 10xxxxxx | -+-----------------------------------+----------------------------------------------+ -| ``U-00000800`` ... ``U-0000FFFF`` | 1110xxxx 10xxxxxx 10xxxxxx | -+-----------------------------------+----------------------------------------------+ -| ``U-00010000`` ... ``U-0010FFFF`` | 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx | -+-----------------------------------+----------------------------------------------+ - -The least significant bit of the Unicode character is the rightmost x bit. - -As UTF-8 is an 8-bit encoding no BOM is required and any ``U+FEFF`` character in -the decoded string (even if it's the first character) is treated as a ``ZERO -WIDTH NO-BREAK SPACE``. - -Without external information it's impossible to reliably determine which -encoding was used for encoding a string. Each charmap encoding can -decode any random byte sequence. However that's not possible with UTF-8, as -UTF-8 byte sequences have a structure that doesn't allow arbitrary byte -sequences. To increase the reliability with which a UTF-8 encoding can be -detected, Microsoft invented a variant of UTF-8 (that Python calls -``"utf-8-sig"``) for its Notepad program: Before any of the Unicode characters -is written to the file, a UTF-8 encoded BOM (which looks like this as a byte -sequence: ``0xef``, ``0xbb``, ``0xbf``) is written. As it's rather improbable -that any charmap encoded file starts with these byte values (which would e.g. -map to - - | LATIN SMALL LETTER I WITH DIAERESIS - | RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK - | INVERTED QUESTION MARK - -in iso-8859-1), this increases the probability that a ``utf-8-sig`` encoding can be -correctly guessed from the byte sequence. So here the BOM is not used to be able -to determine the byte order used for generating the byte sequence, but as a -signature that helps in guessing the encoding. On encoding the utf-8-sig codec -will write ``0xef``, ``0xbb``, ``0xbf`` as the first three bytes to the file. On -decoding ``utf-8-sig`` will skip those three bytes if they appear as the first -three bytes in the file. In UTF-8, the use of the BOM is discouraged and -should generally be avoided. - - -.. _standard-encodings: - -Standard Encodings ------------------- - -Python comes with a number of codecs built-in, either implemented as C functions -or with dictionaries as mapping tables. The following table lists the codecs by -name, together with a few common aliases, and the languages for which the -encoding is likely used. Neither the list of aliases nor the list of languages -is meant to be exhaustive. Notice that spelling alternatives that only differ in -case or use a hyphen instead of an underscore are also valid aliases; therefore, -e.g. ``'utf-8'`` is a valid alias for the ``'utf_8'`` codec. - -.. impl-detail:: - - Some common encodings can bypass the codecs lookup machinery to - improve performance. These optimization opportunities are only - recognized by CPython for a limited set of (case insensitive) - aliases: utf-8, utf8, latin-1, latin1, iso-8859-1, iso8859-1, mbcs - (Windows only), ascii, us-ascii, utf-16, utf16, utf-32, utf32, and - the same using underscores instead of dashes. Using alternative - aliases for these encodings may result in slower execution. - - .. versionchanged:: 3.6 - Optimization opportunity recognized for us-ascii. - -Many of the character sets support the same languages. They vary in individual -characters (e.g. whether the EURO SIGN is supported or not), and in the -assignment of characters to code positions. For the European languages in -particular, the following variants typically exist: - -* an ISO 8859 codeset - -* a Microsoft Windows code page, which is typically derived from an 8859 codeset, - but replaces control characters with additional graphic characters - -* an IBM EBCDIC code page - -* an IBM PC code page, which is ASCII compatible - -.. tabularcolumns:: |l|p{0.3\linewidth}|p{0.3\linewidth}| - -+-----------------+--------------------------------+--------------------------------+ -| Codec | Aliases | Languages | -+=================+================================+================================+ -| ascii | 646, us-ascii | English | -+-----------------+--------------------------------+--------------------------------+ -| big5 | big5-tw, csbig5 | Traditional Chinese | -+-----------------+--------------------------------+--------------------------------+ -| big5hkscs | big5-hkscs, hkscs | Traditional Chinese | -+-----------------+--------------------------------+--------------------------------+ -| cp037 | IBM037, IBM039 | English | -+-----------------+--------------------------------+--------------------------------+ -| cp273 | 273, IBM273, csIBM273 | German | -| | | | -| | | .. versionadded:: 3.4 | -+-----------------+--------------------------------+--------------------------------+ -| cp424 | EBCDIC-CP-HE, IBM424 | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| cp437 | 437, IBM437 | English | -+-----------------+--------------------------------+--------------------------------+ -| cp500 | EBCDIC-CP-BE, EBCDIC-CP-CH, | Western Europe | -| | IBM500 | | -+-----------------+--------------------------------+--------------------------------+ -| cp720 | | Arabic | -+-----------------+--------------------------------+--------------------------------+ -| cp737 | | Greek | -+-----------------+--------------------------------+--------------------------------+ -| cp775 | IBM775 | Baltic languages | -+-----------------+--------------------------------+--------------------------------+ -| cp850 | 850, IBM850 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp852 | 852, IBM852 | Central and Eastern Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp855 | 855, IBM855 | Bulgarian, Byelorussian, | -| | | Macedonian, Russian, Serbian | -+-----------------+--------------------------------+--------------------------------+ -| cp856 | | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| cp857 | 857, IBM857 | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| cp858 | 858, IBM858 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp860 | 860, IBM860 | Portuguese | -+-----------------+--------------------------------+--------------------------------+ -| cp861 | 861, CP-IS, IBM861 | Icelandic | -+-----------------+--------------------------------+--------------------------------+ -| cp862 | 862, IBM862 | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| cp863 | 863, IBM863 | Canadian | -+-----------------+--------------------------------+--------------------------------+ -| cp864 | IBM864 | Arabic | -+-----------------+--------------------------------+--------------------------------+ -| cp865 | 865, IBM865 | Danish, Norwegian | -+-----------------+--------------------------------+--------------------------------+ -| cp866 | 866, IBM866 | Russian | -+-----------------+--------------------------------+--------------------------------+ -| cp869 | 869, CP-GR, IBM869 | Greek | -+-----------------+--------------------------------+--------------------------------+ -| cp874 | | Thai | -+-----------------+--------------------------------+--------------------------------+ -| cp875 | | Greek | -+-----------------+--------------------------------+--------------------------------+ -| cp932 | 932, ms932, mskanji, ms-kanji | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| cp949 | 949, ms949, uhc | Korean | -+-----------------+--------------------------------+--------------------------------+ -| cp950 | 950, ms950 | Traditional Chinese | -+-----------------+--------------------------------+--------------------------------+ -| cp1006 | | Urdu | -+-----------------+--------------------------------+--------------------------------+ -| cp1026 | ibm1026 | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| cp1125 | 1125, ibm1125, cp866u, ruscii | Ukrainian | -| | | | -| | | .. versionadded:: 3.4 | -+-----------------+--------------------------------+--------------------------------+ -| cp1140 | ibm1140 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp1250 | windows-1250 | Central and Eastern Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp1251 | windows-1251 | Bulgarian, Byelorussian, | -| | | Macedonian, Russian, Serbian | -+-----------------+--------------------------------+--------------------------------+ -| cp1252 | windows-1252 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| cp1253 | windows-1253 | Greek | -+-----------------+--------------------------------+--------------------------------+ -| cp1254 | windows-1254 | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| cp1255 | windows-1255 | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| cp1256 | windows-1256 | Arabic | -+-----------------+--------------------------------+--------------------------------+ -| cp1257 | windows-1257 | Baltic languages | -+-----------------+--------------------------------+--------------------------------+ -| cp1258 | windows-1258 | Vietnamese | -+-----------------+--------------------------------+--------------------------------+ -| euc_jp | eucjp, ujis, u-jis | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| euc_jis_2004 | jisx0213, eucjis2004 | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| euc_jisx0213 | eucjisx0213 | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| euc_kr | euckr, korean, ksc5601, | Korean | -| | ks_c-5601, ks_c-5601-1987, | | -| | ksx1001, ks_x-1001 | | -+-----------------+--------------------------------+--------------------------------+ -| gb2312 | chinese, csiso58gb231280, | Simplified Chinese | -| | euc-cn, euccn, eucgb2312-cn, | | -| | gb2312-1980, gb2312-80, | | -| | iso-ir-58 | | -+-----------------+--------------------------------+--------------------------------+ -| gbk | 936, cp936, ms936 | Unified Chinese | -+-----------------+--------------------------------+--------------------------------+ -| gb18030 | gb18030-2000 | Unified Chinese | -+-----------------+--------------------------------+--------------------------------+ -| hz | hzgb, hz-gb, hz-gb-2312 | Simplified Chinese | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp | csiso2022jp, iso2022jp, | Japanese | -| | iso-2022-jp | | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_1 | iso2022jp-1, iso-2022-jp-1 | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_2 | iso2022jp-2, iso-2022-jp-2 | Japanese, Korean, Simplified | -| | | Chinese, Western Europe, Greek | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_2004 | iso2022jp-2004, | Japanese | -| | iso-2022-jp-2004 | | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_3 | iso2022jp-3, iso-2022-jp-3 | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_jp_ext | iso2022jp-ext, iso-2022-jp-ext | Japanese | -+-----------------+--------------------------------+--------------------------------+ -| iso2022_kr | csiso2022kr, iso2022kr, | Korean | -| | iso-2022-kr | | -+-----------------+--------------------------------+--------------------------------+ -| latin_1 | iso-8859-1, iso8859-1, 8859, | Western Europe | -| | cp819, latin, latin1, L1 | | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_2 | iso-8859-2, latin2, L2 | Central and Eastern Europe | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_3 | iso-8859-3, latin3, L3 | Esperanto, Maltese | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_4 | iso-8859-4, latin4, L4 | Baltic languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_5 | iso-8859-5, cyrillic | Bulgarian, Byelorussian, | -| | | Macedonian, Russian, Serbian | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_6 | iso-8859-6, arabic | Arabic | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_7 | iso-8859-7, greek, greek8 | Greek | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_8 | iso-8859-8, hebrew | Hebrew | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_9 | iso-8859-9, latin5, L5 | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_10 | iso-8859-10, latin6, L6 | Nordic languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_11 | iso-8859-11, thai | Thai languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_13 | iso-8859-13, latin7, L7 | Baltic languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_14 | iso-8859-14, latin8, L8 | Celtic languages | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_15 | iso-8859-15, latin9, L9 | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| iso8859_16 | iso-8859-16, latin10, L10 | South-Eastern Europe | -+-----------------+--------------------------------+--------------------------------+ -| johab | cp1361, ms1361 | Korean | -+-----------------+--------------------------------+--------------------------------+ -| koi8_r | | Russian | -+-----------------+--------------------------------+--------------------------------+ -| koi8_t | | Tajik | -| | | | -| | | .. versionadded:: 3.5 | -+-----------------+--------------------------------+--------------------------------+ -| koi8_u | | Ukrainian | -+-----------------+--------------------------------+--------------------------------+ -| kz1048 | kz_1048, strk1048_2002, rk1048 | Kazakh | -| | | | -| | | .. versionadded:: 3.5 | -+-----------------+--------------------------------+--------------------------------+ -| mac_cyrillic | maccyrillic | Bulgarian, Byelorussian, | -| | | Macedonian, Russian, Serbian | -+-----------------+--------------------------------+--------------------------------+ -| mac_greek | macgreek | Greek | -+-----------------+--------------------------------+--------------------------------+ -| mac_iceland | maciceland | Icelandic | -+-----------------+--------------------------------+--------------------------------+ -| mac_latin2 | maclatin2, maccentraleurope, | Central and Eastern Europe | -| | mac_centeuro | | -+-----------------+--------------------------------+--------------------------------+ -| mac_roman | macroman, macintosh | Western Europe | -+-----------------+--------------------------------+--------------------------------+ -| mac_turkish | macturkish | Turkish | -+-----------------+--------------------------------+--------------------------------+ -| ptcp154 | csptcp154, pt154, cp154, | Kazakh | -| | cyrillic-asian | | -+-----------------+--------------------------------+--------------------------------+ -| shift_jis | csshiftjis, shiftjis, sjis, | Japanese | -| | s_jis | | -+-----------------+--------------------------------+--------------------------------+ -| shift_jis_2004 | shiftjis2004, sjis_2004, | Japanese | -| | sjis2004 | | -+-----------------+--------------------------------+--------------------------------+ -| shift_jisx0213 | shiftjisx0213, sjisx0213, | Japanese | -| | s_jisx0213 | | -+-----------------+--------------------------------+--------------------------------+ -| utf_32 | U32, utf32 | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_32_be | UTF-32BE | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_32_le | UTF-32LE | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_16 | U16, utf16 | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_16_be | UTF-16BE | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_16_le | UTF-16LE | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_7 | U7, unicode-1-1-utf-7 | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_8 | U8, UTF, utf8, cp65001 | all languages | -+-----------------+--------------------------------+--------------------------------+ -| utf_8_sig | | all languages | -+-----------------+--------------------------------+--------------------------------+ - -.. versionchanged:: 3.4 - The utf-16\* and utf-32\* encoders no longer allow surrogate code points - (``U+D800``--``U+DFFF``) to be encoded. - The utf-32\* decoders no longer decode - byte sequences that correspond to surrogate code points. - -.. versionchanged:: 3.8 - ``cp65001`` is now an alias to ``utf_8``. - - -Python Specific Encodings -------------------------- - -A number of predefined codecs are specific to Python, so their codec names have -no meaning outside Python. These are listed in the tables below based on the -expected input and output types (note that while text encodings are the most -common use case for codecs, the underlying codec infrastructure supports -arbitrary data transforms rather than just text encodings). For asymmetric -codecs, the stated meaning describes the encoding direction. - -Text Encodings -^^^^^^^^^^^^^^ - -The following codecs provide :class:`str` to :class:`bytes` encoding and -:term:`bytes-like object` to :class:`str` decoding, similar to the Unicode text -encodings. - -.. tabularcolumns:: |l|p{0.3\linewidth}|p{0.3\linewidth}| - -+--------------------+---------+---------------------------+ -| Codec | Aliases | Meaning | -+====================+=========+===========================+ -| idna | | Implement :rfc:`3490`, | -| | | see also | -| | | :mod:`encodings.idna`. | -| | | Only ``errors='strict'`` | -| | | is supported. | -+--------------------+---------+---------------------------+ -| mbcs | ansi, | Windows only: Encode the | -| | dbcs | operand according to the | -| | | ANSI codepage (CP_ACP). | -+--------------------+---------+---------------------------+ -| oem | | Windows only: Encode the | -| | | operand according to the | -| | | OEM codepage (CP_OEMCP). | -| | | | -| | | .. versionadded:: 3.6 | -+--------------------+---------+---------------------------+ -| palmos | | Encoding of PalmOS 3.5. | -+--------------------+---------+---------------------------+ -| punycode | | Implement :rfc:`3492`. | -| | | Stateful codecs are not | -| | | supported. | -+--------------------+---------+---------------------------+ -| raw_unicode_escape | | Latin-1 encoding with | -| | | :samp:`\\u{XXXX}` and | -| | | :samp:`\\U{XXXXXXXX}` | -| | | for other code points. | -| | | Existing | -| | | backslashes are not | -| | | escaped in any way. | -| | | It is used in the Python | -| | | pickle protocol. | -+--------------------+---------+---------------------------+ -| undefined | | Raise an exception for | -| | | all conversions, even | -| | | empty strings. The error | -| | | handler is ignored. | -+--------------------+---------+---------------------------+ -| unicode_escape | | Encoding suitable as the | -| | | contents of a Unicode | -| | | literal in ASCII-encoded | -| | | Python source code, | -| | | except that quotes are | -| | | not escaped. Decode | -| | | from Latin-1 source code. | -| | | Beware that Python source | -| | | code actually uses UTF-8 | -| | | by default. | -+--------------------+---------+---------------------------+ - -.. versionchanged:: 3.8 - "unicode_internal" codec is removed. - - -.. _binary-transforms: - -Binary Transforms -^^^^^^^^^^^^^^^^^ - -The following codecs provide binary transforms: :term:`bytes-like object` -to :class:`bytes` mappings. They are not supported by :meth:`bytes.decode` -(which only produces :class:`str` output). - - -.. tabularcolumns:: |l|L|L|L| - -+----------------------+------------------+------------------------------+------------------------------+ -| Codec | Aliases | Meaning | Encoder / decoder | -+======================+==================+==============================+==============================+ -| base64_codec [#b64]_ | base64, base_64 | Convert the operand to | :meth:`base64.encodebytes` / | -| | | multiline MIME base64 (the | :meth:`base64.decodebytes` | -| | | result always includes a | | -| | | trailing ``'\n'``). | | -| | | | | -| | | .. versionchanged:: 3.4 | | -| | | accepts any | | -| | | :term:`bytes-like object` | | -| | | as input for encoding and | | -| | | decoding | | -+----------------------+------------------+------------------------------+------------------------------+ -| bz2_codec | bz2 | Compress the operand using | :meth:`bz2.compress` / | -| | | bz2. | :meth:`bz2.decompress` | -+----------------------+------------------+------------------------------+------------------------------+ -| hex_codec | hex | Convert the operand to | :meth:`binascii.b2a_hex` / | -| | | hexadecimal | :meth:`binascii.a2b_hex` | -| | | representation, with two | | -| | | digits per byte. | | -+----------------------+------------------+------------------------------+------------------------------+ -| quopri_codec | quopri, | Convert the operand to MIME | :meth:`quopri.encode` with | -| | quotedprintable, | quoted printable. | ``quotetabs=True`` / | -| | quoted_printable | | :meth:`quopri.decode` | -+----------------------+------------------+------------------------------+------------------------------+ -| uu_codec | uu | Convert the operand using | :meth:`!uu.encode` / | -| | | uuencode. | :meth:`!uu.decode` | -| | | | (Note: :mod:`uu` is | -| | | | deprecated.) | -+----------------------+------------------+------------------------------+------------------------------+ -| zlib_codec | zip, zlib | Compress the operand using | :meth:`zlib.compress` / | -| | | gzip. | :meth:`zlib.decompress` | -+----------------------+------------------+------------------------------+------------------------------+ - -.. [#b64] In addition to :term:`bytes-like objects `, - ``'base64_codec'`` also accepts ASCII-only instances of :class:`str` for - decoding - -.. versionadded:: 3.2 - Restoration of the binary transforms. - -.. versionchanged:: 3.4 - Restoration of the aliases for the binary transforms. - - -.. _text-transforms: - -Text Transforms -^^^^^^^^^^^^^^^ - -The following codec provides a text transform: a :class:`str` to :class:`str` -mapping. It is not supported by :meth:`str.encode` (which only produces -:class:`bytes` output). - -.. tabularcolumns:: |l|l|L| - -+--------------------+---------+---------------------------+ -| Codec | Aliases | Meaning | -+====================+=========+===========================+ -| rot_13 | rot13 | Return the Caesar-cypher | -| | | encryption of the | -| | | operand. | -+--------------------+---------+---------------------------+ - -.. versionadded:: 3.2 - Restoration of the ``rot_13`` text transform. - -.. versionchanged:: 3.4 - Restoration of the ``rot13`` alias. - - -:mod:`encodings.idna` --- Internationalized Domain Names in Applications ------------------------------------------------------------------------- - -.. module:: encodings.idna - :synopsis: Internationalized Domain Names implementation -.. moduleauthor:: Martin v. Löwis - -This module implements :rfc:`3490` (Internationalized Domain Names in -Applications) and :rfc:`3492` (Nameprep: A Stringprep Profile for -Internationalized Domain Names (IDN)). It builds upon the ``punycode`` encoding -and :mod:`stringprep`. - -If you need the IDNA 2008 standard from :rfc:`5891` and :rfc:`5895`, use the -third-party `idna module `_. - -These RFCs together define a protocol to support non-ASCII characters in domain -names. A domain name containing non-ASCII characters (such as -``www.Alliancefrançaise.nu``) is converted into an ASCII-compatible encoding -(ACE, such as ``www.xn--alliancefranaise-npb.nu``). The ACE form of the domain -name is then used in all places where arbitrary characters are not allowed by -the protocol, such as DNS queries, HTTP :mailheader:`Host` fields, and so -on. This conversion is carried out in the application; if possible invisible to -the user: The application should transparently convert Unicode domain labels to -IDNA on the wire, and convert back ACE labels to Unicode before presenting them -to the user. - -Python supports this conversion in several ways: the ``idna`` codec performs -conversion between Unicode and ACE, separating an input string into labels -based on the separator characters defined in :rfc:`section 3.1 of RFC 3490 <3490#section-3.1>` -and converting each label to ACE as required, and conversely separating an input -byte string into labels based on the ``.`` separator and converting any ACE -labels found into unicode. Furthermore, the :mod:`socket` module -transparently converts Unicode host names to ACE, so that applications need not -be concerned about converting host names themselves when they pass them to the -socket module. On top of that, modules that have host names as function -parameters, such as :mod:`http.client` and :mod:`ftplib`, accept Unicode host -names (:mod:`http.client` then also transparently sends an IDNA hostname in the -:mailheader:`Host` field if it sends that field at all). - -When receiving host names from the wire (such as in reverse name lookup), no -automatic conversion to Unicode is performed: applications wishing to present -such host names to the user should decode them to Unicode. - -The module :mod:`encodings.idna` also implements the nameprep procedure, which -performs certain normalizations on host names, to achieve case-insensitivity of -international domain names, and to unify similar characters. The nameprep -functions can be used directly if desired. - - -.. function:: nameprep(label) - - Return the nameprepped version of *label*. The implementation currently assumes - query strings, so ``AllowUnassigned`` is true. - - -.. function:: ToASCII(label) - - Convert a label to ASCII, as specified in :rfc:`3490`. ``UseSTD3ASCIIRules`` is - assumed to be false. - - -.. function:: ToUnicode(label) - - Convert a label to Unicode, as specified in :rfc:`3490`. - - -:mod:`encodings.mbcs` --- Windows ANSI codepage ------------------------------------------------ - -.. module:: encodings.mbcs - :synopsis: Windows ANSI codepage - -This module implements the ANSI codepage (CP_ACP). - -.. availability:: Windows. - -.. versionchanged:: 3.3 - Support any error handler. - -.. versionchanged:: 3.2 - Before 3.2, the *errors* argument was ignored; ``'replace'`` was always used - to encode, and ``'ignore'`` to decode. - - -:mod:`encodings.utf_8_sig` --- UTF-8 codec with BOM signature -------------------------------------------------------------- - -.. module:: encodings.utf_8_sig - :synopsis: UTF-8 codec with BOM signature -.. moduleauthor:: Walter Dörwald - -This module implements a variant of the UTF-8 codec. On encoding, a UTF-8 encoded -BOM will be prepended to the UTF-8 encoded bytes. For the stateful encoder this -is only done once (on the first write to the byte stream). On decoding, an -optional UTF-8 encoded BOM at the start of the data will be skipped. diff --git a/Doc/library/codeop.rst b/Doc/library/codeop.rst deleted file mode 100644 index 55606e1c5f09ac..00000000000000 --- a/Doc/library/codeop.rst +++ /dev/null @@ -1,73 +0,0 @@ -:mod:`codeop` --- Compile Python code -===================================== - -.. module:: codeop - :synopsis: Compile (possibly incomplete) Python code. - -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Michael Hudson - -**Source code:** :source:`Lib/codeop.py` - --------------- - -The :mod:`codeop` module provides utilities upon which the Python -read-eval-print loop can be emulated, as is done in the :mod:`code` module. As -a result, you probably don't want to use the module directly; if you want to -include such a loop in your program you probably want to use the :mod:`code` -module instead. - -There are two parts to this job: - -#. Being able to tell if a line of input completes a Python statement: in - short, telling whether to print '``>>>``' or '``...``' next. - -#. Remembering which future statements the user has entered, so subsequent - input can be compiled with these in effect. - -The :mod:`codeop` module provides a way of doing each of these things, and a way -of doing them both. - -To do just the former: - -.. function:: compile_command(source, filename="", symbol="single") - - Tries to compile *source*, which should be a string of Python code and return a - code object if *source* is valid Python code. In that case, the filename - attribute of the code object will be *filename*, which defaults to - ``''``. Returns ``None`` if *source* is *not* valid Python code, but is a - prefix of valid Python code. - - If there is a problem with *source*, an exception will be raised. - :exc:`SyntaxError` is raised if there is invalid Python syntax, and - :exc:`OverflowError` or :exc:`ValueError` if there is an invalid literal. - - The *symbol* argument determines whether *source* is compiled as a statement - (``'single'``, the default), as a sequence of :term:`statement` (``'exec'``) or - as an :term:`expression` (``'eval'``). Any other value will - cause :exc:`ValueError` to be raised. - - .. note:: - - It is possible (but not likely) that the parser stops parsing with a - successful outcome before reaching the end of the source; in this case, - trailing symbols may be ignored instead of causing an error. For example, - a backslash followed by two newlines may be followed by arbitrary garbage. - This will be fixed once the API for the parser is better. - - -.. class:: Compile() - - Instances of this class have :meth:`~object.__call__` methods identical in signature to - the built-in function :func:`compile`, but with the difference that if the - instance compiles program text containing a :mod:`__future__` statement, the - instance 'remembers' and compiles all subsequent program texts with the - statement in force. - - -.. class:: CommandCompiler() - - Instances of this class have :meth:`~object.__call__` methods identical in signature to - :func:`compile_command`; the difference is that if the instance compiles program - text containing a :mod:`__future__` statement, the instance 'remembers' and - compiles all subsequent program texts with the statement in force. diff --git a/Doc/library/collections.abc.rst b/Doc/library/collections.abc.rst deleted file mode 100644 index e4695b44320600..00000000000000 --- a/Doc/library/collections.abc.rst +++ /dev/null @@ -1,418 +0,0 @@ -:mod:`collections.abc` --- Abstract Base Classes for Containers -=============================================================== - -.. module:: collections.abc - :synopsis: Abstract base classes for containers - -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - -.. versionadded:: 3.3 - Formerly, this module was part of the :mod:`collections` module. - -**Source code:** :source:`Lib/_collections_abc.py` - -.. testsetup:: * - - from collections.abc import * - import itertools - __name__ = '' - --------------- - -This module provides :term:`abstract base classes ` that -can be used to test whether a class provides a particular interface; for -example, whether it is :term:`hashable` or whether it is a mapping. - -An :func:`issubclass` or :func:`isinstance` test for an interface works in one -of three ways. - -1) A newly written class can inherit directly from one of the -abstract base classes. The class must supply the required abstract -methods. The remaining mixin methods come from inheritance and can be -overridden if desired. Other methods may be added as needed: - -.. testcode:: - - class C(Sequence): # Direct inheritance - def __init__(self): ... # Extra method not required by the ABC - def __getitem__(self, index): ... # Required abstract method - def __len__(self): ... # Required abstract method - def count(self, value): ... # Optionally override a mixin method - -.. doctest:: - - >>> issubclass(C, Sequence) - True - >>> isinstance(C(), Sequence) - True - -2) Existing classes and built-in classes can be registered as "virtual -subclasses" of the ABCs. Those classes should define the full API -including all of the abstract methods and all of the mixin methods. -This lets users rely on :func:`issubclass` or :func:`isinstance` tests -to determine whether the full interface is supported. The exception to -this rule is for methods that are automatically inferred from the rest -of the API: - -.. testcode:: - - class D: # No inheritance - def __init__(self): ... # Extra method not required by the ABC - def __getitem__(self, index): ... # Abstract method - def __len__(self): ... # Abstract method - def count(self, value): ... # Mixin method - def index(self, value): ... # Mixin method - - Sequence.register(D) # Register instead of inherit - -.. doctest:: - - >>> issubclass(D, Sequence) - True - >>> isinstance(D(), Sequence) - True - -In this example, class :class:`D` does not need to define -``__contains__``, ``__iter__``, and ``__reversed__`` because the -:ref:`in-operator `, the :term:`iteration ` -logic, and the :func:`reversed` function automatically fall back to -using ``__getitem__`` and ``__len__``. - -3) Some simple interfaces are directly recognizable by the presence of -the required methods (unless those methods have been set to -:const:`None`): - -.. testcode:: - - class E: - def __iter__(self): ... - def __next__(next): ... - -.. doctest:: - - >>> issubclass(E, Iterable) - True - >>> isinstance(E(), Iterable) - True - -Complex interfaces do not support this last technique because an -interface is more than just the presence of method names. Interfaces -specify semantics and relationships between methods that cannot be -inferred solely from the presence of specific method names. For -example, knowing that a class supplies ``__getitem__``, ``__len__``, and -``__iter__`` is insufficient for distinguishing a :class:`Sequence` from -a :class:`Mapping`. - -.. versionadded:: 3.9 - These abstract classes now support ``[]``. See :ref:`types-genericalias` - and :pep:`585`. - -.. _collections-abstract-base-classes: - -Collections Abstract Base Classes ---------------------------------- - -The collections module offers the following :term:`ABCs `: - -.. tabularcolumns:: |l|L|L|L| - -============================== ====================== ======================= ==================================================== -ABC Inherits from Abstract Methods Mixin Methods -============================== ====================== ======================= ==================================================== -:class:`Container` [1]_ ``__contains__`` -:class:`Hashable` [1]_ ``__hash__`` -:class:`Iterable` [1]_ [2]_ ``__iter__`` -:class:`Iterator` [1]_ :class:`Iterable` ``__next__`` ``__iter__`` -:class:`Reversible` [1]_ :class:`Iterable` ``__reversed__`` -:class:`Generator` [1]_ :class:`Iterator` ``send``, ``throw`` ``close``, ``__iter__``, ``__next__`` -:class:`Sized` [1]_ ``__len__`` -:class:`Callable` [1]_ ``__call__`` -:class:`Collection` [1]_ :class:`Sized`, ``__contains__``, - :class:`Iterable`, ``__iter__``, - :class:`Container` ``__len__`` - -:class:`Sequence` :class:`Reversible`, ``__getitem__``, ``__contains__``, ``__iter__``, ``__reversed__``, - :class:`Collection` ``__len__`` ``index``, and ``count`` - -:class:`MutableSequence` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods and - ``__setitem__``, ``append``, ``reverse``, ``extend``, ``pop``, - ``__delitem__``, ``remove``, and ``__iadd__`` - ``__len__``, - ``insert`` - -:class:`ByteString` :class:`Sequence` ``__getitem__``, Inherited :class:`Sequence` methods - ``__len__`` - -:class:`Set` :class:`Collection` ``__contains__``, ``__le__``, ``__lt__``, ``__eq__``, ``__ne__``, - ``__iter__``, ``__gt__``, ``__ge__``, ``__and__``, ``__or__``, - ``__len__`` ``__sub__``, ``__xor__``, and ``isdisjoint`` - -:class:`MutableSet` :class:`Set` ``__contains__``, Inherited :class:`Set` methods and - ``__iter__``, ``clear``, ``pop``, ``remove``, ``__ior__``, - ``__len__``, ``__iand__``, ``__ixor__``, and ``__isub__`` - ``add``, - ``discard`` - -:class:`Mapping` :class:`Collection` ``__getitem__``, ``__contains__``, ``keys``, ``items``, ``values``, - ``__iter__``, ``get``, ``__eq__``, and ``__ne__`` - ``__len__`` - -:class:`MutableMapping` :class:`Mapping` ``__getitem__``, Inherited :class:`Mapping` methods and - ``__setitem__``, ``pop``, ``popitem``, ``clear``, ``update``, - ``__delitem__``, and ``setdefault`` - ``__iter__``, - ``__len__`` - - -:class:`MappingView` :class:`Sized` ``__len__`` -:class:`ItemsView` :class:`MappingView`, ``__contains__``, - :class:`Set` ``__iter__`` -:class:`KeysView` :class:`MappingView`, ``__contains__``, - :class:`Set` ``__iter__`` -:class:`ValuesView` :class:`MappingView`, ``__contains__``, ``__iter__`` - :class:`Collection` -:class:`Awaitable` [1]_ ``__await__`` -:class:`Coroutine` [1]_ :class:`Awaitable` ``send``, ``throw`` ``close`` -:class:`AsyncIterable` [1]_ ``__aiter__`` -:class:`AsyncIterator` [1]_ :class:`AsyncIterable` ``__anext__`` ``__aiter__`` -:class:`AsyncGenerator` [1]_ :class:`AsyncIterator` ``asend``, ``athrow`` ``aclose``, ``__aiter__``, ``__anext__`` -============================== ====================== ======================= ==================================================== - - -.. rubric:: Footnotes - -.. [1] These ABCs override :meth:`object.__subclasshook__` to support - testing an interface by verifying the required methods are present - and have not been set to :const:`None`. This only works for simple - interfaces. More complex interfaces require registration or direct - subclassing. - -.. [2] Checking ``isinstance(obj, Iterable)`` detects classes that are - registered as :class:`Iterable` or that have an :meth:`__iter__` - method, but it does not detect classes that iterate with the - :meth:`~object.__getitem__` method. The only reliable way to determine - whether an object is :term:`iterable` is to call ``iter(obj)``. - - -Collections Abstract Base Classes -- Detailed Descriptions ----------------------------------------------------------- - - -.. class:: Container - - ABC for classes that provide the :meth:`__contains__` method. - -.. class:: Hashable - - ABC for classes that provide the :meth:`__hash__` method. - -.. class:: Sized - - ABC for classes that provide the :meth:`__len__` method. - -.. class:: Callable - - ABC for classes that provide the :meth:`__call__` method. - -.. class:: Iterable - - ABC for classes that provide the :meth:`__iter__` method. - - Checking ``isinstance(obj, Iterable)`` detects classes that are registered - as :class:`Iterable` or that have an :meth:`__iter__` method, but it does - not detect classes that iterate with the :meth:`~object.__getitem__` method. - The only reliable way to determine whether an object is :term:`iterable` - is to call ``iter(obj)``. - -.. class:: Collection - - ABC for sized iterable container classes. - - .. versionadded:: 3.6 - -.. class:: Iterator - - ABC for classes that provide the :meth:`~iterator.__iter__` and - :meth:`~iterator.__next__` methods. See also the definition of - :term:`iterator`. - -.. class:: Reversible - - ABC for iterable classes that also provide the :meth:`__reversed__` - method. - - .. versionadded:: 3.6 - -.. class:: Generator - - ABC for generator classes that implement the protocol defined in - :pep:`342` that extends iterators with the :meth:`~generator.send`, - :meth:`~generator.throw` and :meth:`~generator.close` methods. - See also the definition of :term:`generator`. - - .. versionadded:: 3.5 - -.. class:: Sequence - MutableSequence - ByteString - - ABCs for read-only and mutable :term:`sequences `. - - Implementation note: Some of the mixin methods, such as - :meth:`__iter__`, :meth:`__reversed__` and :meth:`index`, make - repeated calls to the underlying :meth:`~object.__getitem__` method. - Consequently, if :meth:`~object.__getitem__` is implemented with constant - access speed, the mixin methods will have linear performance; - however, if the underlying method is linear (as it would be with a - linked list), the mixins will have quadratic performance and will - likely need to be overridden. - - .. versionchanged:: 3.5 - The index() method added support for *stop* and *start* - arguments. - -.. class:: Set - MutableSet - - ABCs for read-only and mutable sets. - -.. class:: Mapping - MutableMapping - - ABCs for read-only and mutable :term:`mappings `. - -.. class:: MappingView - ItemsView - KeysView - ValuesView - - ABCs for mapping, items, keys, and values :term:`views `. - -.. class:: Awaitable - - ABC for :term:`awaitable` objects, which can be used in :keyword:`await` - expressions. Custom implementations must provide the :meth:`__await__` - method. - - :term:`Coroutine ` objects and instances of the - :class:`~collections.abc.Coroutine` ABC are all instances of this ABC. - - .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. - Using ``isinstance(gencoro, Awaitable)`` for them will return ``False``. - Use :func:`inspect.isawaitable` to detect them. - - .. versionadded:: 3.5 - -.. class:: Coroutine - - ABC for coroutine compatible classes. These implement the - following methods, defined in :ref:`coroutine-objects`: - :meth:`~coroutine.send`, :meth:`~coroutine.throw`, and - :meth:`~coroutine.close`. Custom implementations must also implement - :meth:`__await__`. All :class:`Coroutine` instances are also instances of - :class:`Awaitable`. See also the definition of :term:`coroutine`. - - .. note:: - In CPython, generator-based coroutines (generators decorated with - :func:`types.coroutine`) are - *awaitables*, even though they do not have an :meth:`__await__` method. - Using ``isinstance(gencoro, Coroutine)`` for them will return ``False``. - Use :func:`inspect.isawaitable` to detect them. - - .. versionadded:: 3.5 - -.. class:: AsyncIterable - - ABC for classes that provide ``__aiter__`` method. See also the - definition of :term:`asynchronous iterable`. - - .. versionadded:: 3.5 - -.. class:: AsyncIterator - - ABC for classes that provide ``__aiter__`` and ``__anext__`` - methods. See also the definition of :term:`asynchronous iterator`. - - .. versionadded:: 3.5 - -.. class:: AsyncGenerator - - ABC for asynchronous generator classes that implement the protocol - defined in :pep:`525` and :pep:`492`. - - .. versionadded:: 3.6 - -Examples and Recipes --------------------- - -ABCs allow us to ask classes or instances if they provide -particular functionality, for example:: - - size = None - if isinstance(myvar, collections.abc.Sized): - size = len(myvar) - -Several of the ABCs are also useful as mixins that make it easier to develop -classes supporting container APIs. For example, to write a class supporting -the full :class:`Set` API, it is only necessary to supply the three underlying -abstract methods: :meth:`__contains__`, :meth:`__iter__`, and :meth:`__len__`. -The ABC supplies the remaining methods such as :meth:`__and__` and -:meth:`isdisjoint`:: - - class ListBasedSet(collections.abc.Set): - ''' Alternate set implementation favoring space over speed - and not requiring the set elements to be hashable. ''' - def __init__(self, iterable): - self.elements = lst = [] - for value in iterable: - if value not in lst: - lst.append(value) - - def __iter__(self): - return iter(self.elements) - - def __contains__(self, value): - return value in self.elements - - def __len__(self): - return len(self.elements) - - s1 = ListBasedSet('abcdef') - s2 = ListBasedSet('defghi') - overlap = s1 & s2 # The __and__() method is supported automatically - -Notes on using :class:`Set` and :class:`MutableSet` as a mixin: - -(1) - Since some set operations create new sets, the default mixin methods need - a way to create new instances from an iterable. The class constructor is - assumed to have a signature in the form ``ClassName(iterable)``. - That assumption is factored-out to an internal classmethod called - :meth:`_from_iterable` which calls ``cls(iterable)`` to produce a new set. - If the :class:`Set` mixin is being used in a class with a different - constructor signature, you will need to override :meth:`_from_iterable` - with a classmethod or regular method that can construct new instances from - an iterable argument. - -(2) - To override the comparisons (presumably for speed, as the - semantics are fixed), redefine :meth:`__le__` and :meth:`__ge__`, - then the other operations will automatically follow suit. - -(3) - The :class:`Set` mixin provides a :meth:`_hash` method to compute a hash value - for the set; however, :meth:`__hash__` is not defined because not all sets - are :term:`hashable` or immutable. To add set hashability using mixins, - inherit from both :meth:`Set` and :meth:`Hashable`, then define - ``__hash__ = Set._hash``. - -.. seealso:: - - * `OrderedSet recipe `_ for an - example built on :class:`MutableSet`. - - * For more about ABCs, see the :mod:`abc` module and :pep:`3119`. diff --git a/Doc/library/collections.rst b/Doc/library/collections.rst deleted file mode 100644 index a29cc9530390bc..00000000000000 --- a/Doc/library/collections.rst +++ /dev/null @@ -1,1391 +0,0 @@ -:mod:`collections` --- Container datatypes -========================================== - -.. module:: collections - :synopsis: Container datatypes - -.. moduleauthor:: Raymond Hettinger -.. sectionauthor:: Raymond Hettinger - -**Source code:** :source:`Lib/collections/__init__.py` - -.. testsetup:: * - - from collections import * - import itertools - __name__ = '' - --------------- - -This module implements specialized container datatypes providing alternatives to -Python's general purpose built-in containers, :class:`dict`, :class:`list`, -:class:`set`, and :class:`tuple`. - -===================== ==================================================================== -:func:`namedtuple` factory function for creating tuple subclasses with named fields -:class:`deque` list-like container with fast appends and pops on either end -:class:`ChainMap` dict-like class for creating a single view of multiple mappings -:class:`Counter` dict subclass for counting :term:`hashable` objects -:class:`OrderedDict` dict subclass that remembers the order entries were added -:class:`defaultdict` dict subclass that calls a factory function to supply missing values -:class:`UserDict` wrapper around dictionary objects for easier dict subclassing -:class:`UserList` wrapper around list objects for easier list subclassing -:class:`UserString` wrapper around string objects for easier string subclassing -===================== ==================================================================== - - -:class:`ChainMap` objects -------------------------- - -.. versionadded:: 3.3 - -A :class:`ChainMap` class is provided for quickly linking a number of mappings -so they can be treated as a single unit. It is often much faster than creating -a new dictionary and running multiple :meth:`~dict.update` calls. - -The class can be used to simulate nested scopes and is useful in templating. - -.. class:: ChainMap(*maps) - - A :class:`ChainMap` groups multiple dicts or other mappings together to - create a single, updateable view. If no *maps* are specified, a single empty - dictionary is provided so that a new chain always has at least one mapping. - - The underlying mappings are stored in a list. That list is public and can - be accessed or updated using the *maps* attribute. There is no other state. - - Lookups search the underlying mappings successively until a key is found. In - contrast, writes, updates, and deletions only operate on the first mapping. - - A :class:`ChainMap` incorporates the underlying mappings by reference. So, if - one of the underlying mappings gets updated, those changes will be reflected - in :class:`ChainMap`. - - All of the usual dictionary methods are supported. In addition, there is a - *maps* attribute, a method for creating new subcontexts, and a property for - accessing all but the first mapping: - - .. attribute:: maps - - A user updateable list of mappings. The list is ordered from - first-searched to last-searched. It is the only stored state and can - be modified to change which mappings are searched. The list should - always contain at least one mapping. - - .. method:: new_child(m=None, **kwargs) - - Returns a new :class:`ChainMap` containing a new map followed by - all of the maps in the current instance. If ``m`` is specified, - it becomes the new map at the front of the list of mappings; if not - specified, an empty dict is used, so that a call to ``d.new_child()`` - is equivalent to: ``ChainMap({}, *d.maps)``. If any keyword arguments - are specified, they update passed map or new empty dict. This method - is used for creating subcontexts that can be updated without altering - values in any of the parent mappings. - - .. versionchanged:: 3.4 - The optional ``m`` parameter was added. - - .. versionchanged:: 3.10 - Keyword arguments support was added. - - .. attribute:: parents - - Property returning a new :class:`ChainMap` containing all of the maps in - the current instance except the first one. This is useful for skipping - the first map in the search. Use cases are similar to those for the - :keyword:`nonlocal` keyword used in :term:`nested scopes `. The use cases also parallel those for the built-in - :func:`super` function. A reference to ``d.parents`` is equivalent to: - ``ChainMap(*d.maps[1:])``. - - Note, the iteration order of a :class:`ChainMap()` is determined by - scanning the mappings last to first:: - - >>> baseline = {'music': 'bach', 'art': 'rembrandt'} - >>> adjustments = {'art': 'van gogh', 'opera': 'carmen'} - >>> list(ChainMap(adjustments, baseline)) - ['music', 'art', 'opera'] - - This gives the same ordering as a series of :meth:`dict.update` calls - starting with the last mapping:: - - >>> combined = baseline.copy() - >>> combined.update(adjustments) - >>> list(combined) - ['music', 'art', 'opera'] - - .. versionchanged:: 3.9 - Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. - -.. seealso:: - - * The `MultiContext class - `_ - in the Enthought `CodeTools package - `_ has options to support - writing to any mapping in the chain. - - * Django's `Context class - `_ - for templating is a read-only chain of mappings. It also features - pushing and popping of contexts similar to the - :meth:`~collections.ChainMap.new_child` method and the - :attr:`~collections.ChainMap.parents` property. - - * The `Nested Contexts recipe - `_ has options to control - whether writes and other mutations apply only to the first mapping or to - any mapping in the chain. - - * A `greatly simplified read-only version of Chainmap - `_. - - -:class:`ChainMap` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with chained maps. - - -Example of simulating Python's internal lookup chain:: - - import builtins - pylookup = ChainMap(locals(), globals(), vars(builtins)) - -Example of letting user specified command-line arguments take precedence over -environment variables which in turn take precedence over default values:: - - import os, argparse - - defaults = {'color': 'red', 'user': 'guest'} - - parser = argparse.ArgumentParser() - parser.add_argument('-u', '--user') - parser.add_argument('-c', '--color') - namespace = parser.parse_args() - command_line_args = {k: v for k, v in vars(namespace).items() if v is not None} - - combined = ChainMap(command_line_args, os.environ, defaults) - print(combined['color']) - print(combined['user']) - -Example patterns for using the :class:`ChainMap` class to simulate nested -contexts:: - - c = ChainMap() # Create root context - d = c.new_child() # Create nested child context - e = c.new_child() # Child of c, independent from d - e.maps[0] # Current context dictionary -- like Python's locals() - e.maps[-1] # Root context -- like Python's globals() - e.parents # Enclosing context chain -- like Python's nonlocals - - d['x'] = 1 # Set value in current context - d['x'] # Get first key in the chain of contexts - del d['x'] # Delete from current context - list(d) # All nested values - k in d # Check all nested values - len(d) # Number of nested values - d.items() # All nested items - dict(d) # Flatten into a regular dictionary - -The :class:`ChainMap` class only makes updates (writes and deletions) to the -first mapping in the chain while lookups will search the full chain. However, -if deep writes and deletions are desired, it is easy to make a subclass that -updates keys found deeper in the chain:: - - class DeepChainMap(ChainMap): - 'Variant of ChainMap that allows direct updates to inner scopes' - - def __setitem__(self, key, value): - for mapping in self.maps: - if key in mapping: - mapping[key] = value - return - self.maps[0][key] = value - - def __delitem__(self, key): - for mapping in self.maps: - if key in mapping: - del mapping[key] - return - raise KeyError(key) - - >>> d = DeepChainMap({'zebra': 'black'}, {'elephant': 'blue'}, {'lion': 'yellow'}) - >>> d['lion'] = 'orange' # update an existing key two levels down - >>> d['snake'] = 'red' # new keys get added to the topmost dict - >>> del d['elephant'] # remove an existing key one level down - >>> d # display result - DeepChainMap({'zebra': 'black', 'snake': 'red'}, {}, {'lion': 'orange'}) - - -:class:`Counter` objects ------------------------- - -A counter tool is provided to support convenient and rapid tallies. -For example:: - - >>> # Tally occurrences of words in a list - >>> cnt = Counter() - >>> for word in ['red', 'blue', 'red', 'green', 'blue', 'blue']: - ... cnt[word] += 1 - >>> cnt - Counter({'blue': 3, 'red': 2, 'green': 1}) - - >>> # Find the ten most common words in Hamlet - >>> import re - >>> words = re.findall(r'\w+', open('hamlet.txt').read().lower()) - >>> Counter(words).most_common(10) - [('the', 1143), ('and', 966), ('to', 762), ('of', 669), ('i', 631), - ('you', 554), ('a', 546), ('my', 514), ('hamlet', 471), ('in', 451)] - -.. class:: Counter([iterable-or-mapping]) - - A :class:`Counter` is a :class:`dict` subclass for counting :term:`hashable` objects. - It is a collection where elements are stored as dictionary keys - and their counts are stored as dictionary values. Counts are allowed to be - any integer value including zero or negative counts. The :class:`Counter` - class is similar to bags or multisets in other languages. - - Elements are counted from an *iterable* or initialized from another - *mapping* (or counter): - - >>> c = Counter() # a new, empty counter - >>> c = Counter('gallahad') # a new counter from an iterable - >>> c = Counter({'red': 4, 'blue': 2}) # a new counter from a mapping - >>> c = Counter(cats=4, dogs=8) # a new counter from keyword args - - Counter objects have a dictionary interface except that they return a zero - count for missing items instead of raising a :exc:`KeyError`: - - >>> c = Counter(['eggs', 'ham']) - >>> c['bacon'] # count of a missing element is zero - 0 - - Setting a count to zero does not remove an element from a counter. - Use ``del`` to remove it entirely: - - >>> c['sausage'] = 0 # counter entry with a zero count - >>> del c['sausage'] # del actually removes the entry - - .. versionadded:: 3.1 - - .. versionchanged:: 3.7 As a :class:`dict` subclass, :class:`Counter` - inherited the capability to remember insertion order. Math operations - on *Counter* objects also preserve order. Results are ordered - according to when an element is first encountered in the left operand - and then by the order encountered in the right operand. - - Counter objects support additional methods beyond those available for all - dictionaries: - - .. method:: elements() - - Return an iterator over elements repeating each as many times as its - count. Elements are returned in the order first encountered. If an - element's count is less than one, :meth:`elements` will ignore it. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> sorted(c.elements()) - ['a', 'a', 'a', 'a', 'b', 'b'] - - .. method:: most_common([n]) - - Return a list of the *n* most common elements and their counts from the - most common to the least. If *n* is omitted or ``None``, - :meth:`most_common` returns *all* elements in the counter. - Elements with equal counts are ordered in the order first encountered: - - >>> Counter('abracadabra').most_common(3) - [('a', 5), ('b', 2), ('r', 2)] - - .. method:: subtract([iterable-or-mapping]) - - Elements are subtracted from an *iterable* or from another *mapping* - (or counter). Like :meth:`dict.update` but subtracts counts instead - of replacing them. Both inputs and outputs may be zero or negative. - - >>> c = Counter(a=4, b=2, c=0, d=-2) - >>> d = Counter(a=1, b=2, c=3, d=4) - >>> c.subtract(d) - >>> c - Counter({'a': 3, 'b': 0, 'c': -3, 'd': -6}) - - .. versionadded:: 3.2 - - .. method:: total() - - Compute the sum of the counts. - - >>> c = Counter(a=10, b=5, c=0) - >>> c.total() - 15 - - .. versionadded:: 3.10 - - The usual dictionary methods are available for :class:`Counter` objects - except for two which work differently for counters. - - .. method:: fromkeys(iterable) - - This class method is not implemented for :class:`Counter` objects. - - .. method:: update([iterable-or-mapping]) - - Elements are counted from an *iterable* or added-in from another - *mapping* (or counter). Like :meth:`dict.update` but adds counts - instead of replacing them. Also, the *iterable* is expected to be a - sequence of elements, not a sequence of ``(key, value)`` pairs. - -Counters support rich comparison operators for equality, subset, and -superset relationships: ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. -All of those tests treat missing elements as having zero counts so that -``Counter(a=1) == Counter(a=1, b=0)`` returns true. - -.. versionadded:: 3.10 - Rich comparison operations were added. - -.. versionchanged:: 3.10 - In equality tests, missing elements are treated as having zero counts. - Formerly, ``Counter(a=3)`` and ``Counter(a=3, b=0)`` were considered - distinct. - -Common patterns for working with :class:`Counter` objects:: - - c.total() # total of all counts - c.clear() # reset all counts - list(c) # list unique elements - set(c) # convert to a set - dict(c) # convert to a regular dictionary - c.items() # convert to a list of (elem, cnt) pairs - Counter(dict(list_of_pairs)) # convert from a list of (elem, cnt) pairs - c.most_common()[:-n-1:-1] # n least common elements - +c # remove zero and negative counts - -Several mathematical operations are provided for combining :class:`Counter` -objects to produce multisets (counters that have counts greater than zero). -Addition and subtraction combine counters by adding or subtracting the counts -of corresponding elements. Intersection and union return the minimum and -maximum of corresponding counts. Equality and inclusion compare -corresponding counts. Each operation can accept inputs with signed -counts, but the output will exclude results with counts of zero or less. - -.. doctest:: - - >>> c = Counter(a=3, b=1) - >>> d = Counter(a=1, b=2) - >>> c + d # add two counters together: c[x] + d[x] - Counter({'a': 4, 'b': 3}) - >>> c - d # subtract (keeping only positive counts) - Counter({'a': 2}) - >>> c & d # intersection: min(c[x], d[x]) - Counter({'a': 1, 'b': 1}) - >>> c | d # union: max(c[x], d[x]) - Counter({'a': 3, 'b': 2}) - >>> c == d # equality: c[x] == d[x] - False - >>> c <= d # inclusion: c[x] <= d[x] - False - -Unary addition and subtraction are shortcuts for adding an empty counter -or subtracting from an empty counter. - - >>> c = Counter(a=2, b=-4) - >>> +c - Counter({'a': 2}) - >>> -c - Counter({'b': 4}) - -.. versionadded:: 3.3 - Added support for unary plus, unary minus, and in-place multiset operations. - -.. note:: - - Counters were primarily designed to work with positive integers to represent - running counts; however, care was taken to not unnecessarily preclude use - cases needing other types or negative values. To help with those use cases, - this section documents the minimum range and type restrictions. - - * The :class:`Counter` class itself is a dictionary subclass with no - restrictions on its keys and values. The values are intended to be numbers - representing counts, but you *could* store anything in the value field. - - * The :meth:`~Counter.most_common` method requires only that the values be orderable. - - * For in-place operations such as ``c[key] += 1``, the value type need only - support addition and subtraction. So fractions, floats, and decimals would - work and negative values are supported. The same is also true for - :meth:`~Counter.update` and :meth:`~Counter.subtract` which allow negative and zero values - for both inputs and outputs. - - * The multiset methods are designed only for use cases with positive values. - The inputs may be negative or zero, but only outputs with positive values - are created. There are no type restrictions, but the value type needs to - support addition, subtraction, and comparison. - - * The :meth:`~Counter.elements` method requires integer counts. It ignores zero and - negative counts. - -.. seealso:: - - * `Bag class `_ - in Smalltalk. - - * Wikipedia entry for `Multisets `_. - - * `C++ multisets `_ - tutorial with examples. - - * For mathematical operations on multisets and their use cases, see - *Knuth, Donald. The Art of Computer Programming Volume II, - Section 4.6.3, Exercise 19*. - - * To enumerate all distinct multisets of a given size over a given set of - elements, see :func:`itertools.combinations_with_replacement`:: - - map(Counter, combinations_with_replacement('ABC', 2)) # --> AA AB AC BB BC CC - - -:class:`deque` objects ----------------------- - -.. class:: deque([iterable, [maxlen]]) - - Returns a new deque object initialized left-to-right (using :meth:`append`) with - data from *iterable*. If *iterable* is not specified, the new deque is empty. - - Deques are a generalization of stacks and queues (the name is pronounced "deck" - and is short for "double-ended queue"). Deques support thread-safe, memory - efficient appends and pops from either side of the deque with approximately the - same O(1) performance in either direction. - - Though :class:`list` objects support similar operations, they are optimized for - fast fixed-length operations and incur O(n) memory movement costs for - ``pop(0)`` and ``insert(0, v)`` operations which change both the size and - position of the underlying data representation. - - - If *maxlen* is not specified or is ``None``, deques may grow to an - arbitrary length. Otherwise, the deque is bounded to the specified maximum - length. Once a bounded length deque is full, when new items are added, a - corresponding number of items are discarded from the opposite end. Bounded - length deques provide functionality similar to the ``tail`` filter in - Unix. They are also useful for tracking transactions and other pools of data - where only the most recent activity is of interest. - - - Deque objects support the following methods: - - .. method:: append(x) - - Add *x* to the right side of the deque. - - - .. method:: appendleft(x) - - Add *x* to the left side of the deque. - - - .. method:: clear() - - Remove all elements from the deque leaving it with length 0. - - - .. method:: copy() - - Create a shallow copy of the deque. - - .. versionadded:: 3.5 - - - .. method:: count(x) - - Count the number of deque elements equal to *x*. - - .. versionadded:: 3.2 - - - .. method:: extend(iterable) - - Extend the right side of the deque by appending elements from the iterable - argument. - - - .. method:: extendleft(iterable) - - Extend the left side of the deque by appending elements from *iterable*. - Note, the series of left appends results in reversing the order of - elements in the iterable argument. - - - .. method:: index(x[, start[, stop]]) - - Return the position of *x* in the deque (at or after index *start* - and before index *stop*). Returns the first match or raises - :exc:`ValueError` if not found. - - .. versionadded:: 3.5 - - - .. method:: insert(i, x) - - Insert *x* into the deque at position *i*. - - If the insertion would cause a bounded deque to grow beyond *maxlen*, - an :exc:`IndexError` is raised. - - .. versionadded:: 3.5 - - - .. method:: pop() - - Remove and return an element from the right side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: popleft() - - Remove and return an element from the left side of the deque. If no - elements are present, raises an :exc:`IndexError`. - - - .. method:: remove(value) - - Remove the first occurrence of *value*. If not found, raises a - :exc:`ValueError`. - - - .. method:: reverse() - - Reverse the elements of the deque in-place and then return ``None``. - - .. versionadded:: 3.2 - - - .. method:: rotate(n=1) - - Rotate the deque *n* steps to the right. If *n* is negative, rotate - to the left. - - When the deque is not empty, rotating one step to the right is equivalent - to ``d.appendleft(d.pop())``, and rotating one step to the left is - equivalent to ``d.append(d.popleft())``. - - - Deque objects also provide one read-only attribute: - - .. attribute:: maxlen - - Maximum size of a deque or ``None`` if unbounded. - - .. versionadded:: 3.1 - - -In addition to the above, deques support iteration, pickling, ``len(d)``, -``reversed(d)``, ``copy.copy(d)``, ``copy.deepcopy(d)``, membership testing with -the :keyword:`in` operator, and subscript references such as ``d[0]`` to access -the first element. Indexed access is O(1) at both ends but slows to O(n) in -the middle. For fast random access, use lists instead. - -Starting in version 3.5, deques support ``__add__()``, ``__mul__()``, -and ``__imul__()``. - -Example: - -.. doctest:: - - >>> from collections import deque - >>> d = deque('ghi') # make a new deque with three items - >>> for elem in d: # iterate over the deque's elements - ... print(elem.upper()) - G - H - I - - >>> d.append('j') # add a new entry to the right side - >>> d.appendleft('f') # add a new entry to the left side - >>> d # show the representation of the deque - deque(['f', 'g', 'h', 'i', 'j']) - - >>> d.pop() # return and remove the rightmost item - 'j' - >>> d.popleft() # return and remove the leftmost item - 'f' - >>> list(d) # list the contents of the deque - ['g', 'h', 'i'] - >>> d[0] # peek at leftmost item - 'g' - >>> d[-1] # peek at rightmost item - 'i' - - >>> list(reversed(d)) # list the contents of a deque in reverse - ['i', 'h', 'g'] - >>> 'h' in d # search the deque - True - >>> d.extend('jkl') # add multiple elements at once - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - >>> d.rotate(1) # right rotation - >>> d - deque(['l', 'g', 'h', 'i', 'j', 'k']) - >>> d.rotate(-1) # left rotation - >>> d - deque(['g', 'h', 'i', 'j', 'k', 'l']) - - >>> deque(reversed(d)) # make a new deque in reverse order - deque(['l', 'k', 'j', 'i', 'h', 'g']) - >>> d.clear() # empty the deque - >>> d.pop() # cannot pop from an empty deque - Traceback (most recent call last): - File "", line 1, in -toplevel- - d.pop() - IndexError: pop from an empty deque - - >>> d.extendleft('abc') # extendleft() reverses the input order - >>> d - deque(['c', 'b', 'a']) - - -:class:`deque` Recipes -^^^^^^^^^^^^^^^^^^^^^^ - -This section shows various approaches to working with deques. - -Bounded length deques provide functionality similar to the ``tail`` filter -in Unix:: - - def tail(filename, n=10): - 'Return the last n lines of a file' - with open(filename) as f: - return deque(f, n) - -Another approach to using deques is to maintain a sequence of recently -added elements by appending to the right and popping to the left:: - - def moving_average(iterable, n=3): - # moving_average([40, 30, 50, 46, 39, 44]) --> 40.0 42.0 45.0 43.0 - # https://en.wikipedia.org/wiki/Moving_average - it = iter(iterable) - d = deque(itertools.islice(it, n-1)) - d.appendleft(0) - s = sum(d) - for elem in it: - s += elem - d.popleft() - d.append(elem) - yield s / n - -A `round-robin scheduler -`_ can be implemented with -input iterators stored in a :class:`deque`. Values are yielded from the active -iterator in position zero. If that iterator is exhausted, it can be removed -with :meth:`~deque.popleft`; otherwise, it can be cycled back to the end with -the :meth:`~deque.rotate` method:: - - def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" - iterators = deque(map(iter, iterables)) - while iterators: - try: - while True: - yield next(iterators[0]) - iterators.rotate(-1) - except StopIteration: - # Remove an exhausted iterator. - iterators.popleft() - -The :meth:`~deque.rotate` method provides a way to implement :class:`deque` slicing and -deletion. For example, a pure Python implementation of ``del d[n]`` relies on -the ``rotate()`` method to position elements to be popped:: - - def delete_nth(d, n): - d.rotate(-n) - d.popleft() - d.rotate(n) - -To implement :class:`deque` slicing, use a similar approach applying -:meth:`~deque.rotate` to bring a target element to the left side of the deque. Remove -old entries with :meth:`~deque.popleft`, add new entries with :meth:`~deque.extend`, and then -reverse the rotation. -With minor variations on that approach, it is easy to implement Forth style -stack manipulations such as ``dup``, ``drop``, ``swap``, ``over``, ``pick``, -``rot``, and ``roll``. - - -:class:`defaultdict` objects ----------------------------- - -.. class:: defaultdict(default_factory=None, /, [...]) - - Return a new dictionary-like object. :class:`defaultdict` is a subclass of the - built-in :class:`dict` class. It overrides one method and adds one writable - instance variable. The remaining functionality is the same as for the - :class:`dict` class and is not documented here. - - The first argument provides the initial value for the :attr:`default_factory` - attribute; it defaults to ``None``. All remaining arguments are treated the same - as if they were passed to the :class:`dict` constructor, including keyword - arguments. - - - :class:`defaultdict` objects support the following method in addition to the - standard :class:`dict` operations: - - .. method:: __missing__(key) - - If the :attr:`default_factory` attribute is ``None``, this raises a - :exc:`KeyError` exception with the *key* as argument. - - If :attr:`default_factory` is not ``None``, it is called without arguments - to provide a default value for the given *key*, this value is inserted in - the dictionary for the *key*, and returned. - - If calling :attr:`default_factory` raises an exception this exception is - propagated unchanged. - - This method is called by the :meth:`~object.__getitem__` method of the - :class:`dict` class when the requested key is not found; whatever it - returns or raises is then returned or raised by :meth:`~object.__getitem__`. - - Note that :meth:`__missing__` is *not* called for any operations besides - :meth:`~object.__getitem__`. This means that :meth:`get` will, like normal - dictionaries, return ``None`` as a default rather than using - :attr:`default_factory`. - - - :class:`defaultdict` objects support the following instance variable: - - - .. attribute:: default_factory - - This attribute is used by the :meth:`__missing__` method; it is - initialized from the first argument to the constructor, if present, or to - ``None``, if absent. - - .. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in - :pep:`584`. - - -:class:`defaultdict` Examples -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Using :class:`list` as the :attr:`~defaultdict.default_factory`, it is easy to group a -sequence of key-value pairs into a dictionary of lists: - - >>> s = [('yellow', 1), ('blue', 2), ('yellow', 3), ('blue', 4), ('red', 1)] - >>> d = defaultdict(list) - >>> for k, v in s: - ... d[k].append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -When each key is encountered for the first time, it is not already in the -mapping; so an entry is automatically created using the :attr:`~defaultdict.default_factory` -function which returns an empty :class:`list`. The :meth:`list.append` -operation then attaches the value to the new list. When keys are encountered -again, the look-up proceeds normally (returning the list for that key) and the -:meth:`list.append` operation adds another value to the list. This technique is -simpler and faster than an equivalent technique using :meth:`dict.setdefault`: - - >>> d = {} - >>> for k, v in s: - ... d.setdefault(k, []).append(v) - ... - >>> sorted(d.items()) - [('blue', [2, 4]), ('red', [1]), ('yellow', [1, 3])] - -Setting the :attr:`~defaultdict.default_factory` to :class:`int` makes the -:class:`defaultdict` useful for counting (like a bag or multiset in other -languages): - - >>> s = 'mississippi' - >>> d = defaultdict(int) - >>> for k in s: - ... d[k] += 1 - ... - >>> sorted(d.items()) - [('i', 4), ('m', 1), ('p', 2), ('s', 4)] - -When a letter is first encountered, it is missing from the mapping, so the -:attr:`~defaultdict.default_factory` function calls :func:`int` to supply a default count of -zero. The increment operation then builds up the count for each letter. - -The function :func:`int` which always returns zero is just a special case of -constant functions. A faster and more flexible way to create constant functions -is to use a lambda function which can supply any constant value (not just -zero): - - >>> def constant_factory(value): - ... return lambda: value - >>> d = defaultdict(constant_factory('')) - >>> d.update(name='John', action='ran') - >>> '%(name)s %(action)s to %(object)s' % d - 'John ran to ' - -Setting the :attr:`~defaultdict.default_factory` to :class:`set` makes the -:class:`defaultdict` useful for building a dictionary of sets: - - >>> s = [('red', 1), ('blue', 2), ('red', 3), ('blue', 4), ('red', 1), ('blue', 4)] - >>> d = defaultdict(set) - >>> for k, v in s: - ... d[k].add(v) - ... - >>> sorted(d.items()) - [('blue', {2, 4}), ('red', {1, 3})] - - -:func:`namedtuple` Factory Function for Tuples with Named Fields ----------------------------------------------------------------- - -Named tuples assign meaning to each position in a tuple and allow for more readable, -self-documenting code. They can be used wherever regular tuples are used, and -they add the ability to access fields by name instead of position index. - -.. function:: namedtuple(typename, field_names, *, rename=False, defaults=None, module=None) - - Returns a new tuple subclass named *typename*. The new subclass is used to - create tuple-like objects that have fields accessible by attribute lookup as - well as being indexable and iterable. Instances of the subclass also have a - helpful docstring (with typename and field_names) and a helpful :meth:`__repr__` - method which lists the tuple contents in a ``name=value`` format. - - The *field_names* are a sequence of strings such as ``['x', 'y']``. - Alternatively, *field_names* can be a single string with each fieldname - separated by whitespace and/or commas, for example ``'x y'`` or ``'x, y'``. - - Any valid Python identifier may be used for a fieldname except for names - starting with an underscore. Valid identifiers consist of letters, digits, - and underscores but do not start with a digit or underscore and cannot be - a :mod:`keyword` such as *class*, *for*, *return*, *global*, *pass*, - or *raise*. - - If *rename* is true, invalid fieldnames are automatically replaced - with positional names. For example, ``['abc', 'def', 'ghi', 'abc']`` is - converted to ``['abc', '_1', 'ghi', '_3']``, eliminating the keyword - ``def`` and the duplicate fieldname ``abc``. - - *defaults* can be ``None`` or an :term:`iterable` of default values. - Since fields with a default value must come after any fields without a - default, the *defaults* are applied to the rightmost parameters. For - example, if the fieldnames are ``['x', 'y', 'z']`` and the defaults are - ``(1, 2)``, then ``x`` will be a required argument, ``y`` will default to - ``1``, and ``z`` will default to ``2``. - - If *module* is defined, the ``__module__`` attribute of the named tuple is - set to that value. - - Named tuple instances do not have per-instance dictionaries, so they are - lightweight and require no more memory than regular tuples. - - To support pickling, the named tuple class should be assigned to a variable - that matches *typename*. - - .. versionchanged:: 3.1 - Added support for *rename*. - - .. versionchanged:: 3.6 - The *verbose* and *rename* parameters became - :ref:`keyword-only arguments `. - - .. versionchanged:: 3.6 - Added the *module* parameter. - - .. versionchanged:: 3.7 - Removed the *verbose* parameter and the :attr:`_source` attribute. - - .. versionchanged:: 3.7 - Added the *defaults* parameter and the :attr:`_field_defaults` - attribute. - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> # Basic example - >>> Point = namedtuple('Point', ['x', 'y']) - >>> p = Point(11, y=22) # instantiate with positional or keyword arguments - >>> p[0] + p[1] # indexable like the plain tuple (11, 22) - 33 - >>> x, y = p # unpack like a regular tuple - >>> x, y - (11, 22) - >>> p.x + p.y # fields also accessible by name - 33 - >>> p # readable __repr__ with a name=value style - Point(x=11, y=22) - -Named tuples are especially useful for assigning field names to result tuples returned -by the :mod:`csv` or :mod:`sqlite3` modules:: - - EmployeeRecord = namedtuple('EmployeeRecord', 'name, age, title, department, paygrade') - - import csv - for emp in map(EmployeeRecord._make, csv.reader(open("employees.csv", "rb"))): - print(emp.name, emp.title) - - import sqlite3 - conn = sqlite3.connect('/companydata') - cursor = conn.cursor() - cursor.execute('SELECT name, age, title, department, paygrade FROM employees') - for emp in map(EmployeeRecord._make, cursor.fetchall()): - print(emp.name, emp.title) - -In addition to the methods inherited from tuples, named tuples support -three additional methods and two attributes. To prevent conflicts with -field names, the method and attribute names start with an underscore. - -.. classmethod:: somenamedtuple._make(iterable) - - Class method that makes a new instance from an existing sequence or iterable. - - .. doctest:: - - >>> t = [11, 22] - >>> Point._make(t) - Point(x=11, y=22) - -.. method:: somenamedtuple._asdict() - - Return a new :class:`dict` which maps field names to their corresponding - values: - - .. doctest:: - - >>> p = Point(x=11, y=22) - >>> p._asdict() - {'x': 11, 'y': 22} - - .. versionchanged:: 3.1 - Returns an :class:`OrderedDict` instead of a regular :class:`dict`. - - .. versionchanged:: 3.8 - Returns a regular :class:`dict` instead of an :class:`OrderedDict`. - As of Python 3.7, regular dicts are guaranteed to be ordered. If the - extra features of :class:`OrderedDict` are required, the suggested - remediation is to cast the result to the desired type: - ``OrderedDict(nt._asdict())``. - -.. method:: somenamedtuple._replace(**kwargs) - - Return a new instance of the named tuple replacing specified fields with new - values:: - - >>> p = Point(x=11, y=22) - >>> p._replace(x=33) - Point(x=33, y=22) - - >>> for partnum, record in inventory.items(): - ... inventory[partnum] = record._replace(price=newprices[partnum], timestamp=time.now()) - -.. attribute:: somenamedtuple._fields - - Tuple of strings listing the field names. Useful for introspection - and for creating new named tuple types from existing named tuples. - - .. doctest:: - - >>> p._fields # view the field names - ('x', 'y') - - >>> Color = namedtuple('Color', 'red green blue') - >>> Pixel = namedtuple('Pixel', Point._fields + Color._fields) - >>> Pixel(11, 22, 128, 255, 0) - Pixel(x=11, y=22, red=128, green=255, blue=0) - -.. attribute:: somenamedtuple._field_defaults - - Dictionary mapping field names to default values. - - .. doctest:: - - >>> Account = namedtuple('Account', ['type', 'balance'], defaults=[0]) - >>> Account._field_defaults - {'balance': 0} - >>> Account('premium') - Account(type='premium', balance=0) - -To retrieve a field whose name is stored in a string, use the :func:`getattr` -function: - - >>> getattr(p, 'x') - 11 - -To convert a dictionary to a named tuple, use the double-star-operator -(as described in :ref:`tut-unpacking-arguments`): - - >>> d = {'x': 11, 'y': 22} - >>> Point(**d) - Point(x=11, y=22) - -Since a named tuple is a regular Python class, it is easy to add or change -functionality with a subclass. Here is how to add a calculated field and -a fixed-width print format: - -.. doctest:: - - >>> class Point(namedtuple('Point', ['x', 'y'])): - ... __slots__ = () - ... @property - ... def hypot(self): - ... return (self.x ** 2 + self.y ** 2) ** 0.5 - ... def __str__(self): - ... return 'Point: x=%6.3f y=%6.3f hypot=%6.3f' % (self.x, self.y, self.hypot) - - >>> for p in Point(3, 4), Point(14, 5/7): - ... print(p) - Point: x= 3.000 y= 4.000 hypot= 5.000 - Point: x=14.000 y= 0.714 hypot=14.018 - -The subclass shown above sets ``__slots__`` to an empty tuple. This helps -keep memory requirements low by preventing the creation of instance dictionaries. - -Subclassing is not useful for adding new, stored fields. Instead, simply -create a new named tuple type from the :attr:`~somenamedtuple._fields` attribute: - - >>> Point3D = namedtuple('Point3D', Point._fields + ('z',)) - -Docstrings can be customized by making direct assignments to the ``__doc__`` -fields: - - >>> Book = namedtuple('Book', ['id', 'title', 'authors']) - >>> Book.__doc__ += ': Hardcover book in active collection' - >>> Book.id.__doc__ = '13-digit ISBN' - >>> Book.title.__doc__ = 'Title of first printing' - >>> Book.authors.__doc__ = 'List of authors sorted by last name' - -.. versionchanged:: 3.5 - Property docstrings became writeable. - -.. seealso:: - - * See :class:`typing.NamedTuple` for a way to add type hints for named - tuples. It also provides an elegant notation using the :keyword:`class` - keyword:: - - class Component(NamedTuple): - part_number: int - weight: float - description: Optional[str] = None - - * See :meth:`types.SimpleNamespace` for a mutable namespace based on an - underlying dictionary instead of a tuple. - - * The :mod:`dataclasses` module provides a decorator and functions for - automatically adding generated special methods to user-defined classes. - - -:class:`OrderedDict` objects ----------------------------- - -Ordered dictionaries are just like regular dictionaries but have some extra -capabilities relating to ordering operations. They have become less -important now that the built-in :class:`dict` class gained the ability -to remember insertion order (this new behavior became guaranteed in -Python 3.7). - -Some differences from :class:`dict` still remain: - -* The regular :class:`dict` was designed to be very good at mapping - operations. Tracking insertion order was secondary. - -* The :class:`OrderedDict` was designed to be good at reordering operations. - Space efficiency, iteration speed, and the performance of update - operations were secondary. - -* The :class:`OrderedDict` algorithm can handle frequent reordering operations - better than :class:`dict`. As shown in the recipes below, this makes it - suitable for implementing various kinds of LRU caches. - -* The equality operation for :class:`OrderedDict` checks for matching order. - - A regular :class:`dict` can emulate the order sensitive equality test with - ``p == q and all(k1 == k2 for k1, k2 in zip(p, q))``. - -* The :meth:`popitem` method of :class:`OrderedDict` has a different - signature. It accepts an optional argument to specify which item is popped. - - A regular :class:`dict` can emulate OrderedDict's ``od.popitem(last=True)`` - with ``d.popitem()`` which is guaranteed to pop the rightmost (last) item. - - A regular :class:`dict` can emulate OrderedDict's ``od.popitem(last=False)`` - with ``(k := next(iter(d)), d.pop(k))`` which will return and remove the - leftmost (first) item if it exists. - -* :class:`OrderedDict` has a :meth:`move_to_end` method to efficiently - reposition an element to an endpoint. - - A regular :class:`dict` can emulate OrderedDict's ``od.move_to_end(k, - last=True)`` with ``d[k] = d.pop(k)`` which will move the key and its - associated value to the rightmost (last) position. - - A regular :class:`dict` does not have an efficient equivalent for - OrderedDict's ``od.move_to_end(k, last=False)`` which moves the key - and its associated value to the leftmost (first) position. - -* Until Python 3.8, :class:`dict` lacked a :meth:`__reversed__` method. - - -.. class:: OrderedDict([items]) - - Return an instance of a :class:`dict` subclass that has methods - specialized for rearranging dictionary order. - - .. versionadded:: 3.1 - - .. method:: popitem(last=True) - - The :meth:`popitem` method for ordered dictionaries returns and removes a - (key, value) pair. The pairs are returned in - :abbr:`LIFO (last-in, first-out)` order if *last* is true - or :abbr:`FIFO (first-in, first-out)` order if false. - - .. method:: move_to_end(key, last=True) - - Move an existing *key* to either end of an ordered dictionary. The item - is moved to the right end if *last* is true (the default) or to the - beginning if *last* is false. Raises :exc:`KeyError` if the *key* does - not exist: - - .. doctest:: - - >>> d = OrderedDict.fromkeys('abcde') - >>> d.move_to_end('b') - >>> ''.join(d) - 'acdeb' - >>> d.move_to_end('b', last=False) - >>> ''.join(d) - 'bacde' - - .. versionadded:: 3.2 - -In addition to the usual mapping methods, ordered dictionaries also support -reverse iteration using :func:`reversed`. - -Equality tests between :class:`OrderedDict` objects are order-sensitive -and are implemented as ``list(od1.items())==list(od2.items())``. -Equality tests between :class:`OrderedDict` objects and other -:class:`~collections.abc.Mapping` objects are order-insensitive like regular -dictionaries. This allows :class:`OrderedDict` objects to be substituted -anywhere a regular dictionary is used. - -.. versionchanged:: 3.5 - The items, keys, and values :term:`views ` - of :class:`OrderedDict` now support reverse iteration using :func:`reversed`. - -.. versionchanged:: 3.6 - With the acceptance of :pep:`468`, order is retained for keyword arguments - passed to the :class:`OrderedDict` constructor and its :meth:`update` - method. - -.. versionchanged:: 3.9 - Added merge (``|``) and update (``|=``) operators, specified in :pep:`584`. - - -:class:`OrderedDict` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is straightforward to create an ordered dictionary variant -that remembers the order the keys were *last* inserted. -If a new entry overwrites an existing entry, the -original insertion position is changed and moved to the end:: - - class LastUpdatedOrderedDict(OrderedDict): - 'Store items in the order the keys were last added' - - def __setitem__(self, key, value): - super().__setitem__(key, value) - self.move_to_end(key) - -An :class:`OrderedDict` would also be useful for implementing -variants of :func:`functools.lru_cache`: - -.. testcode:: - - from time import time - - class TimeBoundedLRU: - "LRU Cache that invalidates and refreshes old entries." - - def __init__(self, func, maxsize=128, maxage=30): - self.cache = OrderedDict() # { args : (timestamp, result)} - self.func = func - self.maxsize = maxsize - self.maxage = maxage - - def __call__(self, *args): - if args in self.cache: - self.cache.move_to_end(args) - timestamp, result = self.cache[args] - if time() - timestamp <= self.maxage: - return result - result = self.func(*args) - self.cache[args] = time(), result - if len(self.cache) > self.maxsize: - self.cache.popitem(0) - return result - - -.. testcode:: - - class MultiHitLRUCache: - """ LRU cache that defers caching a result until - it has been requested multiple times. - - To avoid flushing the LRU cache with one-time requests, - we don't cache until a request has been made more than once. - - """ - - def __init__(self, func, maxsize=128, maxrequests=4096, cache_after=1): - self.requests = OrderedDict() # { uncached_key : request_count } - self.cache = OrderedDict() # { cached_key : function_result } - self.func = func - self.maxrequests = maxrequests # max number of uncached requests - self.maxsize = maxsize # max number of stored return values - self.cache_after = cache_after - - def __call__(self, *args): - if args in self.cache: - self.cache.move_to_end(args) - return self.cache[args] - result = self.func(*args) - self.requests[args] = self.requests.get(args, 0) + 1 - if self.requests[args] <= self.cache_after: - self.requests.move_to_end(args) - if len(self.requests) > self.maxrequests: - self.requests.popitem(0) - else: - self.requests.pop(args, None) - self.cache[args] = result - if len(self.cache) > self.maxsize: - self.cache.popitem(0) - return result - -.. doctest:: - :hide: - - >>> def square(x): - ... return x * x - ... - >>> f = MultiHitLRUCache(square, maxsize=4, maxrequests=6) - >>> list(map(f, range(10))) # First requests, don't cache - [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] - >>> f(4) # Cache the second request - 16 - >>> f(6) # Cache the second request - 36 - >>> f(2) # The first request aged out, so don't cache - 4 - >>> f(6) # Cache hit - 36 - >>> f(4) # Cache hit and move to front - 16 - >>> list(f.cache.values()) - [36, 16] - >>> set(f.requests).isdisjoint(f.cache) - True - >>> list(map(f, [9, 8, 7])) # Cache these second requests - [81, 64, 49] - >>> list(map(f, [7, 9])) # Cache hits - [49, 81] - >>> list(f.cache.values()) - [16, 64, 49, 81] - >>> set(f.requests).isdisjoint(f.cache) - True - -:class:`UserDict` objects -------------------------- - -The class, :class:`UserDict` acts as a wrapper around dictionary objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`dict`; however, this class can be easier -to work with because the underlying dictionary is accessible as an -attribute. - -.. class:: UserDict([initialdata]) - - Class that simulates a dictionary. The instance's contents are kept in a - regular dictionary, which is accessible via the :attr:`data` attribute of - :class:`UserDict` instances. If *initialdata* is provided, :attr:`data` is - initialized with its contents; note that a reference to *initialdata* will not - be kept, allowing it to be used for other purposes. - - In addition to supporting the methods and operations of mappings, - :class:`UserDict` instances provide the following attribute: - - .. attribute:: data - - A real dictionary used to store the contents of the :class:`UserDict` - class. - - - -:class:`UserList` objects -------------------------- - -This class acts as a wrapper around list objects. It is a useful base class -for your own list-like classes which can inherit from them and override -existing methods or add new ones. In this way, one can add new behaviors to -lists. - -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`list`; however, this class can be easier -to work with because the underlying list is accessible as an attribute. - -.. class:: UserList([list]) - - Class that simulates a list. The instance's contents are kept in a regular - list, which is accessible via the :attr:`data` attribute of :class:`UserList` - instances. The instance's contents are initially set to a copy of *list*, - defaulting to the empty list ``[]``. *list* can be any iterable, for - example a real Python list or a :class:`UserList` object. - - In addition to supporting the methods and operations of mutable sequences, - :class:`UserList` instances provide the following attribute: - - .. attribute:: data - - A real :class:`list` object used to store the contents of the - :class:`UserList` class. - -**Subclassing requirements:** Subclasses of :class:`UserList` are expected to -offer a constructor which can be called with either no arguments or one -argument. List operations which return a new sequence attempt to create an -instance of the actual implementation class. To do so, it assumes that the -constructor can be called with a single parameter, which is a sequence object -used as a data source. - -If a derived class does not wish to comply with this requirement, all of the -special methods supported by this class will need to be overridden; please -consult the sources for information about the methods which need to be provided -in that case. - -:class:`UserString` objects ---------------------------- - -The class, :class:`UserString` acts as a wrapper around string objects. -The need for this class has been partially supplanted by the ability to -subclass directly from :class:`str`; however, this class can be easier -to work with because the underlying string is accessible as an -attribute. - -.. class:: UserString(seq) - - Class that simulates a string object. The instance's - content is kept in a regular string object, which is accessible via the - :attr:`data` attribute of :class:`UserString` instances. The instance's - contents are initially set to a copy of *seq*. The *seq* argument can - be any object which can be converted into a string using the built-in - :func:`str` function. - - In addition to supporting the methods and operations of strings, - :class:`UserString` instances provide the following attribute: - - .. attribute:: data - - A real :class:`str` object used to store the contents of the - :class:`UserString` class. - - .. versionchanged:: 3.5 - New methods ``__getnewargs__``, ``__rmod__``, ``casefold``, - ``format_map``, ``isprintable``, and ``maketrans``. diff --git a/Doc/library/colorsys.rst b/Doc/library/colorsys.rst deleted file mode 100644 index b672a05b39145d..00000000000000 --- a/Doc/library/colorsys.rst +++ /dev/null @@ -1,65 +0,0 @@ -:mod:`colorsys` --- Conversions between color systems -===================================================== - -.. module:: colorsys - :synopsis: Conversion functions between RGB and other color systems. - -.. sectionauthor:: David Ascher - -**Source code:** :source:`Lib/colorsys.py` - --------------- - -The :mod:`colorsys` module defines bidirectional conversions of color values -between colors expressed in the RGB (Red Green Blue) color space used in -computer monitors and three other coordinate systems: YIQ, HLS (Hue Lightness -Saturation) and HSV (Hue Saturation Value). Coordinates in all of these color -spaces are floating point values. In the YIQ space, the Y coordinate is between -0 and 1, but the I and Q coordinates can be positive or negative. In all other -spaces, the coordinates are all between 0 and 1. - -.. seealso:: - - More information about color spaces can be found at - https://poynton.ca/ColorFAQ.html and - https://www.cambridgeincolour.com/tutorials/color-spaces.htm. - -The :mod:`colorsys` module defines the following functions: - - -.. function:: rgb_to_yiq(r, g, b) - - Convert the color from RGB coordinates to YIQ coordinates. - - -.. function:: yiq_to_rgb(y, i, q) - - Convert the color from YIQ coordinates to RGB coordinates. - - -.. function:: rgb_to_hls(r, g, b) - - Convert the color from RGB coordinates to HLS coordinates. - - -.. function:: hls_to_rgb(h, l, s) - - Convert the color from HLS coordinates to RGB coordinates. - - -.. function:: rgb_to_hsv(r, g, b) - - Convert the color from RGB coordinates to HSV coordinates. - - -.. function:: hsv_to_rgb(h, s, v) - - Convert the color from HSV coordinates to RGB coordinates. - -Example:: - - >>> import colorsys - >>> colorsys.rgb_to_hsv(0.2, 0.4, 0.4) - (0.5, 0.5, 0.4) - >>> colorsys.hsv_to_rgb(0.5, 0.5, 0.4) - (0.2, 0.4, 0.4) diff --git a/Doc/library/compileall.rst b/Doc/library/compileall.rst deleted file mode 100644 index 6d16734ddca21e..00000000000000 --- a/Doc/library/compileall.rst +++ /dev/null @@ -1,341 +0,0 @@ -:mod:`compileall` --- Byte-compile Python libraries -=================================================== - -.. module:: compileall - :synopsis: Tools for byte-compiling all Python source files in a directory tree. - -**Source code:** :source:`Lib/compileall.py` - --------------- - -This module provides some utility functions to support installing Python -libraries. These functions compile Python source files in a directory tree. -This module can be used to create the cached byte-code files at library -installation time, which makes them available for use even by users who don't -have write permission to the library directories. - -.. include:: ../includes/wasm-notavail.rst - -.. _compileall-cli: - -Command-line use ----------------- - -This module can work as a script (using :program:`python -m compileall`) to -compile Python sources. - -.. program:: compileall - -.. option:: directory ... - file ... - - Positional arguments are files to compile or directories that contain - source files, traversed recursively. If no argument is given, behave as if - the command line was :samp:`-l {}`. - -.. option:: -l - - Do not recurse into subdirectories, only compile source code files directly - contained in the named or implied directories. - -.. option:: -f - - Force rebuild even if timestamps are up-to-date. - -.. option:: -q - - Do not print the list of files compiled. If passed once, error messages will - still be printed. If passed twice (``-qq``), all output is suppressed. - -.. option:: -d destdir - - Directory prepended to the path to each file being compiled. This will - appear in compilation time tracebacks, and is also compiled in to the - byte-code file, where it will be used in tracebacks and other messages in - cases where the source file does not exist at the time the byte-code file is - executed. - -.. option:: -s strip_prefix -.. option:: -p prepend_prefix - - Remove (``-s``) or append (``-p``) the given prefix of paths - recorded in the ``.pyc`` files. - Cannot be combined with ``-d``. - -.. option:: -x regex - - regex is used to search the full path to each file considered for - compilation, and if the regex produces a match, the file is skipped. - -.. option:: -i list - - Read the file ``list`` and add each line that it contains to the list of - files and directories to compile. If ``list`` is ``-``, read lines from - ``stdin``. - -.. option:: -b - - Write the byte-code files to their legacy locations and names, which may - overwrite byte-code files created by another version of Python. The default - is to write files to their :pep:`3147` locations and names, which allows - byte-code files from multiple versions of Python to coexist. - -.. option:: -r - - Control the maximum recursion level for subdirectories. - If this is given, then ``-l`` option will not be taken into account. - :program:`python -m compileall -r 0` is equivalent to - :program:`python -m compileall -l`. - -.. option:: -j N - - Use *N* workers to compile the files within the given directory. - If ``0`` is used, then the result of :func:`os.cpu_count()` - will be used. - -.. option:: --invalidation-mode [timestamp|checked-hash|unchecked-hash] - - Control how the generated byte-code files are invalidated at runtime. - The ``timestamp`` value, means that ``.pyc`` files with the source timestamp - and size embedded will be generated. The ``checked-hash`` and - ``unchecked-hash`` values cause hash-based pycs to be generated. Hash-based - pycs embed a hash of the source file contents rather than a timestamp. See - :ref:`pyc-invalidation` for more information on how Python validates - bytecode cache files at runtime. - The default is ``timestamp`` if the :envvar:`SOURCE_DATE_EPOCH` environment - variable is not set, and ``checked-hash`` if the ``SOURCE_DATE_EPOCH`` - environment variable is set. - -.. option:: -o level - - Compile with the given optimization level. May be used multiple times - to compile for multiple levels at a time (for example, - ``compileall -o 1 -o 2``). - -.. option:: -e dir - - Ignore symlinks pointing outside the given directory. - -.. option:: --hardlink-dupes - - If two ``.pyc`` files with different optimization level have - the same content, use hard links to consolidate duplicate files. - -.. versionchanged:: 3.2 - Added the ``-i``, ``-b`` and ``-h`` options. - -.. versionchanged:: 3.5 - Added the ``-j``, ``-r``, and ``-qq`` options. ``-q`` option - was changed to a multilevel value. ``-b`` will always produce a - byte-code file ending in ``.pyc``, never ``.pyo``. - -.. versionchanged:: 3.7 - Added the ``--invalidation-mode`` option. - -.. versionchanged:: 3.9 - Added the ``-s``, ``-p``, ``-e`` and ``--hardlink-dupes`` options. - Raised the default recursion limit from 10 to - :py:func:`sys.getrecursionlimit()`. - Added the possibility to specify the ``-o`` option multiple times. - - -There is no command-line option to control the optimization level used by the -:func:`compile` function, because the Python interpreter itself already -provides the option: :program:`python -O -m compileall`. - -Similarly, the :func:`compile` function respects the :data:`sys.pycache_prefix` -setting. The generated bytecode cache will only be useful if :func:`compile` is -run with the same :data:`sys.pycache_prefix` (if any) that will be used at -runtime. - -Public functions ----------------- - -.. function:: compile_dir(dir, maxlevels=sys.getrecursionlimit(), ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, workers=1, invalidation_mode=None, *, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) - - Recursively descend the directory tree named by *dir*, compiling all :file:`.py` - files along the way. Return a true value if all the files compiled successfully, - and a false value otherwise. - - The *maxlevels* parameter is used to limit the depth of the recursion; it - defaults to ``sys.getrecursionlimit()``. - - If *ddir* is given, it is prepended to the path to each file being compiled - for use in compilation time tracebacks, and is also compiled in to the - byte-code file, where it will be used in tracebacks and other messages in - cases where the source file does not exist at the time the byte-code file is - executed. - - If *force* is true, modules are re-compiled even if the timestamps are up to - date. - - If *rx* is given, its ``search`` method is called on the complete path to each - file considered for compilation, and if it returns a true value, the file - is skipped. This can be used to exclude files matching a regular expression, - given as a :ref:`re.Pattern ` object. - - If *quiet* is ``False`` or ``0`` (the default), the filenames and other - information are printed to standard out. Set to ``1``, only errors are - printed. Set to ``2``, all output is suppressed. - - If *legacy* is true, byte-code files are written to their legacy locations - and names, which may overwrite byte-code files created by another version of - Python. The default is to write files to their :pep:`3147` locations and - names, which allows byte-code files from multiple versions of Python to - coexist. - - *optimize* specifies the optimization level for the compiler. It is passed to - the built-in :func:`compile` function. Accepts also a sequence of optimization - levels which lead to multiple compilations of one :file:`.py` file in one call. - - The argument *workers* specifies how many workers are used to - compile files in parallel. The default is to not use multiple workers. - If the platform can't use multiple workers and *workers* argument is given, - then sequential compilation will be used as a fallback. If *workers* - is 0, the number of cores in the system is used. If *workers* is - lower than ``0``, a :exc:`ValueError` will be raised. - - *invalidation_mode* should be a member of the - :class:`py_compile.PycInvalidationMode` enum and controls how the generated - pycs are invalidated at runtime. - - The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to - the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str`` or :py:class:`os.PathLike`. - - If *hardlink_dupes* is true and two ``.pyc`` files with different optimization - level have the same content, use hard links to consolidate duplicate files. - - .. versionchanged:: 3.2 - Added the *legacy* and *optimize* parameter. - - .. versionchanged:: 3.5 - Added the *workers* parameter. - - .. versionchanged:: 3.5 - *quiet* parameter was changed to a multilevel value. - - .. versionchanged:: 3.5 - The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files - no matter what the value of *optimize* is. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. versionchanged:: 3.7 - The *invalidation_mode* parameter was added. - - .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. - - .. versionchanged:: 3.8 - Setting *workers* to 0 now chooses the optimal number of cores. - - .. versionchanged:: 3.9 - Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. - Default value of *maxlevels* was changed from ``10`` to ``sys.getrecursionlimit()`` - -.. function:: compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1, invalidation_mode=None, *, stripdir=None, prependdir=None, limit_sl_dest=None, hardlink_dupes=False) - - Compile the file with path *fullname*. Return a true value if the file - compiled successfully, and a false value otherwise. - - If *ddir* is given, it is prepended to the path to the file being compiled - for use in compilation time tracebacks, and is also compiled in to the - byte-code file, where it will be used in tracebacks and other messages in - cases where the source file does not exist at the time the byte-code file is - executed. - - If *rx* is given, its ``search`` method is passed the full path name to the - file being compiled, and if it returns a true value, the file is not - compiled and ``True`` is returned. This can be used to exclude files matching - a regular expression, given as a :ref:`re.Pattern ` object. - - If *quiet* is ``False`` or ``0`` (the default), the filenames and other - information are printed to standard out. Set to ``1``, only errors are - printed. Set to ``2``, all output is suppressed. - - If *legacy* is true, byte-code files are written to their legacy locations - and names, which may overwrite byte-code files created by another version of - Python. The default is to write files to their :pep:`3147` locations and - names, which allows byte-code files from multiple versions of Python to - coexist. - - *optimize* specifies the optimization level for the compiler. It is passed to - the built-in :func:`compile` function. Accepts also a sequence of optimization - levels which lead to multiple compilations of one :file:`.py` file in one call. - - *invalidation_mode* should be a member of the - :class:`py_compile.PycInvalidationMode` enum and controls how the generated - pycs are invalidated at runtime. - - The *stripdir*, *prependdir* and *limit_sl_dest* arguments correspond to - the ``-s``, ``-p`` and ``-e`` options described above. - They may be specified as ``str`` or :py:class:`os.PathLike`. - - If *hardlink_dupes* is true and two ``.pyc`` files with different optimization - level have the same content, use hard links to consolidate duplicate files. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.5 - *quiet* parameter was changed to a multilevel value. - - .. versionchanged:: 3.5 - The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files - no matter what the value of *optimize* is. - - .. versionchanged:: 3.7 - The *invalidation_mode* parameter was added. - - .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. - - .. versionchanged:: 3.9 - Added *stripdir*, *prependdir*, *limit_sl_dest* and *hardlink_dupes* arguments. - -.. function:: compile_path(skip_curdir=True, maxlevels=0, force=False, quiet=0, legacy=False, optimize=-1, invalidation_mode=None) - - Byte-compile all the :file:`.py` files found along ``sys.path``. Return a - true value if all the files compiled successfully, and a false value otherwise. - - If *skip_curdir* is true (the default), the current directory is not included - in the search. All other parameters are passed to the :func:`compile_dir` - function. Note that unlike the other compile functions, ``maxlevels`` - defaults to ``0``. - - .. versionchanged:: 3.2 - Added the *legacy* and *optimize* parameter. - - .. versionchanged:: 3.5 - *quiet* parameter was changed to a multilevel value. - - .. versionchanged:: 3.5 - The *legacy* parameter only writes out ``.pyc`` files, not ``.pyo`` files - no matter what the value of *optimize* is. - - .. versionchanged:: 3.7 - The *invalidation_mode* parameter was added. - - .. versionchanged:: 3.7.2 - The *invalidation_mode* parameter's default value is updated to None. - -To force a recompile of all the :file:`.py` files in the :file:`Lib/` -subdirectory and all its subdirectories:: - - import compileall - - compileall.compile_dir('Lib/', force=True) - - # Perform same compilation, excluding files in .svn directories. - import re - compileall.compile_dir('Lib/', rx=re.compile(r'[/\\][.]svn'), force=True) - - # pathlib.Path objects can also be used. - import pathlib - compileall.compile_dir(pathlib.Path('Lib/'), force=True) - -.. seealso:: - - Module :mod:`py_compile` - Byte-compile a single source file. diff --git a/Doc/library/concurrency.rst b/Doc/library/concurrency.rst deleted file mode 100644 index 5be1a1106b09a0..00000000000000 --- a/Doc/library/concurrency.rst +++ /dev/null @@ -1,31 +0,0 @@ -.. _concurrency: - -******************** -Concurrent Execution -******************** - -The modules described in this chapter provide support for concurrent -execution of code. The appropriate choice of tool will depend on the -task to be executed (CPU bound vs IO bound) and preferred style of -development (event driven cooperative multitasking vs preemptive -multitasking). Here's an overview: - - -.. toctree:: - - threading.rst - multiprocessing.rst - multiprocessing.shared_memory.rst - concurrent.rst - concurrent.futures.rst - subprocess.rst - sched.rst - queue.rst - contextvars.rst - - -The following are support modules for some of the above services: - -.. toctree:: - - _thread.rst diff --git a/Doc/library/concurrent.futures.rst b/Doc/library/concurrent.futures.rst deleted file mode 100644 index 8ab2c3a2be2d65..00000000000000 --- a/Doc/library/concurrent.futures.rst +++ /dev/null @@ -1,563 +0,0 @@ -:mod:`concurrent.futures` --- Launching parallel tasks -====================================================== - -.. module:: concurrent.futures - :synopsis: Execute computations concurrently using threads or processes. - -.. versionadded:: 3.2 - -**Source code:** :source:`Lib/concurrent/futures/thread.py` -and :source:`Lib/concurrent/futures/process.py` - --------------- - -The :mod:`concurrent.futures` module provides a high-level interface for -asynchronously executing callables. - -The asynchronous execution can be performed with threads, using -:class:`ThreadPoolExecutor`, or separate processes, using -:class:`ProcessPoolExecutor`. Both implement the same interface, which is -defined by the abstract :class:`Executor` class. - -.. include:: ../includes/wasm-notavail.rst - -Executor Objects ----------------- - -.. class:: Executor - - An abstract class that provides methods to execute calls asynchronously. It - should not be used directly, but through its concrete subclasses. - - .. method:: submit(fn, /, *args, **kwargs) - - Schedules the callable, *fn*, to be executed as ``fn(*args, **kwargs)`` - and returns a :class:`Future` object representing the execution of the - callable. :: - - with ThreadPoolExecutor(max_workers=1) as executor: - future = executor.submit(pow, 323, 1235) - print(future.result()) - - .. method:: map(func, *iterables, timeout=None, chunksize=1) - - Similar to :func:`map(func, *iterables) ` except: - - * the *iterables* are collected immediately rather than lazily; - - * *func* is executed asynchronously and several calls to - *func* may be made concurrently. - - The returned iterator raises a :exc:`TimeoutError` - if :meth:`~iterator.__next__` is called and the result isn't available - after *timeout* seconds from the original call to :meth:`Executor.map`. - *timeout* can be an int or a float. If *timeout* is not specified or - ``None``, there is no limit to the wait time. - - If a *func* call raises an exception, then that exception will be - raised when its value is retrieved from the iterator. - - When using :class:`ProcessPoolExecutor`, this method chops *iterables* - into a number of chunks which it submits to the pool as separate - tasks. The (approximate) size of these chunks can be specified by - setting *chunksize* to a positive integer. For very long iterables, - using a large value for *chunksize* can significantly improve - performance compared to the default size of 1. With - :class:`ThreadPoolExecutor`, *chunksize* has no effect. - - .. versionchanged:: 3.5 - Added the *chunksize* argument. - - .. method:: shutdown(wait=True, *, cancel_futures=False) - - Signal the executor that it should free any resources that it is using - when the currently pending futures are done executing. Calls to - :meth:`Executor.submit` and :meth:`Executor.map` made after shutdown will - raise :exc:`RuntimeError`. - - If *wait* is ``True`` then this method will not return until all the - pending futures are done executing and the resources associated with the - executor have been freed. If *wait* is ``False`` then this method will - return immediately and the resources associated with the executor will be - freed when all pending futures are done executing. Regardless of the - value of *wait*, the entire Python program will not exit until all - pending futures are done executing. - - If *cancel_futures* is ``True``, this method will cancel all pending - futures that the executor has not started running. Any futures that - are completed or running won't be cancelled, regardless of the value - of *cancel_futures*. - - If both *cancel_futures* and *wait* are ``True``, all futures that the - executor has started running will be completed prior to this method - returning. The remaining futures are cancelled. - - You can avoid having to call this method explicitly if you use the - :keyword:`with` statement, which will shutdown the :class:`Executor` - (waiting as if :meth:`Executor.shutdown` were called with *wait* set to - ``True``):: - - import shutil - with ThreadPoolExecutor(max_workers=4) as e: - e.submit(shutil.copy, 'src1.txt', 'dest1.txt') - e.submit(shutil.copy, 'src2.txt', 'dest2.txt') - e.submit(shutil.copy, 'src3.txt', 'dest3.txt') - e.submit(shutil.copy, 'src4.txt', 'dest4.txt') - - .. versionchanged:: 3.9 - Added *cancel_futures*. - - -ThreadPoolExecutor ------------------- - -:class:`ThreadPoolExecutor` is an :class:`Executor` subclass that uses a pool of -threads to execute calls asynchronously. - -Deadlocks can occur when the callable associated with a :class:`Future` waits on -the results of another :class:`Future`. For example:: - - import time - def wait_on_b(): - time.sleep(5) - print(b.result()) # b will never complete because it is waiting on a. - return 5 - - def wait_on_a(): - time.sleep(5) - print(a.result()) # a will never complete because it is waiting on b. - return 6 - - - executor = ThreadPoolExecutor(max_workers=2) - a = executor.submit(wait_on_b) - b = executor.submit(wait_on_a) - -And:: - - def wait_on_future(): - f = executor.submit(pow, 5, 2) - # This will never complete because there is only one worker thread and - # it is executing this function. - print(f.result()) - - executor = ThreadPoolExecutor(max_workers=1) - executor.submit(wait_on_future) - - -.. class:: ThreadPoolExecutor(max_workers=None, thread_name_prefix='', initializer=None, initargs=()) - - An :class:`Executor` subclass that uses a pool of at most *max_workers* - threads to execute calls asynchronously. - - All threads enqueued to ``ThreadPoolExecutor`` will be joined before the - interpreter can exit. Note that the exit handler which does this is - executed *before* any exit handlers added using ``atexit``. This means - exceptions in the main thread must be caught and handled in order to - signal threads to exit gracefully. For this reason, it is recommended - that ``ThreadPoolExecutor`` not be used for long-running tasks. - - *initializer* is an optional callable that is called at the start of - each worker thread; *initargs* is a tuple of arguments passed to the - initializer. Should *initializer* raise an exception, all currently - pending jobs will raise a :exc:`~concurrent.futures.thread.BrokenThreadPool`, - as well as any attempt to submit more jobs to the pool. - - .. versionchanged:: 3.5 - If *max_workers* is ``None`` or - not given, it will default to the number of processors on the machine, - multiplied by ``5``, assuming that :class:`ThreadPoolExecutor` is often - used to overlap I/O instead of CPU work and the number of workers - should be higher than the number of workers - for :class:`ProcessPoolExecutor`. - - .. versionadded:: 3.6 - The *thread_name_prefix* argument was added to allow users to - control the :class:`threading.Thread` names for worker threads created by - the pool for easier debugging. - - .. versionchanged:: 3.7 - Added the *initializer* and *initargs* arguments. - - .. versionchanged:: 3.8 - Default value of *max_workers* is changed to ``min(32, os.cpu_count() + 4)``. - This default value preserves at least 5 workers for I/O bound tasks. - It utilizes at most 32 CPU cores for CPU bound tasks which release the GIL. - And it avoids using very large resources implicitly on many-core machines. - - ThreadPoolExecutor now reuses idle worker threads before starting - *max_workers* worker threads too. - - -.. _threadpoolexecutor-example: - -ThreadPoolExecutor Example -~~~~~~~~~~~~~~~~~~~~~~~~~~ -:: - - import concurrent.futures - import urllib.request - - URLS = ['http://www.foxnews.com/', - 'http://www.cnn.com/', - 'http://europe.wsj.com/', - 'http://www.bbc.co.uk/', - 'http://nonexistant-subdomain.python.org/'] - - # Retrieve a single page and report the URL and contents - def load_url(url, timeout): - with urllib.request.urlopen(url, timeout=timeout) as conn: - return conn.read() - - # We can use a with statement to ensure threads are cleaned up promptly - with concurrent.futures.ThreadPoolExecutor(max_workers=5) as executor: - # Start the load operations and mark each future with its URL - future_to_url = {executor.submit(load_url, url, 60): url for url in URLS} - for future in concurrent.futures.as_completed(future_to_url): - url = future_to_url[future] - try: - data = future.result() - except Exception as exc: - print('%r generated an exception: %s' % (url, exc)) - else: - print('%r page is %d bytes' % (url, len(data))) - - -ProcessPoolExecutor -------------------- - -The :class:`ProcessPoolExecutor` class is an :class:`Executor` subclass that -uses a pool of processes to execute calls asynchronously. -:class:`ProcessPoolExecutor` uses the :mod:`multiprocessing` module, which -allows it to side-step the :term:`Global Interpreter Lock -` but also means that -only picklable objects can be executed and returned. - -The ``__main__`` module must be importable by worker subprocesses. This means -that :class:`ProcessPoolExecutor` will not work in the interactive interpreter. - -Calling :class:`Executor` or :class:`Future` methods from a callable submitted -to a :class:`ProcessPoolExecutor` will result in deadlock. - -.. class:: ProcessPoolExecutor(max_workers=None, mp_context=None, initializer=None, initargs=(), max_tasks_per_child=None) - - An :class:`Executor` subclass that executes calls asynchronously using a pool - of at most *max_workers* processes. If *max_workers* is ``None`` or not - given, it will default to the number of processors on the machine. - If *max_workers* is less than or equal to ``0``, then a :exc:`ValueError` - will be raised. - On Windows, *max_workers* must be less than or equal to ``61``. If it is not - then :exc:`ValueError` will be raised. If *max_workers* is ``None``, then - the default chosen will be at most ``61``, even if more processors are - available. - *mp_context* can be a multiprocessing context or None. It will be used to - launch the workers. If *mp_context* is ``None`` or not given, the default - multiprocessing context is used. - - *initializer* is an optional callable that is called at the start of - each worker process; *initargs* is a tuple of arguments passed to the - initializer. Should *initializer* raise an exception, all currently - pending jobs will raise a :exc:`~concurrent.futures.process.BrokenProcessPool`, - as well as any attempt to submit more jobs to the pool. - - *max_tasks_per_child* is an optional argument that specifies the maximum - number of tasks a single process can execute before it will exit and be - replaced with a fresh worker process. By default *max_tasks_per_child* is - ``None`` which means worker processes will live as long as the pool. When - a max is specified, the "spawn" multiprocessing start method will be used by - default in absence of a *mp_context* parameter. This feature is incompatible - with the "fork" start method. - - .. versionchanged:: 3.3 - When one of the worker processes terminates abruptly, a - :exc:`BrokenProcessPool` error is now raised. Previously, behaviour - was undefined but operations on the executor or its futures would often - freeze or deadlock. - - .. versionchanged:: 3.7 - The *mp_context* argument was added to allow users to control the - start_method for worker processes created by the pool. - - Added the *initializer* and *initargs* arguments. - - .. versionchanged:: 3.11 - The *max_tasks_per_child* argument was added to allow users to - control the lifetime of workers in the pool. - - -.. _processpoolexecutor-example: - -ProcessPoolExecutor Example -~~~~~~~~~~~~~~~~~~~~~~~~~~~ -:: - - import concurrent.futures - import math - - PRIMES = [ - 112272535095293, - 112582705942171, - 112272535095293, - 115280095190773, - 115797848077099, - 1099726899285419] - - def is_prime(n): - if n < 2: - return False - if n == 2: - return True - if n % 2 == 0: - return False - - sqrt_n = int(math.floor(math.sqrt(n))) - for i in range(3, sqrt_n + 1, 2): - if n % i == 0: - return False - return True - - def main(): - with concurrent.futures.ProcessPoolExecutor() as executor: - for number, prime in zip(PRIMES, executor.map(is_prime, PRIMES)): - print('%d is prime: %s' % (number, prime)) - - if __name__ == '__main__': - main() - - -Future Objects --------------- - -The :class:`Future` class encapsulates the asynchronous execution of a callable. -:class:`Future` instances are created by :meth:`Executor.submit`. - -.. class:: Future - - Encapsulates the asynchronous execution of a callable. :class:`Future` - instances are created by :meth:`Executor.submit` and should not be created - directly except for testing. - - .. method:: cancel() - - Attempt to cancel the call. If the call is currently being executed or - finished running and cannot be cancelled then the method will return - ``False``, otherwise the call will be cancelled and the method will - return ``True``. - - .. method:: cancelled() - - Return ``True`` if the call was successfully cancelled. - - .. method:: running() - - Return ``True`` if the call is currently being executed and cannot be - cancelled. - - .. method:: done() - - Return ``True`` if the call was successfully cancelled or finished - running. - - .. method:: result(timeout=None) - - Return the value returned by the call. If the call hasn't yet completed - then this method will wait up to *timeout* seconds. If the call hasn't - completed in *timeout* seconds, then a - :exc:`TimeoutError` will be raised. *timeout* can be - an int or float. If *timeout* is not specified or ``None``, there is no - limit to the wait time. - - If the future is cancelled before completing then :exc:`.CancelledError` - will be raised. - - If the call raised an exception, this method will raise the same exception. - - .. method:: exception(timeout=None) - - Return the exception raised by the call. If the call hasn't yet - completed then this method will wait up to *timeout* seconds. If the - call hasn't completed in *timeout* seconds, then a - :exc:`TimeoutError` will be raised. *timeout* can be - an int or float. If *timeout* is not specified or ``None``, there is no - limit to the wait time. - - If the future is cancelled before completing then :exc:`.CancelledError` - will be raised. - - If the call completed without raising, ``None`` is returned. - - .. method:: add_done_callback(fn) - - Attaches the callable *fn* to the future. *fn* will be called, with the - future as its only argument, when the future is cancelled or finishes - running. - - Added callables are called in the order that they were added and are - always called in a thread belonging to the process that added them. If - the callable raises an :exc:`Exception` subclass, it will be logged and - ignored. If the callable raises a :exc:`BaseException` subclass, the - behavior is undefined. - - If the future has already completed or been cancelled, *fn* will be - called immediately. - - The following :class:`Future` methods are meant for use in unit tests and - :class:`Executor` implementations. - - .. method:: set_running_or_notify_cancel() - - This method should only be called by :class:`Executor` implementations - before executing the work associated with the :class:`Future` and by unit - tests. - - If the method returns ``False`` then the :class:`Future` was cancelled, - i.e. :meth:`Future.cancel` was called and returned ``True``. Any threads - waiting on the :class:`Future` completing (i.e. through - :func:`as_completed` or :func:`wait`) will be woken up. - - If the method returns ``True`` then the :class:`Future` was not cancelled - and has been put in the running state, i.e. calls to - :meth:`Future.running` will return ``True``. - - This method can only be called once and cannot be called after - :meth:`Future.set_result` or :meth:`Future.set_exception` have been - called. - - .. method:: set_result(result) - - Sets the result of the work associated with the :class:`Future` to - *result*. - - This method should only be used by :class:`Executor` implementations and - unit tests. - - .. versionchanged:: 3.8 - This method raises - :exc:`concurrent.futures.InvalidStateError` if the :class:`Future` is - already done. - - .. method:: set_exception(exception) - - Sets the result of the work associated with the :class:`Future` to the - :class:`Exception` *exception*. - - This method should only be used by :class:`Executor` implementations and - unit tests. - - .. versionchanged:: 3.8 - This method raises - :exc:`concurrent.futures.InvalidStateError` if the :class:`Future` is - already done. - -Module Functions ----------------- - -.. function:: wait(fs, timeout=None, return_when=ALL_COMPLETED) - - Wait for the :class:`Future` instances (possibly created by different - :class:`Executor` instances) given by *fs* to complete. Duplicate futures - given to *fs* are removed and will be returned only once. Returns a named - 2-tuple of sets. The first set, named ``done``, contains the futures that - completed (finished or cancelled futures) before the wait completed. The - second set, named ``not_done``, contains the futures that did not complete - (pending or running futures). - - *timeout* can be used to control the maximum number of seconds to wait before - returning. *timeout* can be an int or float. If *timeout* is not specified - or ``None``, there is no limit to the wait time. - - *return_when* indicates when this function should return. It must be one of - the following constants: - - .. tabularcolumns:: |l|L| - - +-----------------------------+----------------------------------------+ - | Constant | Description | - +=============================+========================================+ - | :const:`FIRST_COMPLETED` | The function will return when any | - | | future finishes or is cancelled. | - +-----------------------------+----------------------------------------+ - | :const:`FIRST_EXCEPTION` | The function will return when any | - | | future finishes by raising an | - | | exception. If no future raises an | - | | exception then it is equivalent to | - | | :const:`ALL_COMPLETED`. | - +-----------------------------+----------------------------------------+ - | :const:`ALL_COMPLETED` | The function will return when all | - | | futures finish or are cancelled. | - +-----------------------------+----------------------------------------+ - -.. function:: as_completed(fs, timeout=None) - - Returns an iterator over the :class:`Future` instances (possibly created by - different :class:`Executor` instances) given by *fs* that yields futures as - they complete (finished or cancelled futures). Any futures given by *fs* that - are duplicated will be returned once. Any futures that completed before - :func:`as_completed` is called will be yielded first. The returned iterator - raises a :exc:`TimeoutError` if :meth:`~iterator.__next__` - is called and the result isn't available after *timeout* seconds from the - original call to :func:`as_completed`. *timeout* can be an int or float. If - *timeout* is not specified or ``None``, there is no limit to the wait time. - - -.. seealso:: - - :pep:`3148` -- futures - execute computations asynchronously - The proposal which described this feature for inclusion in the Python - standard library. - - -Exception classes ------------------ - -.. currentmodule:: concurrent.futures - -.. exception:: CancelledError - - Raised when a future is cancelled. - -.. exception:: TimeoutError - - A deprecated alias of :exc:`TimeoutError`, - raised when a future operation exceeds the given timeout. - - .. versionchanged:: 3.11 - - This class was made an alias of :exc:`TimeoutError`. - - -.. exception:: BrokenExecutor - - Derived from :exc:`RuntimeError`, this exception class is raised - when an executor is broken for some reason, and cannot be used - to submit or execute new tasks. - - .. versionadded:: 3.7 - -.. exception:: InvalidStateError - - Raised when an operation is performed on a future that is not allowed - in the current state. - - .. versionadded:: 3.8 - -.. currentmodule:: concurrent.futures.thread - -.. exception:: BrokenThreadPool - - Derived from :exc:`~concurrent.futures.BrokenExecutor`, this exception - class is raised when one of the workers of a :class:`ThreadPoolExecutor` - has failed initializing. - - .. versionadded:: 3.7 - -.. currentmodule:: concurrent.futures.process - -.. exception:: BrokenProcessPool - - Derived from :exc:`~concurrent.futures.BrokenExecutor` (formerly - :exc:`RuntimeError`), this exception class is raised when one of the - workers of a :class:`ProcessPoolExecutor` has terminated in a non-clean - fashion (for example, if it was killed from the outside). - - .. versionadded:: 3.3 diff --git a/Doc/library/concurrent.rst b/Doc/library/concurrent.rst deleted file mode 100644 index 8caea78bbb57e8..00000000000000 --- a/Doc/library/concurrent.rst +++ /dev/null @@ -1,6 +0,0 @@ -The :mod:`!concurrent` package -============================== - -Currently, there is only one module in this package: - -* :mod:`concurrent.futures` -- Launching parallel tasks diff --git a/Doc/library/configparser.rst b/Doc/library/configparser.rst deleted file mode 100644 index 819c3799b08da7..00000000000000 --- a/Doc/library/configparser.rst +++ /dev/null @@ -1,1379 +0,0 @@ -:mod:`configparser` --- Configuration file parser -================================================= - -.. module:: configparser - :synopsis: Configuration file parser. - -.. moduleauthor:: Ken Manheimer -.. moduleauthor:: Barry Warsaw -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Łukasz Langa -.. sectionauthor:: Christopher G. Petrilli -.. sectionauthor:: Łukasz Langa - -**Source code:** :source:`Lib/configparser.py` - -.. index:: - pair: .ini; file - pair: configuration; file - single: ini file - single: Windows ini file - --------------- - -This module provides the :class:`ConfigParser` class which implements a basic -configuration language which provides a structure similar to what's found in -Microsoft Windows INI files. You can use this to write Python programs which -can be customized by end users easily. - -.. note:: - - This library does *not* interpret or write the value-type prefixes used in - the Windows Registry extended version of INI syntax. - -.. seealso:: - - Module :mod:`tomllib` - TOML is a well-specified format for application configuration files. - It is specifically designed to be an improved version of INI. - - Module :mod:`shlex` - Support for creating Unix shell-like mini-languages which can also - be used for application configuration files. - - Module :mod:`json` - The ``json`` module implements a subset of JavaScript syntax which is - sometimes used for configuration, but does not support comments. - - -.. testsetup:: - - import configparser - -.. testcleanup:: - - import os - os.remove("example.ini") - - -Quick Start ------------ - -Let's take a very basic configuration file that looks like this: - -.. code-block:: ini - - [DEFAULT] - ServerAliveInterval = 45 - Compression = yes - CompressionLevel = 9 - ForwardX11 = yes - - [forge.example] - User = hg - - [topsecret.server.example] - Port = 50022 - ForwardX11 = no - -The structure of INI files is described `in the following section -<#supported-ini-file-structure>`_. Essentially, the file -consists of sections, each of which contains keys with values. -:mod:`configparser` classes can read and write such files. Let's start by -creating the above configuration file programmatically. - -.. doctest:: - - >>> import configparser - >>> config = configparser.ConfigParser() - >>> config['DEFAULT'] = {'ServerAliveInterval': '45', - ... 'Compression': 'yes', - ... 'CompressionLevel': '9'} - >>> config['forge.example'] = {} - >>> config['forge.example']['User'] = 'hg' - >>> config['topsecret.server.example'] = {} - >>> topsecret = config['topsecret.server.example'] - >>> topsecret['Port'] = '50022' # mutates the parser - >>> topsecret['ForwardX11'] = 'no' # same here - >>> config['DEFAULT']['ForwardX11'] = 'yes' - >>> with open('example.ini', 'w') as configfile: - ... config.write(configfile) - ... - -As you can see, we can treat a config parser much like a dictionary. -There are differences, `outlined later <#mapping-protocol-access>`_, but -the behavior is very close to what you would expect from a dictionary. - -Now that we have created and saved a configuration file, let's read it -back and explore the data it holds. - -.. doctest:: - - >>> config = configparser.ConfigParser() - >>> config.sections() - [] - >>> config.read('example.ini') - ['example.ini'] - >>> config.sections() - ['forge.example', 'topsecret.server.example'] - >>> 'forge.example' in config - True - >>> 'python.org' in config - False - >>> config['forge.example']['User'] - 'hg' - >>> config['DEFAULT']['Compression'] - 'yes' - >>> topsecret = config['topsecret.server.example'] - >>> topsecret['ForwardX11'] - 'no' - >>> topsecret['Port'] - '50022' - >>> for key in config['forge.example']: # doctest: +SKIP - ... print(key) - user - compressionlevel - serveraliveinterval - compression - forwardx11 - >>> config['forge.example']['ForwardX11'] - 'yes' - -As we can see above, the API is pretty straightforward. The only bit of magic -involves the ``DEFAULT`` section which provides default values for all other -sections [1]_. Note also that keys in sections are -case-insensitive and stored in lowercase [1]_. - -It is possible to read several configurations into a single -:class:`ConfigParser`, where the most recently added configuration has the -highest priority. Any conflicting keys are taken from the more recent -configuration while the previously existing keys are retained. - -.. doctest:: - - >>> another_config = configparser.ConfigParser() - >>> another_config.read('example.ini') - ['example.ini'] - >>> another_config['topsecret.server.example']['Port'] - '50022' - >>> another_config.read_string("[topsecret.server.example]\nPort=48484") - >>> another_config['topsecret.server.example']['Port'] - '48484' - >>> another_config.read_dict({"topsecret.server.example": {"Port": 21212}}) - >>> another_config['topsecret.server.example']['Port'] - '21212' - >>> another_config['topsecret.server.example']['ForwardX11'] - 'no' - -This behaviour is equivalent to a :meth:`ConfigParser.read` call with several -files passed to the *filenames* parameter. - - -Supported Datatypes -------------------- - -Config parsers do not guess datatypes of values in configuration files, always -storing them internally as strings. This means that if you need other -datatypes, you should convert on your own: - -.. doctest:: - - >>> int(topsecret['Port']) - 50022 - >>> float(topsecret['CompressionLevel']) - 9.0 - -Since this task is so common, config parsers provide a range of handy getter -methods to handle integers, floats and booleans. The last one is the most -interesting because simply passing the value to ``bool()`` would do no good -since ``bool('False')`` is still ``True``. This is why config parsers also -provide :meth:`~ConfigParser.getboolean`. This method is case-insensitive and -recognizes Boolean values from ``'yes'``/``'no'``, ``'on'``/``'off'``, -``'true'``/``'false'`` and ``'1'``/``'0'`` [1]_. For example: - -.. doctest:: - - >>> topsecret.getboolean('ForwardX11') - False - >>> config['forge.example'].getboolean('ForwardX11') - True - >>> config.getboolean('forge.example', 'Compression') - True - -Apart from :meth:`~ConfigParser.getboolean`, config parsers also -provide equivalent :meth:`~ConfigParser.getint` and -:meth:`~ConfigParser.getfloat` methods. You can register your own -converters and customize the provided ones. [1]_ - -Fallback Values ---------------- - -As with a dictionary, you can use a section's :meth:`get` method to -provide fallback values: - -.. doctest:: - - >>> topsecret.get('Port') - '50022' - >>> topsecret.get('CompressionLevel') - '9' - >>> topsecret.get('Cipher') - >>> topsecret.get('Cipher', '3des-cbc') - '3des-cbc' - -Please note that default values have precedence over fallback values. -For instance, in our example the ``'CompressionLevel'`` key was -specified only in the ``'DEFAULT'`` section. If we try to get it from -the section ``'topsecret.server.example'``, we will always get the default, -even if we specify a fallback: - -.. doctest:: - - >>> topsecret.get('CompressionLevel', '3') - '9' - -One more thing to be aware of is that the parser-level :meth:`get` method -provides a custom, more complex interface, maintained for backwards -compatibility. When using this method, a fallback value can be provided via -the ``fallback`` keyword-only argument: - -.. doctest:: - - >>> config.get('forge.example', 'monster', - ... fallback='No such things as monsters') - 'No such things as monsters' - -The same ``fallback`` argument can be used with the -:meth:`~ConfigParser.getint`, :meth:`~ConfigParser.getfloat` and -:meth:`~ConfigParser.getboolean` methods, for example: - -.. doctest:: - - >>> 'BatchMode' in topsecret - False - >>> topsecret.getboolean('BatchMode', fallback=True) - True - >>> config['DEFAULT']['BatchMode'] = 'no' - >>> topsecret.getboolean('BatchMode', fallback=True) - False - - -Supported INI File Structure ----------------------------- - -A configuration file consists of sections, each led by a ``[section]`` header, -followed by key/value entries separated by a specific string (``=`` or ``:`` by -default [1]_). By default, section names are case sensitive but keys are not -[1]_. Leading and trailing whitespace is removed from keys and values. -Values can be omitted if the parser is configured to allow it [1]_, -in which case the key/value delimiter may also be left -out. Values can also span multiple lines, as long as they are indented deeper -than the first line of the value. Depending on the parser's mode, blank lines -may be treated as parts of multiline values or ignored. - -By default, a valid section name can be any string that does not contain '\\n' or ']'. -To change this, see :attr:`ConfigParser.SECTCRE`. - -Configuration files may include comments, prefixed by specific -characters (``#`` and ``;`` by default [1]_). Comments may appear on -their own on an otherwise empty line, possibly indented. [1]_ - -For example: - -.. code-block:: ini - - [Simple Values] - key=value - spaces in keys=allowed - spaces in values=allowed as well - spaces around the delimiter = obviously - you can also use : to delimit keys from values - - [All Values Are Strings] - values like this: 1000000 - or this: 3.14159265359 - are they treated as numbers? : no - integers, floats and booleans are held as: strings - can use the API to get converted values directly: true - - [Multiline Values] - chorus: I'm a lumberjack, and I'm okay - I sleep all night and I work all day - - [No Values] - key_without_value - empty string value here = - - [You can use comments] - # like this - ; or this - - # By default only in an empty line. - # Inline comments can be harmful because they prevent users - # from using the delimiting characters as parts of values. - # That being said, this can be customized. - - [Sections Can Be Indented] - can_values_be_as_well = True - does_that_mean_anything_special = False - purpose = formatting for readability - multiline_values = are - handled just fine as - long as they are indented - deeper than the first line - of a value - # Did I mention we can indent comments, too? - - -Interpolation of values ------------------------ - -On top of the core functionality, :class:`ConfigParser` supports -interpolation. This means values can be preprocessed before returning them -from ``get()`` calls. - -.. index:: single: % (percent); interpolation in configuration files - -.. class:: BasicInterpolation() - - The default implementation used by :class:`ConfigParser`. It enables - values to contain format strings which refer to other values in the same - section, or values in the special default section [1]_. Additional default - values can be provided on initialization. - - For example: - - .. code-block:: ini - - [Paths] - home_dir: /Users - my_dir: %(home_dir)s/lumberjack - my_pictures: %(my_dir)s/Pictures - - [Escape] - # use a %% to escape the % sign (% is the only character that needs to be escaped): - gain: 80%% - - In the example above, :class:`ConfigParser` with *interpolation* set to - ``BasicInterpolation()`` would resolve ``%(home_dir)s`` to the value of - ``home_dir`` (``/Users`` in this case). ``%(my_dir)s`` in effect would - resolve to ``/Users/lumberjack``. All interpolations are done on demand so - keys used in the chain of references do not have to be specified in any - specific order in the configuration file. - - With ``interpolation`` set to ``None``, the parser would simply return - ``%(my_dir)s/Pictures`` as the value of ``my_pictures`` and - ``%(home_dir)s/lumberjack`` as the value of ``my_dir``. - -.. index:: single: $ (dollar); interpolation in configuration files - -.. class:: ExtendedInterpolation() - - An alternative handler for interpolation which implements a more advanced - syntax, used for instance in ``zc.buildout``. Extended interpolation is - using ``${section:option}`` to denote a value from a foreign section. - Interpolation can span multiple levels. For convenience, if the - ``section:`` part is omitted, interpolation defaults to the current section - (and possibly the default values from the special section). - - For example, the configuration specified above with basic interpolation, - would look like this with extended interpolation: - - .. code-block:: ini - - [Paths] - home_dir: /Users - my_dir: ${home_dir}/lumberjack - my_pictures: ${my_dir}/Pictures - - [Escape] - # use a $$ to escape the $ sign ($ is the only character that needs to be escaped): - cost: $$80 - - Values from other sections can be fetched as well: - - .. code-block:: ini - - [Common] - home_dir: /Users - library_dir: /Library - system_dir: /System - macports_dir: /opt/local - - [Frameworks] - Python: 3.2 - path: ${Common:system_dir}/Library/Frameworks/ - - [Arthur] - nickname: Two Sheds - last_name: Jackson - my_dir: ${Common:home_dir}/twosheds - my_pictures: ${my_dir}/Pictures - python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} - -Mapping Protocol Access ------------------------ - -.. versionadded:: 3.2 - -Mapping protocol access is a generic name for functionality that enables using -custom objects as if they were dictionaries. In case of :mod:`configparser`, -the mapping interface implementation is using the -``parser['section']['option']`` notation. - -``parser['section']`` in particular returns a proxy for the section's data in -the parser. This means that the values are not copied but they are taken from -the original parser on demand. What's even more important is that when values -are changed on a section proxy, they are actually mutated in the original -parser. - -:mod:`configparser` objects behave as close to actual dictionaries as possible. -The mapping interface is complete and adheres to the -:class:`~collections.abc.MutableMapping` ABC. -However, there are a few differences that should be taken into account: - -* By default, all keys in sections are accessible in a case-insensitive manner - [1]_. E.g. ``for option in parser["section"]`` yields only ``optionxform``'ed - option key names. This means lowercased keys by default. At the same time, - for a section that holds the key ``'a'``, both expressions return ``True``:: - - "a" in parser["section"] - "A" in parser["section"] - -* All sections include ``DEFAULTSECT`` values as well which means that - ``.clear()`` on a section may not leave the section visibly empty. This is - because default values cannot be deleted from the section (because technically - they are not there). If they are overridden in the section, deleting causes - the default value to be visible again. Trying to delete a default value - causes a :exc:`KeyError`. - -* ``DEFAULTSECT`` cannot be removed from the parser: - - * trying to delete it raises :exc:`ValueError`, - - * ``parser.clear()`` leaves it intact, - - * ``parser.popitem()`` never returns it. - -* ``parser.get(section, option, **kwargs)`` - the second argument is **not** - a fallback value. Note however that the section-level ``get()`` methods are - compatible both with the mapping protocol and the classic configparser API. - -* ``parser.items()`` is compatible with the mapping protocol (returns a list of - *section_name*, *section_proxy* pairs including the DEFAULTSECT). However, - this method can also be invoked with arguments: ``parser.items(section, raw, - vars)``. The latter call returns a list of *option*, *value* pairs for - a specified ``section``, with all interpolations expanded (unless - ``raw=True`` is provided). - -The mapping protocol is implemented on top of the existing legacy API so that -subclasses overriding the original interface still should have mappings working -as expected. - - -Customizing Parser Behaviour ----------------------------- - -There are nearly as many INI format variants as there are applications using it. -:mod:`configparser` goes a long way to provide support for the largest sensible -set of INI styles available. The default functionality is mainly dictated by -historical background and it's very likely that you will want to customize some -of the features. - -The most common way to change the way a specific config parser works is to use -the :meth:`__init__` options: - -* *defaults*, default value: ``None`` - - This option accepts a dictionary of key-value pairs which will be initially - put in the ``DEFAULT`` section. This makes for an elegant way to support - concise configuration files that don't specify values which are the same as - the documented default. - - Hint: if you want to specify default values for a specific section, use - :meth:`read_dict` before you read the actual file. - -* *dict_type*, default value: :class:`dict` - - This option has a major impact on how the mapping protocol will behave and how - the written configuration files look. With the standard dictionary, every - section is stored in the order they were added to the parser. Same goes for - options within sections. - - An alternative dictionary type can be used for example to sort sections and - options on write-back. - - Please note: there are ways to add a set of key-value pairs in a single - operation. When you use a regular dictionary in those operations, the order - of the keys will be ordered. For example: - - .. doctest:: - - >>> parser = configparser.ConfigParser() - >>> parser.read_dict({'section1': {'key1': 'value1', - ... 'key2': 'value2', - ... 'key3': 'value3'}, - ... 'section2': {'keyA': 'valueA', - ... 'keyB': 'valueB', - ... 'keyC': 'valueC'}, - ... 'section3': {'foo': 'x', - ... 'bar': 'y', - ... 'baz': 'z'} - ... }) - >>> parser.sections() - ['section1', 'section2', 'section3'] - >>> [option for option in parser['section3']] - ['foo', 'bar', 'baz'] - -* *allow_no_value*, default value: ``False`` - - Some configuration files are known to include settings without values, but - which otherwise conform to the syntax supported by :mod:`configparser`. The - *allow_no_value* parameter to the constructor can be used to - indicate that such values should be accepted: - - .. doctest:: - - >>> import configparser - - >>> sample_config = """ - ... [mysqld] - ... user = mysql - ... pid-file = /var/run/mysqld/mysqld.pid - ... skip-external-locking - ... old_passwords = 1 - ... skip-bdb - ... # we don't need ACID today - ... skip-innodb - ... """ - >>> config = configparser.ConfigParser(allow_no_value=True) - >>> config.read_string(sample_config) - - >>> # Settings with values are treated as before: - >>> config["mysqld"]["user"] - 'mysql' - - >>> # Settings without values provide None: - >>> config["mysqld"]["skip-bdb"] - - >>> # Settings which aren't specified still raise an error: - >>> config["mysqld"]["does-not-exist"] - Traceback (most recent call last): - ... - KeyError: 'does-not-exist' - -* *delimiters*, default value: ``('=', ':')`` - - Delimiters are substrings that delimit keys from values within a section. - The first occurrence of a delimiting substring on a line is considered - a delimiter. This means values (but not keys) can contain the delimiters. - - See also the *space_around_delimiters* argument to - :meth:`ConfigParser.write`. - -* *comment_prefixes*, default value: ``('#', ';')`` - -* *inline_comment_prefixes*, default value: ``None`` - - Comment prefixes are strings that indicate the start of a valid comment within - a config file. *comment_prefixes* are used only on otherwise empty lines - (optionally indented) whereas *inline_comment_prefixes* can be used after - every valid value (e.g. section names, options and empty lines as well). By - default inline comments are disabled and ``'#'`` and ``';'`` are used as - prefixes for whole line comments. - - .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched - ``comment_prefixes=('#',';')`` and ``inline_comment_prefixes=(';',)``. - - Please note that config parsers don't support escaping of comment prefixes so - using *inline_comment_prefixes* may prevent users from specifying option - values with characters used as comment prefixes. When in doubt, avoid - setting *inline_comment_prefixes*. In any circumstances, the only way of - storing comment prefix characters at the beginning of a line in multiline - values is to interpolate the prefix, for example:: - - >>> from configparser import ConfigParser, ExtendedInterpolation - >>> parser = ConfigParser(interpolation=ExtendedInterpolation()) - >>> # the default BasicInterpolation could be used as well - >>> parser.read_string(""" - ... [DEFAULT] - ... hash = # - ... - ... [hashes] - ... shebang = - ... ${hash}!/usr/bin/env python - ... ${hash} -*- coding: utf-8 -*- - ... - ... extensions = - ... enabled_extension - ... another_extension - ... #disabled_by_comment - ... yet_another_extension - ... - ... interpolation not necessary = if # is not at line start - ... even in multiline values = line #1 - ... line #2 - ... line #3 - ... """) - >>> print(parser['hashes']['shebang']) - - #!/usr/bin/env python - # -*- coding: utf-8 -*- - >>> print(parser['hashes']['extensions']) - - enabled_extension - another_extension - yet_another_extension - >>> print(parser['hashes']['interpolation not necessary']) - if # is not at line start - >>> print(parser['hashes']['even in multiline values']) - line #1 - line #2 - line #3 - -* *strict*, default value: ``True`` - - When set to ``True``, the parser will not allow for any section or option - duplicates while reading from a single source (using :meth:`read_file`, - :meth:`read_string` or :meth:`read_dict`). It is recommended to use strict - parsers in new applications. - - .. versionchanged:: 3.2 - In previous versions of :mod:`configparser` behaviour matched - ``strict=False``. - -* *empty_lines_in_values*, default value: ``True`` - - In config parsers, values can span multiple lines as long as they are - indented more than the key that holds them. By default parsers also let - empty lines to be parts of values. At the same time, keys can be arbitrarily - indented themselves to improve readability. In consequence, when - configuration files get big and complex, it is easy for the user to lose - track of the file structure. Take for instance: - - .. code-block:: ini - - [Section] - key = multiline - value with a gotcha - - this = is still a part of the multiline value of 'key' - - This can be especially problematic for the user to see if she's using a - proportional font to edit the file. That is why when your application does - not need values with empty lines, you should consider disallowing them. This - will make empty lines split keys every time. In the example above, it would - produce two keys, ``key`` and ``this``. - -* *default_section*, default value: ``configparser.DEFAULTSECT`` (that is: - ``"DEFAULT"``) - - The convention of allowing a special section of default values for other - sections or interpolation purposes is a powerful concept of this library, - letting users create complex declarative configurations. This section is - normally called ``"DEFAULT"`` but this can be customized to point to any - other valid section name. Some typical values include: ``"general"`` or - ``"common"``. The name provided is used for recognizing default sections - when reading from any source and is used when writing configuration back to - a file. Its current value can be retrieved using the - ``parser_instance.default_section`` attribute and may be modified at runtime - (i.e. to convert files from one format to another). - -* *interpolation*, default value: ``configparser.BasicInterpolation`` - - Interpolation behaviour may be customized by providing a custom handler - through the *interpolation* argument. ``None`` can be used to turn off - interpolation completely, ``ExtendedInterpolation()`` provides a more - advanced variant inspired by ``zc.buildout``. More on the subject in the - `dedicated documentation section <#interpolation-of-values>`_. - :class:`RawConfigParser` has a default value of ``None``. - -* *converters*, default value: not set - - Config parsers provide option value getters that perform type conversion. By - default :meth:`~ConfigParser.getint`, :meth:`~ConfigParser.getfloat`, and - :meth:`~ConfigParser.getboolean` are implemented. Should other getters be - desirable, users may define them in a subclass or pass a dictionary where each - key is a name of the converter and each value is a callable implementing said - conversion. For instance, passing ``{'decimal': decimal.Decimal}`` would add - :meth:`getdecimal` on both the parser object and all section proxies. In - other words, it will be possible to write both - ``parser_instance.getdecimal('section', 'key', fallback=0)`` and - ``parser_instance['section'].getdecimal('key', 0)``. - - If the converter needs to access the state of the parser, it can be - implemented as a method on a config parser subclass. If the name of this - method starts with ``get``, it will be available on all section proxies, in - the dict-compatible form (see the ``getdecimal()`` example above). - -More advanced customization may be achieved by overriding default values of -these parser attributes. The defaults are defined on the classes, so they may -be overridden by subclasses or by attribute assignment. - -.. attribute:: ConfigParser.BOOLEAN_STATES - - By default when using :meth:`~ConfigParser.getboolean`, config parsers - consider the following values ``True``: ``'1'``, ``'yes'``, ``'true'``, - ``'on'`` and the following values ``False``: ``'0'``, ``'no'``, ``'false'``, - ``'off'``. You can override this by specifying a custom dictionary of strings - and their Boolean outcomes. For example: - - .. doctest:: - - >>> custom = configparser.ConfigParser() - >>> custom['section1'] = {'funky': 'nope'} - >>> custom['section1'].getboolean('funky') - Traceback (most recent call last): - ... - ValueError: Not a boolean: nope - >>> custom.BOOLEAN_STATES = {'sure': True, 'nope': False} - >>> custom['section1'].getboolean('funky') - False - - Other typical Boolean pairs include ``accept``/``reject`` or - ``enabled``/``disabled``. - -.. method:: ConfigParser.optionxform(option) - :noindex: - - This method transforms option names on every read, get, or set - operation. The default converts the name to lowercase. This also - means that when a configuration file gets written, all keys will be - lowercase. Override this method if that's unsuitable. - For example: - - .. doctest:: - - >>> config = """ - ... [Section1] - ... Key = Value - ... - ... [Section2] - ... AnotherKey = Value - ... """ - >>> typical = configparser.ConfigParser() - >>> typical.read_string(config) - >>> list(typical['Section1'].keys()) - ['key'] - >>> list(typical['Section2'].keys()) - ['anotherkey'] - >>> custom = configparser.RawConfigParser() - >>> custom.optionxform = lambda option: option - >>> custom.read_string(config) - >>> list(custom['Section1'].keys()) - ['Key'] - >>> list(custom['Section2'].keys()) - ['AnotherKey'] - - .. note:: - The optionxform function transforms option names to a canonical form. - This should be an idempotent function: if the name is already in - canonical form, it should be returned unchanged. - - -.. attribute:: ConfigParser.SECTCRE - - A compiled regular expression used to parse section headers. The default - matches ``[section]`` to the name ``"section"``. Whitespace is considered - part of the section name, thus ``[ larch ]`` will be read as a section of - name ``" larch "``. Override this attribute if that's unsuitable. For - example: - - .. doctest:: - - >>> import re - >>> config = """ - ... [Section 1] - ... option = value - ... - ... [ Section 2 ] - ... another = val - ... """ - >>> typical = configparser.ConfigParser() - >>> typical.read_string(config) - >>> typical.sections() - ['Section 1', ' Section 2 '] - >>> custom = configparser.ConfigParser() - >>> custom.SECTCRE = re.compile(r"\[ *(?P

    [^]]+?) *\]") - >>> custom.read_string(config) - >>> custom.sections() - ['Section 1', 'Section 2'] - - .. note:: - - While ConfigParser objects also use an ``OPTCRE`` attribute for recognizing - option lines, it's not recommended to override it because that would - interfere with constructor options *allow_no_value* and *delimiters*. - - -Legacy API Examples -------------------- - -Mainly because of backwards compatibility concerns, :mod:`configparser` -provides also a legacy API with explicit ``get``/``set`` methods. While there -are valid use cases for the methods outlined below, mapping protocol access is -preferred for new projects. The legacy API is at times more advanced, -low-level and downright counterintuitive. - -An example of writing to a configuration file:: - - import configparser - - config = configparser.RawConfigParser() - - # Please note that using RawConfigParser's set functions, you can assign - # non-string values to keys internally, but will receive an error when - # attempting to write to a file or when you get it in non-raw mode. Setting - # values using the mapping protocol or ConfigParser's set() does not allow - # such assignments to take place. - config.add_section('Section1') - config.set('Section1', 'an_int', '15') - config.set('Section1', 'a_bool', 'true') - config.set('Section1', 'a_float', '3.1415') - config.set('Section1', 'baz', 'fun') - config.set('Section1', 'bar', 'Python') - config.set('Section1', 'foo', '%(bar)s is %(baz)s!') - - # Writing our configuration file to 'example.cfg' - with open('example.cfg', 'w') as configfile: - config.write(configfile) - -An example of reading the configuration file again:: - - import configparser - - config = configparser.RawConfigParser() - config.read('example.cfg') - - # getfloat() raises an exception if the value is not a float - # getint() and getboolean() also do this for their respective types - a_float = config.getfloat('Section1', 'a_float') - an_int = config.getint('Section1', 'an_int') - print(a_float + an_int) - - # Notice that the next output does not interpolate '%(bar)s' or '%(baz)s'. - # This is because we are using a RawConfigParser(). - if config.getboolean('Section1', 'a_bool'): - print(config.get('Section1', 'foo')) - -To get interpolation, use :class:`ConfigParser`:: - - import configparser - - cfg = configparser.ConfigParser() - cfg.read('example.cfg') - - # Set the optional *raw* argument of get() to True if you wish to disable - # interpolation in a single get operation. - print(cfg.get('Section1', 'foo', raw=False)) # -> "Python is fun!" - print(cfg.get('Section1', 'foo', raw=True)) # -> "%(bar)s is %(baz)s!" - - # The optional *vars* argument is a dict with members that will take - # precedence in interpolation. - print(cfg.get('Section1', 'foo', vars={'bar': 'Documentation', - 'baz': 'evil'})) - - # The optional *fallback* argument can be used to provide a fallback value - print(cfg.get('Section1', 'foo')) - # -> "Python is fun!" - - print(cfg.get('Section1', 'foo', fallback='Monty is not.')) - # -> "Python is fun!" - - print(cfg.get('Section1', 'monster', fallback='No such things as monsters.')) - # -> "No such things as monsters." - - # A bare print(cfg.get('Section1', 'monster')) would raise NoOptionError - # but we can also use: - - print(cfg.get('Section1', 'monster', fallback=None)) - # -> None - -Default values are available in both types of ConfigParsers. They are used in -interpolation if an option used is not defined elsewhere. :: - - import configparser - - # New instance with 'bar' and 'baz' defaulting to 'Life' and 'hard' each - config = configparser.ConfigParser({'bar': 'Life', 'baz': 'hard'}) - config.read('example.cfg') - - print(config.get('Section1', 'foo')) # -> "Python is fun!" - config.remove_option('Section1', 'bar') - config.remove_option('Section1', 'baz') - print(config.get('Section1', 'foo')) # -> "Life is hard!" - - -.. _configparser-objects: - -ConfigParser Objects --------------------- - -.. class:: ConfigParser(defaults=None, dict_type=dict, allow_no_value=False, delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=configparser.DEFAULTSECT, interpolation=BasicInterpolation(), converters={}) - - The main configuration parser. When *defaults* is given, it is initialized - into the dictionary of intrinsic defaults. When *dict_type* is given, it - will be used to create the dictionary objects for the list of sections, for - the options within a section, and for the default values. - - When *delimiters* is given, it is used as the set of substrings that - divide keys from values. When *comment_prefixes* is given, it will be used - as the set of substrings that prefix comments in otherwise empty lines. - Comments can be indented. When *inline_comment_prefixes* is given, it will - be used as the set of substrings that prefix comments in non-empty lines. - - When *strict* is ``True`` (the default), the parser won't allow for - any section or option duplicates while reading from a single source (file, - string or dictionary), raising :exc:`DuplicateSectionError` or - :exc:`DuplicateOptionError`. When *empty_lines_in_values* is ``False`` - (default: ``True``), each empty line marks the end of an option. Otherwise, - internal empty lines of a multiline option are kept as part of the value. - When *allow_no_value* is ``True`` (default: ``False``), options without - values are accepted; the value held for these is ``None`` and they are - serialized without the trailing delimiter. - - When *default_section* is given, it specifies the name for the special - section holding default values for other sections and interpolation purposes - (normally named ``"DEFAULT"``). This value can be retrieved and changed at - runtime using the ``default_section`` instance attribute. This won't - re-evaluate an already parsed config file, but will be used when writing - parsed settings to a new config file. - - Interpolation behaviour may be customized by providing a custom handler - through the *interpolation* argument. ``None`` can be used to turn off - interpolation completely, ``ExtendedInterpolation()`` provides a more - advanced variant inspired by ``zc.buildout``. More on the subject in the - `dedicated documentation section <#interpolation-of-values>`_. - - All option names used in interpolation will be passed through the - :meth:`optionxform` method just like any other option name reference. For - example, using the default implementation of :meth:`optionxform` (which - converts option names to lower case), the values ``foo %(bar)s`` and ``foo - %(BAR)s`` are equivalent. - - When *converters* is given, it should be a dictionary where each key - represents the name of a type converter and each value is a callable - implementing the conversion from string to the desired datatype. Every - converter gets its own corresponding :meth:`get*()` method on the parser - object and section proxies. - - .. versionchanged:: 3.1 - The default *dict_type* is :class:`collections.OrderedDict`. - - .. versionchanged:: 3.2 - *allow_no_value*, *delimiters*, *comment_prefixes*, *strict*, - *empty_lines_in_values*, *default_section* and *interpolation* were - added. - - .. versionchanged:: 3.5 - The *converters* argument was added. - - .. versionchanged:: 3.7 - The *defaults* argument is read with :meth:`read_dict()`, - providing consistent behavior across the parser: non-string - keys and values are implicitly converted to strings. - - .. versionchanged:: 3.8 - The default *dict_type* is :class:`dict`, since it now preserves - insertion order. - - .. method:: defaults() - - Return a dictionary containing the instance-wide defaults. - - - .. method:: sections() - - Return a list of the sections available; the *default section* is not - included in the list. - - - .. method:: add_section(section) - - Add a section named *section* to the instance. If a section by the given - name already exists, :exc:`DuplicateSectionError` is raised. If the - *default section* name is passed, :exc:`ValueError` is raised. The name - of the section must be a string; if not, :exc:`TypeError` is raised. - - .. versionchanged:: 3.2 - Non-string section names raise :exc:`TypeError`. - - - .. method:: has_section(section) - - Indicates whether the named *section* is present in the configuration. - The *default section* is not acknowledged. - - - .. method:: options(section) - - Return a list of options available in the specified *section*. - - - .. method:: has_option(section, option) - - If the given *section* exists, and contains the given *option*, return - :const:`True`; otherwise return :const:`False`. If the specified - *section* is :const:`None` or an empty string, DEFAULT is assumed. - - - .. method:: read(filenames, encoding=None) - - Attempt to read and parse an iterable of filenames, returning a list of - filenames which were successfully parsed. - - If *filenames* is a string, a :class:`bytes` object or a - :term:`path-like object`, it is treated as - a single filename. If a file named in *filenames* cannot be opened, that - file will be ignored. This is designed so that you can specify an - iterable of potential configuration file locations (for example, the - current directory, the user's home directory, and some system-wide - directory), and all existing configuration files in the iterable will be - read. - - If none of the named files exist, the :class:`ConfigParser` - instance will contain an empty dataset. An application which requires - initial values to be loaded from a file should load the required file or - files using :meth:`read_file` before calling :meth:`read` for any - optional files:: - - import configparser, os - - config = configparser.ConfigParser() - config.read_file(open('defaults.cfg')) - config.read(['site.cfg', os.path.expanduser('~/.myapp.cfg')], - encoding='cp1250') - - .. versionadded:: 3.2 - The *encoding* parameter. Previously, all files were read using the - default encoding for :func:`open`. - - .. versionadded:: 3.6.1 - The *filenames* parameter accepts a :term:`path-like object`. - - .. versionadded:: 3.7 - The *filenames* parameter accepts a :class:`bytes` object. - - - .. method:: read_file(f, source=None) - - Read and parse configuration data from *f* which must be an iterable - yielding Unicode strings (for example files opened in text mode). - - Optional argument *source* specifies the name of the file being read. If - not given and *f* has a :attr:`name` attribute, that is used for - *source*; the default is ``''``. - - .. versionadded:: 3.2 - Replaces :meth:`readfp`. - - .. method:: read_string(string, source='') - - Parse configuration data from a string. - - Optional argument *source* specifies a context-specific name of the - string passed. If not given, ``''`` is used. This should - commonly be a filesystem path or a URL. - - .. versionadded:: 3.2 - - - .. method:: read_dict(dictionary, source='') - - Load configuration from any object that provides a dict-like ``items()`` - method. Keys are section names, values are dictionaries with keys and - values that should be present in the section. If the used dictionary - type preserves order, sections and their keys will be added in order. - Values are automatically converted to strings. - - Optional argument *source* specifies a context-specific name of the - dictionary passed. If not given, ```` is used. - - This method can be used to copy state between parsers. - - .. versionadded:: 3.2 - - - .. method:: get(section, option, *, raw=False, vars=None[, fallback]) - - Get an *option* value for the named *section*. If *vars* is provided, it - must be a dictionary. The *option* is looked up in *vars* (if provided), - *section*, and in *DEFAULTSECT* in that order. If the key is not found - and *fallback* is provided, it is used as a fallback value. ``None`` can - be provided as a *fallback* value. - - All the ``'%'`` interpolations are expanded in the return values, unless - the *raw* argument is true. Values for interpolation keys are looked up - in the same manner as the option. - - .. versionchanged:: 3.2 - Arguments *raw*, *vars* and *fallback* are keyword only to protect - users from trying to use the third argument as the *fallback* fallback - (especially when using the mapping protocol). - - - .. method:: getint(section, option, *, raw=False, vars=None[, fallback]) - - A convenience method which coerces the *option* in the specified *section* - to an integer. See :meth:`get` for explanation of *raw*, *vars* and - *fallback*. - - - .. method:: getfloat(section, option, *, raw=False, vars=None[, fallback]) - - A convenience method which coerces the *option* in the specified *section* - to a floating point number. See :meth:`get` for explanation of *raw*, - *vars* and *fallback*. - - - .. method:: getboolean(section, option, *, raw=False, vars=None[, fallback]) - - A convenience method which coerces the *option* in the specified *section* - to a Boolean value. Note that the accepted values for the option are - ``'1'``, ``'yes'``, ``'true'``, and ``'on'``, which cause this method to - return ``True``, and ``'0'``, ``'no'``, ``'false'``, and ``'off'``, which - cause it to return ``False``. These string values are checked in a - case-insensitive manner. Any other value will cause it to raise - :exc:`ValueError`. See :meth:`get` for explanation of *raw*, *vars* and - *fallback*. - - - .. method:: items(raw=False, vars=None) - items(section, raw=False, vars=None) - - When *section* is not given, return a list of *section_name*, - *section_proxy* pairs, including DEFAULTSECT. - - Otherwise, return a list of *name*, *value* pairs for the options in the - given *section*. Optional arguments have the same meaning as for the - :meth:`get` method. - - .. versionchanged:: 3.8 - Items present in *vars* no longer appear in the result. The previous - behaviour mixed actual parser options with variables provided for - interpolation. - - - .. method:: set(section, option, value) - - If the given section exists, set the given option to the specified value; - otherwise raise :exc:`NoSectionError`. *option* and *value* must be - strings; if not, :exc:`TypeError` is raised. - - - .. method:: write(fileobject, space_around_delimiters=True) - - Write a representation of the configuration to the specified :term:`file - object`, which must be opened in text mode (accepting strings). This - representation can be parsed by a future :meth:`read` call. If - *space_around_delimiters* is true, delimiters between - keys and values are surrounded by spaces. - - .. note:: - - Comments in the original configuration file are not preserved when - writing the configuration back. - What is considered a comment, depends on the given values for - *comment_prefix* and *inline_comment_prefix*. - - - .. method:: remove_option(section, option) - - Remove the specified *option* from the specified *section*. If the - section does not exist, raise :exc:`NoSectionError`. If the option - existed to be removed, return :const:`True`; otherwise return - :const:`False`. - - - .. method:: remove_section(section) - - Remove the specified *section* from the configuration. If the section in - fact existed, return ``True``. Otherwise return ``False``. - - - .. method:: optionxform(option) - - Transforms the option name *option* as found in an input file or as passed - in by client code to the form that should be used in the internal - structures. The default implementation returns a lower-case version of - *option*; subclasses may override this or client code can set an attribute - of this name on instances to affect this behavior. - - You don't need to subclass the parser to use this method, you can also - set it on an instance, to a function that takes a string argument and - returns a string. Setting it to ``str``, for example, would make option - names case sensitive:: - - cfgparser = ConfigParser() - cfgparser.optionxform = str - - Note that when reading configuration files, whitespace around the option - names is stripped before :meth:`optionxform` is called. - - - .. method:: readfp(fp, filename=None) - - .. deprecated:: 3.2 - Use :meth:`read_file` instead. - - .. versionchanged:: 3.2 - :meth:`readfp` now iterates on *fp* instead of calling ``fp.readline()``. - - For existing code calling :meth:`readfp` with arguments which don't - support iteration, the following generator may be used as a wrapper - around the file-like object:: - - def readline_generator(fp): - line = fp.readline() - while line: - yield line - line = fp.readline() - - Instead of ``parser.readfp(fp)`` use - ``parser.read_file(readline_generator(fp))``. - - -.. data:: MAX_INTERPOLATION_DEPTH - - The maximum depth for recursive interpolation for :meth:`get` when the *raw* - parameter is false. This is relevant only when the default *interpolation* - is used. - - -.. _rawconfigparser-objects: - -RawConfigParser Objects ------------------------ - -.. class:: RawConfigParser(defaults=None, dict_type=dict, \ - allow_no_value=False, *, delimiters=('=', ':'), \ - comment_prefixes=('#', ';'), \ - inline_comment_prefixes=None, strict=True, \ - empty_lines_in_values=True, \ - default_section=configparser.DEFAULTSECT[, \ - interpolation]) - - Legacy variant of the :class:`ConfigParser`. It has interpolation - disabled by default and allows for non-string section names, option - names, and values via its unsafe ``add_section`` and ``set`` methods, - as well as the legacy ``defaults=`` keyword argument handling. - - .. versionchanged:: 3.8 - The default *dict_type* is :class:`dict`, since it now preserves - insertion order. - - .. note:: - Consider using :class:`ConfigParser` instead which checks types of - the values to be stored internally. If you don't want interpolation, you - can use ``ConfigParser(interpolation=None)``. - - - .. method:: add_section(section) - - Add a section named *section* to the instance. If a section by the given - name already exists, :exc:`DuplicateSectionError` is raised. If the - *default section* name is passed, :exc:`ValueError` is raised. - - Type of *section* is not checked which lets users create non-string named - sections. This behaviour is unsupported and may cause internal errors. - - - .. method:: set(section, option, value) - - If the given section exists, set the given option to the specified value; - otherwise raise :exc:`NoSectionError`. While it is possible to use - :class:`RawConfigParser` (or :class:`ConfigParser` with *raw* parameters - set to true) for *internal* storage of non-string values, full - functionality (including interpolation and output to files) can only be - achieved using string values. - - This method lets users assign non-string values to keys internally. This - behaviour is unsupported and will cause errors when attempting to write - to a file or get it in non-raw mode. **Use the mapping protocol API** - which does not allow such assignments to take place. - - -Exceptions ----------- - -.. exception:: Error - - Base class for all other :mod:`configparser` exceptions. - - -.. exception:: NoSectionError - - Exception raised when a specified section is not found. - - -.. exception:: DuplicateSectionError - - Exception raised if :meth:`add_section` is called with the name of a section - that is already present or in strict parsers when a section if found more - than once in a single input file, string or dictionary. - - .. versionadded:: 3.2 - Optional ``source`` and ``lineno`` attributes and arguments to - :meth:`__init__` were added. - - -.. exception:: DuplicateOptionError - - Exception raised by strict parsers if a single option appears twice during - reading from a single file, string or dictionary. This catches misspellings - and case sensitivity-related errors, e.g. a dictionary may have two keys - representing the same case-insensitive configuration key. - - -.. exception:: NoOptionError - - Exception raised when a specified option is not found in the specified - section. - - -.. exception:: InterpolationError - - Base class for exceptions raised when problems occur performing string - interpolation. - - -.. exception:: InterpolationDepthError - - Exception raised when string interpolation cannot be completed because the - number of iterations exceeds :const:`MAX_INTERPOLATION_DEPTH`. Subclass of - :exc:`InterpolationError`. - - -.. exception:: InterpolationMissingOptionError - - Exception raised when an option referenced from a value does not exist. - Subclass of :exc:`InterpolationError`. - - -.. exception:: InterpolationSyntaxError - - Exception raised when the source text into which substitutions are made does - not conform to the required syntax. Subclass of :exc:`InterpolationError`. - - -.. exception:: MissingSectionHeaderError - - Exception raised when attempting to parse a file which has no section - headers. - - -.. exception:: ParsingError - - Exception raised when errors occur attempting to parse a file. - - .. versionchanged:: 3.2 - The ``filename`` attribute and :meth:`__init__` argument were renamed to - ``source`` for consistency. - - -.. rubric:: Footnotes - -.. [1] Config parsers allow for heavy customization. If you are interested in - changing the behaviour outlined by the footnote reference, consult the - `Customizing Parser Behaviour`_ section. diff --git a/Doc/library/constants.rst b/Doc/library/constants.rst deleted file mode 100644 index 401dc9a320c5e0..00000000000000 --- a/Doc/library/constants.rst +++ /dev/null @@ -1,107 +0,0 @@ -.. _built-in-consts: - -Built-in Constants -================== - -A small number of constants live in the built-in namespace. They are: - -.. data:: False - - The false value of the :class:`bool` type. Assignments to ``False`` - are illegal and raise a :exc:`SyntaxError`. - - -.. data:: True - - The true value of the :class:`bool` type. Assignments to ``True`` - are illegal and raise a :exc:`SyntaxError`. - - -.. data:: None - - An object frequently used to represent the absence of a value, as when - default arguments are not passed to a function. Assignments to ``None`` - are illegal and raise a :exc:`SyntaxError`. - ``None`` is the sole instance of the :data:`~types.NoneType` type. - - -.. data:: NotImplemented - - A special value which should be returned by the binary special methods - (e.g. :meth:`~object.__eq__`, :meth:`~object.__lt__`, :meth:`~object.__add__`, :meth:`~object.__rsub__`, - etc.) to indicate that the operation is not implemented with respect to - the other type; may be returned by the in-place binary special methods - (e.g. :meth:`~object.__imul__`, :meth:`~object.__iand__`, etc.) for the same purpose. - It should not be evaluated in a boolean context. - ``NotImplemented`` is the sole instance of the :data:`types.NotImplementedType` type. - - .. note:: - - When a binary (or in-place) method returns ``NotImplemented`` the - interpreter will try the reflected operation on the other type (or some - other fallback, depending on the operator). If all attempts return - ``NotImplemented``, the interpreter will raise an appropriate exception. - Incorrectly returning ``NotImplemented`` will result in a misleading - error message or the ``NotImplemented`` value being returned to Python code. - - See :ref:`implementing-the-arithmetic-operations` for examples. - - .. note:: - - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, - even though they have similar names and purposes. - See :exc:`NotImplementedError` for details on when to use it. - - .. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. - - -.. index:: single: ...; ellipsis literal -.. data:: Ellipsis - - The same as the ellipsis literal "``...``". Special value used mostly in conjunction - with extended slicing syntax for user-defined container data types. - ``Ellipsis`` is the sole instance of the :data:`types.EllipsisType` type. - - -.. data:: __debug__ - - This constant is true if Python was not started with an :option:`-O` option. - See also the :keyword:`assert` statement. - - -.. note:: - - The names :data:`None`, :data:`False`, :data:`True` and :data:`__debug__` - cannot be reassigned (assignments to them, even as an attribute name, raise - :exc:`SyntaxError`), so they can be considered "true" constants. - - -Constants added by the :mod:`site` module ------------------------------------------ - -The :mod:`site` module (which is imported automatically during startup, except -if the :option:`-S` command-line option is given) adds several constants to the -built-in namespace. They are useful for the interactive interpreter shell and -should not be used in programs. - -.. data:: quit(code=None) - exit(code=None) - - Objects that when printed, print a message like "Use quit() or Ctrl-D - (i.e. EOF) to exit", and when called, raise :exc:`SystemExit` with the - specified exit code. - -.. data:: copyright - credits - - Objects that when printed or called, print the text of copyright or - credits, respectively. - -.. data:: license - - Object that when printed, prints the message "Type license() to see the - full license text", and when called, displays the full license text in a - pager-like fashion (one screen at a time). diff --git a/Doc/library/contextlib.rst b/Doc/library/contextlib.rst deleted file mode 100644 index 80a683d4756fe8..00000000000000 --- a/Doc/library/contextlib.rst +++ /dev/null @@ -1,1025 +0,0 @@ -:mod:`!contextlib` --- Utilities for :keyword:`!with`\ -statement contexts -========================================================================== - -.. module:: contextlib - :synopsis: Utilities for with-statement contexts. - -**Source code:** :source:`Lib/contextlib.py` - --------------- - -This module provides utilities for common tasks involving the :keyword:`with` -statement. For more information see also :ref:`typecontextmanager` and -:ref:`context-managers`. - - -Utilities ---------- - -Functions and classes provided: - -.. class:: AbstractContextManager - - An :term:`abstract base class` for classes that implement - :meth:`object.__enter__` and :meth:`object.__exit__`. A default - implementation for :meth:`object.__enter__` is provided which returns - ``self`` while :meth:`object.__exit__` is an abstract method which by default - returns ``None``. See also the definition of :ref:`typecontextmanager`. - - .. versionadded:: 3.6 - - -.. class:: AbstractAsyncContextManager - - An :term:`abstract base class` for classes that implement - :meth:`object.__aenter__` and :meth:`object.__aexit__`. A default - implementation for :meth:`object.__aenter__` is provided which returns - ``self`` while :meth:`object.__aexit__` is an abstract method which by default - returns ``None``. See also the definition of - :ref:`async-context-managers`. - - .. versionadded:: 3.7 - - -.. decorator:: contextmanager - - This function is a :term:`decorator` that can be used to define a factory - function for :keyword:`with` statement context managers, without needing to - create a class or separate :meth:`~object.__enter__` and :meth:`~object.__exit__` methods. - - While many objects natively support use in with statements, sometimes a - resource needs to be managed that isn't a context manager in its own right, - and doesn't implement a ``close()`` method for use with ``contextlib.closing`` - - An abstract example would be the following to ensure correct resource - management:: - - from contextlib import contextmanager - - @contextmanager - def managed_resource(*args, **kwds): - # Code to acquire resource, e.g.: - resource = acquire_resource(*args, **kwds) - try: - yield resource - finally: - # Code to release resource, e.g.: - release_resource(resource) - - The function can then be used like this:: - - >>> with managed_resource(timeout=3600) as resource: - ... # Resource is released at the end of this block, - ... # even if code in the block raises an exception - - The function being decorated must return a :term:`generator`-iterator when - called. This iterator must yield exactly one value, which will be bound to - the targets in the :keyword:`with` statement's :keyword:`!as` clause, if any. - - At the point where the generator yields, the block nested in the :keyword:`with` - statement is executed. The generator is then resumed after the block is exited. - If an unhandled exception occurs in the block, it is reraised inside the - generator at the point where the yield occurred. Thus, you can use a - :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally` statement to trap - the error (if any), or ensure that some cleanup takes place. If an exception is - trapped merely in order to log it or to perform some action (rather than to - suppress it entirely), the generator must reraise that exception. Otherwise the - generator context manager will indicate to the :keyword:`!with` statement that - the exception has been handled, and execution will resume with the statement - immediately following the :keyword:`!with` statement. - - :func:`contextmanager` uses :class:`ContextDecorator` so the context managers - it creates can be used as decorators as well as in :keyword:`with` statements. - When used as a decorator, a new generator instance is implicitly created on - each function call (this allows the otherwise "one-shot" context managers - created by :func:`contextmanager` to meet the requirement that context - managers support multiple invocations in order to be used as decorators). - - .. versionchanged:: 3.2 - Use of :class:`ContextDecorator`. - - -.. decorator:: asynccontextmanager - - Similar to :func:`~contextlib.contextmanager`, but creates an - :ref:`asynchronous context manager `. - - This function is a :term:`decorator` that can be used to define a factory - function for :keyword:`async with` statement asynchronous context managers, - without needing to create a class or separate :meth:`__aenter__` and - :meth:`__aexit__` methods. It must be applied to an :term:`asynchronous - generator` function. - - A simple example:: - - from contextlib import asynccontextmanager - - @asynccontextmanager - async def get_connection(): - conn = await acquire_db_connection() - try: - yield conn - finally: - await release_db_connection(conn) - - async def get_all_users(): - async with get_connection() as conn: - return conn.query('SELECT ...') - - .. versionadded:: 3.7 - - Context managers defined with :func:`asynccontextmanager` can be used - either as decorators or with :keyword:`async with` statements:: - - import time - from contextlib import asynccontextmanager - - @asynccontextmanager - async def timeit(): - now = time.monotonic() - try: - yield - finally: - print(f'it took {time.monotonic() - now}s to run') - - @timeit() - async def main(): - # ... async code ... - - When used as a decorator, a new generator instance is implicitly created on - each function call. This allows the otherwise "one-shot" context managers - created by :func:`asynccontextmanager` to meet the requirement that context - managers support multiple invocations in order to be used as decorators. - - .. versionchanged:: 3.10 - Async context managers created with :func:`asynccontextmanager` can - be used as decorators. - - -.. function:: closing(thing) - - Return a context manager that closes *thing* upon completion of the block. This - is basically equivalent to:: - - from contextlib import contextmanager - - @contextmanager - def closing(thing): - try: - yield thing - finally: - thing.close() - - And lets you write code like this:: - - from contextlib import closing - from urllib.request import urlopen - - with closing(urlopen('https://www.python.org')) as page: - for line in page: - print(line) - - without needing to explicitly close ``page``. Even if an error occurs, - ``page.close()`` will be called when the :keyword:`with` block is exited. - - -.. function:: aclosing(thing) - - Return an async context manager that calls the ``aclose()`` method of *thing* - upon completion of the block. This is basically equivalent to:: - - from contextlib import asynccontextmanager - - @asynccontextmanager - async def aclosing(thing): - try: - yield thing - finally: - await thing.aclose() - - Significantly, ``aclosing()`` supports deterministic cleanup of async - generators when they happen to exit early by :keyword:`break` or an - exception. For example:: - - from contextlib import aclosing - - async with aclosing(my_generator()) as values: - async for value in values: - if value == 42: - break - - This pattern ensures that the generator's async exit code is executed in - the same context as its iterations (so that exceptions and context - variables work as expected, and the exit code isn't run after the - lifetime of some task it depends on). - - .. versionadded:: 3.10 - - -.. _simplifying-support-for-single-optional-context-managers: - -.. function:: nullcontext(enter_result=None) - - Return a context manager that returns *enter_result* from ``__enter__``, but - otherwise does nothing. It is intended to be used as a stand-in for an - optional context manager, for example:: - - def myfunction(arg, ignore_exceptions=False): - if ignore_exceptions: - # Use suppress to ignore all exceptions. - cm = contextlib.suppress(Exception) - else: - # Do not ignore any exceptions, cm has no effect. - cm = contextlib.nullcontext() - with cm: - # Do something - - An example using *enter_result*:: - - def process_file(file_or_path): - if isinstance(file_or_path, str): - # If string, open file - cm = open(file_or_path) - else: - # Caller is responsible for closing file - cm = nullcontext(file_or_path) - - with cm as file: - # Perform processing on the file - - It can also be used as a stand-in for - :ref:`asynchronous context managers `:: - - async def send_http(session=None): - if not session: - # If no http session, create it with aiohttp - cm = aiohttp.ClientSession() - else: - # Caller is responsible for closing the session - cm = nullcontext(session) - - async with cm as session: - # Send http requests with session - - .. versionadded:: 3.7 - - .. versionchanged:: 3.10 - :term:`asynchronous context manager` support was added. - - - -.. function:: suppress(*exceptions) - - Return a context manager that suppresses any of the specified exceptions - if they occur in the body of a :keyword:`!with` statement and then - resumes execution with the first statement following the end of the - :keyword:`!with` statement. - - As with any other mechanism that completely suppresses exceptions, this - context manager should be used only to cover very specific errors where - silently continuing with program execution is known to be the right - thing to do. - - For example:: - - from contextlib import suppress - - with suppress(FileNotFoundError): - os.remove('somefile.tmp') - - with suppress(FileNotFoundError): - os.remove('someotherfile.tmp') - - This code is equivalent to:: - - try: - os.remove('somefile.tmp') - except FileNotFoundError: - pass - - try: - os.remove('someotherfile.tmp') - except FileNotFoundError: - pass - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.4 - - -.. function:: redirect_stdout(new_target) - - Context manager for temporarily redirecting :data:`sys.stdout` to - another file or file-like object. - - This tool adds flexibility to existing functions or classes whose output - is hardwired to stdout. - - For example, the output of :func:`help` normally is sent to *sys.stdout*. - You can capture that output in a string by redirecting the output to an - :class:`io.StringIO` object. The replacement stream is returned from the - ``__enter__`` method and so is available as the target of the - :keyword:`with` statement:: - - with redirect_stdout(io.StringIO()) as f: - help(pow) - s = f.getvalue() - - To send the output of :func:`help` to a file on disk, redirect the output - to a regular file:: - - with open('help.txt', 'w') as f: - with redirect_stdout(f): - help(pow) - - To send the output of :func:`help` to *sys.stderr*:: - - with redirect_stdout(sys.stderr): - help(pow) - - Note that the global side effect on :data:`sys.stdout` means that this - context manager is not suitable for use in library code and most threaded - applications. It also has no effect on the output of subprocesses. - However, it is still a useful approach for many utility scripts. - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.4 - - -.. function:: redirect_stderr(new_target) - - Similar to :func:`~contextlib.redirect_stdout` but redirecting - :data:`sys.stderr` to another file or file-like object. - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.5 - - -.. function:: chdir(path) - - Non parallel-safe context manager to change the current working directory. - As this changes a global state, the working directory, it is not suitable - for use in most threaded or async contexts. It is also not suitable for most - non-linear code execution, like generators, where the program execution is - temporarily relinquished -- unless explicitly desired, you should not yield - when this context manager is active. - - This is a simple wrapper around :func:`~os.chdir`, it changes the current - working directory upon entering and restores the old one on exit. - - This context manager is :ref:`reentrant `. - - .. versionadded:: 3.11 - - -.. class:: ContextDecorator() - - A base class that enables a context manager to also be used as a decorator. - - Context managers inheriting from ``ContextDecorator`` have to implement - ``__enter__`` and ``__exit__`` as normal. ``__exit__`` retains its optional - exception handling even when used as a decorator. - - ``ContextDecorator`` is used by :func:`contextmanager`, so you get this - functionality automatically. - - Example of ``ContextDecorator``:: - - from contextlib import ContextDecorator - - class mycontext(ContextDecorator): - def __enter__(self): - print('Starting') - return self - - def __exit__(self, *exc): - print('Finishing') - return False - - The class can then be used like this:: - - >>> @mycontext() - ... def function(): - ... print('The bit in the middle') - ... - >>> function() - Starting - The bit in the middle - Finishing - - >>> with mycontext(): - ... print('The bit in the middle') - ... - Starting - The bit in the middle - Finishing - - This change is just syntactic sugar for any construct of the following form:: - - def f(): - with cm(): - # Do stuff - - ``ContextDecorator`` lets you instead write:: - - @cm() - def f(): - # Do stuff - - It makes it clear that the ``cm`` applies to the whole function, rather than - just a piece of it (and saving an indentation level is nice, too). - - Existing context managers that already have a base class can be extended by - using ``ContextDecorator`` as a mixin class:: - - from contextlib import ContextDecorator - - class mycontext(ContextBaseClass, ContextDecorator): - def __enter__(self): - return self - - def __exit__(self, *exc): - return False - - .. note:: - As the decorated function must be able to be called multiple times, the - underlying context manager must support use in multiple :keyword:`with` - statements. If this is not the case, then the original construct with the - explicit :keyword:`!with` statement inside the function should be used. - - .. versionadded:: 3.2 - - -.. class:: AsyncContextDecorator - - Similar to :class:`ContextDecorator` but only for asynchronous functions. - - Example of ``AsyncContextDecorator``:: - - from asyncio import run - from contextlib import AsyncContextDecorator - - class mycontext(AsyncContextDecorator): - async def __aenter__(self): - print('Starting') - return self - - async def __aexit__(self, *exc): - print('Finishing') - return False - - The class can then be used like this:: - - >>> @mycontext() - ... async def function(): - ... print('The bit in the middle') - ... - >>> run(function()) - Starting - The bit in the middle - Finishing - - >>> async def function(): - ... async with mycontext(): - ... print('The bit in the middle') - ... - >>> run(function()) - Starting - The bit in the middle - Finishing - - .. versionadded:: 3.10 - - -.. class:: ExitStack() - - A context manager that is designed to make it easy to programmatically - combine other context managers and cleanup functions, especially those - that are optional or otherwise driven by input data. - - For example, a set of files may easily be handled in a single with - statement as follows:: - - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # All opened files will automatically be closed at the end of - # the with statement, even if attempts to open files later - # in the list raise an exception - - The :meth:`~object.__enter__` method returns the :class:`ExitStack` instance, and - performs no additional operations. - - Each instance maintains a stack of registered callbacks that are called in - reverse order when the instance is closed (either explicitly or implicitly - at the end of a :keyword:`with` statement). Note that callbacks are *not* - invoked implicitly when the context stack instance is garbage collected. - - This stack model is used so that context managers that acquire their - resources in their ``__init__`` method (such as file objects) can be - handled correctly. - - Since registered callbacks are invoked in the reverse order of - registration, this ends up behaving as if multiple nested :keyword:`with` - statements had been used with the registered set of callbacks. This even - extends to exception handling - if an inner callback suppresses or replaces - an exception, then outer callbacks will be passed arguments based on that - updated state. - - This is a relatively low level API that takes care of the details of - correctly unwinding the stack of exit callbacks. It provides a suitable - foundation for higher level context managers that manipulate the exit - stack in application specific ways. - - .. versionadded:: 3.3 - - .. method:: enter_context(cm) - - Enters a new context manager and adds its :meth:`~object.__exit__` method to - the callback stack. The return value is the result of the context - manager's own :meth:`~object.__enter__` method. - - These context managers may suppress exceptions just as they normally - would if used directly as part of a :keyword:`with` statement. - - .. versionchanged:: 3.11 - Raises :exc:`TypeError` instead of :exc:`AttributeError` if *cm* - is not a context manager. - - .. method:: push(exit) - - Adds a context manager's :meth:`~object.__exit__` method to the callback stack. - - As ``__enter__`` is *not* invoked, this method can be used to cover - part of an :meth:`~object.__enter__` implementation with a context manager's own - :meth:`~object.__exit__` method. - - If passed an object that is not a context manager, this method assumes - it is a callback with the same signature as a context manager's - :meth:`~object.__exit__` method and adds it directly to the callback stack. - - By returning true values, these callbacks can suppress exceptions the - same way context manager :meth:`~object.__exit__` methods can. - - The passed in object is returned from the function, allowing this - method to be used as a function decorator. - - .. method:: callback(callback, /, *args, **kwds) - - Accepts an arbitrary callback function and arguments and adds it to - the callback stack. - - Unlike the other methods, callbacks added this way cannot suppress - exceptions (as they are never passed the exception details). - - The passed in callback is returned from the function, allowing this - method to be used as a function decorator. - - .. method:: pop_all() - - Transfers the callback stack to a fresh :class:`ExitStack` instance - and returns it. No callbacks are invoked by this operation - instead, - they will now be invoked when the new stack is closed (either - explicitly or implicitly at the end of a :keyword:`with` statement). - - For example, a group of files can be opened as an "all or nothing" - operation as follows:: - - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # Hold onto the close method, but don't call it yet. - close_files = stack.pop_all().close - # If opening any file fails, all previously opened files will be - # closed automatically. If all files are opened successfully, - # they will remain open even after the with statement ends. - # close_files() can then be invoked explicitly to close them all. - - .. method:: close() - - Immediately unwinds the callback stack, invoking callbacks in the - reverse order of registration. For any context managers and exit - callbacks registered, the arguments passed in will indicate that no - exception occurred. - -.. class:: AsyncExitStack() - - An :ref:`asynchronous context manager `, similar - to :class:`ExitStack`, that supports combining both synchronous and - asynchronous context managers, as well as having coroutines for - cleanup logic. - - The :meth:`close` method is not implemented, :meth:`aclose` must be used - instead. - - .. coroutinemethod:: enter_async_context(cm) - - Similar to :meth:`enter_context` but expects an asynchronous context - manager. - - .. versionchanged:: 3.11 - Raises :exc:`TypeError` instead of :exc:`AttributeError` if *cm* - is not an asynchronous context manager. - - .. method:: push_async_exit(exit) - - Similar to :meth:`push` but expects either an asynchronous context manager - or a coroutine function. - - .. method:: push_async_callback(callback, /, *args, **kwds) - - Similar to :meth:`callback` but expects a coroutine function. - - .. coroutinemethod:: aclose() - - Similar to :meth:`close` but properly handles awaitables. - - Continuing the example for :func:`asynccontextmanager`:: - - async with AsyncExitStack() as stack: - connections = [await stack.enter_async_context(get_connection()) - for i in range(5)] - # All opened connections will automatically be released at the end of - # the async with statement, even if attempts to open a connection - # later in the list raise an exception. - - .. versionadded:: 3.7 - -Examples and Recipes --------------------- - -This section describes some examples and recipes for making effective use of -the tools provided by :mod:`contextlib`. - - -Supporting a variable number of context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The primary use case for :class:`ExitStack` is the one given in the class -documentation: supporting a variable number of context managers and other -cleanup operations in a single :keyword:`with` statement. The variability -may come from the number of context managers needed being driven by user -input (such as opening a user specified collection of files), or from -some of the context managers being optional:: - - with ExitStack() as stack: - for resource in resources: - stack.enter_context(resource) - if need_special_resource(): - special = acquire_special_resource() - stack.callback(release_special_resource, special) - # Perform operations that use the acquired resources - -As shown, :class:`ExitStack` also makes it quite easy to use :keyword:`with` -statements to manage arbitrary resources that don't natively support the -context management protocol. - - -Catching exceptions from ``__enter__`` methods -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is occasionally desirable to catch exceptions from an ``__enter__`` -method implementation, *without* inadvertently catching exceptions from -the :keyword:`with` statement body or the context manager's ``__exit__`` -method. By using :class:`ExitStack` the steps in the context management -protocol can be separated slightly in order to allow this:: - - stack = ExitStack() - try: - x = stack.enter_context(cm) - except Exception: - # handle __enter__ exception - else: - with stack: - # Handle normal case - -Actually needing to do this is likely to indicate that the underlying API -should be providing a direct resource management interface for use with -:keyword:`try`/:keyword:`except`/:keyword:`finally` statements, but not -all APIs are well designed in that regard. When a context manager is the -only resource management API provided, then :class:`ExitStack` can make it -easier to handle various situations that can't be handled directly in a -:keyword:`with` statement. - - -Cleaning up in an ``__enter__`` implementation -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As noted in the documentation of :meth:`ExitStack.push`, this -method can be useful in cleaning up an already allocated resource if later -steps in the :meth:`~object.__enter__` implementation fail. - -Here's an example of doing this for a context manager that accepts resource -acquisition and release functions, along with an optional validation function, -and maps them to the context management protocol:: - - from contextlib import contextmanager, AbstractContextManager, ExitStack - - class ResourceManager(AbstractContextManager): - - def __init__(self, acquire_resource, release_resource, check_resource_ok=None): - self.acquire_resource = acquire_resource - self.release_resource = release_resource - if check_resource_ok is None: - def check_resource_ok(resource): - return True - self.check_resource_ok = check_resource_ok - - @contextmanager - def _cleanup_on_error(self): - with ExitStack() as stack: - stack.push(self) - yield - # The validation check passed and didn't raise an exception - # Accordingly, we want to keep the resource, and pass it - # back to our caller - stack.pop_all() - - def __enter__(self): - resource = self.acquire_resource() - with self._cleanup_on_error(): - if not self.check_resource_ok(resource): - msg = "Failed validation for {!r}" - raise RuntimeError(msg.format(resource)) - return resource - - def __exit__(self, *exc_details): - # We don't need to duplicate any of our resource release logic - self.release_resource() - - -Replacing any use of ``try-finally`` and flag variables -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A pattern you will sometimes see is a ``try-finally`` statement with a flag -variable to indicate whether or not the body of the ``finally`` clause should -be executed. In its simplest form (that can't already be handled just by -using an ``except`` clause instead), it looks something like this:: - - cleanup_needed = True - try: - result = perform_operation() - if result: - cleanup_needed = False - finally: - if cleanup_needed: - cleanup_resources() - -As with any ``try`` statement based code, this can cause problems for -development and review, because the setup code and the cleanup code can end -up being separated by arbitrarily long sections of code. - -:class:`ExitStack` makes it possible to instead register a callback for -execution at the end of a ``with`` statement, and then later decide to skip -executing that callback:: - - from contextlib import ExitStack - - with ExitStack() as stack: - stack.callback(cleanup_resources) - result = perform_operation() - if result: - stack.pop_all() - -This allows the intended cleanup up behaviour to be made explicit up front, -rather than requiring a separate flag variable. - -If a particular application uses this pattern a lot, it can be simplified -even further by means of a small helper class:: - - from contextlib import ExitStack - - class Callback(ExitStack): - def __init__(self, callback, /, *args, **kwds): - super().__init__() - self.callback(callback, *args, **kwds) - - def cancel(self): - self.pop_all() - - with Callback(cleanup_resources) as cb: - result = perform_operation() - if result: - cb.cancel() - -If the resource cleanup isn't already neatly bundled into a standalone -function, then it is still possible to use the decorator form of -:meth:`ExitStack.callback` to declare the resource cleanup in -advance:: - - from contextlib import ExitStack - - with ExitStack() as stack: - @stack.callback - def cleanup_resources(): - ... - result = perform_operation() - if result: - stack.pop_all() - -Due to the way the decorator protocol works, a callback function -declared this way cannot take any parameters. Instead, any resources to -be released must be accessed as closure variables. - - -Using a context manager as a function decorator -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`ContextDecorator` makes it possible to use a context manager in -both an ordinary ``with`` statement and also as a function decorator. - -For example, it is sometimes useful to wrap functions or groups of statements -with a logger that can track the time of entry and time of exit. Rather than -writing both a function decorator and a context manager for the task, -inheriting from :class:`ContextDecorator` provides both capabilities in a -single definition:: - - from contextlib import ContextDecorator - import logging - - logging.basicConfig(level=logging.INFO) - - class track_entry_and_exit(ContextDecorator): - def __init__(self, name): - self.name = name - - def __enter__(self): - logging.info('Entering: %s', self.name) - - def __exit__(self, exc_type, exc, exc_tb): - logging.info('Exiting: %s', self.name) - -Instances of this class can be used as both a context manager:: - - with track_entry_and_exit('widget loader'): - print('Some time consuming activity goes here') - load_widget() - -And also as a function decorator:: - - @track_entry_and_exit('widget loader') - def activity(): - print('Some time consuming activity goes here') - load_widget() - -Note that there is one additional limitation when using context managers -as function decorators: there's no way to access the return value of -:meth:`~object.__enter__`. If that value is needed, then it is still necessary to use -an explicit ``with`` statement. - -.. seealso:: - - :pep:`343` - The "with" statement - The specification, background, and examples for the Python :keyword:`with` - statement. - -.. _single-use-reusable-and-reentrant-cms: - -Single use, reusable and reentrant context managers ---------------------------------------------------- - -Most context managers are written in a way that means they can only be -used effectively in a :keyword:`with` statement once. These single use -context managers must be created afresh each time they're used - -attempting to use them a second time will trigger an exception or -otherwise not work correctly. - -This common limitation means that it is generally advisable to create -context managers directly in the header of the :keyword:`with` statement -where they are used (as shown in all of the usage examples above). - -Files are an example of effectively single use context managers, since -the first :keyword:`with` statement will close the file, preventing any -further IO operations using that file object. - -Context managers created using :func:`contextmanager` are also single use -context managers, and will complain about the underlying generator failing -to yield if an attempt is made to use them a second time:: - - >>> from contextlib import contextmanager - >>> @contextmanager - ... def singleuse(): - ... print("Before") - ... yield - ... print("After") - ... - >>> cm = singleuse() - >>> with cm: - ... pass - ... - Before - After - >>> with cm: - ... pass - ... - Traceback (most recent call last): - ... - RuntimeError: generator didn't yield - - -.. _reentrant-cms: - -Reentrant context managers -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -More sophisticated context managers may be "reentrant". These context -managers can not only be used in multiple :keyword:`with` statements, -but may also be used *inside* a :keyword:`!with` statement that is already -using the same context manager. - -:class:`threading.RLock` is an example of a reentrant context manager, as are -:func:`suppress`, :func:`redirect_stdout`, and :func:`chdir`. Here's a very -simple example of reentrant use:: - - >>> from contextlib import redirect_stdout - >>> from io import StringIO - >>> stream = StringIO() - >>> write_to_stream = redirect_stdout(stream) - >>> with write_to_stream: - ... print("This is written to the stream rather than stdout") - ... with write_to_stream: - ... print("This is also written to the stream") - ... - >>> print("This is written directly to stdout") - This is written directly to stdout - >>> print(stream.getvalue()) - This is written to the stream rather than stdout - This is also written to the stream - -Real world examples of reentrancy are more likely to involve multiple -functions calling each other and hence be far more complicated than this -example. - -Note also that being reentrant is *not* the same thing as being thread safe. -:func:`redirect_stdout`, for example, is definitely not thread safe, as it -makes a global modification to the system state by binding :data:`sys.stdout` -to a different stream. - - -.. _reusable-cms: - -Reusable context managers -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Distinct from both single use and reentrant context managers are "reusable" -context managers (or, to be completely explicit, "reusable, but not -reentrant" context managers, since reentrant context managers are also -reusable). These context managers support being used multiple times, but -will fail (or otherwise not work correctly) if the specific context manager -instance has already been used in a containing with statement. - -:class:`threading.Lock` is an example of a reusable, but not reentrant, -context manager (for a reentrant lock, it is necessary to use -:class:`threading.RLock` instead). - -Another example of a reusable, but not reentrant, context manager is -:class:`ExitStack`, as it invokes *all* currently registered callbacks -when leaving any with statement, regardless of where those callbacks -were added:: - - >>> from contextlib import ExitStack - >>> stack = ExitStack() - >>> with stack: - ... stack.callback(print, "Callback: from first context") - ... print("Leaving first context") - ... - Leaving first context - Callback: from first context - >>> with stack: - ... stack.callback(print, "Callback: from second context") - ... print("Leaving second context") - ... - Leaving second context - Callback: from second context - >>> with stack: - ... stack.callback(print, "Callback: from outer context") - ... with stack: - ... stack.callback(print, "Callback: from inner context") - ... print("Leaving inner context") - ... print("Leaving outer context") - ... - Leaving inner context - Callback: from inner context - Callback: from outer context - Leaving outer context - -As the output from the example shows, reusing a single stack object across -multiple with statements works correctly, but attempting to nest them -will cause the stack to be cleared at the end of the innermost with -statement, which is unlikely to be desirable behaviour. - -Using separate :class:`ExitStack` instances instead of reusing a single -instance avoids that problem:: - - >>> from contextlib import ExitStack - >>> with ExitStack() as outer_stack: - ... outer_stack.callback(print, "Callback: from outer context") - ... with ExitStack() as inner_stack: - ... inner_stack.callback(print, "Callback: from inner context") - ... print("Leaving inner context") - ... print("Leaving outer context") - ... - Leaving inner context - Callback: from inner context - Leaving outer context - Callback: from outer context diff --git a/Doc/library/contextvars.rst b/Doc/library/contextvars.rst deleted file mode 100644 index 0ac2f3d85749b7..00000000000000 --- a/Doc/library/contextvars.rst +++ /dev/null @@ -1,286 +0,0 @@ -:mod:`contextvars` --- Context Variables -======================================== - -.. module:: contextvars - :synopsis: Context Variables - -.. sectionauthor:: Yury Selivanov - --------------- - -This module provides APIs to manage, store, and access context-local -state. The :class:`~contextvars.ContextVar` class is used to declare -and work with *Context Variables*. The :func:`~contextvars.copy_context` -function and the :class:`~contextvars.Context` class should be used to -manage the current context in asynchronous frameworks. - -Context managers that have state should use Context Variables -instead of :func:`threading.local()` to prevent their state from -bleeding to other code unexpectedly, when used in concurrent code. - -See also :pep:`567` for additional details. - -.. versionadded:: 3.7 - - -Context Variables ------------------ - -.. class:: ContextVar(name, [*, default]) - - This class is used to declare a new Context Variable, e.g.:: - - var: ContextVar[int] = ContextVar('var', default=42) - - The required *name* parameter is used for introspection and debug - purposes. - - The optional keyword-only *default* parameter is returned by - :meth:`ContextVar.get` when no value for the variable is found - in the current context. - - **Important:** Context Variables should be created at the top module - level and never in closures. :class:`Context` objects hold strong - references to context variables which prevents context variables - from being properly garbage collected. - - .. attribute:: ContextVar.name - - The name of the variable. This is a read-only property. - - .. versionadded:: 3.7.1 - - .. method:: get([default]) - - Return a value for the context variable for the current context. - - If there is no value for the variable in the current context, - the method will: - - * return the value of the *default* argument of the method, - if provided; or - - * return the default value for the context variable, - if it was created with one; or - - * raise a :exc:`LookupError`. - - .. method:: set(value) - - Call to set a new value for the context variable in the current - context. - - The required *value* argument is the new value for the context - variable. - - Returns a :class:`~contextvars.Token` object that can be used - to restore the variable to its previous value via the - :meth:`ContextVar.reset` method. - - .. method:: reset(token) - - Reset the context variable to the value it had before the - :meth:`ContextVar.set` that created the *token* was used. - - For example:: - - var = ContextVar('var') - - token = var.set('new value') - # code that uses 'var'; var.get() returns 'new value'. - var.reset(token) - - # After the reset call the var has no value again, so - # var.get() would raise a LookupError. - - -.. class:: Token - - *Token* objects are returned by the :meth:`ContextVar.set` method. - They can be passed to the :meth:`ContextVar.reset` method to revert - the value of the variable to what it was before the corresponding - *set*. - - .. attribute:: Token.var - - A read-only property. Points to the :class:`ContextVar` object - that created the token. - - .. attribute:: Token.old_value - - A read-only property. Set to the value the variable had before - the :meth:`ContextVar.set` method call that created the token. - It points to :attr:`Token.MISSING` if the variable was not set - before the call. - - .. attribute:: Token.MISSING - - A marker object used by :attr:`Token.old_value`. - - -Manual Context Management -------------------------- - -.. function:: copy_context() - - Returns a copy of the current :class:`~contextvars.Context` object. - - The following snippet gets a copy of the current context and prints - all variables and their values that are set in it:: - - ctx: Context = copy_context() - print(list(ctx.items())) - - The function has an O(1) complexity, i.e. works equally fast for - contexts with a few context variables and for contexts that have - a lot of them. - - -.. class:: Context() - - A mapping of :class:`ContextVars ` to their values. - - ``Context()`` creates an empty context with no values in it. - To get a copy of the current context use the - :func:`~contextvars.copy_context` function. - - Every thread will have a different top-level :class:`~contextvars.Context` - object. This means that a :class:`ContextVar` object behaves in a similar - fashion to :func:`threading.local()` when values are assigned in different - threads. - - Context implements the :class:`collections.abc.Mapping` interface. - - .. method:: run(callable, *args, **kwargs) - - Execute ``callable(*args, **kwargs)`` code in the context object - the *run* method is called on. Return the result of the execution - or propagate an exception if one occurred. - - Any changes to any context variables that *callable* makes will - be contained in the context object:: - - var = ContextVar('var') - var.set('spam') - - def main(): - # 'var' was set to 'spam' before - # calling 'copy_context()' and 'ctx.run(main)', so: - # var.get() == ctx[var] == 'spam' - - var.set('ham') - - # Now, after setting 'var' to 'ham': - # var.get() == ctx[var] == 'ham' - - ctx = copy_context() - - # Any changes that the 'main' function makes to 'var' - # will be contained in 'ctx'. - ctx.run(main) - - # The 'main()' function was run in the 'ctx' context, - # so changes to 'var' are contained in it: - # ctx[var] == 'ham' - - # However, outside of 'ctx', 'var' is still set to 'spam': - # var.get() == 'spam' - - The method raises a :exc:`RuntimeError` when called on the same - context object from more than one OS thread, or when called - recursively. - - .. method:: copy() - - Return a shallow copy of the context object. - - .. describe:: var in context - - Return ``True`` if the *context* has a value for *var* set; - return ``False`` otherwise. - - .. describe:: context[var] - - Return the value of the *var* :class:`ContextVar` variable. - If the variable is not set in the context object, a - :exc:`KeyError` is raised. - - .. method:: get(var, [default]) - - Return the value for *var* if *var* has the value in the context - object. Return *default* otherwise. If *default* is not given, - return ``None``. - - .. describe:: iter(context) - - Return an iterator over the variables stored in the context - object. - - .. describe:: len(proxy) - - Return the number of variables set in the context object. - - .. method:: keys() - - Return a list of all variables in the context object. - - .. method:: values() - - Return a list of all variables' values in the context object. - - - .. method:: items() - - Return a list of 2-tuples containing all variables and their - values in the context object. - - -asyncio support ---------------- - -Context variables are natively supported in :mod:`asyncio` and are -ready to be used without any extra configuration. For example, here -is a simple echo server, that uses a context variable to make the -address of a remote client available in the Task that handles that -client:: - - import asyncio - import contextvars - - client_addr_var = contextvars.ContextVar('client_addr') - - def render_goodbye(): - # The address of the currently handled client can be accessed - # without passing it explicitly to this function. - - client_addr = client_addr_var.get() - return f'Good bye, client @ {client_addr}\n'.encode() - - async def handle_request(reader, writer): - addr = writer.transport.get_extra_info('socket').getpeername() - client_addr_var.set(addr) - - # In any code that we call is now possible to get - # client's address by calling 'client_addr_var.get()'. - - while True: - line = await reader.readline() - print(line) - if not line.strip(): - break - writer.write(line) - - writer.write(render_goodbye()) - writer.close() - - async def main(): - srv = await asyncio.start_server( - handle_request, '127.0.0.1', 8081) - - async with srv: - await srv.serve_forever() - - asyncio.run(main()) - - # To test it you can use telnet: - # telnet 127.0.0.1 8081 diff --git a/Doc/library/copy.rst b/Doc/library/copy.rst deleted file mode 100644 index 8f32477ed508c3..00000000000000 --- a/Doc/library/copy.rst +++ /dev/null @@ -1,97 +0,0 @@ -:mod:`copy` --- Shallow and deep copy operations -================================================ - -.. module:: copy - :synopsis: Shallow and deep copy operations. - -**Source code:** :source:`Lib/copy.py` - --------------- - -Assignment statements in Python do not copy objects, they create bindings -between a target and an object. For collections that are mutable or contain -mutable items, a copy is sometimes needed so one can change one copy without -changing the other. This module provides generic shallow and deep copy -operations (explained below). - - -Interface summary: - -.. function:: copy(x) - - Return a shallow copy of *x*. - - -.. function:: deepcopy(x[, memo]) - - Return a deep copy of *x*. - - -.. exception:: Error - - Raised for module specific errors. - -.. _shallow_vs_deep_copy: - -The difference between shallow and deep copying is only relevant for compound -objects (objects that contain other objects, like lists or class instances): - -* A *shallow copy* constructs a new compound object and then (to the extent - possible) inserts *references* into it to the objects found in the original. - -* A *deep copy* constructs a new compound object and then, recursively, inserts - *copies* into it of the objects found in the original. - -Two problems often exist with deep copy operations that don't exist with shallow -copy operations: - -* Recursive objects (compound objects that, directly or indirectly, contain a - reference to themselves) may cause a recursive loop. - -* Because deep copy copies everything it may copy too much, such as data - which is intended to be shared between copies. - -The :func:`deepcopy` function avoids these problems by: - -* keeping a ``memo`` dictionary of objects already copied during the current - copying pass; and - -* letting user-defined classes override the copying operation or the set of - components copied. - -This module does not copy types like module, method, stack trace, stack frame, -file, socket, window, or any similar types. It does "copy" functions and -classes (shallow and deeply), by returning the original object unchanged; this -is compatible with the way these are treated by the :mod:`pickle` module. - -Shallow copies of dictionaries can be made using :meth:`dict.copy`, and -of lists by assigning a slice of the entire list, for example, -``copied_list = original_list[:]``. - -.. index:: pair: module; pickle - -Classes can use the same interfaces to control copying that they use to control -pickling. See the description of module :mod:`pickle` for information on these -methods. In fact, the :mod:`copy` module uses the registered -pickle functions from the :mod:`copyreg` module. - -.. index:: - single: __copy__() (copy protocol) - single: __deepcopy__() (copy protocol) - -In order for a class to define its own copy implementation, it can define -special methods :meth:`__copy__` and :meth:`__deepcopy__`. The former is called -to implement the shallow copy operation; no additional arguments are passed. -The latter is called to implement the deep copy operation; it is passed one -argument, the ``memo`` dictionary. If the :meth:`__deepcopy__` implementation needs -to make a deep copy of a component, it should call the :func:`deepcopy` function -with the component as first argument and the memo dictionary as second argument. -The memo dictionary should be treated as an opaque object. - - -.. seealso:: - - Module :mod:`pickle` - Discussion of the special methods used to support object state retrieval and - restoration. - diff --git a/Doc/library/copyreg.rst b/Doc/library/copyreg.rst deleted file mode 100644 index 2a28c043f80723..00000000000000 --- a/Doc/library/copyreg.rst +++ /dev/null @@ -1,62 +0,0 @@ -:mod:`copyreg` --- Register :mod:`pickle` support functions -=========================================================== - -.. module:: copyreg - :synopsis: Register pickle support functions. - -**Source code:** :source:`Lib/copyreg.py` - -.. index:: - pair: module; pickle - pair: module; copy - --------------- - -The :mod:`copyreg` module offers a way to define functions used while pickling -specific objects. The :mod:`pickle` and :mod:`copy` modules use those functions -when pickling/copying those objects. The module provides configuration -information about object constructors which are not classes. -Such constructors may be factory functions or class instances. - - -.. function:: constructor(object) - - Declares *object* to be a valid constructor. If *object* is not callable (and - hence not valid as a constructor), raises :exc:`TypeError`. - - -.. function:: pickle(type, function, constructor_ob=None) - - Declares that *function* should be used as a "reduction" function for objects - of type *type*. *function* must return either a string or a tuple - containing between two and six elements. See the :attr:`~pickle.Pickler.dispatch_table` - for more details on the interface of *function*. - - The *constructor_ob* parameter is a legacy feature and is now ignored, but if - passed it must be a callable. - - Note that the :attr:`~pickle.Pickler.dispatch_table` attribute of a pickler - object or subclass of :class:`pickle.Pickler` can also be used for - declaring reduction functions. - -Example -------- - -The example below would like to show how to register a pickle function and how -it will be used: - - >>> import copyreg, copy, pickle - >>> class C: - ... def __init__(self, a): - ... self.a = a - ... - >>> def pickle_c(c): - ... print("pickling a C instance...") - ... return C, (c.a,) - ... - >>> copyreg.pickle(C, pickle_c) - >>> c = C(1) - >>> d = copy.copy(c) # doctest: +SKIP - pickling a C instance... - >>> p = pickle.dumps(c) # doctest: +SKIP - pickling a C instance... diff --git a/Doc/library/crypt.rst b/Doc/library/crypt.rst deleted file mode 100644 index 51f91463f5ff93..00000000000000 --- a/Doc/library/crypt.rst +++ /dev/null @@ -1,183 +0,0 @@ -:mod:`crypt` --- Function to check Unix passwords -================================================= - -.. module:: crypt - :platform: Unix - :synopsis: The crypt() function used to check Unix passwords. - :deprecated: - -.. moduleauthor:: Steven D. Majewski -.. sectionauthor:: Steven D. Majewski -.. sectionauthor:: Peter Funk - -**Source code:** :source:`Lib/crypt.py` - -.. index:: - single: crypt(3) - pair: cipher; DES - -.. deprecated-removed:: 3.11 3.13 - The :mod:`crypt` module is deprecated - (see :pep:`PEP 594 <594#crypt>` for details and alternatives). - The :mod:`hashlib` module is a potential replacement for certain use cases. - The `passlib `_ package can replace all use cases of this module. - --------------- - -This module implements an interface to the :manpage:`crypt(3)` routine, which is -a one-way hash function based upon a modified DES algorithm; see the Unix man -page for further details. Possible uses include storing hashed passwords -so you can check passwords without storing the actual password, or attempting -to crack Unix passwords with a dictionary. - -.. index:: single: crypt(3) - -Notice that the behavior of this module depends on the actual implementation of -the :manpage:`crypt(3)` routine in the running system. Therefore, any -extensions available on the current implementation will also be available on -this module. - -.. availability:: Unix, not VxWorks. - -.. include:: ../includes/wasm-notavail.rst - -Hashing Methods ---------------- - -.. versionadded:: 3.3 - -The :mod:`crypt` module defines the list of hashing methods (not all methods -are available on all platforms): - -.. data:: METHOD_SHA512 - - A Modular Crypt Format method with 16 character salt and 86 character - hash based on the SHA-512 hash function. This is the strongest method. - -.. data:: METHOD_SHA256 - - Another Modular Crypt Format method with 16 character salt and 43 - character hash based on the SHA-256 hash function. - -.. data:: METHOD_BLOWFISH - - Another Modular Crypt Format method with 22 character salt and 31 - character hash based on the Blowfish cipher. - - .. versionadded:: 3.7 - -.. data:: METHOD_MD5 - - Another Modular Crypt Format method with 8 character salt and 22 - character hash based on the MD5 hash function. - -.. data:: METHOD_CRYPT - - The traditional method with a 2 character salt and 13 characters of - hash. This is the weakest method. - - -Module Attributes ------------------ - -.. versionadded:: 3.3 - -.. attribute:: methods - - A list of available password hashing algorithms, as - ``crypt.METHOD_*`` objects. This list is sorted from strongest to - weakest. - - -Module Functions ----------------- - -The :mod:`crypt` module defines the following functions: - -.. function:: crypt(word, salt=None) - - *word* will usually be a user's password as typed at a prompt or in a graphical - interface. The optional *salt* is either a string as returned from - :func:`mksalt`, one of the ``crypt.METHOD_*`` values (though not all - may be available on all platforms), or a full encrypted password - including salt, as returned by this function. If *salt* is not - provided, the strongest method available in :attr:`methods` will be used. - - Checking a password is usually done by passing the plain-text password - as *word* and the full results of a previous :func:`crypt` call, - which should be the same as the results of this call. - - *salt* (either a random 2 or 16 character string, possibly prefixed with - ``$digit$`` to indicate the method) which will be used to perturb the - encryption algorithm. The characters in *salt* must be in the set - ``[./a-zA-Z0-9]``, with the exception of Modular Crypt Format which - prefixes a ``$digit$``. - - Returns the hashed password as a string, which will be composed of - characters from the same alphabet as the salt. - - .. index:: single: crypt(3) - - Since a few :manpage:`crypt(3)` extensions allow different values, with - different sizes in the *salt*, it is recommended to use the full crypted - password as salt when checking for a password. - - .. versionchanged:: 3.3 - Accept ``crypt.METHOD_*`` values in addition to strings for *salt*. - - -.. function:: mksalt(method=None, *, rounds=None) - - Return a randomly generated salt of the specified method. If no - *method* is given, the strongest method available in :attr:`methods` is - used. - - The return value is a string suitable for passing as the *salt* argument - to :func:`crypt`. - - *rounds* specifies the number of rounds for ``METHOD_SHA256``, - ``METHOD_SHA512`` and ``METHOD_BLOWFISH``. - For ``METHOD_SHA256`` and ``METHOD_SHA512`` it must be an integer between - ``1000`` and ``999_999_999``, the default is ``5000``. For - ``METHOD_BLOWFISH`` it must be a power of two between ``16`` (2\ :sup:`4`) - and ``2_147_483_648`` (2\ :sup:`31`), the default is ``4096`` - (2\ :sup:`12`). - - .. versionadded:: 3.3 - - .. versionchanged:: 3.7 - Added the *rounds* parameter. - - -Examples --------- - -A simple example illustrating typical use (a constant-time comparison -operation is needed to limit exposure to timing attacks. -:func:`hmac.compare_digest` is suitable for this purpose):: - - import pwd - import crypt - import getpass - from hmac import compare_digest as compare_hash - - def login(): - username = input('Python login: ') - cryptedpasswd = pwd.getpwnam(username)[1] - if cryptedpasswd: - if cryptedpasswd == 'x' or cryptedpasswd == '*': - raise ValueError('no support for shadow passwords') - cleartext = getpass.getpass() - return compare_hash(crypt.crypt(cleartext, cryptedpasswd), cryptedpasswd) - else: - return True - -To generate a hash of a password using the strongest available method and -check it against the original:: - - import crypt - from hmac import compare_digest as compare_hash - - hashed = crypt.crypt(plaintext) - if not compare_hash(hashed, crypt.crypt(plaintext, hashed)): - raise ValueError("hashed version doesn't validate against original") diff --git a/Doc/library/crypto.rst b/Doc/library/crypto.rst deleted file mode 100644 index ae45549a6d8941..00000000000000 --- a/Doc/library/crypto.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. _crypto: - -********************** -Cryptographic Services -********************** - -.. index:: single: cryptography - -The modules described in this chapter implement various algorithms of a -cryptographic nature. They are available at the discretion of the installation. -On Unix systems, the :mod:`crypt` module may also be available. -Here's an overview: - - -.. toctree:: - - hashlib.rst - hmac.rst - secrets.rst diff --git a/Doc/library/csv.rst b/Doc/library/csv.rst deleted file mode 100644 index 52f76304a10529..00000000000000 --- a/Doc/library/csv.rst +++ /dev/null @@ -1,590 +0,0 @@ -:mod:`csv` --- CSV File Reading and Writing -=========================================== - -.. module:: csv - :synopsis: Write and read tabular data to and from delimited files. - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/csv.py` - -.. index:: - single: csv - pair: data; tabular - --------------- - -The so-called CSV (Comma Separated Values) format is the most common import and -export format for spreadsheets and databases. CSV format was used for many -years prior to attempts to describe the format in a standardized way in -:rfc:`4180`. The lack of a well-defined standard means that subtle differences -often exist in the data produced and consumed by different applications. These -differences can make it annoying to process CSV files from multiple sources. -Still, while the delimiters and quoting characters vary, the overall format is -similar enough that it is possible to write a single module which can -efficiently manipulate such data, hiding the details of reading and writing the -data from the programmer. - -The :mod:`csv` module implements classes to read and write tabular data in CSV -format. It allows programmers to say, "write this data in the format preferred -by Excel," or "read data from this file which was generated by Excel," without -knowing the precise details of the CSV format used by Excel. Programmers can -also describe the CSV formats understood by other applications or define their -own special-purpose CSV formats. - -The :mod:`csv` module's :class:`reader` and :class:`writer` objects read and -write sequences. Programmers can also read and write data in dictionary form -using the :class:`DictReader` and :class:`DictWriter` classes. - -.. seealso:: - - :pep:`305` - CSV File API - The Python Enhancement Proposal which proposed this addition to Python. - - -.. _csv-contents: - -Module Contents ---------------- - -The :mod:`csv` module defines the following functions: - - -.. index:: - single: universal newlines; csv.reader function - -.. function:: reader(csvfile, dialect='excel', **fmtparams) - - Return a reader object which will iterate over lines in the given *csvfile*. - *csvfile* can be any object which supports the :term:`iterator` protocol and returns a - string each time its :meth:`!__next__` method is called --- :term:`file objects - ` and list objects are both suitable. If *csvfile* is a file object, - it should be opened with ``newline=''``. [1]_ An optional - *dialect* parameter can be given which is used to define a set of parameters - specific to a particular CSV dialect. It may be an instance of a subclass of - the :class:`Dialect` class or one of the strings returned by the - :func:`list_dialects` function. The other optional *fmtparams* keyword arguments - can be given to override individual formatting parameters in the current - dialect. For full details about the dialect and formatting parameters, see - section :ref:`csv-fmt-params`. - - Each row read from the csv file is returned as a list of strings. No - automatic data type conversion is performed unless the ``QUOTE_NONNUMERIC`` format - option is specified (in which case unquoted fields are transformed into floats). - - A short usage example:: - - >>> import csv - >>> with open('eggs.csv', newline='') as csvfile: - ... spamreader = csv.reader(csvfile, delimiter=' ', quotechar='|') - ... for row in spamreader: - ... print(', '.join(row)) - Spam, Spam, Spam, Spam, Spam, Baked Beans - Spam, Lovely Spam, Wonderful Spam - - -.. function:: writer(csvfile, dialect='excel', **fmtparams) - - Return a writer object responsible for converting the user's data into delimited - strings on the given file-like object. *csvfile* can be any object with a - :func:`write` method. If *csvfile* is a file object, it should be opened with - ``newline=''`` [1]_. An optional *dialect* - parameter can be given which is used to define a set of parameters specific to a - particular CSV dialect. It may be an instance of a subclass of the - :class:`Dialect` class or one of the strings returned by the - :func:`list_dialects` function. The other optional *fmtparams* keyword arguments - can be given to override individual formatting parameters in the current - dialect. For full details about dialects and formatting parameters, see - the :ref:`csv-fmt-params` section. To make it - as easy as possible to interface with modules which implement the DB API, the - value :const:`None` is written as the empty string. While this isn't a - reversible transformation, it makes it easier to dump SQL NULL data values to - CSV files without preprocessing the data returned from a ``cursor.fetch*`` call. - All other non-string data are stringified with :func:`str` before being written. - - A short usage example:: - - import csv - with open('eggs.csv', 'w', newline='') as csvfile: - spamwriter = csv.writer(csvfile, delimiter=' ', - quotechar='|', quoting=csv.QUOTE_MINIMAL) - spamwriter.writerow(['Spam'] * 5 + ['Baked Beans']) - spamwriter.writerow(['Spam', 'Lovely Spam', 'Wonderful Spam']) - - -.. function:: register_dialect(name[, dialect[, **fmtparams]]) - - Associate *dialect* with *name*. *name* must be a string. The - dialect can be specified either by passing a sub-class of :class:`Dialect`, or - by *fmtparams* keyword arguments, or both, with keyword arguments overriding - parameters of the dialect. For full details about dialects and formatting - parameters, see section :ref:`csv-fmt-params`. - - -.. function:: unregister_dialect(name) - - Delete the dialect associated with *name* from the dialect registry. An - :exc:`Error` is raised if *name* is not a registered dialect name. - - -.. function:: get_dialect(name) - - Return the dialect associated with *name*. An :exc:`Error` is raised if - *name* is not a registered dialect name. This function returns an immutable - :class:`Dialect`. - -.. function:: list_dialects() - - Return the names of all registered dialects. - - -.. function:: field_size_limit([new_limit]) - - Returns the current maximum field size allowed by the parser. If *new_limit* is - given, this becomes the new limit. - - -The :mod:`csv` module defines the following classes: - -.. class:: DictReader(f, fieldnames=None, restkey=None, restval=None, \ - dialect='excel', *args, **kwds) - - Create an object that operates like a regular reader but maps the - information in each row to a :class:`dict` whose keys are given by the - optional *fieldnames* parameter. - - The *fieldnames* parameter is a :term:`sequence`. If *fieldnames* is - omitted, the values in the first row of file *f* will be used as the - fieldnames. Regardless of how the fieldnames are determined, the - dictionary preserves their original ordering. - - If a row has more fields than fieldnames, the remaining data is put in a - list and stored with the fieldname specified by *restkey* (which defaults - to ``None``). If a non-blank row has fewer fields than fieldnames, the - missing values are filled-in with the value of *restval* (which defaults - to ``None``). - - All other optional or keyword arguments are passed to the underlying - :class:`reader` instance. - - .. versionchanged:: 3.6 - Returned rows are now of type :class:`OrderedDict`. - - .. versionchanged:: 3.8 - Returned rows are now of type :class:`dict`. - - A short usage example:: - - >>> import csv - >>> with open('names.csv', newline='') as csvfile: - ... reader = csv.DictReader(csvfile) - ... for row in reader: - ... print(row['first_name'], row['last_name']) - ... - Eric Idle - John Cleese - - >>> print(row) - {'first_name': 'John', 'last_name': 'Cleese'} - - -.. class:: DictWriter(f, fieldnames, restval='', extrasaction='raise', \ - dialect='excel', *args, **kwds) - - Create an object which operates like a regular writer but maps dictionaries - onto output rows. The *fieldnames* parameter is a :mod:`sequence - ` of keys that identify the order in which values in the - dictionary passed to the :meth:`writerow` method are written to file - *f*. The optional *restval* parameter specifies the value to be - written if the dictionary is missing a key in *fieldnames*. If the - dictionary passed to the :meth:`writerow` method contains a key not found in - *fieldnames*, the optional *extrasaction* parameter indicates what action to - take. - If it is set to ``'raise'``, the default value, a :exc:`ValueError` - is raised. - If it is set to ``'ignore'``, extra values in the dictionary are ignored. - Any other optional or keyword arguments are passed to the underlying - :class:`writer` instance. - - Note that unlike the :class:`DictReader` class, the *fieldnames* parameter - of the :class:`DictWriter` class is not optional. - - A short usage example:: - - import csv - - with open('names.csv', 'w', newline='') as csvfile: - fieldnames = ['first_name', 'last_name'] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - - writer.writeheader() - writer.writerow({'first_name': 'Baked', 'last_name': 'Beans'}) - writer.writerow({'first_name': 'Lovely', 'last_name': 'Spam'}) - writer.writerow({'first_name': 'Wonderful', 'last_name': 'Spam'}) - - -.. class:: Dialect - - The :class:`Dialect` class is a container class whose attributes contain - information for how to handle doublequotes, whitespace, delimiters, etc. - Due to the lack of a strict CSV specification, different applications - produce subtly different CSV data. :class:`Dialect` instances define how - :class:`reader` and :class:`writer` instances behave. - - All available :class:`Dialect` names are returned by :func:`list_dialects`, - and they can be registered with specific :class:`reader` and :class:`writer` - classes through their initializer (``__init__``) functions like this:: - - import csv - - with open('students.csv', 'w', newline='') as csvfile: - writer = csv.writer(csvfile, dialect='unix') - ^^^^^^^^^^^^^^ - - -.. class:: excel() - - The :class:`excel` class defines the usual properties of an Excel-generated CSV - file. It is registered with the dialect name ``'excel'``. - - -.. class:: excel_tab() - - The :class:`excel_tab` class defines the usual properties of an Excel-generated - TAB-delimited file. It is registered with the dialect name ``'excel-tab'``. - - -.. class:: unix_dialect() - - The :class:`unix_dialect` class defines the usual properties of a CSV file - generated on UNIX systems, i.e. using ``'\n'`` as line terminator and quoting - all fields. It is registered with the dialect name ``'unix'``. - - .. versionadded:: 3.2 - - -.. class:: Sniffer() - - The :class:`Sniffer` class is used to deduce the format of a CSV file. - - The :class:`Sniffer` class provides two methods: - - .. method:: sniff(sample, delimiters=None) - - Analyze the given *sample* and return a :class:`Dialect` subclass - reflecting the parameters found. If the optional *delimiters* parameter - is given, it is interpreted as a string containing possible valid - delimiter characters. - - - .. method:: has_header(sample) - - Analyze the sample text (presumed to be in CSV format) and return - :const:`True` if the first row appears to be a series of column headers. - Inspecting each column, one of two key criteria will be considered to - estimate if the sample contains a header: - - - the second through n-th rows contain numeric values - - the second through n-th rows contain strings where at least one value's - length differs from that of the putative header of that column. - - Twenty rows after the first row are sampled; if more than half of columns + - rows meet the criteria, :const:`True` is returned. - - .. note:: - - This method is a rough heuristic and may produce both false positives and - negatives. - -An example for :class:`Sniffer` use:: - - with open('example.csv', newline='') as csvfile: - dialect = csv.Sniffer().sniff(csvfile.read(1024)) - csvfile.seek(0) - reader = csv.reader(csvfile, dialect) - # ... process CSV file contents here ... - - -The :mod:`csv` module defines the following constants: - -.. data:: QUOTE_ALL - - Instructs :class:`writer` objects to quote all fields. - - -.. data:: QUOTE_MINIMAL - - Instructs :class:`writer` objects to only quote those fields which contain - special characters such as *delimiter*, *quotechar* or any of the characters in - *lineterminator*. - - -.. data:: QUOTE_NONNUMERIC - - Instructs :class:`writer` objects to quote all non-numeric fields. - - Instructs the reader to convert all non-quoted fields to type *float*. - - -.. data:: QUOTE_NONE - - Instructs :class:`writer` objects to never quote fields. When the current - *delimiter* occurs in output data it is preceded by the current *escapechar* - character. If *escapechar* is not set, the writer will raise :exc:`Error` if - any characters that require escaping are encountered. - - Instructs :class:`reader` to perform no special processing of quote characters. - -The :mod:`csv` module defines the following exception: - - -.. exception:: Error - - Raised by any of the functions when an error is detected. - -.. _csv-fmt-params: - -Dialects and Formatting Parameters ----------------------------------- - -To make it easier to specify the format of input and output records, specific -formatting parameters are grouped together into dialects. A dialect is a -subclass of the :class:`Dialect` class having a set of specific methods and a -single :meth:`validate` method. When creating :class:`reader` or -:class:`writer` objects, the programmer can specify a string or a subclass of -the :class:`Dialect` class as the dialect parameter. In addition to, or instead -of, the *dialect* parameter, the programmer can also specify individual -formatting parameters, which have the same names as the attributes defined below -for the :class:`Dialect` class. - -Dialects support the following attributes: - - -.. attribute:: Dialect.delimiter - - A one-character string used to separate fields. It defaults to ``','``. - - -.. attribute:: Dialect.doublequote - - Controls how instances of *quotechar* appearing inside a field should - themselves be quoted. When :const:`True`, the character is doubled. When - :const:`False`, the *escapechar* is used as a prefix to the *quotechar*. It - defaults to :const:`True`. - - On output, if *doublequote* is :const:`False` and no *escapechar* is set, - :exc:`Error` is raised if a *quotechar* is found in a field. - - -.. attribute:: Dialect.escapechar - - A one-character string used by the writer to escape the *delimiter* if *quoting* - is set to :const:`QUOTE_NONE` and the *quotechar* if *doublequote* is - :const:`False`. On reading, the *escapechar* removes any special meaning from - the following character. It defaults to :const:`None`, which disables escaping. - - .. versionchanged:: 3.11 - An empty *escapechar* is not allowed. - -.. attribute:: Dialect.lineterminator - - The string used to terminate lines produced by the :class:`writer`. It defaults - to ``'\r\n'``. - - .. note:: - - The :class:`reader` is hard-coded to recognise either ``'\r'`` or ``'\n'`` as - end-of-line, and ignores *lineterminator*. This behavior may change in the - future. - - -.. attribute:: Dialect.quotechar - - A one-character string used to quote fields containing special characters, such - as the *delimiter* or *quotechar*, or which contain new-line characters. It - defaults to ``'"'``. - - .. versionchanged:: 3.11 - An empty *quotechar* is not allowed. - -.. attribute:: Dialect.quoting - - Controls when quotes should be generated by the writer and recognised by the - reader. It can take on any of the :const:`QUOTE_\*` constants (see section - :ref:`csv-contents`) and defaults to :const:`QUOTE_MINIMAL`. - - -.. attribute:: Dialect.skipinitialspace - - When :const:`True`, spaces immediately following the *delimiter* are ignored. - The default is :const:`False`. - - -.. attribute:: Dialect.strict - - When ``True``, raise exception :exc:`Error` on bad CSV input. - The default is ``False``. - -Reader Objects --------------- - -Reader objects (:class:`DictReader` instances and objects returned by the -:func:`reader` function) have the following public methods: - -.. method:: csvreader.__next__() - - Return the next row of the reader's iterable object as a list (if the object - was returned from :func:`reader`) or a dict (if it is a :class:`DictReader` - instance), parsed according to the current :class:`Dialect`. Usually you - should call this as ``next(reader)``. - - -Reader objects have the following public attributes: - -.. attribute:: csvreader.dialect - - A read-only description of the dialect in use by the parser. - - -.. attribute:: csvreader.line_num - - The number of lines read from the source iterator. This is not the same as the - number of records returned, as records can span multiple lines. - - -DictReader objects have the following public attribute: - -.. attribute:: DictReader.fieldnames - - If not passed as a parameter when creating the object, this attribute is - initialized upon first access or when the first record is read from the - file. - - - -Writer Objects --------------- - -:class:`Writer` objects (:class:`DictWriter` instances and objects returned by -the :func:`writer` function) have the following public methods. A *row* must be -an iterable of strings or numbers for :class:`Writer` objects and a dictionary -mapping fieldnames to strings or numbers (by passing them through :func:`str` -first) for :class:`DictWriter` objects. Note that complex numbers are written -out surrounded by parens. This may cause some problems for other programs which -read CSV files (assuming they support complex numbers at all). - - -.. method:: csvwriter.writerow(row) - - Write the *row* parameter to the writer's file object, formatted according - to the current :class:`Dialect`. Return the return value of the call to the - *write* method of the underlying file object. - - .. versionchanged:: 3.5 - Added support of arbitrary iterables. - -.. method:: csvwriter.writerows(rows) - - Write all elements in *rows* (an iterable of *row* objects as described - above) to the writer's file object, formatted according to the current - dialect. - -Writer objects have the following public attribute: - - -.. attribute:: csvwriter.dialect - - A read-only description of the dialect in use by the writer. - - -DictWriter objects have the following public method: - - -.. method:: DictWriter.writeheader() - - Write a row with the field names (as specified in the constructor) to - the writer's file object, formatted according to the current dialect. Return - the return value of the :meth:`csvwriter.writerow` call used internally. - - .. versionadded:: 3.2 - .. versionchanged:: 3.8 - :meth:`writeheader` now also returns the value returned by - the :meth:`csvwriter.writerow` method it uses internally. - - -.. _csv-examples: - -Examples --------- - -The simplest example of reading a CSV file:: - - import csv - with open('some.csv', newline='') as f: - reader = csv.reader(f) - for row in reader: - print(row) - -Reading a file with an alternate format:: - - import csv - with open('passwd', newline='') as f: - reader = csv.reader(f, delimiter=':', quoting=csv.QUOTE_NONE) - for row in reader: - print(row) - -The corresponding simplest possible writing example is:: - - import csv - with open('some.csv', 'w', newline='') as f: - writer = csv.writer(f) - writer.writerows(someiterable) - -Since :func:`open` is used to open a CSV file for reading, the file -will by default be decoded into unicode using the system default -encoding (see :func:`locale.getencoding`). To decode a file -using a different encoding, use the ``encoding`` argument of open:: - - import csv - with open('some.csv', newline='', encoding='utf-8') as f: - reader = csv.reader(f) - for row in reader: - print(row) - -The same applies to writing in something other than the system default -encoding: specify the encoding argument when opening the output file. - -Registering a new dialect:: - - import csv - csv.register_dialect('unixpwd', delimiter=':', quoting=csv.QUOTE_NONE) - with open('passwd', newline='') as f: - reader = csv.reader(f, 'unixpwd') - -A slightly more advanced use of the reader --- catching and reporting errors:: - - import csv, sys - filename = 'some.csv' - with open(filename, newline='') as f: - reader = csv.reader(f) - try: - for row in reader: - print(row) - except csv.Error as e: - sys.exit('file {}, line {}: {}'.format(filename, reader.line_num, e)) - -And while the module doesn't directly support parsing strings, it can easily be -done:: - - import csv - for row in csv.reader(['one,two,three']): - print(row) - - -.. rubric:: Footnotes - -.. [1] If ``newline=''`` is not specified, newlines embedded inside quoted fields - will not be interpreted correctly, and on platforms that use ``\r\n`` linendings - on write an extra ``\r`` will be added. It should always be safe to specify - ``newline=''``, since the csv module does its own - (:term:`universal `) newline handling. diff --git a/Doc/library/ctypes.rst b/Doc/library/ctypes.rst deleted file mode 100644 index 0088775d2fd2a7..00000000000000 --- a/Doc/library/ctypes.rst +++ /dev/null @@ -1,2593 +0,0 @@ -:mod:`ctypes` --- A foreign function library for Python -======================================================= - -.. module:: ctypes - :synopsis: A foreign function library for Python. - -.. moduleauthor:: Thomas Heller - -**Source code:** :source:`Lib/ctypes` - --------------- - -:mod:`ctypes` is a foreign function library for Python. It provides C compatible -data types, and allows calling functions in DLLs or shared libraries. It can be -used to wrap these libraries in pure Python. - - -.. _ctypes-ctypes-tutorial: - -ctypes tutorial ---------------- - -Note: The code samples in this tutorial use :mod:`doctest` to make sure that -they actually work. Since some code samples behave differently under Linux, -Windows, or macOS, they contain doctest directives in comments. - -Note: Some code samples reference the ctypes :class:`c_int` type. On platforms -where ``sizeof(long) == sizeof(int)`` it is an alias to :class:`c_long`. -So, you should not be confused if :class:`c_long` is printed if you would expect -:class:`c_int` --- they are actually the same type. - -.. _ctypes-loading-dynamic-link-libraries: - -Loading dynamic link libraries -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:mod:`ctypes` exports the *cdll*, and on Windows *windll* and *oledll* -objects, for loading dynamic link libraries. - -You load libraries by accessing them as attributes of these objects. *cdll* -loads libraries which export functions using the standard ``cdecl`` calling -convention, while *windll* libraries call functions using the ``stdcall`` -calling convention. *oledll* also uses the ``stdcall`` calling convention, and -assumes the functions return a Windows :c:type:`HRESULT` error code. The error -code is used to automatically raise an :class:`OSError` exception when the -function call fails. - -.. versionchanged:: 3.3 - Windows errors used to raise :exc:`WindowsError`, which is now an alias - of :exc:`OSError`. - - -Here are some examples for Windows. Note that ``msvcrt`` is the MS standard C -library containing most standard C functions, and uses the cdecl calling -convention:: - - >>> from ctypes import * - >>> print(windll.kernel32) # doctest: +WINDOWS - - >>> print(cdll.msvcrt) # doctest: +WINDOWS - - >>> libc = cdll.msvcrt # doctest: +WINDOWS - >>> - -Windows appends the usual ``.dll`` file suffix automatically. - -.. note:: - Accessing the standard C library through ``cdll.msvcrt`` will use an - outdated version of the library that may be incompatible with the one - being used by Python. Where possible, use native Python functionality, - or else import and use the ``msvcrt`` module. - -On Linux, it is required to specify the filename *including* the extension to -load a library, so attribute access can not be used to load libraries. Either the -:meth:`LoadLibrary` method of the dll loaders should be used, or you should load -the library by creating an instance of CDLL by calling the constructor:: - - >>> cdll.LoadLibrary("libc.so.6") # doctest: +LINUX - - >>> libc = CDLL("libc.so.6") # doctest: +LINUX - >>> libc # doctest: +LINUX - - >>> - -.. XXX Add section for macOS. - - -.. _ctypes-accessing-functions-from-loaded-dlls: - -Accessing functions from loaded dlls -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Functions are accessed as attributes of dll objects:: - - >>> from ctypes import * - >>> libc.printf - <_FuncPtr object at 0x...> - >>> print(windll.kernel32.GetModuleHandleA) # doctest: +WINDOWS - <_FuncPtr object at 0x...> - >>> print(windll.kernel32.MyOwnFunction) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - File "ctypes.py", line 239, in __getattr__ - func = _StdcallFuncPtr(name, self) - AttributeError: function 'MyOwnFunction' not found - >>> - -Note that win32 system dlls like ``kernel32`` and ``user32`` often export ANSI -as well as UNICODE versions of a function. The UNICODE version is exported with -an ``W`` appended to the name, while the ANSI version is exported with an ``A`` -appended to the name. The win32 ``GetModuleHandle`` function, which returns a -*module handle* for a given module name, has the following C prototype, and a -macro is used to expose one of them as ``GetModuleHandle`` depending on whether -UNICODE is defined or not:: - - /* ANSI version */ - HMODULE GetModuleHandleA(LPCSTR lpModuleName); - /* UNICODE version */ - HMODULE GetModuleHandleW(LPCWSTR lpModuleName); - -*windll* does not try to select one of them by magic, you must access the -version you need by specifying ``GetModuleHandleA`` or ``GetModuleHandleW`` -explicitly, and then call it with bytes or string objects respectively. - -Sometimes, dlls export functions with names which aren't valid Python -identifiers, like ``"??2@YAPAXI@Z"``. In this case you have to use -:func:`getattr` to retrieve the function:: - - >>> getattr(cdll.msvcrt, "??2@YAPAXI@Z") # doctest: +WINDOWS - <_FuncPtr object at 0x...> - >>> - -On Windows, some dlls export functions not by name but by ordinal. These -functions can be accessed by indexing the dll object with the ordinal number:: - - >>> cdll.kernel32[1] # doctest: +WINDOWS - <_FuncPtr object at 0x...> - >>> cdll.kernel32[0] # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - File "ctypes.py", line 310, in __getitem__ - func = _StdcallFuncPtr(name, self) - AttributeError: function ordinal 0 not found - >>> - - -.. _ctypes-calling-functions: - -Calling functions -^^^^^^^^^^^^^^^^^ - -You can call these functions like any other Python callable. This example uses -the ``time()`` function, which returns system time in seconds since the Unix -epoch, and the ``GetModuleHandleA()`` function, which returns a win32 module -handle. - -This example calls both functions with a ``NULL`` pointer (``None`` should be used -as the ``NULL`` pointer):: - - >>> print(libc.time(None)) # doctest: +SKIP - 1150640792 - >>> print(hex(windll.kernel32.GetModuleHandleA(None))) # doctest: +WINDOWS - 0x1d000000 - >>> - -:exc:`ValueError` is raised when you call an ``stdcall`` function with the -``cdecl`` calling convention, or vice versa:: - - >>> cdll.kernel32.GetModuleHandleA(None) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - ValueError: Procedure probably called with not enough arguments (4 bytes missing) - >>> - - >>> windll.msvcrt.printf(b"spam") # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - ValueError: Procedure probably called with too many arguments (4 bytes in excess) - >>> - -To find out the correct calling convention you have to look into the C header -file or the documentation for the function you want to call. - -On Windows, :mod:`ctypes` uses win32 structured exception handling to prevent -crashes from general protection faults when functions are called with invalid -argument values:: - - >>> windll.kernel32.GetModuleHandleA(32) # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - OSError: exception: access violation reading 0x00000020 - >>> - -There are, however, enough ways to crash Python with :mod:`ctypes`, so you -should be careful anyway. The :mod:`faulthandler` module can be helpful in -debugging crashes (e.g. from segmentation faults produced by erroneous C library -calls). - -``None``, integers, bytes objects and (unicode) strings are the only native -Python objects that can directly be used as parameters in these function calls. -``None`` is passed as a C ``NULL`` pointer, bytes objects and strings are passed -as pointer to the memory block that contains their data (:c:expr:`char *` or -:c:expr:`wchar_t *`). Python integers are passed as the platforms default C -:c:expr:`int` type, their value is masked to fit into the C type. - -Before we move on calling functions with other parameter types, we have to learn -more about :mod:`ctypes` data types. - - -.. _ctypes-fundamental-data-types: - -Fundamental data types -^^^^^^^^^^^^^^^^^^^^^^ - -:mod:`ctypes` defines a number of primitive C compatible data types: - -+----------------------+------------------------------------------+----------------------------+ -| ctypes type | C type | Python type | -+======================+==========================================+============================+ -| :class:`c_bool` | :c:expr:`_Bool` | bool (1) | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char` | :c:expr:`char` | 1-character bytes object | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar` | :c:type:`wchar_t` | 1-character string | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_byte` | :c:expr:`char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ubyte` | :c:expr:`unsigned char` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_short` | :c:expr:`short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ushort` | :c:expr:`unsigned short` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_int` | :c:expr:`int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_uint` | :c:expr:`unsigned int` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_long` | :c:expr:`long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulong` | :c:expr:`unsigned long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longlong` | :c:expr:`__int64` or :c:expr:`long long` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ulonglong` | :c:expr:`unsigned __int64` or | int | -| | :c:expr:`unsigned long long` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_size_t` | :c:type:`size_t` | int | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_ssize_t` | :c:type:`ssize_t` or | int | -| | :c:expr:`Py_ssize_t` | | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_float` | :c:expr:`float` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_double` | :c:expr:`double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_longdouble`| :c:expr:`long double` | float | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_char_p` | :c:expr:`char *` (NUL terminated) | bytes object or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_wchar_p` | :c:expr:`wchar_t *` (NUL terminated) | string or ``None`` | -+----------------------+------------------------------------------+----------------------------+ -| :class:`c_void_p` | :c:expr:`void *` | int or ``None`` | -+----------------------+------------------------------------------+----------------------------+ - -(1) - The constructor accepts any object with a truth value. - -All these types can be created by calling them with an optional initializer of -the correct type and value:: - - >>> c_int() - c_long(0) - >>> c_wchar_p("Hello, World") - c_wchar_p(140018365411392) - >>> c_ushort(-3) - c_ushort(65533) - >>> - -Since these types are mutable, their value can also be changed afterwards:: - - >>> i = c_int(42) - >>> print(i) - c_long(42) - >>> print(i.value) - 42 - >>> i.value = -99 - >>> print(i.value) - -99 - >>> - -Assigning a new value to instances of the pointer types :class:`c_char_p`, -:class:`c_wchar_p`, and :class:`c_void_p` changes the *memory location* they -point to, *not the contents* of the memory block (of course not, because Python -bytes objects are immutable):: - - >>> s = "Hello, World" - >>> c_s = c_wchar_p(s) - >>> print(c_s) - c_wchar_p(139966785747344) - >>> print(c_s.value) - Hello World - >>> c_s.value = "Hi, there" - >>> print(c_s) # the memory location has changed - c_wchar_p(139966783348904) - >>> print(c_s.value) - Hi, there - >>> print(s) # first object is unchanged - Hello, World - >>> - -You should be careful, however, not to pass them to functions expecting pointers -to mutable memory. If you need mutable memory blocks, ctypes has a -:func:`create_string_buffer` function which creates these in various ways. The -current memory block contents can be accessed (or changed) with the ``raw`` -property; if you want to access it as NUL terminated string, use the ``value`` -property:: - - >>> from ctypes import * - >>> p = create_string_buffer(3) # create a 3 byte buffer, initialized to NUL bytes - >>> print(sizeof(p), repr(p.raw)) - 3 b'\x00\x00\x00' - >>> p = create_string_buffer(b"Hello") # create a buffer containing a NUL terminated string - >>> print(sizeof(p), repr(p.raw)) - 6 b'Hello\x00' - >>> print(repr(p.value)) - b'Hello' - >>> p = create_string_buffer(b"Hello", 10) # create a 10 byte buffer - >>> print(sizeof(p), repr(p.raw)) - 10 b'Hello\x00\x00\x00\x00\x00' - >>> p.value = b"Hi" - >>> print(sizeof(p), repr(p.raw)) - 10 b'Hi\x00lo\x00\x00\x00\x00\x00' - >>> - -The :func:`create_string_buffer` function replaces the old :func:`c_buffer` -function (which is still available as an alias). To create a mutable memory -block containing unicode characters of the C type :c:type:`wchar_t`, use the -:func:`create_unicode_buffer` function. - - -.. _ctypes-calling-functions-continued: - -Calling functions, continued -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Note that printf prints to the real standard output channel, *not* to -:data:`sys.stdout`, so these examples will only work at the console prompt, not -from within *IDLE* or *PythonWin*:: - - >>> printf = libc.printf - >>> printf(b"Hello, %s\n", b"World!") - Hello, World! - 14 - >>> printf(b"Hello, %S\n", "World!") - Hello, World! - 14 - >>> printf(b"%d bottles of beer\n", 42) - 42 bottles of beer - 19 - >>> printf(b"%f bottles of beer\n", 42.5) - Traceback (most recent call last): - File "", line 1, in - ArgumentError: argument 2: TypeError: Don't know how to convert parameter 2 - >>> - -As has been mentioned before, all Python types except integers, strings, and -bytes objects have to be wrapped in their corresponding :mod:`ctypes` type, so -that they can be converted to the required C data type:: - - >>> printf(b"An int %d, a double %f\n", 1234, c_double(3.14)) - An int 1234, a double 3.140000 - 31 - >>> - -.. _ctypes-calling-variadic-functions: - -Calling variadic functions -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -On a lot of platforms calling variadic functions through ctypes is exactly the same -as calling functions with a fixed number of parameters. On some platforms, and in -particular ARM64 for Apple Platforms, the calling convention for variadic functions -is different than that for regular functions. - -On those platforms it is required to specify the *argtypes* attribute for the -regular, non-variadic, function arguments: - -.. code-block:: python3 - - libc.printf.argtypes = [ctypes.c_char_p] - -Because specifying the attribute does not inhibit portability it is advised to always -specify ``argtypes`` for all variadic functions. - - -.. _ctypes-calling-functions-with-own-custom-data-types: - -Calling functions with your own custom data types -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -You can also customize :mod:`ctypes` argument conversion to allow instances of -your own classes be used as function arguments. :mod:`ctypes` looks for an -:attr:`!_as_parameter_` attribute and uses this as the function argument. The -attribute must be an integer, string, bytes, a :mod:`ctypes` instance, or an -object with an :attr:`!_as_parameter_` attribute:: - - >>> class Bottles: - ... def __init__(self, number): - ... self._as_parameter_ = number - ... - >>> bottles = Bottles(42) - >>> printf(b"%d bottles of beer\n", bottles) - 42 bottles of beer - 19 - >>> - -If you don't want to store the instance's data in the :attr:`_as_parameter_` -instance variable, you could define a :class:`property` which makes the -attribute available on request. - - -.. _ctypes-specifying-required-argument-types: - -Specifying the required argument types (function prototypes) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is possible to specify the required argument types of functions exported from -DLLs by setting the :attr:`argtypes` attribute. - -:attr:`argtypes` must be a sequence of C data types (the ``printf`` function is -probably not a good example here, because it takes a variable number and -different types of parameters depending on the format string, on the other hand -this is quite handy to experiment with this feature):: - - >>> printf.argtypes = [c_char_p, c_char_p, c_int, c_double] - >>> printf(b"String '%s', Int %d, Double %f\n", b"Hi", 10, 2.2) - String 'Hi', Int 10, Double 2.200000 - 37 - >>> - -Specifying a format protects against incompatible argument types (just as a -prototype for a C function), and tries to convert the arguments to valid types:: - - >>> printf(b"%d %d %d", 1, 2, 3) - Traceback (most recent call last): - File "", line 1, in - ArgumentError: argument 2: TypeError: wrong type - >>> printf(b"%s %d %f\n", b"X", 2, 3) - X 2 3.000000 - 13 - >>> - -If you have defined your own classes which you pass to function calls, you have -to implement a :meth:`from_param` class method for them to be able to use them -in the :attr:`argtypes` sequence. The :meth:`from_param` class method receives -the Python object passed to the function call, it should do a typecheck or -whatever is needed to make sure this object is acceptable, and then return the -object itself, its :attr:`_as_parameter_` attribute, or whatever you want to -pass as the C function argument in this case. Again, the result should be an -integer, string, bytes, a :mod:`ctypes` instance, or an object with an -:attr:`_as_parameter_` attribute. - - -.. _ctypes-return-types: - -Return types -^^^^^^^^^^^^ - -By default functions are assumed to return the C :c:expr:`int` type. Other -return types can be specified by setting the :attr:`restype` attribute of the -function object. - -Here is a more advanced example, it uses the ``strchr`` function, which expects -a string pointer and a char, and returns a pointer to a string:: - - >>> strchr = libc.strchr - >>> strchr(b"abcdef", ord("d")) # doctest: +SKIP - 8059983 - >>> strchr.restype = c_char_p # c_char_p is a pointer to a string - >>> strchr(b"abcdef", ord("d")) - b'def' - >>> print(strchr(b"abcdef", ord("x"))) - None - >>> - -If you want to avoid the ``ord("x")`` calls above, you can set the -:attr:`argtypes` attribute, and the second argument will be converted from a -single character Python bytes object into a C char:: - - >>> strchr.restype = c_char_p - >>> strchr.argtypes = [c_char_p, c_char] - >>> strchr(b"abcdef", b"d") - 'def' - >>> strchr(b"abcdef", b"def") - Traceback (most recent call last): - File "", line 1, in - ArgumentError: argument 2: TypeError: one character string expected - >>> print(strchr(b"abcdef", b"x")) - None - >>> strchr(b"abcdef", b"d") - 'def' - >>> - -You can also use a callable Python object (a function or a class for example) as -the :attr:`restype` attribute, if the foreign function returns an integer. The -callable will be called with the *integer* the C function returns, and the -result of this call will be used as the result of your function call. This is -useful to check for error return values and automatically raise an exception:: - - >>> GetModuleHandle = windll.kernel32.GetModuleHandleA # doctest: +WINDOWS - >>> def ValidHandle(value): - ... if value == 0: - ... raise WinError() - ... return value - ... - >>> - >>> GetModuleHandle.restype = ValidHandle # doctest: +WINDOWS - >>> GetModuleHandle(None) # doctest: +WINDOWS - 486539264 - >>> GetModuleHandle("something silly") # doctest: +WINDOWS - Traceback (most recent call last): - File "", line 1, in - File "", line 3, in ValidHandle - OSError: [Errno 126] The specified module could not be found. - >>> - -``WinError`` is a function which will call Windows ``FormatMessage()`` api to -get the string representation of an error code, and *returns* an exception. -``WinError`` takes an optional error code parameter, if no one is used, it calls -:func:`GetLastError` to retrieve it. - -Please note that a much more powerful error checking mechanism is available -through the :attr:`errcheck` attribute; see the reference manual for details. - - -.. _ctypes-passing-pointers: - -Passing pointers (or: passing parameters by reference) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Sometimes a C api function expects a *pointer* to a data type as parameter, -probably to write into the corresponding location, or if the data is too large -to be passed by value. This is also known as *passing parameters by reference*. - -:mod:`ctypes` exports the :func:`byref` function which is used to pass parameters -by reference. The same effect can be achieved with the :func:`pointer` function, -although :func:`pointer` does a lot more work since it constructs a real pointer -object, so it is faster to use :func:`byref` if you don't need the pointer -object in Python itself:: - - >>> i = c_int() - >>> f = c_float() - >>> s = create_string_buffer(b'\000' * 32) - >>> print(i.value, f.value, repr(s.value)) - 0 0.0 b'' - >>> libc.sscanf(b"1 3.14 Hello", b"%d %f %s", - ... byref(i), byref(f), s) - 3 - >>> print(i.value, f.value, repr(s.value)) - 1 3.1400001049 b'Hello' - >>> - - -.. _ctypes-structures-unions: - -Structures and unions -^^^^^^^^^^^^^^^^^^^^^ - -Structures and unions must derive from the :class:`Structure` and :class:`Union` -base classes which are defined in the :mod:`ctypes` module. Each subclass must -define a :attr:`_fields_` attribute. :attr:`_fields_` must be a list of -*2-tuples*, containing a *field name* and a *field type*. - -The field type must be a :mod:`ctypes` type like :class:`c_int`, or any other -derived :mod:`ctypes` type: structure, union, array, pointer. - -Here is a simple example of a POINT structure, which contains two integers named -*x* and *y*, and also shows how to initialize a structure in the constructor:: - - >>> from ctypes import * - >>> class POINT(Structure): - ... _fields_ = [("x", c_int), - ... ("y", c_int)] - ... - >>> point = POINT(10, 20) - >>> print(point.x, point.y) - 10 20 - >>> point = POINT(y=5) - >>> print(point.x, point.y) - 0 5 - >>> POINT(1, 2, 3) - Traceback (most recent call last): - File "", line 1, in - TypeError: too many initializers - >>> - -You can, however, build much more complicated structures. A structure can -itself contain other structures by using a structure as a field type. - -Here is a RECT structure which contains two POINTs named *upperleft* and -*lowerright*:: - - >>> class RECT(Structure): - ... _fields_ = [("upperleft", POINT), - ... ("lowerright", POINT)] - ... - >>> rc = RECT(point) - >>> print(rc.upperleft.x, rc.upperleft.y) - 0 5 - >>> print(rc.lowerright.x, rc.lowerright.y) - 0 0 - >>> - -Nested structures can also be initialized in the constructor in several ways:: - - >>> r = RECT(POINT(1, 2), POINT(3, 4)) - >>> r = RECT((1, 2), (3, 4)) - -Field :term:`descriptor`\s can be retrieved from the *class*, they are useful -for debugging because they can provide useful information:: - - >>> print(POINT.x) - - >>> print(POINT.y) - - >>> - - -.. _ctypes-structureunion-alignment-byte-order: - -.. warning:: - - :mod:`ctypes` does not support passing unions or structures with bit-fields - to functions by value. While this may work on 32-bit x86, it's not - guaranteed by the library to work in the general case. Unions and - structures with bit-fields should always be passed to functions by pointer. - -Structure/union alignment and byte order -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, Structure and Union fields are aligned in the same way the C -compiler does it. It is possible to override this behavior by specifying a -:attr:`_pack_` class attribute in the subclass definition. This must be set to a -positive integer and specifies the maximum alignment for the fields. This is -what ``#pragma pack(n)`` also does in MSVC. - -:mod:`ctypes` uses the native byte order for Structures and Unions. To build -structures with non-native byte order, you can use one of the -:class:`BigEndianStructure`, :class:`LittleEndianStructure`, -:class:`BigEndianUnion`, and :class:`LittleEndianUnion` base classes. These -classes cannot contain pointer fields. - - -.. _ctypes-bit-fields-in-structures-unions: - -Bit fields in structures and unions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -It is possible to create structures and unions containing bit fields. Bit fields -are only possible for integer fields, the bit width is specified as the third -item in the :attr:`_fields_` tuples:: - - >>> class Int(Structure): - ... _fields_ = [("first_16", c_int, 16), - ... ("second_16", c_int, 16)] - ... - >>> print(Int.first_16) - - >>> print(Int.second_16) - - >>> - - -.. _ctypes-arrays: - -Arrays -^^^^^^ - -Arrays are sequences, containing a fixed number of instances of the same type. - -The recommended way to create array types is by multiplying a data type with a -positive integer:: - - TenPointsArrayType = POINT * 10 - -Here is an example of a somewhat artificial data type, a structure containing 4 -POINTs among other stuff:: - - >>> from ctypes import * - >>> class POINT(Structure): - ... _fields_ = ("x", c_int), ("y", c_int) - ... - >>> class MyStruct(Structure): - ... _fields_ = [("a", c_int), - ... ("b", c_float), - ... ("point_array", POINT * 4)] - >>> - >>> print(len(MyStruct().point_array)) - 4 - >>> - -Instances are created in the usual way, by calling the class:: - - arr = TenPointsArrayType() - for pt in arr: - print(pt.x, pt.y) - -The above code print a series of ``0 0`` lines, because the array contents is -initialized to zeros. - -Initializers of the correct type can also be specified:: - - >>> from ctypes import * - >>> TenIntegers = c_int * 10 - >>> ii = TenIntegers(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) - >>> print(ii) - - >>> for i in ii: print(i, end=" ") - ... - 1 2 3 4 5 6 7 8 9 10 - >>> - - -.. _ctypes-pointers: - -Pointers -^^^^^^^^ - -Pointer instances are created by calling the :func:`pointer` function on a -:mod:`ctypes` type:: - - >>> from ctypes import * - >>> i = c_int(42) - >>> pi = pointer(i) - >>> - -Pointer instances have a :attr:`~_Pointer.contents` attribute which -returns the object to which the pointer points, the ``i`` object above:: - - >>> pi.contents - c_long(42) - >>> - -Note that :mod:`ctypes` does not have OOR (original object return), it constructs a -new, equivalent object each time you retrieve an attribute:: - - >>> pi.contents is i - False - >>> pi.contents is pi.contents - False - >>> - -Assigning another :class:`c_int` instance to the pointer's contents attribute -would cause the pointer to point to the memory location where this is stored:: - - >>> i = c_int(99) - >>> pi.contents = i - >>> pi.contents - c_long(99) - >>> - -.. XXX Document dereferencing pointers, and that it is preferred over the - .contents attribute. - -Pointer instances can also be indexed with integers:: - - >>> pi[0] - 99 - >>> - -Assigning to an integer index changes the pointed to value:: - - >>> print(i) - c_long(99) - >>> pi[0] = 22 - >>> print(i) - c_long(22) - >>> - -It is also possible to use indexes different from 0, but you must know what -you're doing, just as in C: You can access or change arbitrary memory locations. -Generally you only use this feature if you receive a pointer from a C function, -and you *know* that the pointer actually points to an array instead of a single -item. - -Behind the scenes, the :func:`pointer` function does more than simply create -pointer instances, it has to create pointer *types* first. This is done with the -:func:`POINTER` function, which accepts any :mod:`ctypes` type, and returns a -new type:: - - >>> PI = POINTER(c_int) - >>> PI - - >>> PI(42) - Traceback (most recent call last): - File "", line 1, in - TypeError: expected c_long instead of int - >>> PI(c_int(42)) - - >>> - -Calling the pointer type without an argument creates a ``NULL`` pointer. -``NULL`` pointers have a ``False`` boolean value:: - - >>> null_ptr = POINTER(c_int)() - >>> print(bool(null_ptr)) - False - >>> - -:mod:`ctypes` checks for ``NULL`` when dereferencing pointers (but dereferencing -invalid non-\ ``NULL`` pointers would crash Python):: - - >>> null_ptr[0] - Traceback (most recent call last): - .... - ValueError: NULL pointer access - >>> - - >>> null_ptr[0] = 1234 - Traceback (most recent call last): - .... - ValueError: NULL pointer access - >>> - - -.. _ctypes-type-conversions: - -Type conversions -^^^^^^^^^^^^^^^^ - -Usually, ctypes does strict type checking. This means, if you have -``POINTER(c_int)`` in the :attr:`argtypes` list of a function or as the type of -a member field in a structure definition, only instances of exactly the same -type are accepted. There are some exceptions to this rule, where ctypes accepts -other objects. For example, you can pass compatible array instances instead of -pointer types. So, for ``POINTER(c_int)``, ctypes accepts an array of c_int:: - - >>> class Bar(Structure): - ... _fields_ = [("count", c_int), ("values", POINTER(c_int))] - ... - >>> bar = Bar() - >>> bar.values = (c_int * 3)(1, 2, 3) - >>> bar.count = 3 - >>> for i in range(bar.count): - ... print(bar.values[i]) - ... - 1 - 2 - 3 - >>> - -In addition, if a function argument is explicitly declared to be a pointer type -(such as ``POINTER(c_int)``) in :attr:`argtypes`, an object of the pointed -type (``c_int`` in this case) can be passed to the function. ctypes will apply -the required :func:`byref` conversion in this case automatically. - -To set a POINTER type field to ``NULL``, you can assign ``None``:: - - >>> bar.values = None - >>> - -.. XXX list other conversions... - -Sometimes you have instances of incompatible types. In C, you can cast one type -into another type. :mod:`ctypes` provides a :func:`cast` function which can be -used in the same way. The ``Bar`` structure defined above accepts -``POINTER(c_int)`` pointers or :class:`c_int` arrays for its ``values`` field, -but not instances of other types:: - - >>> bar.values = (c_byte * 4)() - Traceback (most recent call last): - File "", line 1, in - TypeError: incompatible types, c_byte_Array_4 instance instead of LP_c_long instance - >>> - -For these cases, the :func:`cast` function is handy. - -The :func:`cast` function can be used to cast a ctypes instance into a pointer -to a different ctypes data type. :func:`cast` takes two parameters, a ctypes -object that is or can be converted to a pointer of some kind, and a ctypes -pointer type. It returns an instance of the second argument, which references -the same memory block as the first argument:: - - >>> a = (c_byte * 4)() - >>> cast(a, POINTER(c_int)) - - >>> - -So, :func:`cast` can be used to assign to the ``values`` field of ``Bar`` the -structure:: - - >>> bar = Bar() - >>> bar.values = cast((c_byte * 4)(), POINTER(c_int)) - >>> print(bar.values[0]) - 0 - >>> - - -.. _ctypes-incomplete-types: - -Incomplete Types -^^^^^^^^^^^^^^^^ - -*Incomplete Types* are structures, unions or arrays whose members are not yet -specified. In C, they are specified by forward declarations, which are defined -later:: - - struct cell; /* forward declaration */ - - struct cell { - char *name; - struct cell *next; - }; - -The straightforward translation into ctypes code would be this, but it does not -work:: - - >>> class cell(Structure): - ... _fields_ = [("name", c_char_p), - ... ("next", POINTER(cell))] - ... - Traceback (most recent call last): - File "", line 1, in - File "", line 2, in cell - NameError: name 'cell' is not defined - >>> - -because the new ``class cell`` is not available in the class statement itself. -In :mod:`ctypes`, we can define the ``cell`` class and set the :attr:`_fields_` -attribute later, after the class statement:: - - >>> from ctypes import * - >>> class cell(Structure): - ... pass - ... - >>> cell._fields_ = [("name", c_char_p), - ... ("next", POINTER(cell))] - >>> - -Let's try it. We create two instances of ``cell``, and let them point to each -other, and finally follow the pointer chain a few times:: - - >>> c1 = cell() - >>> c1.name = b"foo" - >>> c2 = cell() - >>> c2.name = b"bar" - >>> c1.next = pointer(c2) - >>> c2.next = pointer(c1) - >>> p = c1 - >>> for i in range(8): - ... print(p.name, end=" ") - ... p = p.next[0] - ... - foo bar foo bar foo bar foo bar - >>> - - -.. _ctypes-callback-functions: - -Callback functions -^^^^^^^^^^^^^^^^^^ - -:mod:`ctypes` allows creating C callable function pointers from Python callables. -These are sometimes called *callback functions*. - -First, you must create a class for the callback function. The class knows the -calling convention, the return type, and the number and types of arguments this -function will receive. - -The :func:`CFUNCTYPE` factory function creates types for callback functions -using the ``cdecl`` calling convention. On Windows, the :func:`WINFUNCTYPE` -factory function creates types for callback functions using the ``stdcall`` -calling convention. - -Both of these factory functions are called with the result type as first -argument, and the callback functions expected argument types as the remaining -arguments. - -I will present an example here which uses the standard C library's -:c:func:`qsort` function, that is used to sort items with the help of a callback -function. :c:func:`qsort` will be used to sort an array of integers:: - - >>> IntArray5 = c_int * 5 - >>> ia = IntArray5(5, 1, 7, 33, 99) - >>> qsort = libc.qsort - >>> qsort.restype = None - >>> - -:func:`qsort` must be called with a pointer to the data to sort, the number of -items in the data array, the size of one item, and a pointer to the comparison -function, the callback. The callback will then be called with two pointers to -items, and it must return a negative integer if the first item is smaller than -the second, a zero if they are equal, and a positive integer otherwise. - -So our callback function receives pointers to integers, and must return an -integer. First we create the ``type`` for the callback function:: - - >>> CMPFUNC = CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int)) - >>> - -To get started, here is a simple callback that shows the values it gets -passed:: - - >>> def py_cmp_func(a, b): - ... print("py_cmp_func", a[0], b[0]) - ... return 0 - ... - >>> cmp_func = CMPFUNC(py_cmp_func) - >>> - -The result:: - - >>> qsort(ia, len(ia), sizeof(c_int), cmp_func) # doctest: +LINUX - py_cmp_func 5 1 - py_cmp_func 33 99 - py_cmp_func 7 33 - py_cmp_func 5 7 - py_cmp_func 1 7 - >>> - -Now we can actually compare the two items and return a useful result:: - - >>> def py_cmp_func(a, b): - ... print("py_cmp_func", a[0], b[0]) - ... return a[0] - b[0] - ... - >>> - >>> qsort(ia, len(ia), sizeof(c_int), CMPFUNC(py_cmp_func)) # doctest: +LINUX - py_cmp_func 5 1 - py_cmp_func 33 99 - py_cmp_func 7 33 - py_cmp_func 1 7 - py_cmp_func 5 7 - >>> - -As we can easily check, our array is sorted now:: - - >>> for i in ia: print(i, end=" ") - ... - 1 5 7 33 99 - >>> - -The function factories can be used as decorator factories, so we may as well -write:: - - >>> @CFUNCTYPE(c_int, POINTER(c_int), POINTER(c_int)) - ... def py_cmp_func(a, b): - ... print("py_cmp_func", a[0], b[0]) - ... return a[0] - b[0] - ... - >>> qsort(ia, len(ia), sizeof(c_int), py_cmp_func) - py_cmp_func 5 1 - py_cmp_func 33 99 - py_cmp_func 7 33 - py_cmp_func 1 7 - py_cmp_func 5 7 - >>> - -.. note:: - - Make sure you keep references to :func:`CFUNCTYPE` objects as long as they - are used from C code. :mod:`ctypes` doesn't, and if you don't, they may be - garbage collected, crashing your program when a callback is made. - - Also, note that if the callback function is called in a thread created - outside of Python's control (e.g. by the foreign code that calls the - callback), ctypes creates a new dummy Python thread on every invocation. This - behavior is correct for most purposes, but it means that values stored with - :class:`threading.local` will *not* survive across different callbacks, even when - those calls are made from the same C thread. - -.. _ctypes-accessing-values-exported-from-dlls: - -Accessing values exported from dlls -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Some shared libraries not only export functions, they also export variables. An -example in the Python library itself is the :c:data:`Py_OptimizeFlag`, an integer -set to 0, 1, or 2, depending on the :option:`-O` or :option:`-OO` flag given on -startup. - -:mod:`ctypes` can access values like this with the :meth:`in_dll` class methods of -the type. *pythonapi* is a predefined symbol giving access to the Python C -api:: - - >>> opt_flag = c_int.in_dll(pythonapi, "Py_OptimizeFlag") - >>> print(opt_flag) - c_long(0) - >>> - -If the interpreter would have been started with :option:`-O`, the sample would -have printed ``c_long(1)``, or ``c_long(2)`` if :option:`-OO` would have been -specified. - -An extended example which also demonstrates the use of pointers accesses the -:c:data:`PyImport_FrozenModules` pointer exported by Python. - -Quoting the docs for that value: - - This pointer is initialized to point to an array of :c:struct:`_frozen` - records, terminated by one whose members are all ``NULL`` or zero. When a frozen - module is imported, it is searched in this table. Third-party code could play - tricks with this to provide a dynamically created collection of frozen modules. - -So manipulating this pointer could even prove useful. To restrict the example -size, we show only how this table can be read with :mod:`ctypes`:: - - >>> from ctypes import * - >>> - >>> class struct_frozen(Structure): - ... _fields_ = [("name", c_char_p), - ... ("code", POINTER(c_ubyte)), - ... ("size", c_int), - ... ("get_code", POINTER(c_ubyte)), # Function pointer - ... ] - ... - >>> - -We have defined the :c:struct:`_frozen` data type, so we can get the pointer -to the table:: - - >>> FrozenTable = POINTER(struct_frozen) - >>> table = FrozenTable.in_dll(pythonapi, "_PyImport_FrozenBootstrap") - >>> - -Since ``table`` is a ``pointer`` to the array of ``struct_frozen`` records, we -can iterate over it, but we just have to make sure that our loop terminates, -because pointers have no size. Sooner or later it would probably crash with an -access violation or whatever, so it's better to break out of the loop when we -hit the ``NULL`` entry:: - - >>> for item in table: - ... if item.name is None: - ... break - ... print(item.name.decode("ascii"), item.size) - ... - _frozen_importlib 31764 - _frozen_importlib_external 41499 - zipimport 12345 - >>> - -The fact that standard Python has a frozen module and a frozen package -(indicated by the negative ``size`` member) is not well known, it is only used -for testing. Try it out with ``import __hello__`` for example. - - -.. _ctypes-surprises: - -Surprises -^^^^^^^^^ - -There are some edges in :mod:`ctypes` where you might expect something other -than what actually happens. - -Consider the following example:: - - >>> from ctypes import * - >>> class POINT(Structure): - ... _fields_ = ("x", c_int), ("y", c_int) - ... - >>> class RECT(Structure): - ... _fields_ = ("a", POINT), ("b", POINT) - ... - >>> p1 = POINT(1, 2) - >>> p2 = POINT(3, 4) - >>> rc = RECT(p1, p2) - >>> print(rc.a.x, rc.a.y, rc.b.x, rc.b.y) - 1 2 3 4 - >>> # now swap the two points - >>> rc.a, rc.b = rc.b, rc.a - >>> print(rc.a.x, rc.a.y, rc.b.x, rc.b.y) - 3 4 3 4 - >>> - -Hm. We certainly expected the last statement to print ``3 4 1 2``. What -happened? Here are the steps of the ``rc.a, rc.b = rc.b, rc.a`` line above:: - - >>> temp0, temp1 = rc.b, rc.a - >>> rc.a = temp0 - >>> rc.b = temp1 - >>> - -Note that ``temp0`` and ``temp1`` are objects still using the internal buffer of -the ``rc`` object above. So executing ``rc.a = temp0`` copies the buffer -contents of ``temp0`` into ``rc`` 's buffer. This, in turn, changes the -contents of ``temp1``. So, the last assignment ``rc.b = temp1``, doesn't have -the expected effect. - -Keep in mind that retrieving sub-objects from Structure, Unions, and Arrays -doesn't *copy* the sub-object, instead it retrieves a wrapper object accessing -the root-object's underlying buffer. - -Another example that may behave differently from what one would expect is this:: - - >>> s = c_char_p() - >>> s.value = b"abc def ghi" - >>> s.value - b'abc def ghi' - >>> s.value is s.value - False - >>> - -.. note:: - - Objects instantiated from :class:`c_char_p` can only have their value set to bytes - or integers. - -Why is it printing ``False``? ctypes instances are objects containing a memory -block plus some :term:`descriptor`\s accessing the contents of the memory. -Storing a Python object in the memory block does not store the object itself, -instead the ``contents`` of the object is stored. Accessing the contents again -constructs a new Python object each time! - - -.. _ctypes-variable-sized-data-types: - -Variable-sized data types -^^^^^^^^^^^^^^^^^^^^^^^^^ - -:mod:`ctypes` provides some support for variable-sized arrays and structures. - -The :func:`resize` function can be used to resize the memory buffer of an -existing ctypes object. The function takes the object as first argument, and -the requested size in bytes as the second argument. The memory block cannot be -made smaller than the natural memory block specified by the objects type, a -:exc:`ValueError` is raised if this is tried:: - - >>> short_array = (c_short * 4)() - >>> print(sizeof(short_array)) - 8 - >>> resize(short_array, 4) - Traceback (most recent call last): - ... - ValueError: minimum size is 8 - >>> resize(short_array, 32) - >>> sizeof(short_array) - 32 - >>> sizeof(type(short_array)) - 8 - >>> - -This is nice and fine, but how would one access the additional elements -contained in this array? Since the type still only knows about 4 elements, we -get errors accessing other elements:: - - >>> short_array[:] - [0, 0, 0, 0] - >>> short_array[7] - Traceback (most recent call last): - ... - IndexError: invalid index - >>> - -Another way to use variable-sized data types with :mod:`ctypes` is to use the -dynamic nature of Python, and (re-)define the data type after the required size -is already known, on a case by case basis. - - -.. _ctypes-ctypes-reference: - -ctypes reference ----------------- - - -.. _ctypes-finding-shared-libraries: - -Finding shared libraries -^^^^^^^^^^^^^^^^^^^^^^^^ - -When programming in a compiled language, shared libraries are accessed when -compiling/linking a program, and when the program is run. - -The purpose of the :func:`find_library` function is to locate a library in a way -similar to what the compiler or runtime loader does (on platforms with several -versions of a shared library the most recent should be loaded), while the ctypes -library loaders act like when a program is run, and call the runtime loader -directly. - -The :mod:`ctypes.util` module provides a function which can help to determine -the library to load. - - -.. data:: find_library(name) - :module: ctypes.util - :noindex: - - Try to find a library and return a pathname. *name* is the library name without - any prefix like *lib*, suffix like ``.so``, ``.dylib`` or version number (this - is the form used for the posix linker option :option:`!-l`). If no library can - be found, returns ``None``. - -The exact functionality is system dependent. - -On Linux, :func:`find_library` tries to run external programs -(``/sbin/ldconfig``, ``gcc``, ``objdump`` and ``ld``) to find the library file. -It returns the filename of the library file. - -.. versionchanged:: 3.6 - On Linux, the value of the environment variable ``LD_LIBRARY_PATH`` is used - when searching for libraries, if a library cannot be found by any other means. - -Here are some examples:: - - >>> from ctypes.util import find_library - >>> find_library("m") - 'libm.so.6' - >>> find_library("c") - 'libc.so.6' - >>> find_library("bz2") - 'libbz2.so.1.0' - >>> - -On macOS, :func:`find_library` tries several predefined naming schemes and paths -to locate the library, and returns a full pathname if successful:: - - >>> from ctypes.util import find_library - >>> find_library("c") - '/usr/lib/libc.dylib' - >>> find_library("m") - '/usr/lib/libm.dylib' - >>> find_library("bz2") - '/usr/lib/libbz2.dylib' - >>> find_library("AGL") - '/System/Library/Frameworks/AGL.framework/AGL' - >>> - -On Windows, :func:`find_library` searches along the system search path, and -returns the full pathname, but since there is no predefined naming scheme a call -like ``find_library("c")`` will fail and return ``None``. - -If wrapping a shared library with :mod:`ctypes`, it *may* be better to determine -the shared library name at development time, and hardcode that into the wrapper -module instead of using :func:`find_library` to locate the library at runtime. - - -.. _ctypes-loading-shared-libraries: - -Loading shared libraries -^^^^^^^^^^^^^^^^^^^^^^^^ - -There are several ways to load shared libraries into the Python process. One -way is to instantiate one of the following classes: - - -.. class:: CDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - - Instances of this class represent loaded shared libraries. Functions in these - libraries use the standard C calling convention, and are assumed to return - :c:expr:`int`. - - On Windows creating a :class:`CDLL` instance may fail even if the DLL name - exists. When a dependent DLL of the loaded DLL is not found, a - :exc:`OSError` error is raised with the message *"[WinError 126] The - specified module could not be found".* This error message does not contain - the name of the missing DLL because the Windows API does not return this - information making this error hard to diagnose. To resolve this error and - determine which DLL is not found, you need to find the list of dependent - DLLs and determine which one is not found using Windows debugging and - tracing tools. - -.. seealso:: - - `Microsoft DUMPBIN tool `_ - -- A tool to find DLL dependents. - - -.. class:: OleDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - - Windows only: Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are - assumed to return the windows specific :class:`HRESULT` code. :class:`HRESULT` - values contain information specifying whether the function call failed or - succeeded, together with additional error code. If the return value signals a - failure, an :class:`OSError` is automatically raised. - - .. versionchanged:: 3.3 - :exc:`WindowsError` used to be raised, - which is now an alias of :exc:`OSError`. - - -.. class:: WinDLL(name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None) - - Windows only: Instances of this class represent loaded shared libraries, - functions in these libraries use the ``stdcall`` calling convention, and are - assumed to return :c:expr:`int` by default. - -The Python :term:`global interpreter lock` is released before calling any -function exported by these libraries, and reacquired afterwards. - - -.. class:: PyDLL(name, mode=DEFAULT_MODE, handle=None) - - Instances of this class behave like :class:`CDLL` instances, except that the - Python GIL is *not* released during the function call, and after the function - execution the Python error flag is checked. If the error flag is set, a Python - exception is raised. - - Thus, this is only useful to call Python C api functions directly. - -All these classes can be instantiated by calling them with at least one -argument, the pathname of the shared library. If you have an existing handle to -an already loaded shared library, it can be passed as the ``handle`` named -parameter, otherwise the underlying platforms ``dlopen`` or ``LoadLibrary`` -function is used to load the library into the process, and to get a handle to -it. - -The *mode* parameter can be used to specify how the library is loaded. For -details, consult the :manpage:`dlopen(3)` manpage. On Windows, *mode* is -ignored. On posix systems, RTLD_NOW is always added, and is not -configurable. - -The *use_errno* parameter, when set to true, enables a ctypes mechanism that -allows accessing the system :data:`errno` error number in a safe way. -:mod:`ctypes` maintains a thread-local copy of the systems :data:`errno` -variable; if you call foreign functions created with ``use_errno=True`` then the -:data:`errno` value before the function call is swapped with the ctypes private -copy, the same happens immediately after the function call. - -The function :func:`ctypes.get_errno` returns the value of the ctypes private -copy, and the function :func:`ctypes.set_errno` changes the ctypes private copy -to a new value and returns the former value. - -The *use_last_error* parameter, when set to true, enables the same mechanism for -the Windows error code which is managed by the :func:`GetLastError` and -:func:`SetLastError` Windows API functions; :func:`ctypes.get_last_error` and -:func:`ctypes.set_last_error` are used to request and change the ctypes private -copy of the windows error code. - -The *winmode* parameter is used on Windows to specify how the library is loaded -(since *mode* is ignored). It takes any value that is valid for the Win32 API -``LoadLibraryEx`` flags parameter. When omitted, the default is to use the flags -that result in the most secure DLL load to avoiding issues such as DLL -hijacking. Passing the full path to the DLL is the safest way to ensure the -correct library and dependencies are loaded. - -.. versionchanged:: 3.8 - Added *winmode* parameter. - - -.. data:: RTLD_GLOBAL - :noindex: - - Flag to use as *mode* parameter. On platforms where this flag is not available, - it is defined as the integer zero. - - -.. data:: RTLD_LOCAL - :noindex: - - Flag to use as *mode* parameter. On platforms where this is not available, it - is the same as *RTLD_GLOBAL*. - - -.. data:: DEFAULT_MODE - :noindex: - - The default mode which is used to load shared libraries. On OSX 10.3, this is - *RTLD_GLOBAL*, otherwise it is the same as *RTLD_LOCAL*. - -Instances of these classes have no public methods. Functions exported by the -shared library can be accessed as attributes or by index. Please note that -accessing the function through an attribute caches the result and therefore -accessing it repeatedly returns the same object each time. On the other hand, -accessing it through an index returns a new object each time:: - - >>> from ctypes import CDLL - >>> libc = CDLL("libc.so.6") # On Linux - >>> libc.time == libc.time - True - >>> libc['time'] == libc['time'] - False - -The following public attributes are available, their name starts with an -underscore to not clash with exported function names: - - -.. attribute:: PyDLL._handle - - The system handle used to access the library. - - -.. attribute:: PyDLL._name - - The name of the library passed in the constructor. - -Shared libraries can also be loaded by using one of the prefabricated objects, -which are instances of the :class:`LibraryLoader` class, either by calling the -:meth:`LoadLibrary` method, or by retrieving the library as attribute of the -loader instance. - - -.. class:: LibraryLoader(dlltype) - - Class which loads shared libraries. *dlltype* should be one of the - :class:`CDLL`, :class:`PyDLL`, :class:`WinDLL`, or :class:`OleDLL` types. - - :meth:`__getattr__` has special behavior: It allows loading a shared library by - accessing it as attribute of a library loader instance. The result is cached, - so repeated attribute accesses return the same library each time. - - .. method:: LoadLibrary(name) - - Load a shared library into the process and return it. This method always - returns a new instance of the library. - - -These prefabricated library loaders are available: - -.. data:: cdll - :noindex: - - Creates :class:`CDLL` instances. - - -.. data:: windll - :noindex: - - Windows only: Creates :class:`WinDLL` instances. - - -.. data:: oledll - :noindex: - - Windows only: Creates :class:`OleDLL` instances. - - -.. data:: pydll - :noindex: - - Creates :class:`PyDLL` instances. - - -For accessing the C Python api directly, a ready-to-use Python shared library -object is available: - -.. data:: pythonapi - :noindex: - - An instance of :class:`PyDLL` that exposes Python C API functions as - attributes. Note that all these functions are assumed to return C - :c:expr:`int`, which is of course not always the truth, so you have to assign - the correct :attr:`restype` attribute to use these functions. - -.. audit-event:: ctypes.dlopen name ctypes.LibraryLoader - - Loading a library through any of these objects raises an - :ref:`auditing event ` ``ctypes.dlopen`` with string argument - ``name``, the name used to load the library. - -.. audit-event:: ctypes.dlsym library,name ctypes.LibraryLoader - - Accessing a function on a loaded library raises an auditing event - ``ctypes.dlsym`` with arguments ``library`` (the library object) and ``name`` - (the symbol's name as a string or integer). - -.. audit-event:: ctypes.dlsym/handle handle,name ctypes.LibraryLoader - - In cases when only the library handle is available rather than the object, - accessing a function raises an auditing event ``ctypes.dlsym/handle`` with - arguments ``handle`` (the raw library handle) and ``name``. - -.. _ctypes-foreign-functions: - -Foreign functions -^^^^^^^^^^^^^^^^^ - -As explained in the previous section, foreign functions can be accessed as -attributes of loaded shared libraries. The function objects created in this way -by default accept any number of arguments, accept any ctypes data instances as -arguments, and return the default result type specified by the library loader. -They are instances of a private class: - - -.. class:: _FuncPtr - - Base class for C callable foreign functions. - - Instances of foreign functions are also C compatible data types; they - represent C function pointers. - - This behavior can be customized by assigning to special attributes of the - foreign function object. - - .. attribute:: restype - - Assign a ctypes type to specify the result type of the foreign function. - Use ``None`` for :c:expr:`void`, a function not returning anything. - - It is possible to assign a callable Python object that is not a ctypes - type, in this case the function is assumed to return a C :c:expr:`int`, and - the callable will be called with this integer, allowing further - processing or error checking. Using this is deprecated, for more flexible - post processing or error checking use a ctypes data type as - :attr:`restype` and assign a callable to the :attr:`errcheck` attribute. - - .. attribute:: argtypes - - Assign a tuple of ctypes types to specify the argument types that the - function accepts. Functions using the ``stdcall`` calling convention can - only be called with the same number of arguments as the length of this - tuple; functions using the C calling convention accept additional, - unspecified arguments as well. - - When a foreign function is called, each actual argument is passed to the - :meth:`from_param` class method of the items in the :attr:`argtypes` - tuple, this method allows adapting the actual argument to an object that - the foreign function accepts. For example, a :class:`c_char_p` item in - the :attr:`argtypes` tuple will convert a string passed as argument into - a bytes object using ctypes conversion rules. - - New: It is now possible to put items in argtypes which are not ctypes - types, but each item must have a :meth:`from_param` method which returns a - value usable as argument (integer, string, ctypes instance). This allows - defining adapters that can adapt custom objects as function parameters. - - .. attribute:: errcheck - - Assign a Python function or another callable to this attribute. The - callable will be called with three or more arguments: - - .. function:: callable(result, func, arguments) - :noindex: - :module: - - *result* is what the foreign function returns, as specified by the - :attr:`restype` attribute. - - *func* is the foreign function object itself, this allows reusing the - same callable object to check or post process the results of several - functions. - - *arguments* is a tuple containing the parameters originally passed to - the function call, this allows specializing the behavior on the - arguments used. - - The object that this function returns will be returned from the - foreign function call, but it can also check the result value - and raise an exception if the foreign function call failed. - - -.. exception:: ArgumentError - - This exception is raised when a foreign function call cannot convert one of the - passed arguments. - - -.. audit-event:: ctypes.seh_exception code foreign-functions - - On Windows, when a foreign function call raises a system exception (for - example, due to an access violation), it will be captured and replaced with - a suitable Python exception. Further, an auditing event - ``ctypes.seh_exception`` with argument ``code`` will be raised, allowing an - audit hook to replace the exception with its own. - -.. audit-event:: ctypes.call_function func_pointer,arguments foreign-functions - - Some ways to invoke foreign function calls may raise an auditing event - ``ctypes.call_function`` with arguments ``function pointer`` and ``arguments``. - -.. _ctypes-function-prototypes: - -Function prototypes -^^^^^^^^^^^^^^^^^^^ - -Foreign functions can also be created by instantiating function prototypes. -Function prototypes are similar to function prototypes in C; they describe a -function (return type, argument types, calling convention) without defining an -implementation. The factory functions must be called with the desired result -type and the argument types of the function, and can be used as decorator -factories, and as such, be applied to functions through the ``@wrapper`` syntax. -See :ref:`ctypes-callback-functions` for examples. - - -.. function:: CFUNCTYPE(restype, *argtypes, use_errno=False, use_last_error=False) - - The returned function prototype creates functions that use the standard C - calling convention. The function will release the GIL during the call. If - *use_errno* is set to true, the ctypes private copy of the system - :data:`errno` variable is exchanged with the real :data:`errno` value before - and after the call; *use_last_error* does the same for the Windows error - code. - - -.. function:: WINFUNCTYPE(restype, *argtypes, use_errno=False, use_last_error=False) - - Windows only: The returned function prototype creates functions that use the - ``stdcall`` calling convention. The function will - release the GIL during the call. *use_errno* and *use_last_error* have the - same meaning as above. - - -.. function:: PYFUNCTYPE(restype, *argtypes) - - The returned function prototype creates functions that use the Python calling - convention. The function will *not* release the GIL during the call. - -Function prototypes created by these factory functions can be instantiated in -different ways, depending on the type and number of the parameters in the call: - -.. function:: prototype(address) - :noindex: - :module: - - Returns a foreign function at the specified address which must be an integer. - - -.. function:: prototype(callable) - :noindex: - :module: - - Create a C callable function (a callback function) from a Python *callable*. - - -.. function:: prototype(func_spec[, paramflags]) - :noindex: - :module: - - Returns a foreign function exported by a shared library. *func_spec* must - be a 2-tuple ``(name_or_ordinal, library)``. The first item is the name of - the exported function as string, or the ordinal of the exported function - as small integer. The second item is the shared library instance. - - -.. function:: prototype(vtbl_index, name[, paramflags[, iid]]) - :noindex: - :module: - - Returns a foreign function that will call a COM method. *vtbl_index* is - the index into the virtual function table, a small non-negative - integer. *name* is name of the COM method. *iid* is an optional pointer to - the interface identifier which is used in extended error reporting. - - COM methods use a special calling convention: They require a pointer to - the COM interface as first argument, in addition to those parameters that - are specified in the :attr:`!argtypes` tuple. - -The optional *paramflags* parameter creates foreign function wrappers with much -more functionality than the features described above. - -*paramflags* must be a tuple of the same length as :attr:`~_FuncPtr.argtypes`. - -Each item in this tuple contains further information about a parameter, it must -be a tuple containing one, two, or three items. - -The first item is an integer containing a combination of direction -flags for the parameter: - - 1 - Specifies an input parameter to the function. - - 2 - Output parameter. The foreign function fills in a value. - - 4 - Input parameter which defaults to the integer zero. - -The optional second item is the parameter name as string. If this is specified, -the foreign function can be called with named parameters. - -The optional third item is the default value for this parameter. - - -The following example demonstrates how to wrap the Windows ``MessageBoxW`` function so -that it supports default parameters and named arguments. The C declaration from -the windows header file is this:: - - WINUSERAPI int WINAPI - MessageBoxW( - HWND hWnd, - LPCWSTR lpText, - LPCWSTR lpCaption, - UINT uType); - -Here is the wrapping with :mod:`ctypes`:: - - >>> from ctypes import c_int, WINFUNCTYPE, windll - >>> from ctypes.wintypes import HWND, LPCWSTR, UINT - >>> prototype = WINFUNCTYPE(c_int, HWND, LPCWSTR, LPCWSTR, UINT) - >>> paramflags = (1, "hwnd", 0), (1, "text", "Hi"), (1, "caption", "Hello from ctypes"), (1, "flags", 0) - >>> MessageBox = prototype(("MessageBoxW", windll.user32), paramflags) - -The ``MessageBox`` foreign function can now be called in these ways:: - - >>> MessageBox() - >>> MessageBox(text="Spam, spam, spam") - >>> MessageBox(flags=2, text="foo bar") - -A second example demonstrates output parameters. The win32 ``GetWindowRect`` -function retrieves the dimensions of a specified window by copying them into -``RECT`` structure that the caller has to supply. Here is the C declaration:: - - WINUSERAPI BOOL WINAPI - GetWindowRect( - HWND hWnd, - LPRECT lpRect); - -Here is the wrapping with :mod:`ctypes`:: - - >>> from ctypes import POINTER, WINFUNCTYPE, windll, WinError - >>> from ctypes.wintypes import BOOL, HWND, RECT - >>> prototype = WINFUNCTYPE(BOOL, HWND, POINTER(RECT)) - >>> paramflags = (1, "hwnd"), (2, "lprect") - >>> GetWindowRect = prototype(("GetWindowRect", windll.user32), paramflags) - >>> - -Functions with output parameters will automatically return the output parameter -value if there is a single one, or a tuple containing the output parameter -values when there are more than one, so the GetWindowRect function now returns a -RECT instance, when called. - -Output parameters can be combined with the :attr:`errcheck` protocol to do -further output processing and error checking. The win32 ``GetWindowRect`` api -function returns a ``BOOL`` to signal success or failure, so this function could -do the error checking, and raises an exception when the api call failed:: - - >>> def errcheck(result, func, args): - ... if not result: - ... raise WinError() - ... return args - ... - >>> GetWindowRect.errcheck = errcheck - >>> - -If the :attr:`errcheck` function returns the argument tuple it receives -unchanged, :mod:`ctypes` continues the normal processing it does on the output -parameters. If you want to return a tuple of window coordinates instead of a -``RECT`` instance, you can retrieve the fields in the function and return them -instead, the normal processing will no longer take place:: - - >>> def errcheck(result, func, args): - ... if not result: - ... raise WinError() - ... rc = args[1] - ... return rc.left, rc.top, rc.bottom, rc.right - ... - >>> GetWindowRect.errcheck = errcheck - >>> - - -.. _ctypes-utility-functions: - -Utility functions -^^^^^^^^^^^^^^^^^ - -.. function:: addressof(obj) - - Returns the address of the memory buffer as integer. *obj* must be an - instance of a ctypes type. - - .. audit-event:: ctypes.addressof obj ctypes.addressof - - -.. function:: alignment(obj_or_type) - - Returns the alignment requirements of a ctypes type. *obj_or_type* must be a - ctypes type or instance. - - -.. function:: byref(obj[, offset]) - - Returns a light-weight pointer to *obj*, which must be an instance of a - ctypes type. *offset* defaults to zero, and must be an integer that will be - added to the internal pointer value. - - ``byref(obj, offset)`` corresponds to this C code:: - - (((char *)&obj) + offset) - - The returned object can only be used as a foreign function call parameter. - It behaves similar to ``pointer(obj)``, but the construction is a lot faster. - - -.. function:: cast(obj, type) - - This function is similar to the cast operator in C. It returns a new instance - of *type* which points to the same memory block as *obj*. *type* must be a - pointer type, and *obj* must be an object that can be interpreted as a - pointer. - - -.. function:: create_string_buffer(init_or_size, size=None) - - This function creates a mutable character buffer. The returned object is a - ctypes array of :class:`c_char`. - - *init_or_size* must be an integer which specifies the size of the array, or a - bytes object which will be used to initialize the array items. - - If a bytes object is specified as first argument, the buffer is made one item - larger than its length so that the last element in the array is a NUL - termination character. An integer can be passed as second argument which allows - specifying the size of the array if the length of the bytes should not be used. - - .. audit-event:: ctypes.create_string_buffer init,size ctypes.create_string_buffer - - -.. function:: create_unicode_buffer(init_or_size, size=None) - - This function creates a mutable unicode character buffer. The returned object is - a ctypes array of :class:`c_wchar`. - - *init_or_size* must be an integer which specifies the size of the array, or a - string which will be used to initialize the array items. - - If a string is specified as first argument, the buffer is made one item - larger than the length of the string so that the last element in the array is a - NUL termination character. An integer can be passed as second argument which - allows specifying the size of the array if the length of the string should not - be used. - - .. audit-event:: ctypes.create_unicode_buffer init,size ctypes.create_unicode_buffer - - -.. function:: DllCanUnloadNow() - - Windows only: This function is a hook which allows implementing in-process - COM servers with ctypes. It is called from the DllCanUnloadNow function that - the _ctypes extension dll exports. - - -.. function:: DllGetClassObject() - - Windows only: This function is a hook which allows implementing in-process - COM servers with ctypes. It is called from the DllGetClassObject function - that the ``_ctypes`` extension dll exports. - - -.. function:: find_library(name) - :module: ctypes.util - - Try to find a library and return a pathname. *name* is the library name - without any prefix like ``lib``, suffix like ``.so``, ``.dylib`` or version - number (this is the form used for the posix linker option :option:`!-l`). If - no library can be found, returns ``None``. - - The exact functionality is system dependent. - - -.. function:: find_msvcrt() - :module: ctypes.util - - Windows only: return the filename of the VC runtime library used by Python, - and by the extension modules. If the name of the library cannot be - determined, ``None`` is returned. - - If you need to free memory, for example, allocated by an extension module - with a call to the ``free(void *)``, it is important that you use the - function in the same library that allocated the memory. - - -.. function:: FormatError([code]) - - Windows only: Returns a textual description of the error code *code*. If no - error code is specified, the last error code is used by calling the Windows - api function GetLastError. - - -.. function:: GetLastError() - - Windows only: Returns the last error code set by Windows in the calling thread. - This function calls the Windows ``GetLastError()`` function directly, - it does not return the ctypes-private copy of the error code. - -.. function:: get_errno() - - Returns the current value of the ctypes-private copy of the system - :data:`errno` variable in the calling thread. - - .. audit-event:: ctypes.get_errno "" ctypes.get_errno - -.. function:: get_last_error() - - Windows only: returns the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread. - - .. audit-event:: ctypes.get_last_error "" ctypes.get_last_error - -.. function:: memmove(dst, src, count) - - Same as the standard C memmove library function: copies *count* bytes from - *src* to *dst*. *dst* and *src* must be integers or ctypes instances that can - be converted to pointers. - - -.. function:: memset(dst, c, count) - - Same as the standard C memset library function: fills the memory block at - address *dst* with *count* bytes of value *c*. *dst* must be an integer - specifying an address, or a ctypes instance. - - -.. function:: POINTER(type, /) - - Create and return a new ctypes pointer type. Pointer types are cached and - reused internally, so calling this function repeatedly is cheap. - *type* must be a ctypes type. - - -.. function:: pointer(obj, /) - - Create a new pointer instance, pointing to *obj*. - The returned object is of the type ``POINTER(type(obj))``. - - Note: If you just want to pass a pointer to an object to a foreign function - call, you should use ``byref(obj)`` which is much faster. - - -.. function:: resize(obj, size) - - This function resizes the internal memory buffer of *obj*, which must be an - instance of a ctypes type. It is not possible to make the buffer smaller - than the native size of the objects type, as given by ``sizeof(type(obj))``, - but it is possible to enlarge the buffer. - - -.. function:: set_errno(value) - - Set the current value of the ctypes-private copy of the system :data:`errno` - variable in the calling thread to *value* and return the previous value. - - .. audit-event:: ctypes.set_errno errno ctypes.set_errno - - -.. function:: set_last_error(value) - - Windows only: set the current value of the ctypes-private copy of the system - :data:`LastError` variable in the calling thread to *value* and return the - previous value. - - .. audit-event:: ctypes.set_last_error error ctypes.set_last_error - - -.. function:: sizeof(obj_or_type) - - Returns the size in bytes of a ctypes type or instance memory buffer. - Does the same as the C ``sizeof`` operator. - - -.. function:: string_at(address, size=-1) - - This function returns the C string starting at memory address *address* as a bytes - object. If size is specified, it is used as size, otherwise the string is assumed - to be zero-terminated. - - .. audit-event:: ctypes.string_at address,size ctypes.string_at - - -.. function:: WinError(code=None, descr=None) - - Windows only: this function is probably the worst-named thing in ctypes. It - creates an instance of :exc:`OSError`. If *code* is not specified, - ``GetLastError`` is called to determine the error code. If *descr* is not - specified, :func:`FormatError` is called to get a textual description of the - error. - - .. versionchanged:: 3.3 - An instance of :exc:`WindowsError` used to be created, which is now an - alias of :exc:`OSError`. - - -.. function:: wstring_at(address, size=-1) - - This function returns the wide character string starting at memory address - *address* as a string. If *size* is specified, it is used as the number of - characters of the string, otherwise the string is assumed to be - zero-terminated. - - .. audit-event:: ctypes.wstring_at address,size ctypes.wstring_at - - -.. _ctypes-data-types: - -Data types -^^^^^^^^^^ - - -.. class:: _CData - - This non-public class is the common base class of all ctypes data types. - Among other things, all ctypes type instances contain a memory block that - hold C compatible data; the address of the memory block is returned by the - :func:`addressof` helper function. Another instance variable is exposed as - :attr:`_objects`; this contains other Python objects that need to be kept - alive in case the memory block contains pointers. - - Common methods of ctypes data types, these are all class methods (to be - exact, they are methods of the :term:`metaclass`): - - .. method:: _CData.from_buffer(source[, offset]) - - This method returns a ctypes instance that shares the buffer of the - *source* object. The *source* object must support the writeable buffer - interface. The optional *offset* parameter specifies an offset into the - source buffer in bytes; the default is zero. If the source buffer is not - large enough a :exc:`ValueError` is raised. - - .. audit-event:: ctypes.cdata/buffer pointer,size,offset ctypes._CData.from_buffer - - .. method:: _CData.from_buffer_copy(source[, offset]) - - This method creates a ctypes instance, copying the buffer from the - *source* object buffer which must be readable. The optional *offset* - parameter specifies an offset into the source buffer in bytes; the default - is zero. If the source buffer is not large enough a :exc:`ValueError` is - raised. - - .. audit-event:: ctypes.cdata/buffer pointer,size,offset ctypes._CData.from_buffer_copy - - .. method:: from_address(address) - - This method returns a ctypes type instance using the memory specified by - *address* which must be an integer. - - .. audit-event:: ctypes.cdata address ctypes._CData.from_address - - This method, and others that indirectly call this method, raises an - :ref:`auditing event ` ``ctypes.cdata`` with argument - ``address``. - - .. method:: from_param(obj) - - This method adapts *obj* to a ctypes type. It is called with the actual - object used in a foreign function call when the type is present in the - foreign function's :attr:`argtypes` tuple; it must return an object that - can be used as a function call parameter. - - All ctypes data types have a default implementation of this classmethod - that normally returns *obj* if that is an instance of the type. Some - types accept other objects as well. - - .. method:: in_dll(library, name) - - This method returns a ctypes type instance exported by a shared - library. *name* is the name of the symbol that exports the data, *library* - is the loaded shared library. - - Common instance variables of ctypes data types: - - .. attribute:: _b_base_ - - Sometimes ctypes data instances do not own the memory block they contain, - instead they share part of the memory block of a base object. The - :attr:`_b_base_` read-only member is the root ctypes object that owns the - memory block. - - .. attribute:: _b_needsfree_ - - This read-only variable is true when the ctypes data instance has - allocated the memory block itself, false otherwise. - - .. attribute:: _objects - - This member is either ``None`` or a dictionary containing Python objects - that need to be kept alive so that the memory block contents is kept - valid. This object is only exposed for debugging; never modify the - contents of this dictionary. - - -.. _ctypes-fundamental-data-types-2: - -Fundamental data types -^^^^^^^^^^^^^^^^^^^^^^ - -.. class:: _SimpleCData - - This non-public class is the base class of all fundamental ctypes data - types. It is mentioned here because it contains the common attributes of the - fundamental ctypes data types. :class:`_SimpleCData` is a subclass of - :class:`_CData`, so it inherits their methods and attributes. ctypes data - types that are not and do not contain pointers can now be pickled. - - Instances have a single attribute: - - .. attribute:: value - - This attribute contains the actual value of the instance. For integer and - pointer types, it is an integer, for character types, it is a single - character bytes object or string, for character pointer types it is a - Python bytes object or string. - - When the ``value`` attribute is retrieved from a ctypes instance, usually - a new object is returned each time. :mod:`ctypes` does *not* implement - original object return, always a new object is constructed. The same is - true for all other ctypes object instances. - - -Fundamental data types, when returned as foreign function call results, or, for -example, by retrieving structure field members or array items, are transparently -converted to native Python types. In other words, if a foreign function has a -:attr:`restype` of :class:`c_char_p`, you will always receive a Python bytes -object, *not* a :class:`c_char_p` instance. - -.. XXX above is false, it actually returns a Unicode string - -Subclasses of fundamental data types do *not* inherit this behavior. So, if a -foreign functions :attr:`restype` is a subclass of :class:`c_void_p`, you will -receive an instance of this subclass from the function call. Of course, you can -get the value of the pointer by accessing the ``value`` attribute. - -These are the fundamental ctypes data types: - -.. class:: c_byte - - Represents the C :c:expr:`signed char` datatype, and interprets the value as - small integer. The constructor accepts an optional integer initializer; no - overflow checking is done. - - -.. class:: c_char - - Represents the C :c:expr:`char` datatype, and interprets the value as a single - character. The constructor accepts an optional string initializer, the - length of the string must be exactly one character. - - -.. class:: c_char_p - - Represents the C :c:expr:`char *` datatype when it points to a zero-terminated - string. For a general character pointer that may also point to binary data, - ``POINTER(c_char)`` must be used. The constructor accepts an integer - address, or a bytes object. - - -.. class:: c_double - - Represents the C :c:expr:`double` datatype. The constructor accepts an - optional float initializer. - - -.. class:: c_longdouble - - Represents the C :c:expr:`long double` datatype. The constructor accepts an - optional float initializer. On platforms where ``sizeof(long double) == - sizeof(double)`` it is an alias to :class:`c_double`. - -.. class:: c_float - - Represents the C :c:expr:`float` datatype. The constructor accepts an - optional float initializer. - - -.. class:: c_int - - Represents the C :c:expr:`signed int` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. On platforms - where ``sizeof(int) == sizeof(long)`` it is an alias to :class:`c_long`. - - -.. class:: c_int8 - - Represents the C 8-bit :c:expr:`signed int` datatype. Usually an alias for - :class:`c_byte`. - - -.. class:: c_int16 - - Represents the C 16-bit :c:expr:`signed int` datatype. Usually an alias for - :class:`c_short`. - - -.. class:: c_int32 - - Represents the C 32-bit :c:expr:`signed int` datatype. Usually an alias for - :class:`c_int`. - - -.. class:: c_int64 - - Represents the C 64-bit :c:expr:`signed int` datatype. Usually an alias for - :class:`c_longlong`. - - -.. class:: c_long - - Represents the C :c:expr:`signed long` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. - - -.. class:: c_longlong - - Represents the C :c:expr:`signed long long` datatype. The constructor accepts - an optional integer initializer; no overflow checking is done. - - -.. class:: c_short - - Represents the C :c:expr:`signed short` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. - - -.. class:: c_size_t - - Represents the C :c:type:`size_t` datatype. - - -.. class:: c_ssize_t - - Represents the C :c:type:`ssize_t` datatype. - - .. versionadded:: 3.2 - - -.. class:: c_ubyte - - Represents the C :c:expr:`unsigned char` datatype, it interprets the value as - small integer. The constructor accepts an optional integer initializer; no - overflow checking is done. - - -.. class:: c_uint - - Represents the C :c:expr:`unsigned int` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. On platforms - where ``sizeof(int) == sizeof(long)`` it is an alias for :class:`c_ulong`. - - -.. class:: c_uint8 - - Represents the C 8-bit :c:expr:`unsigned int` datatype. Usually an alias for - :class:`c_ubyte`. - - -.. class:: c_uint16 - - Represents the C 16-bit :c:expr:`unsigned int` datatype. Usually an alias for - :class:`c_ushort`. - - -.. class:: c_uint32 - - Represents the C 32-bit :c:expr:`unsigned int` datatype. Usually an alias for - :class:`c_uint`. - - -.. class:: c_uint64 - - Represents the C 64-bit :c:expr:`unsigned int` datatype. Usually an alias for - :class:`c_ulonglong`. - - -.. class:: c_ulong - - Represents the C :c:expr:`unsigned long` datatype. The constructor accepts an - optional integer initializer; no overflow checking is done. - - -.. class:: c_ulonglong - - Represents the C :c:expr:`unsigned long long` datatype. The constructor - accepts an optional integer initializer; no overflow checking is done. - - -.. class:: c_ushort - - Represents the C :c:expr:`unsigned short` datatype. The constructor accepts - an optional integer initializer; no overflow checking is done. - - -.. class:: c_void_p - - Represents the C :c:expr:`void *` type. The value is represented as integer. - The constructor accepts an optional integer initializer. - - -.. class:: c_wchar - - Represents the C :c:type:`wchar_t` datatype, and interprets the value as a - single character unicode string. The constructor accepts an optional string - initializer, the length of the string must be exactly one character. - - -.. class:: c_wchar_p - - Represents the C :c:expr:`wchar_t *` datatype, which must be a pointer to a - zero-terminated wide character string. The constructor accepts an integer - address, or a string. - - -.. class:: c_bool - - Represent the C :c:expr:`bool` datatype (more accurately, :c:expr:`_Bool` from - C99). Its value can be ``True`` or ``False``, and the constructor accepts any object - that has a truth value. - - -.. class:: HRESULT - - Windows only: Represents a :c:type:`HRESULT` value, which contains success or - error information for a function or method call. - - -.. class:: py_object - - Represents the C :c:expr:`PyObject *` datatype. Calling this without an - argument creates a ``NULL`` :c:expr:`PyObject *` pointer. - -The :mod:`ctypes.wintypes` module provides quite some other Windows specific -data types, for example :c:type:`HWND`, :c:type:`WPARAM`, or :c:type:`DWORD`. Some -useful structures like :c:type:`MSG` or :c:type:`RECT` are also defined. - - -.. _ctypes-structured-data-types: - -Structured data types -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: Union(*args, **kw) - - Abstract base class for unions in native byte order. - - -.. class:: BigEndianUnion(*args, **kw) - - Abstract base class for unions in *big endian* byte order. - - .. versionadded:: 3.11 - -.. class:: LittleEndianUnion(*args, **kw) - - Abstract base class for unions in *little endian* byte order. - - .. versionadded:: 3.11 - -.. class:: BigEndianStructure(*args, **kw) - - Abstract base class for structures in *big endian* byte order. - - -.. class:: LittleEndianStructure(*args, **kw) - - Abstract base class for structures in *little endian* byte order. - -Structures and unions with non-native byte order cannot contain pointer type -fields, or any other data types containing pointer type fields. - - -.. class:: Structure(*args, **kw) - - Abstract base class for structures in *native* byte order. - - Concrete structure and union types must be created by subclassing one of these - types, and at least define a :attr:`_fields_` class variable. :mod:`ctypes` will - create :term:`descriptor`\s which allow reading and writing the fields by direct - attribute accesses. These are the - - - .. attribute:: _fields_ - - A sequence defining the structure fields. The items must be 2-tuples or - 3-tuples. The first item is the name of the field, the second item - specifies the type of the field; it can be any ctypes data type. - - For integer type fields like :class:`c_int`, a third optional item can be - given. It must be a small positive integer defining the bit width of the - field. - - Field names must be unique within one structure or union. This is not - checked, only one field can be accessed when names are repeated. - - It is possible to define the :attr:`_fields_` class variable *after* the - class statement that defines the Structure subclass, this allows creating - data types that directly or indirectly reference themselves:: - - class List(Structure): - pass - List._fields_ = [("pnext", POINTER(List)), - ... - ] - - The :attr:`_fields_` class variable must, however, be defined before the - type is first used (an instance is created, :func:`sizeof` is called on it, - and so on). Later assignments to the :attr:`_fields_` class variable will - raise an AttributeError. - - It is possible to define sub-subclasses of structure types, they inherit - the fields of the base class plus the :attr:`_fields_` defined in the - sub-subclass, if any. - - - .. attribute:: _pack_ - - An optional small integer that allows overriding the alignment of - structure fields in the instance. :attr:`_pack_` must already be defined - when :attr:`_fields_` is assigned, otherwise it will have no effect. - - - .. attribute:: _anonymous_ - - An optional sequence that lists the names of unnamed (anonymous) fields. - :attr:`_anonymous_` must be already defined when :attr:`_fields_` is - assigned, otherwise it will have no effect. - - The fields listed in this variable must be structure or union type fields. - :mod:`ctypes` will create descriptors in the structure type that allows - accessing the nested fields directly, without the need to create the - structure or union field. - - Here is an example type (Windows):: - - class _U(Union): - _fields_ = [("lptdesc", POINTER(TYPEDESC)), - ("lpadesc", POINTER(ARRAYDESC)), - ("hreftype", HREFTYPE)] - - class TYPEDESC(Structure): - _anonymous_ = ("u",) - _fields_ = [("u", _U), - ("vt", VARTYPE)] - - - The ``TYPEDESC`` structure describes a COM data type, the ``vt`` field - specifies which one of the union fields is valid. Since the ``u`` field - is defined as anonymous field, it is now possible to access the members - directly off the TYPEDESC instance. ``td.lptdesc`` and ``td.u.lptdesc`` - are equivalent, but the former is faster since it does not need to create - a temporary union instance:: - - td = TYPEDESC() - td.vt = VT_PTR - td.lptdesc = POINTER(some_type) - td.u.lptdesc = POINTER(some_type) - - It is possible to define sub-subclasses of structures, they inherit the - fields of the base class. If the subclass definition has a separate - :attr:`_fields_` variable, the fields specified in this are appended to the - fields of the base class. - - Structure and union constructors accept both positional and keyword - arguments. Positional arguments are used to initialize member fields in the - same order as they are appear in :attr:`_fields_`. Keyword arguments in the - constructor are interpreted as attribute assignments, so they will initialize - :attr:`_fields_` with the same name, or create new attributes for names not - present in :attr:`_fields_`. - - -.. _ctypes-arrays-pointers: - -Arrays and pointers -^^^^^^^^^^^^^^^^^^^ - -.. class:: Array(*args) - - Abstract base class for arrays. - - The recommended way to create concrete array types is by multiplying any - :mod:`ctypes` data type with a non-negative integer. Alternatively, you can subclass - this type and define :attr:`_length_` and :attr:`_type_` class variables. - Array elements can be read and written using standard - subscript and slice accesses; for slice reads, the resulting object is - *not* itself an :class:`Array`. - - - .. attribute:: _length_ - - A positive integer specifying the number of elements in the array. - Out-of-range subscripts result in an :exc:`IndexError`. Will be - returned by :func:`len`. - - - .. attribute:: _type_ - - Specifies the type of each element in the array. - - - Array subclass constructors accept positional arguments, used to - initialize the elements in order. - - -.. class:: _Pointer - - Private, abstract base class for pointers. - - Concrete pointer types are created by calling :func:`POINTER` with the - type that will be pointed to; this is done automatically by - :func:`pointer`. - - If a pointer points to an array, its elements can be read and - written using standard subscript and slice accesses. Pointer objects - have no size, so :func:`len` will raise :exc:`TypeError`. Negative - subscripts will read from the memory *before* the pointer (as in C), and - out-of-range subscripts will probably crash with an access violation (if - you're lucky). - - - .. attribute:: _type_ - - Specifies the type pointed to. - - .. attribute:: contents - - Returns the object to which to pointer points. Assigning to this - attribute changes the pointer to point to the assigned object. diff --git a/Doc/library/curses.ascii.rst b/Doc/library/curses.ascii.rst deleted file mode 100644 index 410b76e77c025b..00000000000000 --- a/Doc/library/curses.ascii.rst +++ /dev/null @@ -1,231 +0,0 @@ -:mod:`curses.ascii` --- Utilities for ASCII characters -====================================================== - -.. module:: curses.ascii - :synopsis: Constants and set-membership functions for ASCII characters. - -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - -**Source code:** :source:`Lib/curses/ascii.py` - --------------- - -The :mod:`curses.ascii` module supplies name constants for ASCII characters and -functions to test membership in various ASCII character classes. The constants -supplied are names for control characters as follows: - -+---------------+----------------------------------------------+ -| Name | Meaning | -+===============+==============================================+ -| .. data:: NUL | | -+---------------+----------------------------------------------+ -| .. data:: SOH | Start of heading, console interrupt | -+---------------+----------------------------------------------+ -| .. data:: STX | Start of text | -+---------------+----------------------------------------------+ -| .. data:: ETX | End of text | -+---------------+----------------------------------------------+ -| .. data:: EOT | End of transmission | -+---------------+----------------------------------------------+ -| .. data:: ENQ | Enquiry, goes with :const:`ACK` flow control | -+---------------+----------------------------------------------+ -| .. data:: ACK | Acknowledgement | -+---------------+----------------------------------------------+ -| .. data:: BEL | Bell | -+---------------+----------------------------------------------+ -| .. data:: BS | Backspace | -+---------------+----------------------------------------------+ -| .. data:: TAB | Tab | -+---------------+----------------------------------------------+ -| .. data:: HT | Alias for :const:`TAB`: "Horizontal tab" | -+---------------+----------------------------------------------+ -| .. data:: LF | Line feed | -+---------------+----------------------------------------------+ -| .. data:: NL | Alias for :const:`LF`: "New line" | -+---------------+----------------------------------------------+ -| .. data:: VT | Vertical tab | -+---------------+----------------------------------------------+ -| .. data:: FF | Form feed | -+---------------+----------------------------------------------+ -| .. data:: CR | Carriage return | -+---------------+----------------------------------------------+ -| .. data:: SO | Shift-out, begin alternate character set | -+---------------+----------------------------------------------+ -| .. data:: SI | Shift-in, resume default character set | -+---------------+----------------------------------------------+ -| .. data:: DLE | Data-link escape | -+---------------+----------------------------------------------+ -| .. data:: DC1 | XON, for flow control | -+---------------+----------------------------------------------+ -| .. data:: DC2 | Device control 2, block-mode flow control | -+---------------+----------------------------------------------+ -| .. data:: DC3 | XOFF, for flow control | -+---------------+----------------------------------------------+ -| .. data:: DC4 | Device control 4 | -+---------------+----------------------------------------------+ -| .. data:: NAK | Negative acknowledgement | -+---------------+----------------------------------------------+ -| .. data:: SYN | Synchronous idle | -+---------------+----------------------------------------------+ -| .. data:: ETB | End transmission block | -+---------------+----------------------------------------------+ -| .. data:: CAN | Cancel | -+---------------+----------------------------------------------+ -| .. data:: EM | End of medium | -+---------------+----------------------------------------------+ -| .. data:: SUB | Substitute | -+---------------+----------------------------------------------+ -| .. data:: ESC | Escape | -+---------------+----------------------------------------------+ -| .. data:: FS | File separator | -+---------------+----------------------------------------------+ -| .. data:: GS | Group separator | -+---------------+----------------------------------------------+ -| .. data:: RS | Record separator, block-mode terminator | -+---------------+----------------------------------------------+ -| .. data:: US | Unit separator | -+---------------+----------------------------------------------+ -| .. data:: SP | Space | -+---------------+----------------------------------------------+ -| .. data:: DEL | Delete | -+---------------+----------------------------------------------+ - -Note that many of these have little practical significance in modern usage. The -mnemonics derive from teleprinter conventions that predate digital computers. - -The module supplies the following functions, patterned on those in the standard -C library: - - -.. function:: isalnum(c) - - Checks for an ASCII alphanumeric character; it is equivalent to ``isalpha(c) or - isdigit(c)``. - - -.. function:: isalpha(c) - - Checks for an ASCII alphabetic character; it is equivalent to ``isupper(c) or - islower(c)``. - - -.. function:: isascii(c) - - Checks for a character value that fits in the 7-bit ASCII set. - - -.. function:: isblank(c) - - Checks for an ASCII whitespace character; space or horizontal tab. - - -.. function:: iscntrl(c) - - Checks for an ASCII control character (in the range 0x00 to 0x1f or 0x7f). - - -.. function:: isdigit(c) - - Checks for an ASCII decimal digit, ``'0'`` through ``'9'``. This is equivalent - to ``c in string.digits``. - - -.. function:: isgraph(c) - - Checks for ASCII any printable character except space. - - -.. function:: islower(c) - - Checks for an ASCII lower-case character. - - -.. function:: isprint(c) - - Checks for any ASCII printable character including space. - - -.. function:: ispunct(c) - - Checks for any printable ASCII character which is not a space or an alphanumeric - character. - - -.. function:: isspace(c) - - Checks for ASCII white-space characters; space, line feed, carriage return, form - feed, horizontal tab, vertical tab. - - -.. function:: isupper(c) - - Checks for an ASCII uppercase letter. - - -.. function:: isxdigit(c) - - Checks for an ASCII hexadecimal digit. This is equivalent to ``c in - string.hexdigits``. - - -.. function:: isctrl(c) - - Checks for an ASCII control character (ordinal values 0 to 31). - - -.. function:: ismeta(c) - - Checks for a non-ASCII character (ordinal values 0x80 and above). - -These functions accept either integers or single-character strings; when the argument is a -string, it is first converted using the built-in function :func:`ord`. - -Note that all these functions check ordinal bit values derived from the -character of the string you pass in; they do not actually know anything about -the host machine's character encoding. - -The following two functions take either a single-character string or integer -byte value; they return a value of the same type. - - -.. function:: ascii(c) - - Return the ASCII value corresponding to the low 7 bits of *c*. - - -.. function:: ctrl(c) - - Return the control character corresponding to the given character (the character - bit value is bitwise-anded with 0x1f). - - -.. function:: alt(c) - - Return the 8-bit character corresponding to the given ASCII character (the - character bit value is bitwise-ored with 0x80). - -The following function takes either a single-character string or integer value; -it returns a string. - - -.. index:: - single: ^ (caret); in curses module - single: ! (exclamation); in curses module - -.. function:: unctrl(c) - - Return a string representation of the ASCII character *c*. If *c* is printable, - this string is the character itself. If the character is a control character - (0x00--0x1f) the string consists of a caret (``'^'``) followed by the - corresponding uppercase letter. If the character is an ASCII delete (0x7f) the - string is ``'^?'``. If the character has its meta bit (0x80) set, the meta bit - is stripped, the preceding rules applied, and ``'!'`` prepended to the result. - - -.. data:: controlnames - - A 33-element string array that contains the ASCII mnemonics for the thirty-two - ASCII control characters from 0 (NUL) to 0x1f (US), in order, plus the mnemonic - ``SP`` for the space character. - diff --git a/Doc/library/curses.panel.rst b/Doc/library/curses.panel.rst deleted file mode 100644 index d770c03c8375f4..00000000000000 --- a/Doc/library/curses.panel.rst +++ /dev/null @@ -1,120 +0,0 @@ -:mod:`curses.panel` --- A panel stack extension for curses -========================================================== - -.. module:: curses.panel - :synopsis: A panel stack extension that adds depth to curses windows. - -.. sectionauthor:: A.M. Kuchling - --------------- - -Panels are windows with the added feature of depth, so they can be stacked on -top of each other, and only the visible portions of each window will be -displayed. Panels can be added, moved up or down in the stack, and removed. - - -.. _cursespanel-functions: - -Functions ---------- - -The module :mod:`curses.panel` defines the following functions: - - -.. function:: bottom_panel() - - Returns the bottom panel in the panel stack. - - -.. function:: new_panel(win) - - Returns a panel object, associating it with the given window *win*. Be aware - that you need to keep the returned panel object referenced explicitly. If you - don't, the panel object is garbage collected and removed from the panel stack. - - -.. function:: top_panel() - - Returns the top panel in the panel stack. - - -.. function:: update_panels() - - Updates the virtual screen after changes in the panel stack. This does not call - :func:`curses.doupdate`, so you'll have to do this yourself. - - -.. _curses-panel-objects: - -Panel Objects -------------- - -Panel objects, as returned by :func:`new_panel` above, are windows with a -stacking order. There's always a window associated with a panel which determines -the content, while the panel methods are responsible for the window's depth in -the panel stack. - -Panel objects have the following methods: - - -.. method:: Panel.above() - - Returns the panel above the current panel. - - -.. method:: Panel.below() - - Returns the panel below the current panel. - - -.. method:: Panel.bottom() - - Push the panel to the bottom of the stack. - - -.. method:: Panel.hidden() - - Returns ``True`` if the panel is hidden (not visible), ``False`` otherwise. - - -.. method:: Panel.hide() - - Hide the panel. This does not delete the object, it just makes the window on - screen invisible. - - -.. method:: Panel.move(y, x) - - Move the panel to the screen coordinates ``(y, x)``. - - -.. method:: Panel.replace(win) - - Change the window associated with the panel to the window *win*. - - -.. method:: Panel.set_userptr(obj) - - Set the panel's user pointer to *obj*. This is used to associate an arbitrary - piece of data with the panel, and can be any Python object. - - -.. method:: Panel.show() - - Display the panel (which might have been hidden). - - -.. method:: Panel.top() - - Push panel to the top of the stack. - - -.. method:: Panel.userptr() - - Returns the user pointer for the panel. This might be any Python object. - - -.. method:: Panel.window() - - Returns the window object associated with the panel. - diff --git a/Doc/library/curses.rst b/Doc/library/curses.rst deleted file mode 100644 index 0d743efd28efff..00000000000000 --- a/Doc/library/curses.rst +++ /dev/null @@ -1,1937 +0,0 @@ -:mod:`curses` --- Terminal handling for character-cell displays -=============================================================== - -.. module:: curses - :synopsis: An interface to the curses library, providing portable - terminal handling. - :platform: Unix - -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Eric Raymond - -**Source code:** :source:`Lib/curses` - --------------- - -The :mod:`curses` module provides an interface to the curses library, the -de-facto standard for portable advanced terminal handling. - -While curses is most widely used in the Unix environment, versions are available -for Windows, DOS, and possibly other systems as well. This extension module is -designed to match the API of ncurses, an open-source curses library hosted on -Linux and the BSD variants of Unix. - -.. note:: - - Whenever the documentation mentions a *character* it can be specified - as an integer, a one-character Unicode string or a one-byte byte string. - - Whenever the documentation mentions a *character string* it can be specified - as a Unicode string or a byte string. - -.. seealso:: - - Module :mod:`curses.ascii` - Utilities for working with ASCII characters, regardless of your locale settings. - - Module :mod:`curses.panel` - A panel stack extension that adds depth to curses windows. - - Module :mod:`curses.textpad` - Editable text widget for curses supporting :program:`Emacs`\ -like bindings. - - :ref:`curses-howto` - Tutorial material on using curses with Python, by Andrew Kuchling and Eric - Raymond. - - The :source:`Tools/demo/` directory in the Python source distribution contains - some example programs using the curses bindings provided by this module. - - -.. _curses-functions: - -Functions ---------- - -The module :mod:`curses` defines the following exception: - - -.. exception:: error - - Exception raised when a curses library function returns an error. - -.. note:: - - Whenever *x* or *y* arguments to a function or a method are optional, they - default to the current cursor location. Whenever *attr* is optional, it defaults - to :const:`A_NORMAL`. - -The module :mod:`curses` defines the following functions: - - -.. function:: baudrate() - - Return the output speed of the terminal in bits per second. On software - terminal emulators it will have a fixed high value. Included for historical - reasons; in former times, it was used to write output loops for time delays and - occasionally to change interfaces depending on the line speed. - - -.. function:: beep() - - Emit a short attention sound. - - -.. function:: can_change_color() - - Return ``True`` or ``False``, depending on whether the programmer can change the colors - displayed by the terminal. - - -.. function:: cbreak() - - Enter cbreak mode. In cbreak mode (sometimes called "rare" mode) normal tty - line buffering is turned off and characters are available to be read one by one. - However, unlike raw mode, special characters (interrupt, quit, suspend, and flow - control) retain their effects on the tty driver and calling program. Calling - first :func:`raw` then :func:`cbreak` leaves the terminal in cbreak mode. - - -.. function:: color_content(color_number) - - Return the intensity of the red, green, and blue (RGB) components in the color - *color_number*, which must be between ``0`` and ``COLORS - 1``. Return a 3-tuple, - containing the R,G,B values for the given color, which will be between - ``0`` (no component) and ``1000`` (maximum amount of component). - - -.. function:: color_pair(pair_number) - - Return the attribute value for displaying text in the specified color pair. - Only the first 256 color pairs are supported. This - attribute value can be combined with :const:`A_STANDOUT`, :const:`A_REVERSE`, - and the other :const:`!A_\*` attributes. :func:`pair_number` is the counterpart - to this function. - - -.. function:: curs_set(visibility) - - Set the cursor state. *visibility* can be set to ``0``, ``1``, or ``2``, for invisible, - normal, or very visible. If the terminal supports the visibility requested, return the - previous cursor state; otherwise raise an exception. On many - terminals, the "visible" mode is an underline cursor and the "very visible" mode - is a block cursor. - - -.. function:: def_prog_mode() - - Save the current terminal mode as the "program" mode, the mode when the running - program is using curses. (Its counterpart is the "shell" mode, for when the - program is not in curses.) Subsequent calls to :func:`reset_prog_mode` will - restore this mode. - - -.. function:: def_shell_mode() - - Save the current terminal mode as the "shell" mode, the mode when the running - program is not using curses. (Its counterpart is the "program" mode, when the - program is using curses capabilities.) Subsequent calls to - :func:`reset_shell_mode` will restore this mode. - - -.. function:: delay_output(ms) - - Insert an *ms* millisecond pause in output. - - -.. function:: doupdate() - - Update the physical screen. The curses library keeps two data structures, one - representing the current physical screen contents and a virtual screen - representing the desired next state. The :func:`doupdate` ground updates the - physical screen to match the virtual screen. - - The virtual screen may be updated by a :meth:`~window.noutrefresh` call after write - operations such as :meth:`~window.addstr` have been performed on a window. The normal - :meth:`~window.refresh` call is simply :meth:`!noutrefresh` followed by :func:`!doupdate`; - if you have to update multiple windows, you can speed performance and perhaps - reduce screen flicker by issuing :meth:`!noutrefresh` calls on all windows, - followed by a single :func:`!doupdate`. - - -.. function:: echo() - - Enter echo mode. In echo mode, each character input is echoed to the screen as - it is entered. - - -.. function:: endwin() - - De-initialize the library, and return terminal to normal status. - - -.. function:: erasechar() - - Return the user's current erase character as a one-byte bytes object. Under Unix operating systems this - is a property of the controlling tty of the curses program, and is not set by - the curses library itself. - - -.. function:: filter() - - The :func:`.filter` routine, if used, must be called before :func:`initscr` is - called. The effect is that, during those calls, :envvar:`LINES` is set to ``1``; the - capabilities ``clear``, ``cup``, ``cud``, ``cud1``, ``cuu1``, ``cuu``, ``vpa`` are disabled; and the ``home`` - string is set to the value of ``cr``. The effect is that the cursor is confined to - the current line, and so are screen updates. This may be used for enabling - character-at-a-time line editing without touching the rest of the screen. - - -.. function:: flash() - - Flash the screen. That is, change it to reverse-video and then change it back - in a short interval. Some people prefer such as 'visible bell' to the audible - attention signal produced by :func:`beep`. - - -.. function:: flushinp() - - Flush all input buffers. This throws away any typeahead that has been typed - by the user and has not yet been processed by the program. - - -.. function:: getmouse() - - After :meth:`~window.getch` returns :const:`KEY_MOUSE` to signal a mouse event, this - method should be called to retrieve the queued mouse event, represented as a - 5-tuple ``(id, x, y, z, bstate)``. *id* is an ID value used to distinguish - multiple devices, and *x*, *y*, *z* are the event's coordinates. (*z* is - currently unused.) *bstate* is an integer value whose bits will be set to - indicate the type of event, and will be the bitwise OR of one or more of the - following constants, where *n* is the button number from 1 to 5: - :const:`BUTTONn_PRESSED`, :const:`BUTTONn_RELEASED`, :const:`BUTTONn_CLICKED`, - :const:`BUTTONn_DOUBLE_CLICKED`, :const:`BUTTONn_TRIPLE_CLICKED`, - :const:`BUTTON_SHIFT`, :const:`BUTTON_CTRL`, :const:`BUTTON_ALT`. - - .. versionchanged:: 3.10 - The ``BUTTON5_*`` constants are now exposed if they are provided by the - underlying curses library. - - -.. function:: getsyx() - - Return the current coordinates of the virtual screen cursor as a tuple - ``(y, x)``. If :meth:`leaveok ` is currently ``True``, then return ``(-1, -1)``. - - -.. function:: getwin(file) - - Read window related data stored in the file by an earlier :func:`window.putwin` call. - The routine then creates and initializes a new window using that data, returning - the new window object. - - -.. function:: has_colors() - - Return ``True`` if the terminal can display colors; otherwise, return ``False``. - -.. function:: has_extended_color_support() - - Return ``True`` if the module supports extended colors; otherwise, return - ``False``. Extended color support allows more than 256 color pairs for - terminals that support more than 16 colors (e.g. xterm-256color). - - Extended color support requires ncurses version 6.1 or later. - - .. versionadded:: 3.10 - -.. function:: has_ic() - - Return ``True`` if the terminal has insert- and delete-character capabilities. - This function is included for historical reasons only, as all modern software - terminal emulators have such capabilities. - - -.. function:: has_il() - - Return ``True`` if the terminal has insert- and delete-line capabilities, or can - simulate them using scrolling regions. This function is included for - historical reasons only, as all modern software terminal emulators have such - capabilities. - - -.. function:: has_key(ch) - - Take a key value *ch*, and return ``True`` if the current terminal type recognizes - a key with that value. - - -.. function:: halfdelay(tenths) - - Used for half-delay mode, which is similar to cbreak mode in that characters - typed by the user are immediately available to the program. However, after - blocking for *tenths* tenths of seconds, raise an exception if nothing has - been typed. The value of *tenths* must be a number between ``1`` and ``255``. Use - :func:`nocbreak` to leave half-delay mode. - - -.. function:: init_color(color_number, r, g, b) - - Change the definition of a color, taking the number of the color to be changed - followed by three RGB values (for the amounts of red, green, and blue - components). The value of *color_number* must be between ``0`` and - ``COLORS - 1``. Each of *r*, *g*, *b*, must be a value between ``0`` and - ``1000``. When :func:`init_color` is used, all occurrences of that color on the - screen immediately change to the new definition. This function is a no-op on - most terminals; it is active only if :func:`can_change_color` returns ``True``. - - -.. function:: init_pair(pair_number, fg, bg) - - Change the definition of a color-pair. It takes three arguments: the number of - the color-pair to be changed, the foreground color number, and the background - color number. The value of *pair_number* must be between ``1`` and - ``COLOR_PAIRS - 1`` (the ``0`` color pair is wired to white on black and cannot - be changed). The value of *fg* and *bg* arguments must be between ``0`` and - ``COLORS - 1``, or, after calling :func:`use_default_colors`, ``-1``. - If the color-pair was previously initialized, the screen is - refreshed and all occurrences of that color-pair are changed to the new - definition. - - -.. function:: initscr() - - Initialize the library. Return a :ref:`window ` object - which represents the whole screen. - - .. note:: - - If there is an error opening the terminal, the underlying curses library may - cause the interpreter to exit. - - -.. function:: is_term_resized(nlines, ncols) - - Return ``True`` if :func:`resize_term` would modify the window structure, - ``False`` otherwise. - - -.. function:: isendwin() - - Return ``True`` if :func:`endwin` has been called (that is, the curses library has - been deinitialized). - - -.. function:: keyname(k) - - Return the name of the key numbered *k* as a bytes object. The name of a key generating printable - ASCII character is the key's character. The name of a control-key combination - is a two-byte bytes object consisting of a caret (``b'^'``) followed by the corresponding - printable ASCII character. The name of an alt-key combination (128--255) is a - bytes object consisting of the prefix ``b'M-'`` followed by the name of the corresponding - ASCII character. - - -.. function:: killchar() - - Return the user's current line kill character as a one-byte bytes object. Under Unix operating systems - this is a property of the controlling tty of the curses program, and is not set - by the curses library itself. - - -.. function:: longname() - - Return a bytes object containing the terminfo long name field describing the current - terminal. The maximum length of a verbose description is 128 characters. It is - defined only after the call to :func:`initscr`. - - -.. function:: meta(flag) - - If *flag* is ``True``, allow 8-bit characters to be input. If - *flag* is ``False``, allow only 7-bit chars. - - -.. function:: mouseinterval(interval) - - Set the maximum time in milliseconds that can elapse between press and release - events in order for them to be recognized as a click, and return the previous - interval value. The default value is 200 milliseconds, or one fifth of a second. - - -.. function:: mousemask(mousemask) - - Set the mouse events to be reported, and return a tuple ``(availmask, - oldmask)``. *availmask* indicates which of the specified mouse events can be - reported; on complete failure it returns ``0``. *oldmask* is the previous value of - the given window's mouse event mask. If this function is never called, no mouse - events are ever reported. - - -.. function:: napms(ms) - - Sleep for *ms* milliseconds. - - -.. function:: newpad(nlines, ncols) - - Create and return a pointer to a new pad data structure with the given number - of lines and columns. Return a pad as a window object. - - A pad is like a window, except that it is not restricted by the screen size, and - is not necessarily associated with a particular part of the screen. Pads can be - used when a large window is needed, and only a part of the window will be on the - screen at one time. Automatic refreshes of pads (such as from scrolling or - echoing of input) do not occur. The :meth:`~window.refresh` and :meth:`~window.noutrefresh` - methods of a pad require 6 arguments to specify the part of the pad to be - displayed and the location on the screen to be used for the display. The - arguments are *pminrow*, *pmincol*, *sminrow*, *smincol*, *smaxrow*, *smaxcol*; the *p* - arguments refer to the upper left corner of the pad region to be displayed and - the *s* arguments define a clipping box on the screen within which the pad region - is to be displayed. - - -.. function:: newwin(nlines, ncols) - newwin(nlines, ncols, begin_y, begin_x) - - Return a new :ref:`window `, whose left-upper corner - is at ``(begin_y, begin_x)``, and whose height/width is *nlines*/*ncols*. - - By default, the window will extend from the specified position to the lower - right corner of the screen. - - -.. function:: nl() - - Enter newline mode. This mode translates the return key into newline on input, - and translates newline into return and line-feed on output. Newline mode is - initially on. - - -.. function:: nocbreak() - - Leave cbreak mode. Return to normal "cooked" mode with line buffering. - - -.. function:: noecho() - - Leave echo mode. Echoing of input characters is turned off. - - -.. function:: nonl() - - Leave newline mode. Disable translation of return into newline on input, and - disable low-level translation of newline into newline/return on output (but this - does not change the behavior of ``addch('\n')``, which always does the - equivalent of return and line feed on the virtual screen). With translation - off, curses can sometimes speed up vertical motion a little; also, it will be - able to detect the return key on input. - - -.. function:: noqiflush() - - When the :func:`!noqiflush` routine is used, normal flush of input and output queues - associated with the ``INTR``, ``QUIT`` and ``SUSP`` characters will not be done. You may - want to call :func:`!noqiflush` in a signal handler if you want output to - continue as though the interrupt had not occurred, after the handler exits. - - -.. function:: noraw() - - Leave raw mode. Return to normal "cooked" mode with line buffering. - - -.. function:: pair_content(pair_number) - - Return a tuple ``(fg, bg)`` containing the colors for the requested color pair. - The value of *pair_number* must be between ``0`` and ``COLOR_PAIRS - 1``. - - -.. function:: pair_number(attr) - - Return the number of the color-pair set by the attribute value *attr*. - :func:`color_pair` is the counterpart to this function. - - -.. function:: putp(str) - - Equivalent to ``tputs(str, 1, putchar)``; emit the value of a specified - terminfo capability for the current terminal. Note that the output of :func:`putp` - always goes to standard output. - - -.. function:: qiflush([flag]) - - If *flag* is ``False``, the effect is the same as calling :func:`noqiflush`. If - *flag* is ``True``, or no argument is provided, the queues will be flushed when - these control characters are read. - - -.. function:: raw() - - Enter raw mode. In raw mode, normal line buffering and processing of - interrupt, quit, suspend, and flow control keys are turned off; characters are - presented to curses input functions one by one. - - -.. function:: reset_prog_mode() - - Restore the terminal to "program" mode, as previously saved by - :func:`def_prog_mode`. - - -.. function:: reset_shell_mode() - - Restore the terminal to "shell" mode, as previously saved by - :func:`def_shell_mode`. - - -.. function:: resetty() - - Restore the state of the terminal modes to what it was at the last call to - :func:`savetty`. - - -.. function:: resize_term(nlines, ncols) - - Backend function used by :func:`resizeterm`, performing most of the work; - when resizing the windows, :func:`resize_term` blank-fills the areas that are - extended. The calling application should fill in these areas with - appropriate data. The :func:`!resize_term` function attempts to resize all - windows. However, due to the calling convention of pads, it is not possible - to resize these without additional interaction with the application. - - -.. function:: resizeterm(nlines, ncols) - - Resize the standard and current windows to the specified dimensions, and - adjusts other bookkeeping data used by the curses library that record the - window dimensions (in particular the SIGWINCH handler). - - -.. function:: savetty() - - Save the current state of the terminal modes in a buffer, usable by - :func:`resetty`. - -.. function:: get_escdelay() - - Retrieves the value set by :func:`set_escdelay`. - - .. versionadded:: 3.9 - -.. function:: set_escdelay(ms) - - Sets the number of milliseconds to wait after reading an escape character, - to distinguish between an individual escape character entered on the - keyboard from escape sequences sent by cursor and function keys. - - .. versionadded:: 3.9 - -.. function:: get_tabsize() - - Retrieves the value set by :func:`set_tabsize`. - - .. versionadded:: 3.9 - -.. function:: set_tabsize(size) - - Sets the number of columns used by the curses library when converting a tab - character to spaces as it adds the tab to a window. - - .. versionadded:: 3.9 - -.. function:: setsyx(y, x) - - Set the virtual screen cursor to *y*, *x*. If *y* and *x* are both ``-1``, then - :meth:`leaveok ` is set ``True``. - - -.. function:: setupterm(term=None, fd=-1) - - Initialize the terminal. *term* is a string giving - the terminal name, or ``None``; if omitted or ``None``, the value of the - :envvar:`TERM` environment variable will be used. *fd* is the - file descriptor to which any initialization sequences will be sent; if not - supplied or ``-1``, the file descriptor for ``sys.stdout`` will be used. - - -.. function:: start_color() - - Must be called if the programmer wants to use colors, and before any other color - manipulation routine is called. It is good practice to call this routine right - after :func:`initscr`. - - :func:`start_color` initializes eight basic colors (black, red, green, yellow, - blue, magenta, cyan, and white), and two global variables in the :mod:`curses` - module, :const:`COLORS` and :const:`COLOR_PAIRS`, containing the maximum number - of colors and color-pairs the terminal can support. It also restores the colors - on the terminal to the values they had when the terminal was just turned on. - - -.. function:: termattrs() - - Return a logical OR of all video attributes supported by the terminal. This - information is useful when a curses program needs complete control over the - appearance of the screen. - - -.. function:: termname() - - Return the value of the environment variable :envvar:`TERM`, as a bytes object, - truncated to 14 characters. - - -.. function:: tigetflag(capname) - - Return the value of the Boolean capability corresponding to the terminfo - capability name *capname* as an integer. Return the value ``-1`` if *capname* is not a - Boolean capability, or ``0`` if it is canceled or absent from the terminal - description. - - -.. function:: tigetnum(capname) - - Return the value of the numeric capability corresponding to the terminfo - capability name *capname* as an integer. Return the value ``-2`` if *capname* is not a - numeric capability, or ``-1`` if it is canceled or absent from the terminal - description. - - -.. function:: tigetstr(capname) - - Return the value of the string capability corresponding to the terminfo - capability name *capname* as a bytes object. Return ``None`` if *capname* - is not a terminfo "string capability", or is canceled or absent from the - terminal description. - - -.. function:: tparm(str[, ...]) - - Instantiate the bytes object *str* with the supplied parameters, where *str* should - be a parameterized string obtained from the terminfo database. E.g. - ``tparm(tigetstr("cup"), 5, 3)`` could result in ``b'\033[6;4H'``, the exact - result depending on terminal type. - - -.. function:: typeahead(fd) - - Specify that the file descriptor *fd* be used for typeahead checking. If *fd* - is ``-1``, then no typeahead checking is done. - - The curses library does "line-breakout optimization" by looking for typeahead - periodically while updating the screen. If input is found, and it is coming - from a tty, the current update is postponed until refresh or doupdate is called - again, allowing faster response to commands typed in advance. This function - allows specifying a different file descriptor for typeahead checking. - - -.. function:: unctrl(ch) - - Return a bytes object which is a printable representation of the character *ch*. - Control characters are represented as a caret followed by the character, for - example as ``b'^C'``. Printing characters are left as they are. - - -.. function:: ungetch(ch) - - Push *ch* so the next :meth:`~window.getch` will return it. - - .. note:: - - Only one *ch* can be pushed before :meth:`!getch` is called. - - -.. function:: update_lines_cols() - - Update the :const:`LINES` and :const:`COLS` module variables. - Useful for detecting manual screen resize. - - .. versionadded:: 3.5 - - -.. function:: unget_wch(ch) - - Push *ch* so the next :meth:`~window.get_wch` will return it. - - .. note:: - - Only one *ch* can be pushed before :meth:`!get_wch` is called. - - .. versionadded:: 3.3 - - -.. function:: ungetmouse(id, x, y, z, bstate) - - Push a :const:`KEY_MOUSE` event onto the input queue, associating the given - state data with it. - - -.. function:: use_env(flag) - - If used, this function should be called before :func:`initscr` or newterm are - called. When *flag* is ``False``, the values of lines and columns specified in the - terminfo database will be used, even if environment variables :envvar:`LINES` - and :envvar:`COLUMNS` (used by default) are set, or if curses is running in a - window (in which case default behavior would be to use the window size if - :envvar:`LINES` and :envvar:`COLUMNS` are not set). - - -.. function:: use_default_colors() - - Allow use of default values for colors on terminals supporting this feature. Use - this to support transparency in your application. The default color is assigned - to the color number ``-1``. After calling this function, ``init_pair(x, - curses.COLOR_RED, -1)`` initializes, for instance, color pair *x* to a red - foreground color on the default background. - - -.. function:: wrapper(func, /, *args, **kwargs) - - Initialize curses and call another callable object, *func*, which should be the - rest of your curses-using application. If the application raises an exception, - this function will restore the terminal to a sane state before re-raising the - exception and generating a traceback. The callable object *func* is then passed - the main window 'stdscr' as its first argument, followed by any other arguments - passed to :func:`!wrapper`. Before calling *func*, :func:`!wrapper` turns on - cbreak mode, turns off echo, enables the terminal keypad, and initializes colors - if the terminal has color support. On exit (whether normally or by exception) - it restores cooked mode, turns on echo, and disables the terminal keypad. - - -.. _curses-window-objects: - -Window Objects --------------- - -Window objects, as returned by :func:`initscr` and :func:`newwin` above, have -the following methods and attributes: - - -.. method:: window.addch(ch[, attr]) - window.addch(y, x, ch[, attr]) - - Paint character *ch* at ``(y, x)`` with attributes *attr*, overwriting any - character previously painted at that location. By default, the character - position and attributes are the current settings for the window object. - - .. note:: - - Writing outside the window, subwindow, or pad raises a :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, - or pad will cause an exception to be raised after the character is printed. - - -.. method:: window.addnstr(str, n[, attr]) - window.addnstr(y, x, str, n[, attr]) - - Paint at most *n* characters of the character string *str* at - ``(y, x)`` with attributes - *attr*, overwriting anything previously on the display. - - -.. method:: window.addstr(str[, attr]) - window.addstr(y, x, str[, attr]) - - Paint the character string *str* at ``(y, x)`` with attributes - *attr*, overwriting anything previously on the display. - - .. note:: - - * Writing outside the window, subwindow, or pad raises :exc:`curses.error`. - Attempting to write to the lower right corner of a window, subwindow, - or pad will cause an exception to be raised after the string is printed. - - * A `bug in ncurses `_, the backend - for this Python module, can cause SegFaults when resizing windows. This - is fixed in ncurses-6.1-20190511. If you are stuck with an earlier - ncurses, you can avoid triggering this if you do not call :func:`addstr` - with a *str* that has embedded newlines. Instead, call :func:`addstr` - separately for each line. - - -.. method:: window.attroff(attr) - - Remove attribute *attr* from the "background" set applied to all writes to the - current window. - - -.. method:: window.attron(attr) - - Add attribute *attr* from the "background" set applied to all writes to the - current window. - - -.. method:: window.attrset(attr) - - Set the "background" set of attributes to *attr*. This set is initially - ``0`` (no attributes). - - -.. method:: window.bkgd(ch[, attr]) - - Set the background property of the window to the character *ch*, with - attributes *attr*. The change is then applied to every character position in - that window: - - * The attribute of every character in the window is changed to the new - background attribute. - - * Wherever the former background character appears, it is changed to the new - background character. - - -.. method:: window.bkgdset(ch[, attr]) - - Set the window's background. A window's background consists of a character and - any combination of attributes. The attribute part of the background is combined - (OR'ed) with all non-blank characters that are written into the window. Both - the character and attribute parts of the background are combined with the blank - characters. The background becomes a property of the character and moves with - the character through any scrolling and insert/delete line/character operations. - - -.. method:: window.border([ls[, rs[, ts[, bs[, tl[, tr[, bl[, br]]]]]]]]) - - Draw a border around the edges of the window. Each parameter specifies the - character to use for a specific part of the border; see the table below for more - details. - - .. note:: - - A ``0`` value for any parameter will cause the default character to be used for - that parameter. Keyword parameters can *not* be used. The defaults are listed - in this table: - - +-----------+---------------------+-----------------------+ - | Parameter | Description | Default value | - +===========+=====================+=======================+ - | *ls* | Left side | :const:`ACS_VLINE` | - +-----------+---------------------+-----------------------+ - | *rs* | Right side | :const:`ACS_VLINE` | - +-----------+---------------------+-----------------------+ - | *ts* | Top | :const:`ACS_HLINE` | - +-----------+---------------------+-----------------------+ - | *bs* | Bottom | :const:`ACS_HLINE` | - +-----------+---------------------+-----------------------+ - | *tl* | Upper-left corner | :const:`ACS_ULCORNER` | - +-----------+---------------------+-----------------------+ - | *tr* | Upper-right corner | :const:`ACS_URCORNER` | - +-----------+---------------------+-----------------------+ - | *bl* | Bottom-left corner | :const:`ACS_LLCORNER` | - +-----------+---------------------+-----------------------+ - | *br* | Bottom-right corner | :const:`ACS_LRCORNER` | - +-----------+---------------------+-----------------------+ - - -.. method:: window.box([vertch, horch]) - - Similar to :meth:`border`, but both *ls* and *rs* are *vertch* and both *ts* and - *bs* are *horch*. The default corner characters are always used by this function. - - -.. method:: window.chgat(attr) - window.chgat(num, attr) - window.chgat(y, x, attr) - window.chgat(y, x, num, attr) - - Set the attributes of *num* characters at the current cursor position, or at - position ``(y, x)`` if supplied. If *num* is not given or is ``-1``, - the attribute will be set on all the characters to the end of the line. This - function moves cursor to position ``(y, x)`` if supplied. The changed line - will be touched using the :meth:`touchline` method so that the contents will - be redisplayed by the next window refresh. - - -.. method:: window.clear() - - Like :meth:`erase`, but also cause the whole window to be repainted upon next - call to :meth:`refresh`. - - -.. method:: window.clearok(flag) - - If *flag* is ``True``, the next call to :meth:`refresh` will clear the window - completely. - - -.. method:: window.clrtobot() - - Erase from cursor to the end of the window: all lines below the cursor are - deleted, and then the equivalent of :meth:`clrtoeol` is performed. - - -.. method:: window.clrtoeol() - - Erase from cursor to the end of the line. - - -.. method:: window.cursyncup() - - Update the current cursor position of all the ancestors of the window to - reflect the current cursor position of the window. - - -.. method:: window.delch([y, x]) - - Delete any character at ``(y, x)``. - - -.. method:: window.deleteln() - - Delete the line under the cursor. All following lines are moved up by one line. - - -.. method:: window.derwin(begin_y, begin_x) - window.derwin(nlines, ncols, begin_y, begin_x) - - An abbreviation for "derive window", :meth:`derwin` is the same as calling - :meth:`subwin`, except that *begin_y* and *begin_x* are relative to the origin - of the window, rather than relative to the entire screen. Return a window - object for the derived window. - - -.. method:: window.echochar(ch[, attr]) - - Add character *ch* with attribute *attr*, and immediately call :meth:`refresh` - on the window. - - -.. method:: window.enclose(y, x) - - Test whether the given pair of screen-relative character-cell coordinates are - enclosed by the given window, returning ``True`` or ``False``. It is useful for - determining what subset of the screen windows enclose the location of a mouse - event. - - .. versionchanged:: 3.10 - Previously it returned ``1`` or ``0`` instead of ``True`` or ``False``. - - -.. attribute:: window.encoding - - Encoding used to encode method arguments (Unicode strings and characters). - The encoding attribute is inherited from the parent window when a subwindow - is created, for example with :meth:`window.subwin`. - By default, current locale encoding is used (see :func:`locale.getencoding`). - - .. versionadded:: 3.3 - - -.. method:: window.erase() - - Clear the window. - - -.. method:: window.getbegyx() - - Return a tuple ``(y, x)`` of co-ordinates of upper-left corner. - - -.. method:: window.getbkgd() - - Return the given window's current background character/attribute pair. - - -.. method:: window.getch([y, x]) - - Get a character. Note that the integer returned does *not* have to be in ASCII - range: function keys, keypad keys and so on are represented by numbers higher - than 255. In no-delay mode, return ``-1`` if there is no input, otherwise - wait until a key is pressed. - - -.. method:: window.get_wch([y, x]) - - Get a wide character. Return a character for most keys, or an integer for - function keys, keypad keys, and other special keys. - In no-delay mode, raise an exception if there is no input. - - .. versionadded:: 3.3 - - -.. method:: window.getkey([y, x]) - - Get a character, returning a string instead of an integer, as :meth:`getch` - does. Function keys, keypad keys and other special keys return a multibyte - string containing the key name. In no-delay mode, raise an exception if - there is no input. - - -.. method:: window.getmaxyx() - - Return a tuple ``(y, x)`` of the height and width of the window. - - -.. method:: window.getparyx() - - Return the beginning coordinates of this window relative to its parent window - as a tuple ``(y, x)``. Return ``(-1, -1)`` if this window has no - parent. - - -.. method:: window.getstr() - window.getstr(n) - window.getstr(y, x) - window.getstr(y, x, n) - - Read a bytes object from the user, with primitive line editing capacity. - - -.. method:: window.getyx() - - Return a tuple ``(y, x)`` of current cursor position relative to the window's - upper-left corner. - - -.. method:: window.hline(ch, n) - window.hline(y, x, ch, n) - - Display a horizontal line starting at ``(y, x)`` with length *n* consisting of - the character *ch*. - - -.. method:: window.idcok(flag) - - If *flag* is ``False``, curses no longer considers using the hardware insert/delete - character feature of the terminal; if *flag* is ``True``, use of character insertion - and deletion is enabled. When curses is first initialized, use of character - insert/delete is enabled by default. - - -.. method:: window.idlok(flag) - - If *flag* is ``True``, :mod:`curses` will try and use hardware line - editing facilities. Otherwise, line insertion/deletion are disabled. - - -.. method:: window.immedok(flag) - - If *flag* is ``True``, any change in the window image automatically causes the - window to be refreshed; you no longer have to call :meth:`refresh` yourself. - However, it may degrade performance considerably, due to repeated calls to - wrefresh. This option is disabled by default. - - -.. method:: window.inch([y, x]) - - Return the character at the given position in the window. The bottom 8 bits are - the character proper, and upper bits are the attributes. - - -.. method:: window.insch(ch[, attr]) - window.insch(y, x, ch[, attr]) - - Paint character *ch* at ``(y, x)`` with attributes *attr*, moving the line from - position *x* right by one character. - - -.. method:: window.insdelln(nlines) - - Insert *nlines* lines into the specified window above the current line. The - *nlines* bottom lines are lost. For negative *nlines*, delete *nlines* lines - starting with the one under the cursor, and move the remaining lines up. The - bottom *nlines* lines are cleared. The current cursor position remains the - same. - - -.. method:: window.insertln() - - Insert a blank line under the cursor. All following lines are moved down by one - line. - - -.. method:: window.insnstr(str, n[, attr]) - window.insnstr(y, x, str, n[, attr]) - - Insert a character string (as many characters as will fit on the line) before - the character under the cursor, up to *n* characters. If *n* is zero or - negative, the entire string is inserted. All characters to the right of the - cursor are shifted right, with the rightmost characters on the line being lost. - The cursor position does not change (after moving to *y*, *x*, if specified). - - -.. method:: window.insstr(str[, attr]) - window.insstr(y, x, str[, attr]) - - Insert a character string (as many characters as will fit on the line) before - the character under the cursor. All characters to the right of the cursor are - shifted right, with the rightmost characters on the line being lost. The cursor - position does not change (after moving to *y*, *x*, if specified). - - -.. method:: window.instr([n]) - window.instr(y, x[, n]) - - Return a bytes object of characters, extracted from the window starting at the - current cursor position, or at *y*, *x* if specified. Attributes are stripped - from the characters. If *n* is specified, :meth:`instr` returns a string - at most *n* characters long (exclusive of the trailing NUL). - - -.. method:: window.is_linetouched(line) - - Return ``True`` if the specified line was modified since the last call to - :meth:`refresh`; otherwise return ``False``. Raise a :exc:`curses.error` - exception if *line* is not valid for the given window. - - -.. method:: window.is_wintouched() - - Return ``True`` if the specified window was modified since the last call to - :meth:`refresh`; otherwise return ``False``. - - -.. method:: window.keypad(flag) - - If *flag* is ``True``, escape sequences generated by some keys (keypad, function keys) - will be interpreted by :mod:`curses`. If *flag* is ``False``, escape sequences will be - left as is in the input stream. - - -.. method:: window.leaveok(flag) - - If *flag* is ``True``, cursor is left where it is on update, instead of being at "cursor - position." This reduces cursor movement where possible. If possible the cursor - will be made invisible. - - If *flag* is ``False``, cursor will always be at "cursor position" after an update. - - -.. method:: window.move(new_y, new_x) - - Move cursor to ``(new_y, new_x)``. - - -.. method:: window.mvderwin(y, x) - - Move the window inside its parent window. The screen-relative parameters of - the window are not changed. This routine is used to display different parts of - the parent window at the same physical position on the screen. - - -.. method:: window.mvwin(new_y, new_x) - - Move the window so its upper-left corner is at ``(new_y, new_x)``. - - -.. method:: window.nodelay(flag) - - If *flag* is ``True``, :meth:`getch` will be non-blocking. - - -.. method:: window.notimeout(flag) - - If *flag* is ``True``, escape sequences will not be timed out. - - If *flag* is ``False``, after a few milliseconds, an escape sequence will not be - interpreted, and will be left in the input stream as is. - - -.. method:: window.noutrefresh() - - Mark for refresh but wait. This function updates the data structure - representing the desired state of the window, but does not force an update of - the physical screen. To accomplish that, call :func:`doupdate`. - - -.. method:: window.overlay(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) - - Overlay the window on top of *destwin*. The windows need not be the same size, - only the overlapping region is copied. This copy is non-destructive, which means - that the current background character does not overwrite the old contents of - *destwin*. - - To get fine-grained control over the copied region, the second form of - :meth:`overlay` can be used. *sminrow* and *smincol* are the upper-left - coordinates of the source window, and the other variables mark a rectangle in - the destination window. - - -.. method:: window.overwrite(destwin[, sminrow, smincol, dminrow, dmincol, dmaxrow, dmaxcol]) - - Overwrite the window on top of *destwin*. The windows need not be the same size, - in which case only the overlapping region is copied. This copy is destructive, - which means that the current background character overwrites the old contents of - *destwin*. - - To get fine-grained control over the copied region, the second form of - :meth:`overwrite` can be used. *sminrow* and *smincol* are the upper-left - coordinates of the source window, the other variables mark a rectangle in the - destination window. - - -.. method:: window.putwin(file) - - Write all data associated with the window into the provided file object. This - information can be later retrieved using the :func:`getwin` function. - - -.. method:: window.redrawln(beg, num) - - Indicate that the *num* screen lines, starting at line *beg*, are corrupted and - should be completely redrawn on the next :meth:`refresh` call. - - -.. method:: window.redrawwin() - - Touch the entire window, causing it to be completely redrawn on the next - :meth:`refresh` call. - - -.. method:: window.refresh([pminrow, pmincol, sminrow, smincol, smaxrow, smaxcol]) - - Update the display immediately (sync actual screen with previous - drawing/deleting methods). - - The 6 optional arguments can only be specified when the window is a pad created - with :func:`newpad`. The additional parameters are needed to indicate what part - of the pad and screen are involved. *pminrow* and *pmincol* specify the upper - left-hand corner of the rectangle to be displayed in the pad. *sminrow*, - *smincol*, *smaxrow*, and *smaxcol* specify the edges of the rectangle to be - displayed on the screen. The lower right-hand corner of the rectangle to be - displayed in the pad is calculated from the screen coordinates, since the - rectangles must be the same size. Both rectangles must be entirely contained - within their respective structures. Negative values of *pminrow*, *pmincol*, - *sminrow*, or *smincol* are treated as if they were zero. - - -.. method:: window.resize(nlines, ncols) - - Reallocate storage for a curses window to adjust its dimensions to the - specified values. If either dimension is larger than the current values, the - window's data is filled with blanks that have the current background - rendition (as set by :meth:`bkgdset`) merged into them. - - -.. method:: window.scroll([lines=1]) - - Scroll the screen or scrolling region upward by *lines* lines. - - -.. method:: window.scrollok(flag) - - Control what happens when the cursor of a window is moved off the edge of the - window or scrolling region, either as a result of a newline action on the bottom - line, or typing the last character of the last line. If *flag* is ``False``, the - cursor is left on the bottom line. If *flag* is ``True``, the window is scrolled up - one line. Note that in order to get the physical scrolling effect on the - terminal, it is also necessary to call :meth:`idlok`. - - -.. method:: window.setscrreg(top, bottom) - - Set the scrolling region from line *top* to line *bottom*. All scrolling actions - will take place in this region. - - -.. method:: window.standend() - - Turn off the standout attribute. On some terminals this has the side effect of - turning off all attributes. - - -.. method:: window.standout() - - Turn on attribute *A_STANDOUT*. - - -.. method:: window.subpad(begin_y, begin_x) - window.subpad(nlines, ncols, begin_y, begin_x) - - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. - - -.. method:: window.subwin(begin_y, begin_x) - window.subwin(nlines, ncols, begin_y, begin_x) - - Return a sub-window, whose upper-left corner is at ``(begin_y, begin_x)``, and - whose width/height is *ncols*/*nlines*. - - By default, the sub-window will extend from the specified position to the lower - right corner of the window. - - -.. method:: window.syncdown() - - Touch each location in the window that has been touched in any of its ancestor - windows. This routine is called by :meth:`refresh`, so it should almost never - be necessary to call it manually. - - -.. method:: window.syncok(flag) - - If *flag* is ``True``, then :meth:`syncup` is called automatically - whenever there is a change in the window. - - -.. method:: window.syncup() - - Touch all locations in ancestors of the window that have been changed in the - window. - - -.. method:: window.timeout(delay) - - Set blocking or non-blocking read behavior for the window. If *delay* is - negative, blocking read is used (which will wait indefinitely for input). If - *delay* is zero, then non-blocking read is used, and :meth:`getch` will - return ``-1`` if no input is waiting. If *delay* is positive, then - :meth:`getch` will block for *delay* milliseconds, and return ``-1`` if there is - still no input at the end of that time. - - -.. method:: window.touchline(start, count[, changed]) - - Pretend *count* lines have been changed, starting with line *start*. If - *changed* is supplied, it specifies whether the affected lines are marked as - having been changed (*changed*\ ``=True``) or unchanged (*changed*\ ``=False``). - - -.. method:: window.touchwin() - - Pretend the whole window has been changed, for purposes of drawing - optimizations. - - -.. method:: window.untouchwin() - - Mark all lines in the window as unchanged since the last call to - :meth:`refresh`. - - -.. method:: window.vline(ch, n[, attr]) - window.vline(y, x, ch, n[, attr]) - - Display a vertical line starting at ``(y, x)`` with length *n* consisting of the - character *ch* with attributes *attr*. - - -Constants ---------- - -The :mod:`curses` module defines the following data members: - - -.. data:: ERR - - Some curses routines that return an integer, such as :meth:`~window.getch`, return - :const:`ERR` upon failure. - - -.. data:: OK - - Some curses routines that return an integer, such as :func:`napms`, return - :const:`OK` upon success. - - -.. data:: version -.. data:: __version__ - - A bytes object representing the current version of the module. - - -.. data:: ncurses_version - - A named tuple containing the three components of the ncurses library - version: *major*, *minor*, and *patch*. All values are integers. The - components can also be accessed by name, so ``curses.ncurses_version[0]`` - is equivalent to ``curses.ncurses_version.major`` and so on. - - Availability: if the ncurses library is used. - - .. versionadded:: 3.8 - -.. data:: COLORS - - The maximum number of colors the terminal can support. - It is defined only after the call to :func:`start_color`. - -.. data:: COLOR_PAIRS - - The maximum number of color pairs the terminal can support. - It is defined only after the call to :func:`start_color`. - -.. data:: COLS - - The width of the screen, i.e., the number of columns. - It is defined only after the call to :func:`initscr`. - Updated by :func:`update_lines_cols`, :func:`resizeterm` and - :func:`resize_term`. - -.. data:: LINES - - The height of the screen, i.e., the number of lines. - It is defined only after the call to :func:`initscr`. - Updated by :func:`update_lines_cols`, :func:`resizeterm` and - :func:`resize_term`. - - -Some constants are available to specify character cell attributes. -The exact constants available are system dependent. - -+------------------------+-------------------------------+ -| Attribute | Meaning | -+========================+===============================+ -| .. data:: A_ALTCHARSET | Alternate character set mode | -+------------------------+-------------------------------+ -| .. data:: A_BLINK | Blink mode | -+------------------------+-------------------------------+ -| .. data:: A_BOLD | Bold mode | -+------------------------+-------------------------------+ -| .. data:: A_DIM | Dim mode | -+------------------------+-------------------------------+ -| .. data:: A_INVIS | Invisible or blank mode | -+------------------------+-------------------------------+ -| .. data:: A_ITALIC | Italic mode | -+------------------------+-------------------------------+ -| .. data:: A_NORMAL | Normal attribute | -+------------------------+-------------------------------+ -| .. data:: A_PROTECT | Protected mode | -+------------------------+-------------------------------+ -| .. data:: A_REVERSE | Reverse background and | -| | foreground colors | -+------------------------+-------------------------------+ -| .. data:: A_STANDOUT | Standout mode | -+------------------------+-------------------------------+ -| .. data:: A_UNDERLINE | Underline mode | -+------------------------+-------------------------------+ -| .. data:: A_HORIZONTAL | Horizontal highlight | -+------------------------+-------------------------------+ -| .. data:: A_LEFT | Left highlight | -+------------------------+-------------------------------+ -| .. data:: A_LOW | Low highlight | -+------------------------+-------------------------------+ -| .. data:: A_RIGHT | Right highlight | -+------------------------+-------------------------------+ -| .. data:: A_TOP | Top highlight | -+------------------------+-------------------------------+ -| .. data:: A_VERTICAL | Vertical highlight | -+------------------------+-------------------------------+ - -.. versionadded:: 3.7 - ``A_ITALIC`` was added. - -Several constants are available to extract corresponding attributes returned -by some methods. - -+-------------------------+-------------------------------+ -| Bit-mask | Meaning | -+=========================+===============================+ -| .. data:: A_ATTRIBUTES | Bit-mask to extract | -| | attributes | -+-------------------------+-------------------------------+ -| .. data:: A_CHARTEXT | Bit-mask to extract a | -| | character | -+-------------------------+-------------------------------+ -| .. data:: A_COLOR | Bit-mask to extract | -| | color-pair field information | -+-------------------------+-------------------------------+ - -Keys are referred to by integer constants with names starting with ``KEY_``. -The exact keycaps available are system dependent. - -.. XXX this table is far too large! should it be alphabetized? - -+-------------------------+--------------------------------------------+ -| Key constant | Key | -+=========================+============================================+ -| .. data:: KEY_MIN | Minimum key value | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_BREAK | Break key (unreliable) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_DOWN | Down-arrow | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_UP | Up-arrow | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_LEFT | Left-arrow | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_RIGHT | Right-arrow | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_HOME | Home key (upward+left arrow) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_BACKSPACE | Backspace (unreliable) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_F0 | Function keys. Up to 64 function keys are | -| | supported. | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_Fn | Value of function key *n* | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_DL | Delete line | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_IL | Insert line | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_DC | Delete character | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_IC | Insert char or enter insert mode | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_EIC | Exit insert char mode | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_CLEAR | Clear screen | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_EOS | Clear to end of screen | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_EOL | Clear to end of line | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SF | Scroll 1 line forward | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SR | Scroll 1 line backward (reverse) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_NPAGE | Next page | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_PPAGE | Previous page | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_STAB | Set tab | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_CTAB | Clear tab | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_CATAB | Clear all tabs | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_ENTER | Enter or send (unreliable) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SRESET | Soft (partial) reset (unreliable) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_RESET | Reset or hard reset (unreliable) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_PRINT | Print | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_LL | Home down or bottom (lower left) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_A1 | Upper left of keypad | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_A3 | Upper right of keypad | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_B2 | Center of keypad | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_C1 | Lower left of keypad | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_C3 | Lower right of keypad | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_BTAB | Back tab | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_BEG | Beg (beginning) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_CANCEL | Cancel | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_CLOSE | Close | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_COMMAND | Cmd (command) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_COPY | Copy | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_CREATE | Create | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_END | End | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_EXIT | Exit | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_FIND | Find | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_HELP | Help | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_MARK | Mark | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_MESSAGE | Message | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_MOVE | Move | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_NEXT | Next | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_OPEN | Open | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_OPTIONS | Options | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_PREVIOUS | Prev (previous) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_REDO | Redo | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_REFERENCE | Ref (reference) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_REFRESH | Refresh | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_REPLACE | Replace | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_RESTART | Restart | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_RESUME | Resume | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SAVE | Save | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SBEG | Shifted Beg (beginning) | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SCANCEL | Shifted Cancel | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SCOMMAND | Shifted Command | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SCOPY | Shifted Copy | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SCREATE | Shifted Create | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SDC | Shifted Delete char | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SDL | Shifted Delete line | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SELECT | Select | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SEND | Shifted End | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SEOL | Shifted Clear line | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SEXIT | Shifted Exit | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SFIND | Shifted Find | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SHELP | Shifted Help | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SHOME | Shifted Home | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SIC | Shifted Input | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SLEFT | Shifted Left arrow | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SMESSAGE | Shifted Message | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SMOVE | Shifted Move | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SNEXT | Shifted Next | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SOPTIONS | Shifted Options | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SPREVIOUS | Shifted Prev | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SPRINT | Shifted Print | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SREDO | Shifted Redo | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SREPLACE | Shifted Replace | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SRIGHT | Shifted Right arrow | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SRSUME | Shifted Resume | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SSAVE | Shifted Save | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SSUSPEND | Shifted Suspend | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SUNDO | Shifted Undo | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_SUSPEND | Suspend | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_UNDO | Undo | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_MOUSE | Mouse event has occurred | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_RESIZE | Terminal resize event | -+-------------------------+--------------------------------------------+ -| .. data:: KEY_MAX | Maximum key value | -+-------------------------+--------------------------------------------+ - -On VT100s and their software emulations, such as X terminal emulators, there are -normally at least four function keys (:const:`KEY_F1 `, :const:`KEY_F2 `, -:const:`KEY_F3 `, :const:`KEY_F4 `) available, and the arrow keys mapped to -:const:`KEY_UP`, :const:`KEY_DOWN`, :const:`KEY_LEFT` and :const:`KEY_RIGHT` in -the obvious way. If your machine has a PC keyboard, it is safe to expect arrow -keys and twelve function keys (older PC keyboards may have only ten function -keys); also, the following keypad mappings are standard: - -+------------------+-----------+ -| Keycap | Constant | -+==================+===========+ -| :kbd:`Insert` | KEY_IC | -+------------------+-----------+ -| :kbd:`Delete` | KEY_DC | -+------------------+-----------+ -| :kbd:`Home` | KEY_HOME | -+------------------+-----------+ -| :kbd:`End` | KEY_END | -+------------------+-----------+ -| :kbd:`Page Up` | KEY_PPAGE | -+------------------+-----------+ -| :kbd:`Page Down` | KEY_NPAGE | -+------------------+-----------+ - -.. _curses-acs-codes: - -The following table lists characters from the alternate character set. These are -inherited from the VT100 terminal, and will generally be available on software -emulations such as X terminals. When there is no graphic available, curses -falls back on a crude printable ASCII approximation. - -.. note:: - - These are available only after :func:`initscr` has been called. - -+------------------------+------------------------------------------+ -| ACS code | Meaning | -+========================+==========================================+ -| .. data:: ACS_BBSS | alternate name for upper right corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_BLOCK | solid square block | -+------------------------+------------------------------------------+ -| .. data:: ACS_BOARD | board of squares | -+------------------------+------------------------------------------+ -| .. data:: ACS_BSBS | alternate name for horizontal line | -+------------------------+------------------------------------------+ -| .. data:: ACS_BSSB | alternate name for upper left corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_BSSS | alternate name for top tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_BTEE | bottom tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_BULLET | bullet | -+------------------------+------------------------------------------+ -| .. data:: ACS_CKBOARD | checker board (stipple) | -+------------------------+------------------------------------------+ -| .. data:: ACS_DARROW | arrow pointing down | -+------------------------+------------------------------------------+ -| .. data:: ACS_DEGREE | degree symbol | -+------------------------+------------------------------------------+ -| .. data:: ACS_DIAMOND | diamond | -+------------------------+------------------------------------------+ -| .. data:: ACS_GEQUAL | greater-than-or-equal-to | -+------------------------+------------------------------------------+ -| .. data:: ACS_HLINE | horizontal line | -+------------------------+------------------------------------------+ -| .. data:: ACS_LANTERN | lantern symbol | -+------------------------+------------------------------------------+ -| .. data:: ACS_LARROW | left arrow | -+------------------------+------------------------------------------+ -| .. data:: ACS_LEQUAL | less-than-or-equal-to | -+------------------------+------------------------------------------+ -| .. data:: ACS_LLCORNER | lower left-hand corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_LRCORNER | lower right-hand corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_LTEE | left tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_NEQUAL | not-equal sign | -+------------------------+------------------------------------------+ -| .. data:: ACS_PI | letter pi | -+------------------------+------------------------------------------+ -| .. data:: ACS_PLMINUS | plus-or-minus sign | -+------------------------+------------------------------------------+ -| .. data:: ACS_PLUS | big plus sign | -+------------------------+------------------------------------------+ -| .. data:: ACS_RARROW | right arrow | -+------------------------+------------------------------------------+ -| .. data:: ACS_RTEE | right tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_S1 | scan line 1 | -+------------------------+------------------------------------------+ -| .. data:: ACS_S3 | scan line 3 | -+------------------------+------------------------------------------+ -| .. data:: ACS_S7 | scan line 7 | -+------------------------+------------------------------------------+ -| .. data:: ACS_S9 | scan line 9 | -+------------------------+------------------------------------------+ -| .. data:: ACS_SBBS | alternate name for lower right corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_SBSB | alternate name for vertical line | -+------------------------+------------------------------------------+ -| .. data:: ACS_SBSS | alternate name for right tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_SSBB | alternate name for lower left corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_SSBS | alternate name for bottom tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_SSSB | alternate name for left tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_SSSS | alternate name for crossover or big plus | -+------------------------+------------------------------------------+ -| .. data:: ACS_STERLING | pound sterling | -+------------------------+------------------------------------------+ -| .. data:: ACS_TTEE | top tee | -+------------------------+------------------------------------------+ -| .. data:: ACS_UARROW | up arrow | -+------------------------+------------------------------------------+ -| .. data:: ACS_ULCORNER | upper left corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_URCORNER | upper right corner | -+------------------------+------------------------------------------+ -| .. data:: ACS_VLINE | vertical line | -+------------------------+------------------------------------------+ - -The following table lists mouse button constants used by :meth:`getmouse`: - -+----------------------------------+---------------------------------------------+ -| Mouse button constant | Meaning | -+==================================+=============================================+ -| .. data:: BUTTONn_PRESSED | Mouse button *n* pressed | -+----------------------------------+---------------------------------------------+ -| .. data:: BUTTONn_RELEASED | Mouse button *n* released | -+----------------------------------+---------------------------------------------+ -| .. data:: BUTTONn_CLICKED | Mouse button *n* clicked | -+----------------------------------+---------------------------------------------+ -| .. data:: BUTTONn_DOUBLE_CLICKED | Mouse button *n* double clicked | -+----------------------------------+---------------------------------------------+ -| .. data:: BUTTONn_TRIPLE_CLICKED | Mouse button *n* triple clicked | -+----------------------------------+---------------------------------------------+ -| .. data:: BUTTON_SHIFT | Shift was down during button state change | -+----------------------------------+---------------------------------------------+ -| .. data:: BUTTON_CTRL | Control was down during button state change | -+----------------------------------+---------------------------------------------+ -| .. data:: BUTTON_ALT | Control was down during button state change | -+----------------------------------+---------------------------------------------+ - -.. versionchanged:: 3.10 - The ``BUTTON5_*`` constants are now exposed if they are provided by the - underlying curses library. - -The following table lists the predefined colors: - -+-------------------------+----------------------------+ -| Constant | Color | -+=========================+============================+ -| .. data:: COLOR_BLACK | Black | -+-------------------------+----------------------------+ -| .. data:: COLOR_BLUE | Blue | -+-------------------------+----------------------------+ -| .. data:: COLOR_CYAN | Cyan (light greenish blue) | -+-------------------------+----------------------------+ -| .. data:: COLOR_GREEN | Green | -+-------------------------+----------------------------+ -| .. data:: COLOR_MAGENTA | Magenta (purplish red) | -+-------------------------+----------------------------+ -| .. data:: COLOR_RED | Red | -+-------------------------+----------------------------+ -| .. data:: COLOR_WHITE | White | -+-------------------------+----------------------------+ -| .. data:: COLOR_YELLOW | Yellow | -+-------------------------+----------------------------+ - - -:mod:`curses.textpad` --- Text input widget for curses programs -=============================================================== - -.. module:: curses.textpad - :synopsis: Emacs-like input editing in a curses window. -.. moduleauthor:: Eric Raymond -.. sectionauthor:: Eric Raymond - - -The :mod:`curses.textpad` module provides a :class:`Textbox` class that handles -elementary text editing in a curses window, supporting a set of keybindings -resembling those of Emacs (thus, also of Netscape Navigator, BBedit 6.x, -FrameMaker, and many other programs). The module also provides a -rectangle-drawing function useful for framing text boxes or for other purposes. - -The module :mod:`curses.textpad` defines the following function: - - -.. function:: rectangle(win, uly, ulx, lry, lrx) - - Draw a rectangle. The first argument must be a window object; the remaining - arguments are coordinates relative to that window. The second and third - arguments are the y and x coordinates of the upper left hand corner of the - rectangle to be drawn; the fourth and fifth arguments are the y and x - coordinates of the lower right hand corner. The rectangle will be drawn using - VT100/IBM PC forms characters on terminals that make this possible (including - xterm and most other software terminal emulators). Otherwise it will be drawn - with ASCII dashes, vertical bars, and plus signs. - - -.. _curses-textpad-objects: - -Textbox objects ---------------- - -You can instantiate a :class:`Textbox` object as follows: - - -.. class:: Textbox(win) - - Return a textbox widget object. The *win* argument should be a curses - :ref:`window ` object in which the textbox is to - be contained. The edit cursor of the textbox is initially located at the - upper left hand corner of the containing window, with coordinates ``(0, 0)``. - The instance's :attr:`stripspaces` flag is initially on. - - :class:`Textbox` objects have the following methods: - - - .. method:: edit([validator]) - - This is the entry point you will normally use. It accepts editing - keystrokes until one of the termination keystrokes is entered. If - *validator* is supplied, it must be a function. It will be called for - each keystroke entered with the keystroke as a parameter; command dispatch - is done on the result. This method returns the window contents as a - string; whether blanks in the window are included is affected by the - :attr:`stripspaces` attribute. - - - .. method:: do_command(ch) - - Process a single command keystroke. Here are the supported special - keystrokes: - - +------------------+-------------------------------------------+ - | Keystroke | Action | - +==================+===========================================+ - | :kbd:`Control-A` | Go to left edge of window. | - +------------------+-------------------------------------------+ - | :kbd:`Control-B` | Cursor left, wrapping to previous line if | - | | appropriate. | - +------------------+-------------------------------------------+ - | :kbd:`Control-D` | Delete character under cursor. | - +------------------+-------------------------------------------+ - | :kbd:`Control-E` | Go to right edge (stripspaces off) or end | - | | of line (stripspaces on). | - +------------------+-------------------------------------------+ - | :kbd:`Control-F` | Cursor right, wrapping to next line when | - | | appropriate. | - +------------------+-------------------------------------------+ - | :kbd:`Control-G` | Terminate, returning the window contents. | - +------------------+-------------------------------------------+ - | :kbd:`Control-H` | Delete character backward. | - +------------------+-------------------------------------------+ - | :kbd:`Control-J` | Terminate if the window is 1 line, | - | | otherwise insert newline. | - +------------------+-------------------------------------------+ - | :kbd:`Control-K` | If line is blank, delete it, otherwise | - | | clear to end of line. | - +------------------+-------------------------------------------+ - | :kbd:`Control-L` | Refresh screen. | - +------------------+-------------------------------------------+ - | :kbd:`Control-N` | Cursor down; move down one line. | - +------------------+-------------------------------------------+ - | :kbd:`Control-O` | Insert a blank line at cursor location. | - +------------------+-------------------------------------------+ - | :kbd:`Control-P` | Cursor up; move up one line. | - +------------------+-------------------------------------------+ - - Move operations do nothing if the cursor is at an edge where the movement - is not possible. The following synonyms are supported where possible: - - +--------------------------------+------------------+ - | Constant | Keystroke | - +================================+==================+ - | :const:`~curses.KEY_LEFT` | :kbd:`Control-B` | - +--------------------------------+------------------+ - | :const:`~curses.KEY_RIGHT` | :kbd:`Control-F` | - +--------------------------------+------------------+ - | :const:`~curses.KEY_UP` | :kbd:`Control-P` | - +--------------------------------+------------------+ - | :const:`~curses.KEY_DOWN` | :kbd:`Control-N` | - +--------------------------------+------------------+ - | :const:`~curses.KEY_BACKSPACE` | :kbd:`Control-h` | - +--------------------------------+------------------+ - - All other keystrokes are treated as a command to insert the given - character and move right (with line wrapping). - - - .. method:: gather() - - Return the window contents as a string; whether blanks in the - window are included is affected by the :attr:`stripspaces` member. - - - .. attribute:: stripspaces - - This attribute is a flag which controls the interpretation of blanks in - the window. When it is on, trailing blanks on each line are ignored; any - cursor motion that would land the cursor on a trailing blank goes to the - end of that line instead, and trailing blanks are stripped when the window - contents are gathered. diff --git a/Doc/library/custominterp.rst b/Doc/library/custominterp.rst deleted file mode 100644 index 9ea9e901372ee4..00000000000000 --- a/Doc/library/custominterp.rst +++ /dev/null @@ -1,19 +0,0 @@ -.. _custominterp: - -************************** -Custom Python Interpreters -************************** - -The modules described in this chapter allow writing interfaces similar to -Python's interactive interpreter. If you want a Python interpreter that -supports some special feature in addition to the Python language, you should -look at the :mod:`code` module. (The :mod:`codeop` module is lower-level, used -to support compiling a possibly incomplete chunk of Python code.) - -The full list of modules described in this chapter is: - - -.. toctree:: - - code.rst - codeop.rst diff --git a/Doc/library/dataclasses.rst b/Doc/library/dataclasses.rst deleted file mode 100644 index 915ff05d98a711..00000000000000 --- a/Doc/library/dataclasses.rst +++ /dev/null @@ -1,803 +0,0 @@ -:mod:`dataclasses` --- Data Classes -=================================== - -.. module:: dataclasses - :synopsis: Generate special methods on user-defined classes. - -.. moduleauthor:: Eric V. Smith -.. sectionauthor:: Eric V. Smith - -**Source code:** :source:`Lib/dataclasses.py` - --------------- - -This module provides a decorator and functions for automatically -adding generated :term:`special method`\s such as :meth:`~object.__init__` and -:meth:`~object.__repr__` to user-defined classes. It was originally described -in :pep:`557`. - -The member variables to use in these generated methods are defined -using :pep:`526` type annotations. For example, this code:: - - from dataclasses import dataclass - - @dataclass - class InventoryItem: - """Class for keeping track of an item in inventory.""" - name: str - unit_price: float - quantity_on_hand: int = 0 - - def total_cost(self) -> float: - return self.unit_price * self.quantity_on_hand - -will add, among other things, a :meth:`~object.__init__` that looks like:: - - def __init__(self, name: str, unit_price: float, quantity_on_hand: int = 0): - self.name = name - self.unit_price = unit_price - self.quantity_on_hand = quantity_on_hand - -Note that this method is automatically added to the class: it is not -directly specified in the ``InventoryItem`` definition shown above. - -.. versionadded:: 3.7 - -Module contents ---------------- - -.. decorator:: dataclass(*, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) - - This function is a :term:`decorator` that is used to add generated - :term:`special method`\s to classes, as described below. - - The :func:`dataclass` decorator examines the class to find - ``field``\s. A ``field`` is defined as a class variable that has a - :term:`type annotation `. With two - exceptions described below, nothing in :func:`dataclass` - examines the type specified in the variable annotation. - - The order of the fields in all of the generated methods is the - order in which they appear in the class definition. - - The :func:`dataclass` decorator will add various "dunder" methods to - the class, described below. If any of the added methods already - exist in the class, the behavior depends on the parameter, as documented - below. The decorator returns the same class that it is called on; no new - class is created. - - If :func:`dataclass` is used just as a simple decorator with no parameters, - it acts as if it has the default values documented in this - signature. That is, these three uses of :func:`dataclass` are - equivalent:: - - @dataclass - class C: - ... - - @dataclass() - class C: - ... - - @dataclass(init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, - match_args=True, kw_only=False, slots=False, weakref_slot=False) - class C: - ... - - The parameters to :func:`dataclass` are: - - - ``init``: If true (the default), a :meth:`~object.__init__` method will be - generated. - - If the class already defines :meth:`~object.__init__`, this parameter is - ignored. - - - ``repr``: If true (the default), a :meth:`~object.__repr__` method will be - generated. The generated repr string will have the class name and - the name and repr of each field, in the order they are defined in - the class. Fields that are marked as being excluded from the repr - are not included. For example: - ``InventoryItem(name='widget', unit_price=3.0, quantity_on_hand=10)``. - - If the class already defines :meth:`~object.__repr__`, this parameter is - ignored. - - - ``eq``: If true (the default), an :meth:`~object.__eq__` method will be - generated. This method compares the class as if it were a tuple - of its fields, in order. Both instances in the comparison must - be of the identical type. - - If the class already defines :meth:`~object.__eq__`, this parameter is - ignored. - - - ``order``: If true (the default is ``False``), :meth:`~object.__lt__`, - :meth:`~object.__le__`, :meth:`~object.__gt__`, and :meth:`~object.__ge__` methods will be - generated. These compare the class as if it were a tuple of its - fields, in order. Both instances in the comparison must be of the - identical type. If ``order`` is true and ``eq`` is false, a - :exc:`ValueError` is raised. - - If the class already defines any of :meth:`~object.__lt__`, - :meth:`~object.__le__`, :meth:`~object.__gt__`, or :meth:`~object.__ge__`, then - :exc:`TypeError` is raised. - - - ``unsafe_hash``: If ``False`` (the default), a :meth:`~object.__hash__` method - is generated according to how ``eq`` and ``frozen`` are set. - - :meth:`~object.__hash__` is used by built-in :meth:`hash()`, and when objects are - added to hashed collections such as dictionaries and sets. Having a - :meth:`~object.__hash__` implies that instances of the class are immutable. - Mutability is a complicated property that depends on the programmer's - intent, the existence and behavior of :meth:`~object.__eq__`, and the values of - the ``eq`` and ``frozen`` flags in the :func:`dataclass` decorator. - - By default, :func:`dataclass` will not implicitly add a :meth:`~object.__hash__` - method unless it is safe to do so. Neither will it add or change an - existing explicitly defined :meth:`~object.__hash__` method. Setting the class - attribute ``__hash__ = None`` has a specific meaning to Python, as - described in the :meth:`~object.__hash__` documentation. - - If :meth:`~object.__hash__` is not explicitly defined, or if it is set to ``None``, - then :func:`dataclass` *may* add an implicit :meth:`~object.__hash__` method. - Although not recommended, you can force :func:`dataclass` to create a - :meth:`~object.__hash__` method with ``unsafe_hash=True``. This might be the case - if your class is logically immutable but can nonetheless be mutated. - This is a specialized use case and should be considered carefully. - - Here are the rules governing implicit creation of a :meth:`~object.__hash__` - method. Note that you cannot both have an explicit :meth:`~object.__hash__` - method in your dataclass and set ``unsafe_hash=True``; this will result - in a :exc:`TypeError`. - - If ``eq`` and ``frozen`` are both true, by default :func:`dataclass` will - generate a :meth:`~object.__hash__` method for you. If ``eq`` is true and - ``frozen`` is false, :meth:`~object.__hash__` will be set to ``None``, marking it - unhashable (which it is, since it is mutable). If ``eq`` is false, - :meth:`~object.__hash__` will be left untouched meaning the :meth:`~object.__hash__` - method of the superclass will be used (if the superclass is - :class:`object`, this means it will fall back to id-based hashing). - - - ``frozen``: If true (the default is ``False``), assigning to fields will - generate an exception. This emulates read-only frozen instances. If - :meth:`~object.__setattr__` or :meth:`~object.__delattr__` is defined in the class, then - :exc:`TypeError` is raised. See the discussion below. - - - ``match_args``: If true (the default is ``True``), the - ``__match_args__`` tuple will be created from the list of - parameters to the generated :meth:`~object.__init__` method (even if - :meth:`~object.__init__` is not generated, see above). If false, or if - ``__match_args__`` is already defined in the class, then - ``__match_args__`` will not be generated. - - .. versionadded:: 3.10 - - - ``kw_only``: If true (the default value is ``False``), then all - fields will be marked as keyword-only. If a field is marked as - keyword-only, then the only effect is that the :meth:`~object.__init__` - parameter generated from a keyword-only field must be specified - with a keyword when :meth:`~object.__init__` is called. There is no - effect on any other aspect of dataclasses. See the - :term:`parameter` glossary entry for details. Also see the - :const:`KW_ONLY` section. - - .. versionadded:: 3.10 - - - ``slots``: If true (the default is ``False``), :attr:`~object.__slots__` attribute - will be generated and new class will be returned instead of the original one. - If :attr:`~object.__slots__` is already defined in the class, then :exc:`TypeError` - is raised. - - .. versionadded:: 3.10 - - .. versionchanged:: 3.11 - If a field name is already included in the ``__slots__`` - of a base class, it will not be included in the generated ``__slots__`` - to prevent :ref:`overriding them `. - Therefore, do not use ``__slots__`` to retrieve the field names of a - dataclass. Use :func:`fields` instead. - To be able to determine inherited slots, - base class ``__slots__`` may be any iterable, but *not* an iterator. - - - - ``weakref_slot``: If true (the default is ``False``), add a slot - named "__weakref__", which is required to make an instance - weakref-able. It is an error to specify ``weakref_slot=True`` - without also specifying ``slots=True``. - - .. versionadded:: 3.11 - - ``field``\s may optionally specify a default value, using normal - Python syntax:: - - @dataclass - class C: - a: int # 'a' has no default value - b: int = 0 # assign a default value for 'b' - - In this example, both ``a`` and ``b`` will be included in the added - :meth:`~object.__init__` method, which will be defined as:: - - def __init__(self, a: int, b: int = 0): - - :exc:`TypeError` will be raised if a field without a default value - follows a field with a default value. This is true whether this - occurs in a single class, or as a result of class inheritance. - -.. function:: field(*, default=MISSING, default_factory=MISSING, init=True, repr=True, hash=None, compare=True, metadata=None, kw_only=MISSING) - - For common and simple use cases, no other functionality is - required. There are, however, some dataclass features that - require additional per-field information. To satisfy this need for - additional information, you can replace the default field value - with a call to the provided :func:`field` function. For example:: - - @dataclass - class C: - mylist: list[int] = field(default_factory=list) - - c = C() - c.mylist += [1, 2, 3] - - As shown above, the :const:`MISSING` value is a sentinel object used to - detect if some parameters are provided by the user. This sentinel is - used because ``None`` is a valid value for some parameters with - a distinct meaning. No code should directly use the :const:`MISSING` value. - - The parameters to :func:`field` are: - - - ``default``: If provided, this will be the default value for this - field. This is needed because the :meth:`field` call itself - replaces the normal position of the default value. - - - ``default_factory``: If provided, it must be a zero-argument - callable that will be called when a default value is needed for - this field. Among other purposes, this can be used to specify - fields with mutable default values, as discussed below. It is an - error to specify both ``default`` and ``default_factory``. - - - ``init``: If true (the default), this field is included as a - parameter to the generated :meth:`~object.__init__` method. - - - ``repr``: If true (the default), this field is included in the - string returned by the generated :meth:`~object.__repr__` method. - - - ``hash``: This can be a bool or ``None``. If true, this field is - included in the generated :meth:`~object.__hash__` method. If ``None`` (the - default), use the value of ``compare``: this would normally be - the expected behavior. A field should be considered in the hash - if it's used for comparisons. Setting this value to anything - other than ``None`` is discouraged. - - One possible reason to set ``hash=False`` but ``compare=True`` - would be if a field is expensive to compute a hash value for, - that field is needed for equality testing, and there are other - fields that contribute to the type's hash value. Even if a field - is excluded from the hash, it will still be used for comparisons. - - - ``compare``: If true (the default), this field is included in the - generated equality and comparison methods (:meth:`~object.__eq__`, - :meth:`~object.__gt__`, et al.). - - - ``metadata``: This can be a mapping or None. None is treated as - an empty dict. This value is wrapped in - :func:`~types.MappingProxyType` to make it read-only, and exposed - on the :class:`Field` object. It is not used at all by Data - Classes, and is provided as a third-party extension mechanism. - Multiple third-parties can each have their own key, to use as a - namespace in the metadata. - - - ``kw_only``: If true, this field will be marked as keyword-only. - This is used when the generated :meth:`~object.__init__` method's - parameters are computed. - - .. versionadded:: 3.10 - - If the default value of a field is specified by a call to - :func:`field()`, then the class attribute for this field will be - replaced by the specified ``default`` value. If no ``default`` is - provided, then the class attribute will be deleted. The intent is - that after the :func:`dataclass` decorator runs, the class - attributes will all contain the default values for the fields, just - as if the default value itself were specified. For example, - after:: - - @dataclass - class C: - x: int - y: int = field(repr=False) - z: int = field(repr=False, default=10) - t: int = 20 - - The class attribute ``C.z`` will be ``10``, the class attribute - ``C.t`` will be ``20``, and the class attributes ``C.x`` and - ``C.y`` will not be set. - -.. class:: Field - - :class:`Field` objects describe each defined field. These objects - are created internally, and are returned by the :func:`fields` - module-level method (see below). Users should never instantiate a - :class:`Field` object directly. Its documented attributes are: - - - ``name``: The name of the field. - - ``type``: The type of the field. - - ``default``, ``default_factory``, ``init``, ``repr``, ``hash``, - ``compare``, ``metadata``, and ``kw_only`` have the identical - meaning and values as they do in the :func:`field` function. - - Other attributes may exist, but they are private and must not be - inspected or relied on. - -.. function:: fields(class_or_instance) - - Returns a tuple of :class:`Field` objects that define the fields for this - dataclass. Accepts either a dataclass, or an instance of a dataclass. - Raises :exc:`TypeError` if not passed a dataclass or instance of one. - Does not return pseudo-fields which are ``ClassVar`` or ``InitVar``. - -.. function:: asdict(obj, *, dict_factory=dict) - - Converts the dataclass ``obj`` to a dict (by using the - factory function ``dict_factory``). Each dataclass is converted - to a dict of its fields, as ``name: value`` pairs. dataclasses, dicts, - lists, and tuples are recursed into. Other objects are copied with - :func:`copy.deepcopy`. - - Example of using :func:`asdict` on nested dataclasses:: - - @dataclass - class Point: - x: int - y: int - - @dataclass - class C: - mylist: list[Point] - - p = Point(10, 20) - assert asdict(p) == {'x': 10, 'y': 20} - - c = C([Point(0, 0), Point(10, 4)]) - assert asdict(c) == {'mylist': [{'x': 0, 'y': 0}, {'x': 10, 'y': 4}]} - - To create a shallow copy, the following workaround may be used:: - - dict((field.name, getattr(obj, field.name)) for field in fields(obj)) - - :func:`asdict` raises :exc:`TypeError` if ``obj`` is not a dataclass - instance. - -.. function:: astuple(obj, *, tuple_factory=tuple) - - Converts the dataclass ``obj`` to a tuple (by using the - factory function ``tuple_factory``). Each dataclass is converted - to a tuple of its field values. dataclasses, dicts, lists, and - tuples are recursed into. Other objects are copied with - :func:`copy.deepcopy`. - - Continuing from the previous example:: - - assert astuple(p) == (10, 20) - assert astuple(c) == ([(0, 0), (10, 4)],) - - To create a shallow copy, the following workaround may be used:: - - tuple(getattr(obj, field.name) for field in dataclasses.fields(obj)) - - :func:`astuple` raises :exc:`TypeError` if ``obj`` is not a dataclass - instance. - -.. function:: make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, kw_only=False, slots=False, weakref_slot=False) - - Creates a new dataclass with name ``cls_name``, fields as defined - in ``fields``, base classes as given in ``bases``, and initialized - with a namespace as given in ``namespace``. ``fields`` is an - iterable whose elements are each either ``name``, ``(name, type)``, - or ``(name, type, Field)``. If just ``name`` is supplied, - ``typing.Any`` is used for ``type``. The values of ``init``, - ``repr``, ``eq``, ``order``, ``unsafe_hash``, ``frozen``, - ``match_args``, ``kw_only``, ``slots``, and ``weakref_slot`` have - the same meaning as they do in :func:`dataclass`. - - This function is not strictly required, because any Python - mechanism for creating a new class with ``__annotations__`` can - then apply the :func:`dataclass` function to convert that class to - a dataclass. This function is provided as a convenience. For - example:: - - C = make_dataclass('C', - [('x', int), - 'y', - ('z', int, field(default=5))], - namespace={'add_one': lambda self: self.x + 1}) - - Is equivalent to:: - - @dataclass - class C: - x: int - y: 'typing.Any' - z: int = 5 - - def add_one(self): - return self.x + 1 - -.. function:: replace(obj, /, **changes) - - Creates a new object of the same type as ``obj``, replacing - fields with values from ``changes``. If ``obj`` is not a Data - Class, raises :exc:`TypeError`. If values in ``changes`` do not - specify fields, raises :exc:`TypeError`. - - The newly returned object is created by calling the :meth:`~object.__init__` - method of the dataclass. This ensures that - :ref:`__post_init__ `, if present, is also called. - - Init-only variables without default values, if any exist, must be - specified on the call to :func:`replace` so that they can be passed to - :meth:`~object.__init__` and :ref:`__post_init__ `. - - It is an error for ``changes`` to contain any fields that are - defined as having ``init=False``. A :exc:`ValueError` will be raised - in this case. - - Be forewarned about how ``init=False`` fields work during a call to - :func:`replace`. They are not copied from the source object, but - rather are initialized in :ref:`__post_init__ `, if they're - initialized at all. It is expected that ``init=False`` fields will - be rarely and judiciously used. If they are used, it might be wise - to have alternate class constructors, or perhaps a custom - ``replace()`` (or similarly named) method which handles instance - copying. - -.. function:: is_dataclass(obj) - - Return ``True`` if its parameter is a dataclass or an instance of one, - otherwise return ``False``. - - If you need to know if a class is an instance of a dataclass (and - not a dataclass itself), then add a further check for ``not - isinstance(obj, type)``:: - - def is_dataclass_instance(obj): - return is_dataclass(obj) and not isinstance(obj, type) - -.. data:: MISSING - - A sentinel value signifying a missing default or default_factory. - -.. data:: KW_ONLY - - A sentinel value used as a type annotation. Any fields after a - pseudo-field with the type of :const:`KW_ONLY` are marked as - keyword-only fields. Note that a pseudo-field of type - :const:`KW_ONLY` is otherwise completely ignored. This includes the - name of such a field. By convention, a name of ``_`` is used for a - :const:`KW_ONLY` field. Keyword-only fields signify - :meth:`~object.__init__` parameters that must be specified as keywords when - the class is instantiated. - - In this example, the fields ``y`` and ``z`` will be marked as keyword-only fields:: - - @dataclass - class Point: - x: float - _: KW_ONLY - y: float - z: float - - p = Point(0, y=1.5, z=2.0) - - In a single dataclass, it is an error to specify more than one - field whose type is :const:`KW_ONLY`. - - .. versionadded:: 3.10 - -.. exception:: FrozenInstanceError - - Raised when an implicitly defined :meth:`~object.__setattr__` or - :meth:`~object.__delattr__` is called on a dataclass which was defined with - ``frozen=True``. It is a subclass of :exc:`AttributeError`. - -.. _post-init-processing: - -Post-init processing --------------------- - -The generated :meth:`~object.__init__` code will call a method named -:meth:`!__post_init__`, if :meth:`!__post_init__` is defined on the -class. It will normally be called as ``self.__post_init__()``. -However, if any ``InitVar`` fields are defined, they will also be -passed to :meth:`!__post_init__` in the order they were defined in the -class. If no :meth:`~object.__init__` method is generated, then -:meth:`!__post_init__` will not automatically be called. - -Among other uses, this allows for initializing field values that -depend on one or more other fields. For example:: - - @dataclass - class C: - a: float - b: float - c: float = field(init=False) - - def __post_init__(self): - self.c = self.a + self.b - -The :meth:`~object.__init__` method generated by :func:`dataclass` does not call base -class :meth:`~object.__init__` methods. If the base class has an :meth:`~object.__init__` method -that has to be called, it is common to call this method in a -:meth:`!__post_init__` method:: - - @dataclass - class Rectangle: - height: float - width: float - - @dataclass - class Square(Rectangle): - side: float - - def __post_init__(self): - super().__init__(self.side, self.side) - -Note, however, that in general the dataclass-generated :meth:`~object.__init__` methods -don't need to be called, since the derived dataclass will take care of -initializing all fields of any base class that is a dataclass itself. - -See the section below on init-only variables for ways to pass -parameters to :meth:`!__post_init__`. Also see the warning about how -:func:`replace` handles ``init=False`` fields. - -Class variables ---------------- - -One of the few places where :func:`dataclass` actually inspects the type -of a field is to determine if a field is a class variable as defined -in :pep:`526`. It does this by checking if the type of the field is -``typing.ClassVar``. If a field is a ``ClassVar``, it is excluded -from consideration as a field and is ignored by the dataclass -mechanisms. Such ``ClassVar`` pseudo-fields are not returned by the -module-level :func:`fields` function. - -Init-only variables -------------------- - -Another place where :func:`dataclass` inspects a type annotation is to -determine if a field is an init-only variable. It does this by seeing -if the type of a field is of type ``dataclasses.InitVar``. If a field -is an ``InitVar``, it is considered a pseudo-field called an init-only -field. As it is not a true field, it is not returned by the -module-level :func:`fields` function. Init-only fields are added as -parameters to the generated :meth:`~object.__init__` method, and are passed to -the optional :ref:`__post_init__ ` method. They are not otherwise used -by dataclasses. - -For example, suppose a field will be initialized from a database, if a -value is not provided when creating the class:: - - @dataclass - class C: - i: int - j: int | None = None - database: InitVar[DatabaseType | None] = None - - def __post_init__(self, database): - if self.j is None and database is not None: - self.j = database.lookup('j') - - c = C(10, database=my_database) - -In this case, :func:`fields` will return :class:`Field` objects for ``i`` and -``j``, but not for ``database``. - -Frozen instances ----------------- - -It is not possible to create truly immutable Python objects. However, -by passing ``frozen=True`` to the :meth:`dataclass` decorator you can -emulate immutability. In that case, dataclasses will add -:meth:`~object.__setattr__` and :meth:`~object.__delattr__` methods to the class. These -methods will raise a :exc:`FrozenInstanceError` when invoked. - -There is a tiny performance penalty when using ``frozen=True``: -:meth:`~object.__init__` cannot use simple assignment to initialize fields, and -must use :meth:`!object.__setattr__`. - -Inheritance ------------ - -When the dataclass is being created by the :meth:`dataclass` decorator, -it looks through all of the class's base classes in reverse MRO (that -is, starting at :class:`object`) and, for each dataclass that it finds, -adds the fields from that base class to an ordered mapping of fields. -After all of the base class fields are added, it adds its own fields -to the ordered mapping. All of the generated methods will use this -combined, calculated ordered mapping of fields. Because the fields -are in insertion order, derived classes override base classes. An -example:: - - @dataclass - class Base: - x: Any = 15.0 - y: int = 0 - - @dataclass - class C(Base): - z: int = 10 - x: int = 15 - -The final list of fields is, in order, ``x``, ``y``, ``z``. The final -type of ``x`` is ``int``, as specified in class ``C``. - -The generated :meth:`~object.__init__` method for ``C`` will look like:: - - def __init__(self, x: int = 15, y: int = 0, z: int = 10): - -Re-ordering of keyword-only parameters in :meth:`~object.__init__` ------------------------------------------------------------------- - -After the parameters needed for :meth:`~object.__init__` are computed, any -keyword-only parameters are moved to come after all regular -(non-keyword-only) parameters. This is a requirement of how -keyword-only parameters are implemented in Python: they must come -after non-keyword-only parameters. - -In this example, ``Base.y``, ``Base.w``, and ``D.t`` are keyword-only -fields, and ``Base.x`` and ``D.z`` are regular fields:: - - @dataclass - class Base: - x: Any = 15.0 - _: KW_ONLY - y: int = 0 - w: int = 1 - - @dataclass - class D(Base): - z: int = 10 - t: int = field(kw_only=True, default=0) - -The generated :meth:`~object.__init__` method for ``D`` will look like:: - - def __init__(self, x: Any = 15.0, z: int = 10, *, y: int = 0, w: int = 1, t: int = 0): - -Note that the parameters have been re-ordered from how they appear in -the list of fields: parameters derived from regular fields are -followed by parameters derived from keyword-only fields. - -The relative ordering of keyword-only parameters is maintained in the -re-ordered :meth:`~object.__init__` parameter list. - - -Default factory functions -------------------------- - -If a :func:`field` specifies a ``default_factory``, it is called with -zero arguments when a default value for the field is needed. For -example, to create a new instance of a list, use:: - - mylist: list = field(default_factory=list) - -If a field is excluded from :meth:`~object.__init__` (using ``init=False``) -and the field also specifies ``default_factory``, then the default -factory function will always be called from the generated -:meth:`~object.__init__` function. This happens because there is no other -way to give the field an initial value. - -Mutable default values ----------------------- - -Python stores default member variable values in class attributes. -Consider this example, not using dataclasses:: - - class C: - x = [] - def add(self, element): - self.x.append(element) - - o1 = C() - o2 = C() - o1.add(1) - o2.add(2) - assert o1.x == [1, 2] - assert o1.x is o2.x - -Note that the two instances of class ``C`` share the same class -variable ``x``, as expected. - -Using dataclasses, *if* this code was valid:: - - @dataclass - class D: - x: list = [] # This code raises ValueError - def add(self, element): - self.x += element - -it would generate code similar to:: - - class D: - x = [] - def __init__(self, x=x): - self.x = x - def add(self, element): - self.x += element - - assert D().x is D().x - -This has the same issue as the original example using class ``C``. -That is, two instances of class ``D`` that do not specify a value -for ``x`` when creating a class instance will share the same copy -of ``x``. Because dataclasses just use normal Python class -creation they also share this behavior. There is no general way -for Data Classes to detect this condition. Instead, the -:func:`dataclass` decorator will raise a :exc:`ValueError` if it -detects an unhashable default parameter. The assumption is that if -a value is unhashable, it is mutable. This is a partial solution, -but it does protect against many common errors. - -Using default factory functions is a way to create new instances of -mutable types as default values for fields:: - - @dataclass - class D: - x: list = field(default_factory=list) - - assert D().x is not D().x - -.. versionchanged:: 3.11 - Instead of looking for and disallowing objects of type ``list``, - ``dict``, or ``set``, unhashable objects are now not allowed as - default values. Unhashability is used to approximate - mutability. - -Descriptor-typed fields ------------------------ - -Fields that are assigned :ref:`descriptor objects ` as their -default value have the following special behaviors: - -* The value for the field passed to the dataclass's ``__init__`` method is - passed to the descriptor's ``__set__`` method rather than overwriting the - descriptor object. -* Similarly, when getting or setting the field, the descriptor's - ``__get__`` or ``__set__`` method is called rather than returning or - overwriting the descriptor object. -* To determine whether a field contains a default value, ``dataclasses`` - will call the descriptor's ``__get__`` method using its class access - form (i.e. ``descriptor.__get__(obj=None, type=cls)``. If the - descriptor returns a value in this case, it will be used as the - field's default. On the other hand, if the descriptor raises - :exc:`AttributeError` in this situation, no default value will be - provided for the field. - -:: - - class IntConversionDescriptor: - def __init__(self, *, default): - self._default = default - - def __set_name__(self, owner, name): - self._name = "_" + name - - def __get__(self, obj, type): - if obj is None: - return self._default - - return getattr(obj, self._name, self._default) - - def __set__(self, obj, value): - setattr(obj, self._name, int(value)) - - @dataclass - class InventoryItem: - quantity_on_hand: IntConversionDescriptor = IntConversionDescriptor(default=100) - - i = InventoryItem() - print(i.quantity_on_hand) # 100 - i.quantity_on_hand = 2.5 # calls __set__ with 2.5 - print(i.quantity_on_hand) # 2 - -Note that if a field is annotated with a descriptor type, but is not assigned -a descriptor object as its default value, the field will act like a normal -field. diff --git a/Doc/library/datatypes.rst b/Doc/library/datatypes.rst deleted file mode 100644 index ff51b2779e5fa3..00000000000000 --- a/Doc/library/datatypes.rst +++ /dev/null @@ -1,36 +0,0 @@ -.. _datatypes: - -********** -Data Types -********** - -The modules described in this chapter provide a variety of specialized data -types such as dates and times, fixed-type arrays, heap queues, double-ended -queues, and enumerations. - -Python also provides some built-in data types, in particular, -:class:`dict`, :class:`list`, :class:`set` and :class:`frozenset`, and -:class:`tuple`. The :class:`str` class is used to hold -Unicode strings, and the :class:`bytes` and :class:`bytearray` classes are used -to hold binary data. - -The following modules are documented in this chapter: - - -.. toctree:: - - datetime.rst - zoneinfo.rst - calendar.rst - collections.rst - collections.abc.rst - heapq.rst - bisect.rst - array.rst - weakref.rst - types.rst - copy.rst - pprint.rst - reprlib.rst - enum.rst - graphlib.rst diff --git a/Doc/library/datetime.rst b/Doc/library/datetime.rst deleted file mode 100644 index 1c2abc8cf4e30e..00000000000000 --- a/Doc/library/datetime.rst +++ /dev/null @@ -1,2621 +0,0 @@ -:mod:`datetime` --- Basic date and time types -============================================= - -.. module:: datetime - :synopsis: Basic date and time types. - -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: A.M. Kuchling - -**Source code:** :source:`Lib/datetime.py` - --------------- - -.. XXX what order should the types be discussed in? - -The :mod:`datetime` module supplies classes for manipulating dates and times. - -While date and time arithmetic is supported, the focus of the implementation is -on efficient attribute extraction for output formatting and manipulation. - -.. tip:: - - Skip to :ref:`the format codes `. - -.. seealso:: - - Module :mod:`calendar` - General calendar related functions. - - Module :mod:`time` - Time access and conversions. - - Module :mod:`zoneinfo` - Concrete time zones representing the IANA time zone database. - - Package `dateutil `_ - Third-party library with expanded time zone and parsing support. - - Package `DateType `_ - Third-party library that introduces distinct static types to e.g. allow static type checkers - to differentiate between naive and aware datetimes. - -.. _datetime-naive-aware: - -Aware and Naive Objects ------------------------ - -Date and time objects may be categorized as "aware" or "naive" depending on -whether or not they include timezone information. - -With sufficient knowledge of applicable algorithmic and political time -adjustments, such as time zone and daylight saving time information, -an **aware** object can locate itself relative to other aware objects. -An aware object represents a specific moment in time that is not open to -interpretation. [#]_ - -A **naive** object does not contain enough information to unambiguously locate -itself relative to other date/time objects. Whether a naive object represents -Coordinated Universal Time (UTC), local time, or time in some other timezone is -purely up to the program, just like it is up to the program whether a -particular number represents metres, miles, or mass. Naive objects are easy to -understand and to work with, at the cost of ignoring some aspects of reality. - -For applications requiring aware objects, :class:`.datetime` and :class:`.time` -objects have an optional time zone information attribute, :attr:`!tzinfo`, that -can be set to an instance of a subclass of the abstract :class:`tzinfo` class. -These :class:`tzinfo` objects capture information about the offset from UTC -time, the time zone name, and whether daylight saving time is in effect. - -Only one concrete :class:`tzinfo` class, the :class:`timezone` class, is -supplied by the :mod:`datetime` module. The :class:`timezone` class can -represent simple timezones with fixed offsets from UTC, such as UTC itself or -North American EST and EDT timezones. Supporting timezones at deeper levels of -detail is up to the application. The rules for time adjustment across the -world are more political than rational, change frequently, and there is no -standard suitable for every application aside from UTC. - -Constants ---------- - -The :mod:`datetime` module exports the following constants: - -.. data:: MINYEAR - - The smallest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MINYEAR` is ``1``. - - -.. data:: MAXYEAR - - The largest year number allowed in a :class:`date` or :class:`.datetime` object. - :const:`MAXYEAR` is ``9999``. - -.. attribute:: UTC - - Alias for the UTC timezone singleton :attr:`datetime.timezone.utc`. - - .. versionadded:: 3.11 - -Available Types ---------------- - -.. class:: date - :noindex: - - An idealized naive date, assuming the current Gregorian calendar always was, and - always will be, in effect. Attributes: :attr:`year`, :attr:`month`, and - :attr:`day`. - - -.. class:: time - :noindex: - - An idealized time, independent of any particular day, assuming that every day - has exactly 24\*60\*60 seconds. (There is no notion of "leap seconds" here.) - Attributes: :attr:`hour`, :attr:`minute`, :attr:`second`, :attr:`microsecond`, - and :attr:`.tzinfo`. - - -.. class:: datetime - :noindex: - - A combination of a date and a time. Attributes: :attr:`year`, :attr:`month`, - :attr:`day`, :attr:`hour`, :attr:`minute`, :attr:`second`, :attr:`microsecond`, - and :attr:`.tzinfo`. - - -.. class:: timedelta - :noindex: - - A duration expressing the difference between two :class:`date`, :class:`.time`, - or :class:`.datetime` instances to microsecond resolution. - - -.. class:: tzinfo - :noindex: - - An abstract base class for time zone information objects. These are used by the - :class:`.datetime` and :class:`.time` classes to provide a customizable notion of - time adjustment (for example, to account for time zone and/or daylight saving - time). - -.. class:: timezone - :noindex: - - A class that implements the :class:`tzinfo` abstract base class as a - fixed offset from the UTC. - - .. versionadded:: 3.2 - -Objects of these types are immutable. - -Subclass relationships:: - - object - timedelta - tzinfo - timezone - time - date - datetime - -Common Properties -^^^^^^^^^^^^^^^^^ - -The :class:`date`, :class:`.datetime`, :class:`.time`, and :class:`timezone` types -share these common features: - -- Objects of these types are immutable. -- Objects of these types are :term:`hashable`, meaning that they can be used as - dictionary keys. -- Objects of these types support efficient pickling via the :mod:`pickle` module. - -Determining if an Object is Aware or Naive -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Objects of the :class:`date` type are always naive. - -An object of type :class:`.time` or :class:`.datetime` may be aware or naive. - -A :class:`.datetime` object *d* is aware if both of the following hold: - -1. ``d.tzinfo`` is not ``None`` -2. ``d.tzinfo.utcoffset(d)`` does not return ``None`` - -Otherwise, *d* is naive. - -A :class:`.time` object *t* is aware if both of the following hold: - -1. ``t.tzinfo`` is not ``None`` -2. ``t.tzinfo.utcoffset(None)`` does not return ``None``. - -Otherwise, *t* is naive. - -The distinction between aware and naive doesn't apply to :class:`timedelta` -objects. - -.. _datetime-timedelta: - -:class:`timedelta` Objects --------------------------- - -A :class:`timedelta` object represents a duration, the difference between two -dates or times. - -.. class:: timedelta(days=0, seconds=0, microseconds=0, milliseconds=0, minutes=0, hours=0, weeks=0) - - All arguments are optional and default to ``0``. Arguments may be integers - or floats, and may be positive or negative. - - Only *days*, *seconds* and *microseconds* are stored internally. - Arguments are converted to those units: - - * A millisecond is converted to 1000 microseconds. - * A minute is converted to 60 seconds. - * An hour is converted to 3600 seconds. - * A week is converted to 7 days. - - and days, seconds and microseconds are then normalized so that the - representation is unique, with - - * ``0 <= microseconds < 1000000`` - * ``0 <= seconds < 3600*24`` (the number of seconds in one day) - * ``-999999999 <= days <= 999999999`` - - The following example illustrates how any arguments besides - *days*, *seconds* and *microseconds* are "merged" and normalized into those - three resulting attributes:: - - >>> from datetime import timedelta - >>> delta = timedelta( - ... days=50, - ... seconds=27, - ... microseconds=10, - ... milliseconds=29000, - ... minutes=5, - ... hours=8, - ... weeks=2 - ... ) - >>> # Only days, seconds, and microseconds remain - >>> delta - datetime.timedelta(days=64, seconds=29156, microseconds=10) - - If any argument is a float and there are fractional microseconds, - the fractional microseconds left over from all arguments are - combined and their sum is rounded to the nearest microsecond using - round-half-to-even tiebreaker. If no argument is a float, the - conversion and normalization processes are exact (no information is - lost). - - If the normalized value of days lies outside the indicated range, - :exc:`OverflowError` is raised. - - Note that normalization of negative values may be surprising at first. For - example:: - - >>> from datetime import timedelta - >>> d = timedelta(microseconds=-1) - >>> (d.days, d.seconds, d.microseconds) - (-1, 86399, 999999) - - -Class attributes: - -.. attribute:: timedelta.min - - The most negative :class:`timedelta` object, ``timedelta(-999999999)``. - - -.. attribute:: timedelta.max - - The most positive :class:`timedelta` object, ``timedelta(days=999999999, - hours=23, minutes=59, seconds=59, microseconds=999999)``. - - -.. attribute:: timedelta.resolution - - The smallest possible difference between non-equal :class:`timedelta` objects, - ``timedelta(microseconds=1)``. - -Note that, because of normalization, ``timedelta.max`` > ``-timedelta.min``. -``-timedelta.max`` is not representable as a :class:`timedelta` object. - -Instance attributes (read-only): - -+------------------+--------------------------------------------+ -| Attribute | Value | -+==================+============================================+ -| ``days`` | Between -999999999 and 999999999 inclusive | -+------------------+--------------------------------------------+ -| ``seconds`` | Between 0 and 86399 inclusive | -+------------------+--------------------------------------------+ -| ``microseconds`` | Between 0 and 999999 inclusive | -+------------------+--------------------------------------------+ - -Supported operations: - -.. XXX this table is too wide! - -+--------------------------------+-----------------------------------------------+ -| Operation | Result | -+================================+===============================================+ -| ``t1 = t2 + t3`` | Sum of *t2* and *t3*. Afterwards *t1*-*t2* == | -| | *t3* and *t1*-*t3* == *t2* are true. (1) | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 - t3`` | Difference of *t2* and *t3*. Afterwards *t1* | -| | == *t2* - *t3* and *t2* == *t1* + *t3* are | -| | true. (1)(6) | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 * i or t1 = i * t2`` | Delta multiplied by an integer. | -| | Afterwards *t1* // i == *t2* is true, | -| | provided ``i != 0``. | -+--------------------------------+-----------------------------------------------+ -| | In general, *t1* \* i == *t1* \* (i-1) + *t1* | -| | is true. (1) | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 * f or t1 = f * t2`` | Delta multiplied by a float. The result is | -| | rounded to the nearest multiple of | -| | timedelta.resolution using round-half-to-even.| -+--------------------------------+-----------------------------------------------+ -| ``f = t2 / t3`` | Division (3) of overall duration *t2* by | -| | interval unit *t3*. Returns a :class:`float` | -| | object. | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 / f or t1 = t2 / i`` | Delta divided by a float or an int. The result| -| | is rounded to the nearest multiple of | -| | timedelta.resolution using round-half-to-even.| -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 // i`` or | The floor is computed and the remainder (if | -| ``t1 = t2 // t3`` | any) is thrown away. In the second case, an | -| | integer is returned. (3) | -+--------------------------------+-----------------------------------------------+ -| ``t1 = t2 % t3`` | The remainder is computed as a | -| | :class:`timedelta` object. (3) | -+--------------------------------+-----------------------------------------------+ -| ``q, r = divmod(t1, t2)`` | Computes the quotient and the remainder: | -| | ``q = t1 // t2`` (3) and ``r = t1 % t2``. | -| | q is an integer and r is a :class:`timedelta` | -| | object. | -+--------------------------------+-----------------------------------------------+ -| ``+t1`` | Returns a :class:`timedelta` object with the | -| | same value. (2) | -+--------------------------------+-----------------------------------------------+ -| ``-t1`` | equivalent to | -| | :class:`timedelta`\ (-*t1.days*, | -| | -*t1.seconds*, -*t1.microseconds*), | -| | and to *t1*\* -1. (1)(4) | -+--------------------------------+-----------------------------------------------+ -| ``abs(t)`` | equivalent to +\ *t* when ``t.days >= 0``, | -| | and to -*t* when ``t.days < 0``. (2) | -+--------------------------------+-----------------------------------------------+ -| ``str(t)`` | Returns a string in the form | -| | ``[D day[s], ][H]H:MM:SS[.UUUUUU]``, where D | -| | is negative for negative ``t``. (5) | -+--------------------------------+-----------------------------------------------+ -| ``repr(t)`` | Returns a string representation of the | -| | :class:`timedelta` object as a constructor | -| | call with canonical attribute values. | -+--------------------------------+-----------------------------------------------+ - - -Notes: - -(1) - This is exact but may overflow. - -(2) - This is exact and cannot overflow. - -(3) - Division by 0 raises :exc:`ZeroDivisionError`. - -(4) - -*timedelta.max* is not representable as a :class:`timedelta` object. - -(5) - String representations of :class:`timedelta` objects are normalized - similarly to their internal representation. This leads to somewhat - unusual results for negative timedeltas. For example:: - - >>> timedelta(hours=-5) - datetime.timedelta(days=-1, seconds=68400) - >>> print(_) - -1 day, 19:00:00 - -(6) - The expression ``t2 - t3`` will always be equal to the expression ``t2 + (-t3)`` except - when t3 is equal to ``timedelta.max``; in that case the former will produce a result - while the latter will overflow. - -In addition to the operations listed above, :class:`timedelta` objects support -certain additions and subtractions with :class:`date` and :class:`.datetime` -objects (see below). - -.. versionchanged:: 3.2 - Floor division and true division of a :class:`timedelta` object by another - :class:`timedelta` object are now supported, as are remainder operations and - the :func:`divmod` function. True division and multiplication of a - :class:`timedelta` object by a :class:`float` object are now supported. - - -Comparisons of :class:`timedelta` objects are supported, with some caveats. - -The comparisons ``==`` or ``!=`` *always* return a :class:`bool`, no matter -the type of the compared object:: - - >>> from datetime import timedelta - >>> delta1 = timedelta(seconds=57) - >>> delta2 = timedelta(hours=25, seconds=2) - >>> delta2 != delta1 - True - >>> delta2 == 5 - False - -For all other comparisons (such as ``<`` and ``>``), when a :class:`timedelta` -object is compared to an object of a different type, :exc:`TypeError` -is raised:: - - >>> delta2 > delta1 - True - >>> delta2 > 5 - Traceback (most recent call last): - File "", line 1, in - TypeError: '>' not supported between instances of 'datetime.timedelta' and 'int' - -In Boolean contexts, a :class:`timedelta` object is -considered to be true if and only if it isn't equal to ``timedelta(0)``. - -Instance methods: - -.. method:: timedelta.total_seconds() - - Return the total number of seconds contained in the duration. Equivalent to - ``td / timedelta(seconds=1)``. For interval units other than seconds, use the - division form directly (e.g. ``td / timedelta(microseconds=1)``). - - Note that for very large time intervals (greater than 270 years on - most platforms) this method will lose microsecond accuracy. - - .. versionadded:: 3.2 - -Examples of usage: :class:`timedelta` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -An additional example of normalization:: - - >>> # Components of another_year add up to exactly 365 days - >>> from datetime import timedelta - >>> year = timedelta(days=365) - >>> another_year = timedelta(weeks=40, days=84, hours=23, - ... minutes=50, seconds=600) - >>> year == another_year - True - >>> year.total_seconds() - 31536000.0 - -Examples of :class:`timedelta` arithmetic:: - - >>> from datetime import timedelta - >>> year = timedelta(days=365) - >>> ten_years = 10 * year - >>> ten_years - datetime.timedelta(days=3650) - >>> ten_years.days // 365 - 10 - >>> nine_years = ten_years - year - >>> nine_years - datetime.timedelta(days=3285) - >>> three_years = nine_years // 3 - >>> three_years, three_years.days // 365 - (datetime.timedelta(days=1095), 3) - -.. _datetime-date: - -:class:`date` Objects ---------------------- - -A :class:`date` object represents a date (year, month and day) in an idealized -calendar, the current Gregorian calendar indefinitely extended in both -directions. - -January 1 of year 1 is called day number 1, January 2 of year 1 is -called day number 2, and so on. [#]_ - -.. class:: date(year, month, day) - - All arguments are required. Arguments must be integers, in the following - ranges: - - * ``MINYEAR <= year <= MAXYEAR`` - * ``1 <= month <= 12`` - * ``1 <= day <= number of days in the given month and year`` - - If an argument outside those ranges is given, :exc:`ValueError` is raised. - - -Other constructors, all class methods: - -.. classmethod:: date.today() - - Return the current local date. - - This is equivalent to ``date.fromtimestamp(time.time())``. - -.. classmethod:: date.fromtimestamp(timestamp) - - Return the local date corresponding to the POSIX timestamp, such as is - returned by :func:`time.time`. - - This may raise :exc:`OverflowError`, if the timestamp is out - of the range of values supported by the platform C :c:func:`localtime` - function, and :exc:`OSError` on :c:func:`localtime` failure. - It's common for this to be restricted to years from 1970 through 2038. Note - that on non-POSIX systems that include leap seconds in their notion of a - timestamp, leap seconds are ignored by :meth:`fromtimestamp`. - - .. versionchanged:: 3.3 - Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp - is out of the range of values supported by the platform C - :c:func:`localtime` function. Raise :exc:`OSError` instead of - :exc:`ValueError` on :c:func:`localtime` failure. - - -.. classmethod:: date.fromordinal(ordinal) - - Return the date corresponding to the proleptic Gregorian ordinal, where - January 1 of year 1 has ordinal 1. - - :exc:`ValueError` is raised unless ``1 <= ordinal <= - date.max.toordinal()``. For any date *d*, - ``date.fromordinal(d.toordinal()) == d``. - - -.. classmethod:: date.fromisoformat(date_string) - - Return a :class:`date` corresponding to a *date_string* given in any valid - ISO 8601 format, except ordinal dates (e.g. ``YYYY-DDD``):: - - >>> from datetime import date - >>> date.fromisoformat('2019-12-04') - datetime.date(2019, 12, 4) - >>> date.fromisoformat('20191204') - datetime.date(2019, 12, 4) - >>> date.fromisoformat('2021-W01-1') - datetime.date(2021, 1, 4) - - .. versionadded:: 3.7 - .. versionchanged:: 3.11 - Previously, this method only supported the format ``YYYY-MM-DD``. - -.. classmethod:: date.fromisocalendar(year, week, day) - - Return a :class:`date` corresponding to the ISO calendar date specified by - year, week and day. This is the inverse of the function :meth:`date.isocalendar`. - - .. versionadded:: 3.8 - - -Class attributes: - -.. attribute:: date.min - - The earliest representable date, ``date(MINYEAR, 1, 1)``. - - -.. attribute:: date.max - - The latest representable date, ``date(MAXYEAR, 12, 31)``. - - -.. attribute:: date.resolution - - The smallest possible difference between non-equal date objects, - ``timedelta(days=1)``. - - -Instance attributes (read-only): - -.. attribute:: date.year - - Between :const:`MINYEAR` and :const:`MAXYEAR` inclusive. - - -.. attribute:: date.month - - Between 1 and 12 inclusive. - - -.. attribute:: date.day - - Between 1 and the number of days in the given month of the given year. - - -Supported operations: - -+-------------------------------+----------------------------------------------+ -| Operation | Result | -+===============================+==============================================+ -| ``date2 = date1 + timedelta`` | *date2* will be ``timedelta.days`` days | -| | after *date1*. (1) | -+-------------------------------+----------------------------------------------+ -| ``date2 = date1 - timedelta`` | Computes *date2* such that ``date2 + | -| | timedelta == date1``. (2) | -+-------------------------------+----------------------------------------------+ -| ``timedelta = date1 - date2`` | \(3) | -+-------------------------------+----------------------------------------------+ -| ``date1 < date2`` | *date1* is considered less than *date2* when | -| | *date1* precedes *date2* in time. (4) | -+-------------------------------+----------------------------------------------+ - -Notes: - -(1) - *date2* is moved forward in time if ``timedelta.days > 0``, or backward if - ``timedelta.days < 0``. Afterward ``date2 - date1 == timedelta.days``. - ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. - :exc:`OverflowError` is raised if ``date2.year`` would be smaller than - :const:`MINYEAR` or larger than :const:`MAXYEAR`. - -(2) - ``timedelta.seconds`` and ``timedelta.microseconds`` are ignored. - -(3) - This is exact, and cannot overflow. timedelta.seconds and - timedelta.microseconds are 0, and date2 + timedelta == date1 after. - -(4) - In other words, ``date1 < date2`` if and only if ``date1.toordinal() < - date2.toordinal()``. Date comparison raises :exc:`TypeError` if - the other comparand isn't also a :class:`date` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`date` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. - -In Boolean contexts, all :class:`date` objects are considered to be true. - -Instance methods: - -.. method:: date.replace(year=self.year, month=self.month, day=self.day) - - Return a date with the same value, except for those parameters given new - values by whichever keyword arguments are specified. - - Example:: - - >>> from datetime import date - >>> d = date(2002, 12, 31) - >>> d.replace(day=26) - datetime.date(2002, 12, 26) - - -.. method:: date.timetuple() - - Return a :class:`time.struct_time` such as returned by :func:`time.localtime`. - - The hours, minutes and seconds are 0, and the DST flag is -1. - - ``d.timetuple()`` is equivalent to:: - - time.struct_time((d.year, d.month, d.day, 0, 0, 0, d.weekday(), yday, -1)) - - where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` - is the day number within the current year starting with ``1`` for January 1st. - - -.. method:: date.toordinal() - - Return the proleptic Gregorian ordinal of the date, where January 1 of year 1 - has ordinal 1. For any :class:`date` object *d*, - ``date.fromordinal(d.toordinal()) == d``. - - -.. method:: date.weekday() - - Return the day of the week as an integer, where Monday is 0 and Sunday is 6. - For example, ``date(2002, 12, 4).weekday() == 2``, a Wednesday. See also - :meth:`isoweekday`. - - -.. method:: date.isoweekday() - - Return the day of the week as an integer, where Monday is 1 and Sunday is 7. - For example, ``date(2002, 12, 4).isoweekday() == 3``, a Wednesday. See also - :meth:`weekday`, :meth:`isocalendar`. - - -.. method:: date.isocalendar() - - Return a :term:`named tuple` object with three components: ``year``, - ``week`` and ``weekday``. - - The ISO calendar is a widely used variant of the Gregorian calendar. [#]_ - - The ISO year consists of 52 or 53 full weeks, and where a week starts on a - Monday and ends on a Sunday. The first week of an ISO year is the first - (Gregorian) calendar week of a year containing a Thursday. This is called week - number 1, and the ISO year of that Thursday is the same as its Gregorian year. - - For example, 2004 begins on a Thursday, so the first week of ISO year 2004 - begins on Monday, 29 Dec 2003 and ends on Sunday, 4 Jan 2004:: - - >>> from datetime import date - >>> date(2003, 12, 29).isocalendar() - datetime.IsoCalendarDate(year=2004, week=1, weekday=1) - >>> date(2004, 1, 4).isocalendar() - datetime.IsoCalendarDate(year=2004, week=1, weekday=7) - - .. versionchanged:: 3.9 - Result changed from a tuple to a :term:`named tuple`. - -.. method:: date.isoformat() - - Return a string representing the date in ISO 8601 format, ``YYYY-MM-DD``:: - - >>> from datetime import date - >>> date(2002, 12, 4).isoformat() - '2002-12-04' - -.. method:: date.__str__() - - For a date *d*, ``str(d)`` is equivalent to ``d.isoformat()``. - - -.. method:: date.ctime() - - Return a string representing the date:: - - >>> from datetime import date - >>> date(2002, 12, 4).ctime() - 'Wed Dec 4 00:00:00 2002' - - ``d.ctime()`` is equivalent to:: - - time.ctime(time.mktime(d.timetuple())) - - on platforms where the native C - :c:func:`ctime` function (which :func:`time.ctime` invokes, but which - :meth:`date.ctime` does not invoke) conforms to the C standard. - - -.. method:: date.strftime(format) - - Return a string representing the date, controlled by an explicit format string. - Format codes referring to hours, minutes or seconds will see 0 values. - See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. - - -.. method:: date.__format__(format) - - Same as :meth:`.date.strftime`. This makes it possible to specify a format - string for a :class:`.date` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. - See also :ref:`strftime-strptime-behavior` and :meth:`date.isoformat`. - -Examples of Usage: :class:`date` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Example of counting days to an event:: - - >>> import time - >>> from datetime import date - >>> today = date.today() - >>> today - datetime.date(2007, 12, 5) - >>> today == date.fromtimestamp(time.time()) - True - >>> my_birthday = date(today.year, 6, 24) - >>> if my_birthday < today: - ... my_birthday = my_birthday.replace(year=today.year + 1) - >>> my_birthday - datetime.date(2008, 6, 24) - >>> time_to_birthday = abs(my_birthday - today) - >>> time_to_birthday.days - 202 - -More examples of working with :class:`date`: - -.. doctest:: - - >>> from datetime import date - >>> d = date.fromordinal(730920) # 730920th day after 1. 1. 0001 - >>> d - datetime.date(2002, 3, 11) - - >>> # Methods related to formatting string output - >>> d.isoformat() - '2002-03-11' - >>> d.strftime("%d/%m/%y") - '11/03/02' - >>> d.strftime("%A %d. %B %Y") - 'Monday 11. March 2002' - >>> d.ctime() - 'Mon Mar 11 00:00:00 2002' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}.'.format(d, "day", "month") - 'The day is 11, the month is March.' - - >>> # Methods for to extracting 'components' under different calendars - >>> t = d.timetuple() - >>> for i in t: # doctest: +SKIP - ... print(i) - 2002 # year - 3 # month - 11 # day - 0 - 0 - 0 - 0 # weekday (0 = Monday) - 70 # 70th day in the year - -1 - >>> ic = d.isocalendar() - >>> for i in ic: # doctest: +SKIP - ... print(i) - 2002 # ISO year - 11 # ISO week number - 1 # ISO day number ( 1 = Monday ) - - >>> # A date object is immutable; all operations produce a new object - >>> d.replace(year=2005) - datetime.date(2005, 3, 11) - - -.. _datetime-datetime: - -:class:`.datetime` Objects --------------------------- - -A :class:`.datetime` object is a single object containing all the information -from a :class:`date` object and a :class:`.time` object. - -Like a :class:`date` object, :class:`.datetime` assumes the current Gregorian -calendar extended in both directions; like a :class:`.time` object, -:class:`.datetime` assumes there are exactly 3600\*24 seconds in every day. - -Constructor: - -.. class:: datetime(year, month, day, hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) - - The *year*, *month* and *day* arguments are required. *tzinfo* may be ``None``, or an - instance of a :class:`tzinfo` subclass. The remaining arguments must be integers - in the following ranges: - - * ``MINYEAR <= year <= MAXYEAR``, - * ``1 <= month <= 12``, - * ``1 <= day <= number of days in the given month and year``, - * ``0 <= hour < 24``, - * ``0 <= minute < 60``, - * ``0 <= second < 60``, - * ``0 <= microsecond < 1000000``, - * ``fold in [0, 1]``. - - If an argument outside those ranges is given, :exc:`ValueError` is raised. - - .. versionadded:: 3.6 - Added the ``fold`` argument. - -Other constructors, all class methods: - -.. classmethod:: datetime.today() - - Return the current local datetime, with :attr:`.tzinfo` ``None``. - - Equivalent to:: - - datetime.fromtimestamp(time.time()) - - See also :meth:`now`, :meth:`fromtimestamp`. - - This method is functionally equivalent to :meth:`now`, but without a - ``tz`` parameter. - -.. classmethod:: datetime.now(tz=None) - - Return the current local date and time. - - If optional argument *tz* is ``None`` - or not specified, this is like :meth:`today`, but, if possible, supplies more - precision than can be gotten from going through a :func:`time.time` timestamp - (for example, this may be possible on platforms supplying the C - :c:func:`gettimeofday` function). - - If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, - and the current date and time are converted to *tz*’s time zone. - - This function is preferred over :meth:`today` and :meth:`utcnow`. - - -.. classmethod:: datetime.utcnow() - - Return the current UTC date and time, with :attr:`.tzinfo` ``None``. - - This is like :meth:`now`, but returns the current UTC date and time, as a naive - :class:`.datetime` object. An aware current UTC datetime can be obtained by - calling ``datetime.now(timezone.utc)``. See also :meth:`now`. - - .. warning:: - - Because naive ``datetime`` objects are treated by many ``datetime`` methods - as local times, it is preferred to use aware datetimes to represent times - in UTC. As such, the recommended way to create an object representing the - current time in UTC is by calling ``datetime.now(timezone.utc)``. - - -.. classmethod:: datetime.fromtimestamp(timestamp, tz=None) - - Return the local date and time corresponding to the POSIX timestamp, such as is - returned by :func:`time.time`. If optional argument *tz* is ``None`` or not - specified, the timestamp is converted to the platform's local date and time, and - the returned :class:`.datetime` object is naive. - - If *tz* is not ``None``, it must be an instance of a :class:`tzinfo` subclass, and the - timestamp is converted to *tz*’s time zone. - - :meth:`fromtimestamp` may raise :exc:`OverflowError`, if the timestamp is out of - the range of values supported by the platform C :c:func:`localtime` or - :c:func:`gmtime` functions, and :exc:`OSError` on :c:func:`localtime` or - :c:func:`gmtime` failure. - It's common for this to be restricted to years in - 1970 through 2038. Note that on non-POSIX systems that include leap seconds in - their notion of a timestamp, leap seconds are ignored by :meth:`fromtimestamp`, - and then it's possible to have two timestamps differing by a second that yield - identical :class:`.datetime` objects. This method is preferred over - :meth:`utcfromtimestamp`. - - .. versionchanged:: 3.3 - Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp - is out of the range of values supported by the platform C - :c:func:`localtime` or :c:func:`gmtime` functions. Raise :exc:`OSError` - instead of :exc:`ValueError` on :c:func:`localtime` or :c:func:`gmtime` - failure. - - .. versionchanged:: 3.6 - :meth:`fromtimestamp` may return instances with :attr:`.fold` set to 1. - -.. classmethod:: datetime.utcfromtimestamp(timestamp) - - Return the UTC :class:`.datetime` corresponding to the POSIX timestamp, with - :attr:`.tzinfo` ``None``. (The resulting object is naive.) - - This may raise :exc:`OverflowError`, if the timestamp is - out of the range of values supported by the platform C :c:func:`gmtime` function, - and :exc:`OSError` on :c:func:`gmtime` failure. - It's common for this to be restricted to years in 1970 through 2038. - - To get an aware :class:`.datetime` object, call :meth:`fromtimestamp`:: - - datetime.fromtimestamp(timestamp, timezone.utc) - - On the POSIX compliant platforms, it is equivalent to the following - expression:: - - datetime(1970, 1, 1, tzinfo=timezone.utc) + timedelta(seconds=timestamp) - - except the latter formula always supports the full years range: between - :const:`MINYEAR` and :const:`MAXYEAR` inclusive. - - .. warning:: - - Because naive ``datetime`` objects are treated by many ``datetime`` methods - as local times, it is preferred to use aware datetimes to represent times - in UTC. As such, the recommended way to create an object representing a - specific timestamp in UTC is by calling - ``datetime.fromtimestamp(timestamp, tz=timezone.utc)``. - - .. versionchanged:: 3.3 - Raise :exc:`OverflowError` instead of :exc:`ValueError` if the timestamp - is out of the range of values supported by the platform C - :c:func:`gmtime` function. Raise :exc:`OSError` instead of - :exc:`ValueError` on :c:func:`gmtime` failure. - - -.. classmethod:: datetime.fromordinal(ordinal) - - Return the :class:`.datetime` corresponding to the proleptic Gregorian ordinal, - where January 1 of year 1 has ordinal 1. :exc:`ValueError` is raised unless ``1 - <= ordinal <= datetime.max.toordinal()``. The hour, minute, second and - microsecond of the result are all 0, and :attr:`.tzinfo` is ``None``. - - -.. classmethod:: datetime.combine(date, time, tzinfo=self.tzinfo) - - Return a new :class:`.datetime` object whose date components are equal to the - given :class:`date` object's, and whose time components - are equal to the given :class:`.time` object's. If the *tzinfo* - argument is provided, its value is used to set the :attr:`.tzinfo` attribute - of the result, otherwise the :attr:`~.time.tzinfo` attribute of the *time* argument - is used. - - For any :class:`.datetime` object *d*, - ``d == datetime.combine(d.date(), d.time(), d.tzinfo)``. If date is a - :class:`.datetime` object, its time components and :attr:`.tzinfo` attributes - are ignored. - - .. versionchanged:: 3.6 - Added the *tzinfo* argument. - - -.. classmethod:: datetime.fromisoformat(date_string) - - Return a :class:`.datetime` corresponding to a *date_string* in any valid - ISO 8601 format, with the following exceptions: - - 1. Time zone offsets may have fractional seconds. - 2. The ``T`` separator may be replaced by any single unicode character. - 3. Ordinal dates are not currently supported. - 4. Fractional hours and minutes are not supported. - - Examples:: - - >>> from datetime import datetime - >>> datetime.fromisoformat('2011-11-04') - datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('20111104') - datetime.datetime(2011, 11, 4, 0, 0) - >>> datetime.fromisoformat('2011-11-04T00:05:23') - datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-11-04T00:05:23Z') - datetime.datetime(2011, 11, 4, 0, 5, 23, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('20111104T000523') - datetime.datetime(2011, 11, 4, 0, 5, 23) - >>> datetime.fromisoformat('2011-W01-2T00:05:23.283') - datetime.datetime(2011, 1, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283') - datetime.datetime(2011, 11, 4, 0, 5, 23, 283000) - >>> datetime.fromisoformat('2011-11-04 00:05:23.283+00:00') - datetime.datetime(2011, 11, 4, 0, 5, 23, 283000, tzinfo=datetime.timezone.utc) - >>> datetime.fromisoformat('2011-11-04T00:05:23+04:00') # doctest: +NORMALIZE_WHITESPACE - datetime.datetime(2011, 11, 4, 0, 5, 23, - tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - - .. versionadded:: 3.7 - .. versionchanged:: 3.11 - Previously, this method only supported formats that could be emitted by - :meth:`date.isoformat()` or :meth:`datetime.isoformat()`. - - -.. classmethod:: datetime.fromisocalendar(year, week, day) - - Return a :class:`.datetime` corresponding to the ISO calendar date specified - by year, week and day. The non-date components of the datetime are populated - with their normal default values. This is the inverse of the function - :meth:`datetime.isocalendar`. - - .. versionadded:: 3.8 - -.. classmethod:: datetime.strptime(date_string, format) - - Return a :class:`.datetime` corresponding to *date_string*, parsed according to - *format*. - - If *format* does not contain microseconds or timezone information, this is equivalent to:: - - datetime(*(time.strptime(date_string, format)[0:6])) - - :exc:`ValueError` is raised if the date_string and format - can't be parsed by :func:`time.strptime` or if it returns a value which isn't a - time tuple. See also :ref:`strftime-strptime-behavior` and - :meth:`datetime.fromisoformat`. - - - -Class attributes: - -.. attribute:: datetime.min - - The earliest representable :class:`.datetime`, ``datetime(MINYEAR, 1, 1, - tzinfo=None)``. - - -.. attribute:: datetime.max - - The latest representable :class:`.datetime`, ``datetime(MAXYEAR, 12, 31, 23, 59, - 59, 999999, tzinfo=None)``. - - -.. attribute:: datetime.resolution - - The smallest possible difference between non-equal :class:`.datetime` objects, - ``timedelta(microseconds=1)``. - - -Instance attributes (read-only): - -.. attribute:: datetime.year - - Between :const:`MINYEAR` and :const:`MAXYEAR` inclusive. - - -.. attribute:: datetime.month - - Between 1 and 12 inclusive. - - -.. attribute:: datetime.day - - Between 1 and the number of days in the given month of the given year. - - -.. attribute:: datetime.hour - - In ``range(24)``. - - -.. attribute:: datetime.minute - - In ``range(60)``. - - -.. attribute:: datetime.second - - In ``range(60)``. - - -.. attribute:: datetime.microsecond - - In ``range(1000000)``. - - -.. attribute:: datetime.tzinfo - - The object passed as the *tzinfo* argument to the :class:`.datetime` constructor, - or ``None`` if none was passed. - - -.. attribute:: datetime.fold - - In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A - repeated interval occurs when clocks are rolled back at the end of daylight saving - time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. - - .. versionadded:: 3.6 - -Supported operations: - -+---------------------------------------+--------------------------------+ -| Operation | Result | -+=======================================+================================+ -| ``datetime2 = datetime1 + timedelta`` | \(1) | -+---------------------------------------+--------------------------------+ -| ``datetime2 = datetime1 - timedelta`` | \(2) | -+---------------------------------------+--------------------------------+ -| ``timedelta = datetime1 - datetime2`` | \(3) | -+---------------------------------------+--------------------------------+ -| ``datetime1 < datetime2`` | Compares :class:`.datetime` to | -| | :class:`.datetime`. (4) | -+---------------------------------------+--------------------------------+ - -(1) - datetime2 is a duration of timedelta removed from datetime1, moving forward in - time if ``timedelta.days`` > 0, or backward if ``timedelta.days`` < 0. The - result has the same :attr:`~.datetime.tzinfo` attribute as the input datetime, and - datetime2 - datetime1 == timedelta after. :exc:`OverflowError` is raised if - datetime2.year would be smaller than :const:`MINYEAR` or larger than - :const:`MAXYEAR`. Note that no time zone adjustments are done even if the - input is an aware object. - -(2) - Computes the datetime2 such that datetime2 + timedelta == datetime1. As for - addition, the result has the same :attr:`~.datetime.tzinfo` attribute as the input - datetime, and no time zone adjustments are done even if the input is aware. - -(3) - Subtraction of a :class:`.datetime` from a :class:`.datetime` is defined only if - both operands are naive, or if both are aware. If one is aware and the other is - naive, :exc:`TypeError` is raised. - - If both are naive, or both are aware and have the same :attr:`~.datetime.tzinfo` attribute, - the :attr:`~.datetime.tzinfo` attributes are ignored, and the result is a :class:`timedelta` - object *t* such that ``datetime2 + t == datetime1``. No time zone adjustments - are done in this case. - - If both are aware and have different :attr:`~.datetime.tzinfo` attributes, ``a-b`` acts - as if *a* and *b* were first converted to naive UTC datetimes first. The - result is ``(a.replace(tzinfo=None) - a.utcoffset()) - (b.replace(tzinfo=None) - - b.utcoffset())`` except that the implementation never overflows. - -(4) - *datetime1* is considered less than *datetime2* when *datetime1* precedes - *datetime2* in time. - - If one comparand is naive and the other is aware, :exc:`TypeError` - is raised if an order comparison is attempted. For equality - comparisons, naive instances are never equal to aware instances. - - If both comparands are aware, and have the same :attr:`~.datetime.tzinfo` attribute, the - common :attr:`~.datetime.tzinfo` attribute is ignored and the base datetimes are - compared. If both comparands are aware and have different :attr:`~.datetime.tzinfo` - attributes, the comparands are first adjusted by subtracting their UTC - offsets (obtained from ``self.utcoffset()``). - - .. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`.datetime` - instances don't raise :exc:`TypeError`. - - .. note:: - - In order to stop comparison from falling back to the default scheme of comparing - object addresses, datetime comparison normally raises :exc:`TypeError` if the - other comparand isn't also a :class:`.datetime` object. However, - ``NotImplemented`` is returned instead if the other comparand has a - :meth:`timetuple` attribute. This hook gives other kinds of date objects a - chance at implementing mixed-type comparison. If not, when a :class:`.datetime` - object is compared to an object of a different type, :exc:`TypeError` is raised - unless the comparison is ``==`` or ``!=``. The latter cases return - :const:`False` or :const:`True`, respectively. - -Instance methods: - -.. method:: datetime.date() - - Return :class:`date` object with same year, month and day. - - -.. method:: datetime.time() - - Return :class:`.time` object with same hour, minute, second, microsecond and fold. - :attr:`.tzinfo` is ``None``. See also method :meth:`timetz`. - - .. versionchanged:: 3.6 - The fold value is copied to the returned :class:`.time` object. - - -.. method:: datetime.timetz() - - Return :class:`.time` object with same hour, minute, second, microsecond, fold, and - tzinfo attributes. See also method :meth:`time`. - - .. versionchanged:: 3.6 - The fold value is copied to the returned :class:`.time` object. - - -.. method:: datetime.replace(year=self.year, month=self.month, day=self.day, \ - hour=self.hour, minute=self.minute, second=self.second, microsecond=self.microsecond, \ - tzinfo=self.tzinfo, *, fold=0) - - Return a datetime with the same attributes, except for those attributes given - new values by whichever keyword arguments are specified. Note that - ``tzinfo=None`` can be specified to create a naive datetime from an aware - datetime with no conversion of date and time data. - - .. versionadded:: 3.6 - Added the ``fold`` argument. - - -.. method:: datetime.astimezone(tz=None) - - Return a :class:`.datetime` object with new :attr:`.tzinfo` attribute *tz*, - adjusting the date and time data so the result is the same UTC time as - *self*, but in *tz*'s local time. - - If provided, *tz* must be an instance of a :class:`tzinfo` subclass, and its - :meth:`utcoffset` and :meth:`dst` methods must not return ``None``. If *self* - is naive, it is presumed to represent time in the system timezone. - - If called without arguments (or with ``tz=None``) the system local - timezone is assumed for the target timezone. The ``.tzinfo`` attribute of the converted - datetime instance will be set to an instance of :class:`timezone` - with the zone name and offset obtained from the OS. - - If ``self.tzinfo`` is *tz*, ``self.astimezone(tz)`` is equal to *self*: no - adjustment of date or time data is performed. Else the result is local - time in the timezone *tz*, representing the same UTC time as *self*: after - ``astz = dt.astimezone(tz)``, ``astz - astz.utcoffset()`` will have - the same date and time data as ``dt - dt.utcoffset()``. - - If you merely want to attach a time zone object *tz* to a datetime *dt* without - adjustment of date and time data, use ``dt.replace(tzinfo=tz)``. If you - merely want to remove the time zone object from an aware datetime *dt* without - conversion of date and time data, use ``dt.replace(tzinfo=None)``. - - Note that the default :meth:`tzinfo.fromutc` method can be overridden in a - :class:`tzinfo` subclass to affect the result returned by :meth:`astimezone`. - Ignoring error cases, :meth:`astimezone` acts like:: - - def astimezone(self, tz): - if self.tzinfo is tz: - return self - # Convert self to UTC, and attach the new time zone object. - utc = (self - self.utcoffset()).replace(tzinfo=tz) - # Convert from UTC to tz's local time. - return tz.fromutc(utc) - - .. versionchanged:: 3.3 - *tz* now can be omitted. - - .. versionchanged:: 3.6 - The :meth:`astimezone` method can now be called on naive instances that - are presumed to represent system local time. - - -.. method:: datetime.utcoffset() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.utcoffset(self)``, and raises an exception if the latter doesn't - return ``None`` or a :class:`timedelta` object with magnitude less than one day. - - .. versionchanged:: 3.7 - The UTC offset is not restricted to a whole number of minutes. - - -.. method:: datetime.dst() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.dst(self)``, and raises an exception if the latter doesn't return - ``None`` or a :class:`timedelta` object with magnitude less than one day. - - .. versionchanged:: 3.7 - The DST offset is not restricted to a whole number of minutes. - - -.. method:: datetime.tzname() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.tzname(self)``, raises an exception if the latter doesn't return - ``None`` or a string object, - - -.. method:: datetime.timetuple() - - Return a :class:`time.struct_time` such as returned by :func:`time.localtime`. - - ``d.timetuple()`` is equivalent to:: - - time.struct_time((d.year, d.month, d.day, - d.hour, d.minute, d.second, - d.weekday(), yday, dst)) - - where ``yday = d.toordinal() - date(d.year, 1, 1).toordinal() + 1`` - is the day number within the current year starting with ``1`` for January - 1st. The :attr:`tm_isdst` flag of the result is set according to the - :meth:`dst` method: :attr:`.tzinfo` is ``None`` or :meth:`dst` returns - ``None``, :attr:`tm_isdst` is set to ``-1``; else if :meth:`dst` returns a - non-zero value, :attr:`tm_isdst` is set to ``1``; else :attr:`tm_isdst` is - set to ``0``. - - -.. method:: datetime.utctimetuple() - - If :class:`.datetime` instance *d* is naive, this is the same as - ``d.timetuple()`` except that :attr:`tm_isdst` is forced to 0 regardless of what - ``d.dst()`` returns. DST is never in effect for a UTC time. - - If *d* is aware, *d* is normalized to UTC time, by subtracting - ``d.utcoffset()``, and a :class:`time.struct_time` for the - normalized time is returned. :attr:`tm_isdst` is forced to 0. Note - that an :exc:`OverflowError` may be raised if *d*.year was - ``MINYEAR`` or ``MAXYEAR`` and UTC adjustment spills over a year - boundary. - - .. warning:: - - Because naive ``datetime`` objects are treated by many ``datetime`` methods - as local times, it is preferred to use aware datetimes to represent times - in UTC; as a result, using :meth:`datetime.utctimetuple` may give misleading - results. If you have a naive ``datetime`` representing UTC, use - ``datetime.replace(tzinfo=timezone.utc)`` to make it aware, at which point - you can use :meth:`.datetime.timetuple`. - -.. method:: datetime.toordinal() - - Return the proleptic Gregorian ordinal of the date. The same as - ``self.date().toordinal()``. - -.. method:: datetime.timestamp() - - Return POSIX timestamp corresponding to the :class:`.datetime` - instance. The return value is a :class:`float` similar to that - returned by :func:`time.time`. - - Naive :class:`.datetime` instances are assumed to represent local - time and this method relies on the platform C :c:func:`mktime` - function to perform the conversion. Since :class:`.datetime` - supports wider range of values than :c:func:`mktime` on many - platforms, this method may raise :exc:`OverflowError` for times far - in the past or far in the future. - - For aware :class:`.datetime` instances, the return value is computed - as:: - - (dt - datetime(1970, 1, 1, tzinfo=timezone.utc)).total_seconds() - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - The :meth:`timestamp` method uses the :attr:`.fold` attribute to - disambiguate the times during a repeated interval. - - .. note:: - - There is no method to obtain the POSIX timestamp directly from a - naive :class:`.datetime` instance representing UTC time. If your - application uses this convention and your system timezone is not - set to UTC, you can obtain the POSIX timestamp by supplying - ``tzinfo=timezone.utc``:: - - timestamp = dt.replace(tzinfo=timezone.utc).timestamp() - - or by calculating the timestamp directly:: - - timestamp = (dt - datetime(1970, 1, 1)) / timedelta(seconds=1) - -.. method:: datetime.weekday() - - Return the day of the week as an integer, where Monday is 0 and Sunday is 6. - The same as ``self.date().weekday()``. See also :meth:`isoweekday`. - - -.. method:: datetime.isoweekday() - - Return the day of the week as an integer, where Monday is 1 and Sunday is 7. - The same as ``self.date().isoweekday()``. See also :meth:`weekday`, - :meth:`isocalendar`. - - -.. method:: datetime.isocalendar() - - Return a :term:`named tuple` with three components: ``year``, ``week`` - and ``weekday``. The same as ``self.date().isocalendar()``. - - -.. method:: datetime.isoformat(sep='T', timespec='auto') - - Return a string representing the date and time in ISO 8601 format: - - - ``YYYY-MM-DDTHH:MM:SS.ffffff``, if :attr:`microsecond` is not 0 - - ``YYYY-MM-DDTHH:MM:SS``, if :attr:`microsecond` is 0 - - If :meth:`utcoffset` does not return ``None``, a string is - appended, giving the UTC offset: - - - ``YYYY-MM-DDTHH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` - is not 0 - - ``YYYY-MM-DDTHH:MM:SS+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` is 0 - - Examples:: - - >>> from datetime import datetime, timezone - >>> datetime(2019, 5, 18, 15, 17, 8, 132263).isoformat() - '2019-05-18T15:17:08.132263' - >>> datetime(2019, 5, 18, 15, 17, tzinfo=timezone.utc).isoformat() - '2019-05-18T15:17:00+00:00' - - The optional argument *sep* (default ``'T'``) is a one-character separator, - placed between the date and time portions of the result. For example:: - - >>> from datetime import tzinfo, timedelta, datetime - >>> class TZ(tzinfo): - ... """A time zone with an arbitrary, constant -06:39 offset.""" - ... def utcoffset(self, dt): - ... return timedelta(hours=-6, minutes=-39) - ... - >>> datetime(2002, 12, 25, tzinfo=TZ()).isoformat(' ') - '2002-12-25 00:00:00-06:39' - >>> datetime(2009, 11, 27, microsecond=100, tzinfo=TZ()).isoformat() - '2009-11-27T00:00:00.000100-06:39' - - The optional argument *timespec* specifies the number of additional - components of the time to include (the default is ``'auto'``). - It can be one of the following: - - - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, - same as ``'microseconds'`` otherwise. - - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format. - - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format. - - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` - in ``HH:MM:SS`` format. - - ``'milliseconds'``: Include full time, but truncate fractional second - part to milliseconds. ``HH:MM:SS.sss`` format. - - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format. - - .. note:: - - Excluded time components are truncated, not rounded. - - :exc:`ValueError` will be raised on an invalid *timespec* argument:: - - - >>> from datetime import datetime - >>> datetime.now().isoformat(timespec='minutes') # doctest: +SKIP - '2002-12-25T00:00' - >>> dt = datetime(2015, 1, 1, 12, 30, 59, 0) - >>> dt.isoformat(timespec='microseconds') - '2015-01-01T12:30:59.000000' - - .. versionadded:: 3.6 - Added the *timespec* argument. - - -.. method:: datetime.__str__() - - For a :class:`.datetime` instance *d*, ``str(d)`` is equivalent to - ``d.isoformat(' ')``. - - -.. method:: datetime.ctime() - - Return a string representing the date and time:: - - >>> from datetime import datetime - >>> datetime(2002, 12, 4, 20, 30, 40).ctime() - 'Wed Dec 4 20:30:40 2002' - - The output string will *not* include time zone information, regardless - of whether the input is aware or naive. - - ``d.ctime()`` is equivalent to:: - - time.ctime(time.mktime(d.timetuple())) - - on platforms where the native C :c:func:`ctime` function - (which :func:`time.ctime` invokes, but which - :meth:`datetime.ctime` does not invoke) conforms to the C standard. - - -.. method:: datetime.strftime(format) - - Return a string representing the date and time, - controlled by an explicit format string. - See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. - - -.. method:: datetime.__format__(format) - - Same as :meth:`.datetime.strftime`. This makes it possible to specify a format - string for a :class:`.datetime` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. - See also :ref:`strftime-strptime-behavior` and :meth:`datetime.isoformat`. - - -Examples of Usage: :class:`.datetime` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Examples of working with :class:`~datetime.datetime` objects: - -.. doctest:: - - >>> from datetime import datetime, date, time, timezone - - >>> # Using datetime.combine() - >>> d = date(2005, 7, 14) - >>> t = time(12, 30) - >>> datetime.combine(d, t) - datetime.datetime(2005, 7, 14, 12, 30) - - >>> # Using datetime.now() - >>> datetime.now() # doctest: +SKIP - datetime.datetime(2007, 12, 6, 16, 29, 43, 79043) # GMT +1 - >>> datetime.now(timezone.utc) # doctest: +SKIP - datetime.datetime(2007, 12, 6, 15, 29, 43, 79060, tzinfo=datetime.timezone.utc) - - >>> # Using datetime.strptime() - >>> dt = datetime.strptime("21/11/06 16:30", "%d/%m/%y %H:%M") - >>> dt - datetime.datetime(2006, 11, 21, 16, 30) - - >>> # Using datetime.timetuple() to get tuple of all attributes - >>> tt = dt.timetuple() - >>> for it in tt: # doctest: +SKIP - ... print(it) - ... - 2006 # year - 11 # month - 21 # day - 16 # hour - 30 # minute - 0 # second - 1 # weekday (0 = Monday) - 325 # number of days since 1st January - -1 # dst - method tzinfo.dst() returned None - - >>> # Date in ISO format - >>> ic = dt.isocalendar() - >>> for it in ic: # doctest: +SKIP - ... print(it) - ... - 2006 # ISO year - 47 # ISO week - 2 # ISO weekday - - >>> # Formatting a datetime - >>> dt.strftime("%A, %d. %B %Y %I:%M%p") - 'Tuesday, 21. November 2006 04:30PM' - >>> 'The {1} is {0:%d}, the {2} is {0:%B}, the {3} is {0:%I:%M%p}.'.format(dt, "day", "month", "time") - 'The day is 21, the month is November, the time is 04:30PM.' - -The example below defines a :class:`tzinfo` subclass capturing time zone -information for Kabul, Afghanistan, which used +4 UTC until 1945 -and then +4:30 UTC thereafter:: - - from datetime import timedelta, datetime, tzinfo, timezone - - class KabulTz(tzinfo): - # Kabul used +4 until 1945, when they moved to +4:30 - UTC_MOVE_DATE = datetime(1944, 12, 31, 20, tzinfo=timezone.utc) - - def utcoffset(self, dt): - if dt.year < 1945: - return timedelta(hours=4) - elif (1945, 1, 1, 0, 0) <= dt.timetuple()[:5] < (1945, 1, 1, 0, 30): - # An ambiguous ("imaginary") half-hour range representing - # a 'fold' in time due to the shift from +4 to +4:30. - # If dt falls in the imaginary range, use fold to decide how - # to resolve. See PEP495. - return timedelta(hours=4, minutes=(30 if dt.fold else 0)) - else: - return timedelta(hours=4, minutes=30) - - def fromutc(self, dt): - # Follow same validations as in datetime.tzinfo - if not isinstance(dt, datetime): - raise TypeError("fromutc() requires a datetime argument") - if dt.tzinfo is not self: - raise ValueError("dt.tzinfo is not self") - - # A custom implementation is required for fromutc as - # the input to this function is a datetime with utc values - # but with a tzinfo set to self. - # See datetime.astimezone or fromtimestamp. - if dt.replace(tzinfo=timezone.utc) >= self.UTC_MOVE_DATE: - return dt + timedelta(hours=4, minutes=30) - else: - return dt + timedelta(hours=4) - - def dst(self, dt): - # Kabul does not observe daylight saving time. - return timedelta(0) - - def tzname(self, dt): - if dt >= self.UTC_MOVE_DATE: - return "+04:30" - return "+04" - -Usage of ``KabulTz`` from above:: - - >>> tz1 = KabulTz() - - >>> # Datetime before the change - >>> dt1 = datetime(1900, 11, 21, 16, 30, tzinfo=tz1) - >>> print(dt1.utcoffset()) - 4:00:00 - - >>> # Datetime after the change - >>> dt2 = datetime(2006, 6, 14, 13, 0, tzinfo=tz1) - >>> print(dt2.utcoffset()) - 4:30:00 - - >>> # Convert datetime to another time zone - >>> dt3 = dt2.astimezone(timezone.utc) - >>> dt3 - datetime.datetime(2006, 6, 14, 8, 30, tzinfo=datetime.timezone.utc) - >>> dt2 - datetime.datetime(2006, 6, 14, 13, 0, tzinfo=KabulTz()) - >>> dt2 == dt3 - True - -.. _datetime-time: - -:class:`.time` Objects ----------------------- - -A :class:`time` object represents a (local) time of day, independent of any particular -day, and subject to adjustment via a :class:`tzinfo` object. - -.. class:: time(hour=0, minute=0, second=0, microsecond=0, tzinfo=None, *, fold=0) - - All arguments are optional. *tzinfo* may be ``None``, or an instance of a - :class:`tzinfo` subclass. The remaining arguments must be integers in the - following ranges: - - * ``0 <= hour < 24``, - * ``0 <= minute < 60``, - * ``0 <= second < 60``, - * ``0 <= microsecond < 1000000``, - * ``fold in [0, 1]``. - - If an argument outside those ranges is given, :exc:`ValueError` is raised. All - default to ``0`` except *tzinfo*, which defaults to :const:`None`. - -Class attributes: - - -.. attribute:: time.min - - The earliest representable :class:`.time`, ``time(0, 0, 0, 0)``. - - -.. attribute:: time.max - - The latest representable :class:`.time`, ``time(23, 59, 59, 999999)``. - - -.. attribute:: time.resolution - - The smallest possible difference between non-equal :class:`.time` objects, - ``timedelta(microseconds=1)``, although note that arithmetic on - :class:`.time` objects is not supported. - - -Instance attributes (read-only): - -.. attribute:: time.hour - - In ``range(24)``. - - -.. attribute:: time.minute - - In ``range(60)``. - - -.. attribute:: time.second - - In ``range(60)``. - - -.. attribute:: time.microsecond - - In ``range(1000000)``. - - -.. attribute:: time.tzinfo - - The object passed as the tzinfo argument to the :class:`.time` constructor, or - ``None`` if none was passed. - - -.. attribute:: time.fold - - In ``[0, 1]``. Used to disambiguate wall times during a repeated interval. (A - repeated interval occurs when clocks are rolled back at the end of daylight saving - time or when the UTC offset for the current zone is decreased for political reasons.) - The value 0 (1) represents the earlier (later) of the two moments with the same wall - time representation. - - .. versionadded:: 3.6 - -:class:`.time` objects support comparison of :class:`.time` to :class:`.time`, -where *a* is considered less -than *b* when *a* precedes *b* in time. If one comparand is naive and the other -is aware, :exc:`TypeError` is raised if an order comparison is attempted. For equality -comparisons, naive instances are never equal to aware instances. - -If both comparands are aware, and have -the same :attr:`~time.tzinfo` attribute, the common :attr:`~time.tzinfo` attribute is -ignored and the base times are compared. If both comparands are aware and -have different :attr:`~time.tzinfo` attributes, the comparands are first adjusted by -subtracting their UTC offsets (obtained from ``self.utcoffset()``). In order -to stop mixed-type comparisons from falling back to the default comparison by -object address, when a :class:`.time` object is compared to an object of a -different type, :exc:`TypeError` is raised unless the comparison is ``==`` or -``!=``. The latter cases return :const:`False` or :const:`True`, respectively. - -.. versionchanged:: 3.3 - Equality comparisons between aware and naive :class:`~datetime.time` instances - don't raise :exc:`TypeError`. - -In Boolean contexts, a :class:`.time` object is always considered to be true. - -.. versionchanged:: 3.5 - Before Python 3.5, a :class:`.time` object was considered to be false if it - represented midnight in UTC. This behavior was considered obscure and - error-prone and has been removed in Python 3.5. See :issue:`13936` for full - details. - - -Other constructor: - -.. classmethod:: time.fromisoformat(time_string) - - Return a :class:`.time` corresponding to a *time_string* in any valid - ISO 8601 format, with the following exceptions: - - 1. Time zone offsets may have fractional seconds. - 2. The leading ``T``, normally required in cases where there may be ambiguity between - a date and a time, is not required. - 3. Fractional seconds may have any number of digits (anything beyond 6 will - be truncated). - 4. Fractional hours and minutes are not supported. - - Examples:: - - >>> from datetime import time - >>> time.fromisoformat('04:23:01') - datetime.time(4, 23, 1) - >>> time.fromisoformat('T04:23:01') - datetime.time(4, 23, 1) - >>> time.fromisoformat('T042301') - datetime.time(4, 23, 1) - >>> time.fromisoformat('04:23:01.000384') - datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01,000') - datetime.time(4, 23, 1, 384) - >>> time.fromisoformat('04:23:01+04:00') - datetime.time(4, 23, 1, tzinfo=datetime.timezone(datetime.timedelta(seconds=14400))) - >>> time.fromisoformat('04:23:01Z') - datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) - >>> time.fromisoformat('04:23:01+00:00') - datetime.time(4, 23, 1, tzinfo=datetime.timezone.utc) - - - .. versionadded:: 3.7 - .. versionchanged:: 3.11 - Previously, this method only supported formats that could be emitted by - :meth:`time.isoformat()`. - - -Instance methods: - -.. method:: time.replace(hour=self.hour, minute=self.minute, second=self.second, \ - microsecond=self.microsecond, tzinfo=self.tzinfo, *, fold=0) - - Return a :class:`.time` with the same value, except for those attributes given - new values by whichever keyword arguments are specified. Note that - ``tzinfo=None`` can be specified to create a naive :class:`.time` from an - aware :class:`.time`, without conversion of the time data. - - .. versionadded:: 3.6 - Added the ``fold`` argument. - - -.. method:: time.isoformat(timespec='auto') - - Return a string representing the time in ISO 8601 format, one of: - - - ``HH:MM:SS.ffffff``, if :attr:`microsecond` is not 0 - - ``HH:MM:SS``, if :attr:`microsecond` is 0 - - ``HH:MM:SS.ffffff+HH:MM[:SS[.ffffff]]``, if :meth:`utcoffset` does not return ``None`` - - ``HH:MM:SS+HH:MM[:SS[.ffffff]]``, if :attr:`microsecond` is 0 and :meth:`utcoffset` does not return ``None`` - - The optional argument *timespec* specifies the number of additional - components of the time to include (the default is ``'auto'``). - It can be one of the following: - - - ``'auto'``: Same as ``'seconds'`` if :attr:`microsecond` is 0, - same as ``'microseconds'`` otherwise. - - ``'hours'``: Include the :attr:`hour` in the two-digit ``HH`` format. - - ``'minutes'``: Include :attr:`hour` and :attr:`minute` in ``HH:MM`` format. - - ``'seconds'``: Include :attr:`hour`, :attr:`minute`, and :attr:`second` - in ``HH:MM:SS`` format. - - ``'milliseconds'``: Include full time, but truncate fractional second - part to milliseconds. ``HH:MM:SS.sss`` format. - - ``'microseconds'``: Include full time in ``HH:MM:SS.ffffff`` format. - - .. note:: - - Excluded time components are truncated, not rounded. - - :exc:`ValueError` will be raised on an invalid *timespec* argument. - - Example:: - - >>> from datetime import time - >>> time(hour=12, minute=34, second=56, microsecond=123456).isoformat(timespec='minutes') - '12:34' - >>> dt = time(hour=12, minute=34, second=56, microsecond=0) - >>> dt.isoformat(timespec='microseconds') - '12:34:56.000000' - >>> dt.isoformat(timespec='auto') - '12:34:56' - - .. versionadded:: 3.6 - Added the *timespec* argument. - - -.. method:: time.__str__() - - For a time *t*, ``str(t)`` is equivalent to ``t.isoformat()``. - - -.. method:: time.strftime(format) - - Return a string representing the time, controlled by an explicit format - string. See also :ref:`strftime-strptime-behavior` and :meth:`time.isoformat`. - - -.. method:: time.__format__(format) - - Same as :meth:`.time.strftime`. This makes it possible to specify - a format string for a :class:`.time` object in :ref:`formatted string - literals ` and when using :meth:`str.format`. - See also :ref:`strftime-strptime-behavior` and :meth:`time.isoformat`. - - -.. method:: time.utcoffset() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.utcoffset(None)``, and raises an exception if the latter doesn't - return ``None`` or a :class:`timedelta` object with magnitude less than one day. - - .. versionchanged:: 3.7 - The UTC offset is not restricted to a whole number of minutes. - - -.. method:: time.dst() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.dst(None)``, and raises an exception if the latter doesn't return - ``None``, or a :class:`timedelta` object with magnitude less than one day. - - .. versionchanged:: 3.7 - The DST offset is not restricted to a whole number of minutes. - -.. method:: time.tzname() - - If :attr:`.tzinfo` is ``None``, returns ``None``, else returns - ``self.tzinfo.tzname(None)``, or raises an exception if the latter doesn't - return ``None`` or a string object. - -Examples of Usage: :class:`.time` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Examples of working with a :class:`.time` object:: - - >>> from datetime import time, tzinfo, timedelta - >>> class TZ1(tzinfo): - ... def utcoffset(self, dt): - ... return timedelta(hours=1) - ... def dst(self, dt): - ... return timedelta(0) - ... def tzname(self,dt): - ... return "+01:00" - ... def __repr__(self): - ... return f"{self.__class__.__name__}()" - ... - >>> t = time(12, 10, 30, tzinfo=TZ1()) - >>> t - datetime.time(12, 10, 30, tzinfo=TZ1()) - >>> t.isoformat() - '12:10:30+01:00' - >>> t.dst() - datetime.timedelta(0) - >>> t.tzname() - '+01:00' - >>> t.strftime("%H:%M:%S %Z") - '12:10:30 +01:00' - >>> 'The {} is {:%H:%M}.'.format("time", t) - 'The time is 12:10.' - - -.. _datetime-tzinfo: - -:class:`tzinfo` Objects ------------------------ - -.. class:: tzinfo() - - This is an abstract base class, meaning that this class should not be - instantiated directly. Define a subclass of :class:`tzinfo` to capture - information about a particular time zone. - - An instance of (a concrete subclass of) :class:`tzinfo` can be passed to the - constructors for :class:`.datetime` and :class:`.time` objects. The latter objects - view their attributes as being in local time, and the :class:`tzinfo` object - supports methods revealing offset of local time from UTC, the name of the time - zone, and DST offset, all relative to a date or time object passed to them. - - You need to derive a concrete subclass, and (at least) - supply implementations of the standard :class:`tzinfo` methods needed by the - :class:`.datetime` methods you use. The :mod:`datetime` module provides - :class:`timezone`, a simple concrete subclass of :class:`tzinfo` which can - represent timezones with fixed offset from UTC such as UTC itself or North - American EST and EDT. - - Special requirement for pickling: A :class:`tzinfo` subclass must have an - :meth:`__init__` method that can be called with no arguments, otherwise it can be - pickled but possibly not unpickled again. This is a technical requirement that - may be relaxed in the future. - - A concrete subclass of :class:`tzinfo` may need to implement the following - methods. Exactly which methods are needed depends on the uses made of aware - :mod:`datetime` objects. If in doubt, simply implement all of them. - - -.. method:: tzinfo.utcoffset(dt) - - Return offset of local time from UTC, as a :class:`timedelta` object that is - positive east of UTC. If local time is west of UTC, this should be negative. - - This represents the *total* offset from UTC; for example, if a - :class:`tzinfo` object represents both time zone and DST adjustments, - :meth:`utcoffset` should return their sum. If the UTC offset isn't known, - return ``None``. Else the value returned must be a :class:`timedelta` object - strictly between ``-timedelta(hours=24)`` and ``timedelta(hours=24)`` - (the magnitude of the offset must be less than one day). Most implementations - of :meth:`utcoffset` will probably look like one of these two:: - - return CONSTANT # fixed-offset class - return CONSTANT + self.dst(dt) # daylight-aware class - - If :meth:`utcoffset` does not return ``None``, :meth:`dst` should not return - ``None`` either. - - The default implementation of :meth:`utcoffset` raises - :exc:`NotImplementedError`. - - .. versionchanged:: 3.7 - The UTC offset is not restricted to a whole number of minutes. - - -.. method:: tzinfo.dst(dt) - - Return the daylight saving time (DST) adjustment, as a :class:`timedelta` - object or - ``None`` if DST information isn't known. - - Return ``timedelta(0)`` if DST is not in effect. - If DST is in effect, return the offset as a :class:`timedelta` object - (see :meth:`utcoffset` for details). Note that DST offset, if applicable, has - already been added to the UTC offset returned by :meth:`utcoffset`, so there's - no need to consult :meth:`dst` unless you're interested in obtaining DST info - separately. For example, :meth:`datetime.timetuple` calls its :attr:`~.datetime.tzinfo` - attribute's :meth:`dst` method to determine how the :attr:`tm_isdst` flag - should be set, and :meth:`tzinfo.fromutc` calls :meth:`dst` to account for - DST changes when crossing time zones. - - An instance *tz* of a :class:`tzinfo` subclass that models both standard and - daylight times must be consistent in this sense: - - ``tz.utcoffset(dt) - tz.dst(dt)`` - - must return the same result for every :class:`.datetime` *dt* with ``dt.tzinfo == - tz`` For sane :class:`tzinfo` subclasses, this expression yields the time - zone's "standard offset", which should not depend on the date or the time, but - only on geographic location. The implementation of :meth:`datetime.astimezone` - relies on this, but cannot detect violations; it's the programmer's - responsibility to ensure it. If a :class:`tzinfo` subclass cannot guarantee - this, it may be able to override the default implementation of - :meth:`tzinfo.fromutc` to work correctly with :meth:`astimezone` regardless. - - Most implementations of :meth:`dst` will probably look like one of these two:: - - def dst(self, dt): - # a fixed-offset class: doesn't account for DST - return timedelta(0) - - or:: - - def dst(self, dt): - # Code to set dston and dstoff to the time zone's DST - # transition times based on the input dt.year, and expressed - # in standard local time. - - if dston <= dt.replace(tzinfo=None) < dstoff: - return timedelta(hours=1) - else: - return timedelta(0) - - The default implementation of :meth:`dst` raises :exc:`NotImplementedError`. - - .. versionchanged:: 3.7 - The DST offset is not restricted to a whole number of minutes. - - -.. method:: tzinfo.tzname(dt) - - Return the time zone name corresponding to the :class:`.datetime` object *dt*, as - a string. Nothing about string names is defined by the :mod:`datetime` module, - and there's no requirement that it mean anything in particular. For example, - "GMT", "UTC", "-500", "-5:00", "EDT", "US/Eastern", "America/New York" are all - valid replies. Return ``None`` if a string name isn't known. Note that this is - a method rather than a fixed string primarily because some :class:`tzinfo` - subclasses will wish to return different names depending on the specific value - of *dt* passed, especially if the :class:`tzinfo` class is accounting for - daylight time. - - The default implementation of :meth:`tzname` raises :exc:`NotImplementedError`. - - -These methods are called by a :class:`.datetime` or :class:`.time` object, in -response to their methods of the same names. A :class:`.datetime` object passes -itself as the argument, and a :class:`.time` object passes ``None`` as the -argument. A :class:`tzinfo` subclass's methods should therefore be prepared to -accept a *dt* argument of ``None``, or of class :class:`.datetime`. - -When ``None`` is passed, it's up to the class designer to decide the best -response. For example, returning ``None`` is appropriate if the class wishes to -say that time objects don't participate in the :class:`tzinfo` protocols. It -may be more useful for ``utcoffset(None)`` to return the standard UTC offset, as -there is no other convention for discovering the standard offset. - -When a :class:`.datetime` object is passed in response to a :class:`.datetime` -method, ``dt.tzinfo`` is the same object as *self*. :class:`tzinfo` methods can -rely on this, unless user code calls :class:`tzinfo` methods directly. The -intent is that the :class:`tzinfo` methods interpret *dt* as being in local -time, and not need worry about objects in other timezones. - -There is one more :class:`tzinfo` method that a subclass may wish to override: - - -.. method:: tzinfo.fromutc(dt) - - This is called from the default :class:`datetime.astimezone()` - implementation. When called from that, ``dt.tzinfo`` is *self*, and *dt*'s - date and time data are to be viewed as expressing a UTC time. The purpose - of :meth:`fromutc` is to adjust the date and time data, returning an - equivalent datetime in *self*'s local time. - - Most :class:`tzinfo` subclasses should be able to inherit the default - :meth:`fromutc` implementation without problems. It's strong enough to handle - fixed-offset time zones, and time zones accounting for both standard and - daylight time, and the latter even if the DST transition times differ in - different years. An example of a time zone the default :meth:`fromutc` - implementation may not handle correctly in all cases is one where the standard - offset (from UTC) depends on the specific date and time passed, which can happen - for political reasons. The default implementations of :meth:`astimezone` and - :meth:`fromutc` may not produce the result you want if the result is one of the - hours straddling the moment the standard offset changes. - - Skipping code for error cases, the default :meth:`fromutc` implementation acts - like:: - - def fromutc(self, dt): - # raise ValueError error if dt.tzinfo is not self - dtoff = dt.utcoffset() - dtdst = dt.dst() - # raise ValueError if dtoff is None or dtdst is None - delta = dtoff - dtdst # this is self's standard offset - if delta: - dt += delta # convert to standard local time - dtdst = dt.dst() - # raise ValueError if dtdst is None - if dtdst: - return dt + dtdst - else: - return dt - -In the following :download:`tzinfo_examples.py -<../includes/tzinfo_examples.py>` file there are some examples of -:class:`tzinfo` classes: - -.. literalinclude:: ../includes/tzinfo_examples.py - -Note that there are unavoidable subtleties twice per year in a :class:`tzinfo` -subclass accounting for both standard and daylight time, at the DST transition -points. For concreteness, consider US Eastern (UTC -0500), where EDT begins the -minute after 1:59 (EST) on the second Sunday in March, and ends the minute after -1:59 (EDT) on the first Sunday in November:: - - UTC 3:MM 4:MM 5:MM 6:MM 7:MM 8:MM - EST 22:MM 23:MM 0:MM 1:MM 2:MM 3:MM - EDT 23:MM 0:MM 1:MM 2:MM 3:MM 4:MM - - start 22:MM 23:MM 0:MM 1:MM 3:MM 4:MM - - end 23:MM 0:MM 1:MM 1:MM 2:MM 3:MM - -When DST starts (the "start" line), the local wall clock leaps from 1:59 to -3:00. A wall time of the form 2:MM doesn't really make sense on that day, so -``astimezone(Eastern)`` won't deliver a result with ``hour == 2`` on the day DST -begins. For example, at the Spring forward transition of 2016, we get:: - - >>> from datetime import datetime, timezone - >>> from tzinfo_examples import HOUR, Eastern - >>> u0 = datetime(2016, 3, 13, 5, tzinfo=timezone.utc) - >>> for i in range(4): - ... u = u0 + i*HOUR - ... t = u.astimezone(Eastern) - ... print(u.time(), 'UTC =', t.time(), t.tzname()) - ... - 05:00:00 UTC = 00:00:00 EST - 06:00:00 UTC = 01:00:00 EST - 07:00:00 UTC = 03:00:00 EDT - 08:00:00 UTC = 04:00:00 EDT - - -When DST ends (the "end" line), there's a potentially worse problem: there's an -hour that can't be spelled unambiguously in local wall time: the last hour of -daylight time. In Eastern, that's times of the form 5:MM UTC on the day -daylight time ends. The local wall clock leaps from 1:59 (daylight time) back -to 1:00 (standard time) again. Local times of the form 1:MM are ambiguous. -:meth:`astimezone` mimics the local clock's behavior by mapping two adjacent UTC -hours into the same local hour then. In the Eastern example, UTC times of the -form 5:MM and 6:MM both map to 1:MM when converted to Eastern, but earlier times -have the :attr:`~datetime.fold` attribute set to 0 and the later times have it set to 1. -For example, at the Fall back transition of 2016, we get:: - - >>> u0 = datetime(2016, 11, 6, 4, tzinfo=timezone.utc) - >>> for i in range(4): - ... u = u0 + i*HOUR - ... t = u.astimezone(Eastern) - ... print(u.time(), 'UTC =', t.time(), t.tzname(), t.fold) - ... - 04:00:00 UTC = 00:00:00 EDT 0 - 05:00:00 UTC = 01:00:00 EDT 0 - 06:00:00 UTC = 01:00:00 EST 1 - 07:00:00 UTC = 02:00:00 EST 0 - -Note that the :class:`.datetime` instances that differ only by the value of the -:attr:`~datetime.fold` attribute are considered equal in comparisons. - -Applications that can't bear wall-time ambiguities should explicitly check the -value of the :attr:`~datetime.fold` attribute or avoid using hybrid -:class:`tzinfo` subclasses; there are no ambiguities when using :class:`timezone`, -or any other fixed-offset :class:`tzinfo` subclass (such as a class representing -only EST (fixed offset -5 hours), or only EDT (fixed offset -4 hours)). - -.. seealso:: - - :mod:`zoneinfo` - The :mod:`datetime` module has a basic :class:`timezone` class (for - handling arbitrary fixed offsets from UTC) and its :attr:`timezone.utc` - attribute (a UTC timezone instance). - - ``zoneinfo`` brings the *IANA timezone database* (also known as the Olson - database) to Python, and its usage is recommended. - - `IANA timezone database `_ - The Time Zone Database (often called tz, tzdata or zoneinfo) contains code - and data that represent the history of local time for many representative - locations around the globe. It is updated periodically to reflect changes - made by political bodies to time zone boundaries, UTC offsets, and - daylight-saving rules. - - -.. _datetime-timezone: - -:class:`timezone` Objects --------------------------- - -The :class:`timezone` class is a subclass of :class:`tzinfo`, each -instance of which represents a timezone defined by a fixed offset from -UTC. - -Objects of this class cannot be used to represent timezone information in the -locations where different offsets are used in different days of the year or -where historical changes have been made to civil time. - - -.. class:: timezone(offset, name=None) - - The *offset* argument must be specified as a :class:`timedelta` - object representing the difference between the local time and UTC. It must - be strictly between ``-timedelta(hours=24)`` and - ``timedelta(hours=24)``, otherwise :exc:`ValueError` is raised. - - The *name* argument is optional. If specified it must be a string that - will be used as the value returned by the :meth:`datetime.tzname` method. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.7 - The UTC offset is not restricted to a whole number of minutes. - - -.. method:: timezone.utcoffset(dt) - - Return the fixed value specified when the :class:`timezone` instance is - constructed. - - The *dt* argument is ignored. The return value is a :class:`timedelta` - instance equal to the difference between the local time and UTC. - - .. versionchanged:: 3.7 - The UTC offset is not restricted to a whole number of minutes. - -.. method:: timezone.tzname(dt) - - Return the fixed value specified when the :class:`timezone` instance - is constructed. - - If *name* is not provided in the constructor, the name returned by - ``tzname(dt)`` is generated from the value of the ``offset`` as follows. If - *offset* is ``timedelta(0)``, the name is "UTC", otherwise it is a string in - the format ``UTC±HH:MM``, where ± is the sign of ``offset``, HH and MM are - two digits of ``offset.hours`` and ``offset.minutes`` respectively. - - .. versionchanged:: 3.6 - Name generated from ``offset=timedelta(0)`` is now plain ``'UTC'``, not - ``'UTC+00:00'``. - - -.. method:: timezone.dst(dt) - - Always returns ``None``. - -.. method:: timezone.fromutc(dt) - - Return ``dt + offset``. The *dt* argument must be an aware - :class:`.datetime` instance, with ``tzinfo`` set to ``self``. - -Class attributes: - -.. attribute:: timezone.utc - - The UTC timezone, ``timezone(timedelta(0))``. - - -.. index:: - single: % (percent); datetime format - -.. _strftime-strptime-behavior: - -:meth:`strftime` and :meth:`strptime` Behavior ----------------------------------------------- - -:class:`date`, :class:`.datetime`, and :class:`.time` objects all support a -``strftime(format)`` method, to create a string representing the time under the -control of an explicit format string. - -Conversely, the :meth:`datetime.strptime` class method creates a -:class:`.datetime` object from a string representing a date and time and a -corresponding format string. - -The table below provides a high-level comparison of :meth:`strftime` -versus :meth:`strptime`: - -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| | ``strftime`` | ``strptime`` | -+================+========================================================+==============================================================================+ -| Usage | Convert object to a string according to a given format | Parse a string into a :class:`.datetime` object given a corresponding format | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Type of method | Instance method | Class method | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Method of | :class:`date`; :class:`.datetime`; :class:`.time` | :class:`.datetime` | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ -| Signature | ``strftime(format)`` | ``strptime(date_string, format)`` | -+----------------+--------------------------------------------------------+------------------------------------------------------------------------------+ - - - .. _format-codes: - -:meth:`strftime` and :meth:`strptime` Format Codes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -These methods accept format codes that can be used to parse and format dates:: - - >>> datetime.strptime('31/01/22 23:59:59.999999', - ... '%d/%m/%y %H:%M:%S.%f') - datetime.datetime(2022, 1, 31, 23, 59, 59, 999999) - >>> _.strftime('%a %d %b %Y, %I:%M%p') - 'Mon 31 Jan 2022, 11:59PM' - -The following is a list of all the format codes that the 1989 C standard -requires, and these work on all platforms with a standard C implementation. - -+-----------+--------------------------------+------------------------+-------+ -| Directive | Meaning | Example | Notes | -+===========+================================+========================+=======+ -| ``%a`` | Weekday as locale's || Sun, Mon, ..., Sat | \(1) | -| | abbreviated name. | (en_US); | | -| | || So, Mo, ..., Sa | | -| | | (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%A`` | Weekday as locale's full name. || Sunday, Monday, ..., | \(1) | -| | | Saturday (en_US); | | -| | || Sonntag, Montag, ..., | | -| | | Samstag (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%w`` | Weekday as a decimal number, | 0, 1, ..., 6 | | -| | where 0 is Sunday and 6 is | | | -| | Saturday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%d`` | Day of the month as a | 01, 02, ..., 31 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%b`` | Month as locale's abbreviated || Jan, Feb, ..., Dec | \(1) | -| | name. | (en_US); | | -| | || Jan, Feb, ..., Dez | | -| | | (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%B`` | Month as locale's full name. || January, February, | \(1) | -| | | ..., December (en_US);| | -| | || Januar, Februar, ..., | | -| | | Dezember (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%m`` | Month as a zero-padded | 01, 02, ..., 12 | \(9) | -| | decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%y`` | Year without century as a | 00, 01, ..., 99 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%Y`` | Year with century as a decimal | 0001, 0002, ..., 2013, | \(2) | -| | number. | 2014, ..., 9998, 9999 | | -+-----------+--------------------------------+------------------------+-------+ -| ``%H`` | Hour (24-hour clock) as a | 00, 01, ..., 23 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%I`` | Hour (12-hour clock) as a | 01, 02, ..., 12 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%p`` | Locale's equivalent of either || AM, PM (en_US); | \(1), | -| | AM or PM. || am, pm (de_DE) | \(3) | -+-----------+--------------------------------+------------------------+-------+ -| ``%M`` | Minute as a zero-padded | 00, 01, ..., 59 | \(9) | -| | decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%S`` | Second as a zero-padded | 00, 01, ..., 59 | \(4), | -| | decimal number. | | \(9) | -+-----------+--------------------------------+------------------------+-------+ -| ``%f`` | Microsecond as a decimal | 000000, 000001, ..., | \(5) | -| | number, zero-padded to 6 | 999999 | | -| | digits. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%z`` | UTC offset in the form | (empty), +0000, | \(6) | -| | ``±HHMM[SS[.ffffff]]`` (empty | -0400, +1030, | | -| | string if the object is | +063415, | | -| | naive). | -030712.345216 | | -+-----------+--------------------------------+------------------------+-------+ -| ``%Z`` | Time zone name (empty string | (empty), UTC, GMT | \(6) | -| | if the object is naive). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%j`` | Day of the year as a | 001, 002, ..., 366 | \(9) | -| | zero-padded decimal number. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%U`` | Week number of the year | 00, 01, ..., 53 | \(7), | -| | (Sunday as the first day of | | \(9) | -| | the week) as a zero-padded | | | -| | decimal number. All days in a | | | -| | new year preceding the first | | | -| | Sunday are considered to be in | | | -| | week 0. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%W`` | Week number of the year | 00, 01, ..., 53 | \(7), | -| | (Monday as the first day of | | \(9) | -| | the week) as a zero-padded | | | -| | decimal number. All days in a | | | -| | new year preceding the first | | | -| | Monday are considered to be in | | | -| | week 0. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%c`` | Locale's appropriate date and || Tue Aug 16 21:30:00 | \(1) | -| | time representation. | 1988 (en_US); | | -| | || Di 16 Aug 21:30:00 | | -| | | 1988 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%x`` | Locale's appropriate date || 08/16/88 (None); | \(1) | -| | representation. || 08/16/1988 (en_US); | | -| | || 16.08.1988 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%X`` | Locale's appropriate time || 21:30:00 (en_US); | \(1) | -| | representation. || 21:30:00 (de_DE) | | -+-----------+--------------------------------+------------------------+-------+ -| ``%%`` | A literal ``'%'`` character. | % | | -+-----------+--------------------------------+------------------------+-------+ - -Several additional directives not required by the C89 standard are included for -convenience. These parameters all correspond to ISO 8601 date values. - -+-----------+--------------------------------+------------------------+-------+ -| Directive | Meaning | Example | Notes | -+===========+================================+========================+=======+ -| ``%G`` | ISO 8601 year with century | 0001, 0002, ..., 2013, | \(8) | -| | representing the year that | 2014, ..., 9998, 9999 | | -| | contains the greater part of | | | -| | the ISO week (``%V``). | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%u`` | ISO 8601 weekday as a decimal | 1, 2, ..., 7 | | -| | number where 1 is Monday. | | | -+-----------+--------------------------------+------------------------+-------+ -| ``%V`` | ISO 8601 week as a decimal | 01, 02, ..., 53 | \(8), | -| | number with Monday as | | \(9) | -| | the first day of the week. | | | -| | Week 01 is the week containing | | | -| | Jan 4. | | | -+-----------+--------------------------------+------------------------+-------+ - -These may not be available on all platforms when used with the :meth:`strftime` -method. The ISO 8601 year and ISO 8601 week directives are not interchangeable -with the year and week number directives above. Calling :meth:`strptime` with -incomplete or ambiguous ISO 8601 directives will raise a :exc:`ValueError`. - -The full set of format codes supported varies across platforms, because Python -calls the platform C library's :func:`strftime` function, and platform -variations are common. To see the full set of format codes supported on your -platform, consult the :manpage:`strftime(3)` documentation. There are also -differences between platforms in handling of unsupported format specifiers. - -.. versionadded:: 3.6 - ``%G``, ``%u`` and ``%V`` were added. - -Technical Detail -^^^^^^^^^^^^^^^^ - -Broadly speaking, ``d.strftime(fmt)`` acts like the :mod:`time` module's -``time.strftime(fmt, d.timetuple())`` although not all objects support a -:meth:`timetuple` method. - -For the :meth:`datetime.strptime` class method, the default value is -``1900-01-01T00:00:00.000``: any components not specified in the format string -will be pulled from the default value. [#]_ - -Using ``datetime.strptime(date_string, format)`` is equivalent to:: - - datetime(*(time.strptime(date_string, format)[0:6])) - -except when the format includes sub-second components or timezone offset -information, which are supported in ``datetime.strptime`` but are discarded by -``time.strptime``. - -For :class:`.time` objects, the format codes for year, month, and day should not -be used, as :class:`time` objects have no such values. If they're used anyway, -``1900`` is substituted for the year, and ``1`` for the month and day. - -For :class:`date` objects, the format codes for hours, minutes, seconds, and -microseconds should not be used, as :class:`date` objects have no such -values. If they're used anyway, ``0`` is substituted for them. - -For the same reason, handling of format strings containing Unicode code points -that can't be represented in the charset of the current locale is also -platform-dependent. On some platforms such code points are preserved intact in -the output, while on others ``strftime`` may raise :exc:`UnicodeError` or return -an empty string instead. - -Notes: - -(1) - Because the format depends on the current locale, care should be taken when - making assumptions about the output value. Field orderings will vary (for - example, "month/day/year" versus "day/month/year"), and the output may - contain non-ASCII characters. - -(2) - The :meth:`strptime` method can parse years in the full [1, 9999] range, but - years < 1000 must be zero-filled to 4-digit width. - - .. versionchanged:: 3.2 - In previous versions, :meth:`strftime` method was restricted to - years >= 1900. - - .. versionchanged:: 3.3 - In version 3.2, :meth:`strftime` method was restricted to - years >= 1000. - -(3) - When used with the :meth:`strptime` method, the ``%p`` directive only affects - the output hour field if the ``%I`` directive is used to parse the hour. - -(4) - Unlike the :mod:`time` module, the :mod:`datetime` module does not support - leap seconds. - -(5) - When used with the :meth:`strptime` method, the ``%f`` directive - accepts from one to six digits and zero pads on the right. ``%f`` is - an extension to the set of format characters in the C standard (but - implemented separately in datetime objects, and therefore always - available). - -(6) - For a naive object, the ``%z`` and ``%Z`` format codes are replaced by empty - strings. - - For an aware object: - - ``%z`` - :meth:`utcoffset` is transformed into a string of the form - ``±HHMM[SS[.ffffff]]``, where ``HH`` is a 2-digit string giving the number - of UTC offset hours, ``MM`` is a 2-digit string giving the number of UTC - offset minutes, ``SS`` is a 2-digit string giving the number of UTC offset - seconds and ``ffffff`` is a 6-digit string giving the number of UTC - offset microseconds. The ``ffffff`` part is omitted when the offset is a - whole number of seconds and both the ``ffffff`` and the ``SS`` part is - omitted when the offset is a whole number of minutes. For example, if - :meth:`utcoffset` returns ``timedelta(hours=-3, minutes=-30)``, ``%z`` is - replaced with the string ``'-0330'``. - - .. versionchanged:: 3.7 - The UTC offset is not restricted to a whole number of minutes. - - .. versionchanged:: 3.7 - When the ``%z`` directive is provided to the :meth:`strptime` method, - the UTC offsets can have a colon as a separator between hours, minutes - and seconds. - For example, ``'+01:00:00'`` will be parsed as an offset of one hour. - In addition, providing ``'Z'`` is identical to ``'+00:00'``. - - ``%Z`` - In :meth:`strftime`, ``%Z`` is replaced by an empty string if - :meth:`tzname` returns ``None``; otherwise ``%Z`` is replaced by the - returned value, which must be a string. - - :meth:`strptime` only accepts certain values for ``%Z``: - - 1. any value in ``time.tzname`` for your machine's locale - 2. the hard-coded values ``UTC`` and ``GMT`` - - So someone living in Japan may have ``JST``, ``UTC``, and ``GMT`` as - valid values, but probably not ``EST``. It will raise ``ValueError`` for - invalid values. - - .. versionchanged:: 3.2 - When the ``%z`` directive is provided to the :meth:`strptime` method, an - aware :class:`.datetime` object will be produced. The ``tzinfo`` of the - result will be set to a :class:`timezone` instance. - -(7) - When used with the :meth:`strptime` method, ``%U`` and ``%W`` are only used - in calculations when the day of the week and the calendar year (``%Y``) - are specified. - -(8) - Similar to ``%U`` and ``%W``, ``%V`` is only used in calculations when the - day of the week and the ISO year (``%G``) are specified in a - :meth:`strptime` format string. Also note that ``%G`` and ``%Y`` are not - interchangeable. - -(9) - When used with the :meth:`strptime` method, the leading zero is optional - for formats ``%d``, ``%m``, ``%H``, ``%I``, ``%M``, ``%S``, ``%j``, ``%U``, - ``%W``, and ``%V``. Format ``%y`` does require a leading zero. - -.. rubric:: Footnotes - -.. [#] If, that is, we ignore the effects of Relativity - -.. [#] This matches the definition of the "proleptic Gregorian" calendar in - Dershowitz and Reingold's book *Calendrical Calculations*, - where it's the base calendar for all computations. See the book for - algorithms for converting between proleptic Gregorian ordinals and - many other calendar systems. - -.. [#] See R. H. van Gent's `guide to the mathematics of the ISO 8601 calendar - `_ - for a good explanation. - -.. [#] Passing ``datetime.strptime('Feb 29', '%b %d')`` will fail since ``1900`` is not a leap year. diff --git a/Doc/library/dbm.rst b/Doc/library/dbm.rst deleted file mode 100644 index 2be499337a2a15..00000000000000 --- a/Doc/library/dbm.rst +++ /dev/null @@ -1,408 +0,0 @@ -:mod:`dbm` --- Interfaces to Unix "databases" -============================================= - -.. module:: dbm - :synopsis: Interfaces to various Unix "database" formats. - -**Source code:** :source:`Lib/dbm/__init__.py` - --------------- - -:mod:`dbm` is a generic interface to variants of the DBM database --- -:mod:`dbm.gnu` or :mod:`dbm.ndbm`. If none of these modules is installed, the -slow-but-simple implementation in module :mod:`dbm.dumb` will be used. There -is a `third party interface `_ to -the Oracle Berkeley DB. - - -.. exception:: error - - A tuple containing the exceptions that can be raised by each of the supported - modules, with a unique exception also named :exc:`dbm.error` as the first - item --- the latter is used when :exc:`dbm.error` is raised. - - -.. function:: whichdb(filename) - - This function attempts to guess which of the several simple database modules - available --- :mod:`dbm.gnu`, :mod:`dbm.ndbm` or :mod:`dbm.dumb` --- should - be used to open a given file. - - Returns one of the following values: ``None`` if the file can't be opened - because it's unreadable or doesn't exist; the empty string (``''``) if the - file's format can't be guessed; or a string containing the required module - name, such as ``'dbm.ndbm'`` or ``'dbm.gnu'``. - -.. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. - -.. function:: open(file, flag='r', mode=0o666) - - Open the database file *file* and return a corresponding object. - - If the database file already exists, the :func:`whichdb` function is used to - determine its type and the appropriate module is used; if it does not exist, - the first module listed above that can be imported is used. - - The optional *flag* argument can be: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). - - -The object returned by :func:`.open` supports the same basic functionality as -dictionaries; keys and their corresponding values can be stored, retrieved, and -deleted, and the :keyword:`in` operator and the :meth:`keys` method are -available, as well as :meth:`get` and :meth:`setdefault`. - -.. versionchanged:: 3.2 - :meth:`get` and :meth:`setdefault` are now available in all database modules. - -.. versionchanged:: 3.8 - Deleting a key from a read-only database raises database module specific error - instead of :exc:`KeyError`. - -.. versionchanged:: 3.11 - Accepts :term:`path-like object` for file. - -Key and values are always stored as bytes. This means that when -strings are used they are implicitly converted to the default encoding before -being stored. - -These objects also support being used in a :keyword:`with` statement, which -will automatically close them when done. - -.. versionchanged:: 3.4 - Added native support for the context management protocol to the objects - returned by :func:`.open`. - -The following example records some hostnames and a corresponding title, and -then prints out the contents of the database:: - - import dbm - - # Open database, creating it if necessary. - with dbm.open('cache', 'c') as db: - - # Record some values - db[b'hello'] = b'there' - db['www.python.org'] = 'Python Website' - db['www.cnn.com'] = 'Cable News Network' - - # Note that the keys are considered bytes now. - assert db[b'www.python.org'] == b'Python Website' - # Notice how the value is now in bytes. - assert db['www.cnn.com'] == b'Cable News Network' - - # Often-used methods of the dict interface work too. - print(db.get('python.org', b'not present')) - - # Storing a non-string key or value will raise an exception (most - # likely a TypeError). - db['www.yahoo.com'] = 4 - - # db is automatically closed when leaving the with statement. - - -.. seealso:: - - Module :mod:`shelve` - Persistence module which stores non-string data. - - -The individual submodules are described in the following sections. - - -:mod:`dbm.gnu` --- GNU's reinterpretation of dbm ------------------------------------------------- - -.. module:: dbm.gnu - :platform: Unix - :synopsis: GNU's reinterpretation of dbm. - -**Source code:** :source:`Lib/dbm/gnu.py` - --------------- - -This module is quite similar to the :mod:`dbm` module, but uses the GNU library -``gdbm`` instead to provide some additional functionality. Please note that the -file formats created by :mod:`dbm.gnu` and :mod:`dbm.ndbm` are incompatible. - -The :mod:`dbm.gnu` module provides an interface to the GNU DBM library. -``dbm.gnu.gdbm`` objects behave like mappings (dictionaries), except that keys and -values are always converted to bytes before storing. Printing a ``gdbm`` -object doesn't print the -keys and values, and the :meth:`items` and :meth:`values` methods are not -supported. - -.. exception:: error - - Raised on :mod:`dbm.gnu`-specific errors, such as I/O errors. :exc:`KeyError` is - raised for general mapping errors like specifying an incorrect key. - - -.. function:: open(filename[, flag[, mode]]) - - Open a ``gdbm`` database and return a :class:`gdbm` object. The *filename* - argument is the name of the database file. - - The optional *flag* argument can be: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The following additional characters may be appended to the flag to control - how the database is opened: - - +---------+--------------------------------------------+ - | Value | Meaning | - +=========+============================================+ - | ``'f'`` | Open the database in fast mode. Writes | - | | to the database will not be synchronized. | - +---------+--------------------------------------------+ - | ``'s'`` | Synchronized mode. This will cause changes | - | | to the database to be immediately written | - | | to the file. | - +---------+--------------------------------------------+ - | ``'u'`` | Do not lock database. | - +---------+--------------------------------------------+ - - Not all flags are valid for all versions of ``gdbm``. The module constant - :const:`open_flags` is a string of supported flag characters. The exception - :exc:`error` is raised if an invalid flag is specified. - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666``. - - In addition to the dictionary-like methods, ``gdbm`` objects have the - following methods: - - .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. - - .. method:: gdbm.firstkey() - - It's possible to loop over every key in the database using this method and the - :meth:`nextkey` method. The traversal is ordered by ``gdbm``'s internal - hash values, and won't be sorted by the key values. This method returns - the starting key. - - .. method:: gdbm.nextkey(key) - - Returns the key that follows *key* in the traversal. The following code prints - every key in the database ``db``, without having to create a list in memory that - contains them all:: - - k = db.firstkey() - while k is not None: - print(k) - k = db.nextkey(k) - - .. method:: gdbm.reorganize() - - If you have carried out a lot of deletions and would like to shrink the space - used by the ``gdbm`` file, this routine will reorganize the database. ``gdbm`` - objects will not shorten the length of a database file except by using this - reorganization; otherwise, deleted file space will be kept and reused as new - (key, value) pairs are added. - - .. method:: gdbm.sync() - - When the database has been opened in fast mode, this method forces any - unwritten data to be written to the disk. - - .. method:: gdbm.close() - - Close the ``gdbm`` database. - -:mod:`dbm.ndbm` --- Interface based on ndbm -------------------------------------------- - -.. module:: dbm.ndbm - :platform: Unix - :synopsis: The standard "database" interface, based on ndbm. - -**Source code:** :source:`Lib/dbm/ndbm.py` - --------------- - -The :mod:`dbm.ndbm` module provides an interface to the Unix "(n)dbm" library. -Dbm objects behave like mappings (dictionaries), except that keys and values are -always stored as bytes. Printing a ``dbm`` object doesn't print the keys and -values, and the :meth:`items` and :meth:`values` methods are not supported. - -This module can be used with the "classic" ndbm interface or the GNU GDBM -compatibility interface. On Unix, the :program:`configure` script will attempt -to locate the appropriate header file to simplify building this module. - -.. exception:: error - - Raised on :mod:`dbm.ndbm`-specific errors, such as I/O errors. :exc:`KeyError` is raised - for general mapping errors like specifying an incorrect key. - - -.. data:: library - - Name of the ``ndbm`` implementation library used. - - -.. function:: open(filename[, flag[, mode]]) - - Open a dbm database and return a ``ndbm`` object. The *filename* argument is the - name of the database file (without the :file:`.dir` or :file:`.pag` extensions). - - The optional *flag* argument must be one of these values: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be - modified by the prevailing umask). - - In addition to the dictionary-like methods, ``ndbm`` objects - provide the following method: - - .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. - - .. method:: ndbm.close() - - Close the ``ndbm`` database. - - -:mod:`dbm.dumb` --- Portable DBM implementation ------------------------------------------------ - -.. module:: dbm.dumb - :synopsis: Portable implementation of the simple DBM interface. - -**Source code:** :source:`Lib/dbm/dumb.py` - -.. index:: single: databases - -.. note:: - - The :mod:`dbm.dumb` module is intended as a last resort fallback for the - :mod:`dbm` module when a more robust module is not available. The :mod:`dbm.dumb` - module is not written for speed and is not nearly as heavily used as the other - database modules. - --------------- - -The :mod:`dbm.dumb` module provides a persistent dictionary-like interface which -is written entirely in Python. Unlike other modules such as :mod:`dbm.gnu` no -external library is required. As with other persistent mappings, the keys and -values are always stored as bytes. - -The module defines the following: - - -.. exception:: error - - Raised on :mod:`dbm.dumb`-specific errors, such as I/O errors. :exc:`KeyError` is - raised for general mapping errors like specifying an incorrect key. - - -.. function:: open(filename[, flag[, mode]]) - - Open a ``dumbdbm`` database and return a dumbdbm object. The *filename* argument is - the basename of the database file (without any specific extensions). When a - dumbdbm database is created, files with :file:`.dat` and :file:`.dir` extensions - are created. - - The optional *flag* argument can be: - - +---------+-------------------------------------------+ - | Value | Meaning | - +=========+===========================================+ - | ``'r'`` | Open existing database for reading only | - | | (default) | - +---------+-------------------------------------------+ - | ``'w'`` | Open existing database for reading and | - | | writing | - +---------+-------------------------------------------+ - | ``'c'`` | Open database for reading and writing, | - | | creating it if it doesn't exist | - +---------+-------------------------------------------+ - | ``'n'`` | Always create a new, empty database, open | - | | for reading and writing | - +---------+-------------------------------------------+ - - The optional *mode* argument is the Unix mode of the file, used only when the - database has to be created. It defaults to octal ``0o666`` (and will be modified - by the prevailing umask). - - .. warning:: - It is possible to crash the Python interpreter when loading a database - with a sufficiently large/complex entry due to stack depth limitations in - Python's AST compiler. - - .. versionchanged:: 3.5 - :func:`.open` always creates a new database when the flag has the value - ``'n'``. - - .. versionchanged:: 3.8 - A database opened with flags ``'r'`` is now read-only. Opening with - flags ``'r'`` and ``'w'`` no longer creates a database if it does not - exist. - - .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. - - In addition to the methods provided by the - :class:`collections.abc.MutableMapping` class, :class:`dumbdbm` objects - provide the following methods: - - .. method:: dumbdbm.sync() - - Synchronize the on-disk directory and data files. This method is called - by the :meth:`Shelve.sync` method. - - .. method:: dumbdbm.close() - - Close the ``dumbdbm`` database. - diff --git a/Doc/library/debug.rst b/Doc/library/debug.rst deleted file mode 100644 index 60223657a44043..00000000000000 --- a/Doc/library/debug.rst +++ /dev/null @@ -1,21 +0,0 @@ -*********************** -Debugging and Profiling -*********************** - -These libraries help you with Python development: the debugger enables you to -step through code, analyze stack frames and set breakpoints etc., and the -profilers run code and give you a detailed breakdown of execution times, -allowing you to identify bottlenecks in your programs. Auditing events -provide visibility into runtime behaviors that would otherwise require -intrusive debugging or patching. - -.. toctree:: - - audit_events.rst - bdb.rst - faulthandler.rst - pdb.rst - profile.rst - timeit.rst - trace.rst - tracemalloc.rst diff --git a/Doc/library/decimal.rst b/Doc/library/decimal.rst deleted file mode 100644 index b4e3430b1008f1..00000000000000 --- a/Doc/library/decimal.rst +++ /dev/null @@ -1,2247 +0,0 @@ -:mod:`decimal` --- Decimal fixed point and floating point arithmetic -==================================================================== - -.. module:: decimal - :synopsis: Implementation of the General Decimal Arithmetic Specification. - -.. moduleauthor:: Eric Price -.. moduleauthor:: Facundo Batista -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Aahz -.. moduleauthor:: Tim Peters -.. moduleauthor:: Stefan Krah -.. sectionauthor:: Raymond D. Hettinger - -**Source code:** :source:`Lib/decimal.py` - -.. import modules for testing inline doctests with the Sphinx doctest builder -.. testsetup:: * - - import decimal - import math - from decimal import * - # make sure each group gets a fresh context - setcontext(Context()) - -.. testcleanup:: * - - # make sure other tests (outside this file) get a fresh context - setcontext(Context()) - --------------- - -The :mod:`decimal` module provides support for fast correctly rounded -decimal floating point arithmetic. It offers several advantages over the -:class:`float` datatype: - -* Decimal "is based on a floating-point model which was designed with people - in mind, and necessarily has a paramount guiding principle -- computers must - provide an arithmetic that works in the same way as the arithmetic that - people learn at school." -- excerpt from the decimal arithmetic specification. - -* Decimal numbers can be represented exactly. In contrast, numbers like - ``1.1`` and ``2.2`` do not have exact representations in binary - floating point. End users typically would not expect ``1.1 + 2.2`` to display - as ``3.3000000000000003`` as it does with binary floating point. - -* The exactness carries over into arithmetic. In decimal floating point, ``0.1 - + 0.1 + 0.1 - 0.3`` is exactly equal to zero. In binary floating point, the result - is ``5.5511151231257827e-017``. While near to zero, the differences - prevent reliable equality testing and differences can accumulate. For this - reason, decimal is preferred in accounting applications which have strict - equality invariants. - -* The decimal module incorporates a notion of significant places so that ``1.30 - + 1.20`` is ``2.50``. The trailing zero is kept to indicate significance. - This is the customary presentation for monetary applications. For - multiplication, the "schoolbook" approach uses all the figures in the - multiplicands. For instance, ``1.3 * 1.2`` gives ``1.56`` while ``1.30 * - 1.20`` gives ``1.5600``. - -* Unlike hardware based binary floating point, the decimal module has a user - alterable precision (defaulting to 28 places) which can be as large as needed for - a given problem: - - >>> from decimal import * - >>> getcontext().prec = 6 - >>> Decimal(1) / Decimal(7) - Decimal('0.142857') - >>> getcontext().prec = 28 - >>> Decimal(1) / Decimal(7) - Decimal('0.1428571428571428571428571429') - -* Both binary and decimal floating point are implemented in terms of published - standards. While the built-in float type exposes only a modest portion of its - capabilities, the decimal module exposes all required parts of the standard. - When needed, the programmer has full control over rounding and signal handling. - This includes an option to enforce exact arithmetic by using exceptions - to block any inexact operations. - -* The decimal module was designed to support "without prejudice, both exact - unrounded decimal arithmetic (sometimes called fixed-point arithmetic) - and rounded floating-point arithmetic." -- excerpt from the decimal - arithmetic specification. - -The module design is centered around three concepts: the decimal number, the -context for arithmetic, and signals. - -A decimal number is immutable. It has a sign, coefficient digits, and an -exponent. To preserve significance, the coefficient digits do not truncate -trailing zeros. Decimals also include special values such as -``Infinity``, ``-Infinity``, and ``NaN``. The standard also -differentiates ``-0`` from ``+0``. - -The context for arithmetic is an environment specifying precision, rounding -rules, limits on exponents, flags indicating the results of operations, and trap -enablers which determine whether signals are treated as exceptions. Rounding -options include :const:`ROUND_CEILING`, :const:`ROUND_DOWN`, -:const:`ROUND_FLOOR`, :const:`ROUND_HALF_DOWN`, :const:`ROUND_HALF_EVEN`, -:const:`ROUND_HALF_UP`, :const:`ROUND_UP`, and :const:`ROUND_05UP`. - -Signals are groups of exceptional conditions arising during the course of -computation. Depending on the needs of the application, signals may be ignored, -considered as informational, or treated as exceptions. The signals in the -decimal module are: :const:`Clamped`, :const:`InvalidOperation`, -:const:`DivisionByZero`, :const:`Inexact`, :const:`Rounded`, :const:`Subnormal`, -:const:`Overflow`, :const:`Underflow` and :const:`FloatOperation`. - -For each signal there is a flag and a trap enabler. When a signal is -encountered, its flag is set to one, then, if the trap enabler is -set to one, an exception is raised. Flags are sticky, so the user needs to -reset them before monitoring a calculation. - - -.. seealso:: - - * IBM's General Decimal Arithmetic Specification, `The General Decimal Arithmetic - Specification `_. - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-tutorial: - -Quick-start Tutorial --------------------- - -The usual start to using decimals is importing the module, viewing the current -context with :func:`getcontext` and, if necessary, setting new values for -precision, rounding, or enabled traps:: - - >>> from decimal import * - >>> getcontext() - Context(prec=28, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, - capitals=1, clamp=0, flags=[], traps=[Overflow, DivisionByZero, - InvalidOperation]) - - >>> getcontext().prec = 7 # Set a new precision - -Decimal instances can be constructed from integers, strings, floats, or tuples. -Construction from an integer or a float performs an exact conversion of the -value of that integer or float. Decimal numbers include special values such as -``NaN`` which stands for "Not a number", positive and negative -``Infinity``, and ``-0``:: - - >>> getcontext().prec = 28 - >>> Decimal(10) - Decimal('10') - >>> Decimal('3.14') - Decimal('3.14') - >>> Decimal(3.14) - Decimal('3.140000000000000124344978758017532527446746826171875') - >>> Decimal((0, (3, 1, 4), -2)) - Decimal('3.14') - >>> Decimal(str(2.0 ** 0.5)) - Decimal('1.4142135623730951') - >>> Decimal(2) ** Decimal('0.5') - Decimal('1.414213562373095048801688724') - >>> Decimal('NaN') - Decimal('NaN') - >>> Decimal('-Infinity') - Decimal('-Infinity') - -If the :exc:`FloatOperation` signal is trapped, accidental mixing of -decimals and floats in constructors or ordering comparisons raises -an exception:: - - >>> c = getcontext() - >>> c.traps[FloatOperation] = True - >>> Decimal(3.14) - Traceback (most recent call last): - File "", line 1, in - decimal.FloatOperation: [] - >>> Decimal('3.5') < 3.7 - Traceback (most recent call last): - File "", line 1, in - decimal.FloatOperation: [] - >>> Decimal('3.5') == 3.5 - True - -.. versionadded:: 3.3 - -The significance of a new Decimal is determined solely by the number of digits -input. Context precision and rounding only come into play during arithmetic -operations. - -.. doctest:: newcontext - - >>> getcontext().prec = 6 - >>> Decimal('3.0') - Decimal('3.0') - >>> Decimal('3.1415926535') - Decimal('3.1415926535') - >>> Decimal('3.1415926535') + Decimal('2.7182818285') - Decimal('5.85987') - >>> getcontext().rounding = ROUND_UP - >>> Decimal('3.1415926535') + Decimal('2.7182818285') - Decimal('5.85988') - -If the internal limits of the C version are exceeded, constructing -a decimal raises :class:`InvalidOperation`:: - - >>> Decimal("1e9999999999999999999") - Traceback (most recent call last): - File "", line 1, in - decimal.InvalidOperation: [] - -.. versionchanged:: 3.3 - -Decimals interact well with much of the rest of Python. Here is a small decimal -floating point flying circus: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> data = list(map(Decimal, '1.34 1.87 3.45 2.35 1.00 0.03 9.25'.split())) - >>> max(data) - Decimal('9.25') - >>> min(data) - Decimal('0.03') - >>> sorted(data) - [Decimal('0.03'), Decimal('1.00'), Decimal('1.34'), Decimal('1.87'), - Decimal('2.35'), Decimal('3.45'), Decimal('9.25')] - >>> sum(data) - Decimal('19.29') - >>> a,b,c = data[:3] - >>> str(a) - '1.34' - >>> float(a) - 1.34 - >>> round(a, 1) - Decimal('1.3') - >>> int(a) - 1 - >>> a * 5 - Decimal('6.70') - >>> a * b - Decimal('2.5058') - >>> c % a - Decimal('0.77') - -And some mathematical functions are also available to Decimal: - - >>> getcontext().prec = 28 - >>> Decimal(2).sqrt() - Decimal('1.414213562373095048801688724') - >>> Decimal(1).exp() - Decimal('2.718281828459045235360287471') - >>> Decimal('10').ln() - Decimal('2.302585092994045684017991455') - >>> Decimal('10').log10() - Decimal('1') - -The :meth:`~Decimal.quantize` method rounds a number to a fixed exponent. This method is -useful for monetary applications that often round results to a fixed number of -places: - - >>> Decimal('7.325').quantize(Decimal('.01'), rounding=ROUND_DOWN) - Decimal('7.32') - >>> Decimal('7.325').quantize(Decimal('1.'), rounding=ROUND_UP) - Decimal('8') - -As shown above, the :func:`getcontext` function accesses the current context and -allows the settings to be changed. This approach meets the needs of most -applications. - -For more advanced work, it may be useful to create alternate contexts using the -Context() constructor. To make an alternate active, use the :func:`setcontext` -function. - -In accordance with the standard, the :mod:`decimal` module provides two ready to -use standard contexts, :const:`BasicContext` and :const:`ExtendedContext`. The -former is especially useful for debugging because many of the traps are -enabled: - -.. doctest:: newcontext - :options: +NORMALIZE_WHITESPACE - - >>> myothercontext = Context(prec=60, rounding=ROUND_HALF_DOWN) - >>> setcontext(myothercontext) - >>> Decimal(1) / Decimal(7) - Decimal('0.142857142857142857142857142857142857142857142857142857142857') - - >>> ExtendedContext - Context(prec=9, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, - capitals=1, clamp=0, flags=[], traps=[]) - >>> setcontext(ExtendedContext) - >>> Decimal(1) / Decimal(7) - Decimal('0.142857143') - >>> Decimal(42) / Decimal(0) - Decimal('Infinity') - - >>> setcontext(BasicContext) - >>> Decimal(42) / Decimal(0) - Traceback (most recent call last): - File "", line 1, in -toplevel- - Decimal(42) / Decimal(0) - DivisionByZero: x / 0 - -Contexts also have signal flags for monitoring exceptional conditions -encountered during computations. The flags remain set until explicitly cleared, -so it is best to clear the flags before each set of monitored computations by -using the :meth:`~Context.clear_flags` method. :: - - >>> setcontext(ExtendedContext) - >>> getcontext().clear_flags() - >>> Decimal(355) / Decimal(113) - Decimal('3.14159292') - >>> getcontext() - Context(prec=9, rounding=ROUND_HALF_EVEN, Emin=-999999, Emax=999999, - capitals=1, clamp=0, flags=[Inexact, Rounded], traps=[]) - -The *flags* entry shows that the rational approximation to pi was -rounded (digits beyond the context precision were thrown away) and that the -result is inexact (some of the discarded digits were non-zero). - -Individual traps are set using the dictionary in the :attr:`~Context.traps` -attribute of a context: - -.. doctest:: newcontext - - >>> setcontext(ExtendedContext) - >>> Decimal(1) / Decimal(0) - Decimal('Infinity') - >>> getcontext().traps[DivisionByZero] = 1 - >>> Decimal(1) / Decimal(0) - Traceback (most recent call last): - File "", line 1, in -toplevel- - Decimal(1) / Decimal(0) - DivisionByZero: x / 0 - -Most programs adjust the current context only once, at the beginning of the -program. And, in many applications, data is converted to :class:`Decimal` with -a single cast inside a loop. With context set and decimals created, the bulk of -the program manipulates the data no differently than with other Python numeric -types. - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-decimal: - -Decimal objects ---------------- - - -.. class:: Decimal(value="0", context=None) - - Construct a new :class:`Decimal` object based from *value*. - - *value* can be an integer, string, tuple, :class:`float`, or another :class:`Decimal` - object. If no *value* is given, returns ``Decimal('0')``. If *value* is a - string, it should conform to the decimal numeric string syntax after leading - and trailing whitespace characters, as well as underscores throughout, are removed:: - - sign ::= '+' | '-' - digit ::= '0' | '1' | '2' | '3' | '4' | '5' | '6' | '7' | '8' | '9' - indicator ::= 'e' | 'E' - digits ::= digit [digit]... - decimal-part ::= digits '.' [digits] | ['.'] digits - exponent-part ::= indicator [sign] digits - infinity ::= 'Infinity' | 'Inf' - nan ::= 'NaN' [digits] | 'sNaN' [digits] - numeric-value ::= decimal-part [exponent-part] | infinity - numeric-string ::= [sign] numeric-value | [sign] nan - - Other Unicode decimal digits are also permitted where ``digit`` - appears above. These include decimal digits from various other - alphabets (for example, Arabic-Indic and Devanāgarī digits) along - with the fullwidth digits ``'\uff10'`` through ``'\uff19'``. - - If *value* is a :class:`tuple`, it should have three components, a sign - (``0`` for positive or ``1`` for negative), a :class:`tuple` of - digits, and an integer exponent. For example, ``Decimal((0, (1, 4, 1, 4), -3))`` - returns ``Decimal('1.414')``. - - If *value* is a :class:`float`, the binary floating point value is losslessly - converted to its exact decimal equivalent. This conversion can often require - 53 or more digits of precision. For example, ``Decimal(float('1.1'))`` - converts to - ``Decimal('1.100000000000000088817841970012523233890533447265625')``. - - The *context* precision does not affect how many digits are stored. That is - determined exclusively by the number of digits in *value*. For example, - ``Decimal('3.00000')`` records all five zeros even if the context precision is - only three. - - The purpose of the *context* argument is determining what to do if *value* is a - malformed string. If the context traps :const:`InvalidOperation`, an exception - is raised; otherwise, the constructor returns a new Decimal with the value of - ``NaN``. - - Once constructed, :class:`Decimal` objects are immutable. - - .. versionchanged:: 3.2 - The argument to the constructor is now permitted to be a :class:`float` - instance. - - .. versionchanged:: 3.3 - :class:`float` arguments raise an exception if the :exc:`FloatOperation` - trap is set. By default the trap is off. - - .. versionchanged:: 3.6 - Underscores are allowed for grouping, as with integral and floating-point - literals in code. - - Decimal floating point objects share many properties with the other built-in - numeric types such as :class:`float` and :class:`int`. All of the usual math - operations and special methods apply. Likewise, decimal objects can be - copied, pickled, printed, used as dictionary keys, used as set elements, - compared, sorted, and coerced to another type (such as :class:`float` or - :class:`int`). - - There are some small differences between arithmetic on Decimal objects and - arithmetic on integers and floats. When the remainder operator ``%`` is - applied to Decimal objects, the sign of the result is the sign of the - *dividend* rather than the sign of the divisor:: - - >>> (-7) % 4 - 1 - >>> Decimal(-7) % Decimal(4) - Decimal('-3') - - The integer division operator ``//`` behaves analogously, returning the - integer part of the true quotient (truncating towards zero) rather than its - floor, so as to preserve the usual identity ``x == (x // y) * y + x % y``:: - - >>> -7 // 4 - -2 - >>> Decimal(-7) // Decimal(4) - Decimal('-1') - - The ``%`` and ``//`` operators implement the ``remainder`` and - ``divide-integer`` operations (respectively) as described in the - specification. - - Decimal objects cannot generally be combined with floats or - instances of :class:`fractions.Fraction` in arithmetic operations: - an attempt to add a :class:`Decimal` to a :class:`float`, for - example, will raise a :exc:`TypeError`. However, it is possible to - use Python's comparison operators to compare a :class:`Decimal` - instance ``x`` with another number ``y``. This avoids confusing results - when doing equality comparisons between numbers of different types. - - .. versionchanged:: 3.2 - Mixed-type comparisons between :class:`Decimal` instances and other - numeric types are now fully supported. - - In addition to the standard numeric properties, decimal floating point - objects also have a number of specialized methods: - - - .. method:: adjusted() - - Return the adjusted exponent after shifting out the coefficient's - rightmost digits until only the lead digit remains: - ``Decimal('321e+5').adjusted()`` returns seven. Used for determining the - position of the most significant digit with respect to the decimal point. - - .. method:: as_integer_ratio() - - Return a pair ``(n, d)`` of integers that represent the given - :class:`Decimal` instance as a fraction, in lowest terms and - with a positive denominator:: - - >>> Decimal('-3.14').as_integer_ratio() - (-157, 50) - - The conversion is exact. Raise OverflowError on infinities and ValueError - on NaNs. - - .. versionadded:: 3.6 - - .. method:: as_tuple() - - Return a :term:`named tuple` representation of the number: - ``DecimalTuple(sign, digits, exponent)``. - - - .. method:: canonical() - - Return the canonical encoding of the argument. Currently, the encoding of - a :class:`Decimal` instance is always canonical, so this operation returns - its argument unchanged. - - .. method:: compare(other, context=None) - - Compare the values of two Decimal instances. :meth:`compare` returns a - Decimal instance, and if either operand is a NaN then the result is a - NaN:: - - a or b is a NaN ==> Decimal('NaN') - a < b ==> Decimal('-1') - a == b ==> Decimal('0') - a > b ==> Decimal('1') - - .. method:: compare_signal(other, context=None) - - This operation is identical to the :meth:`compare` method, except that all - NaNs signal. That is, if neither operand is a signaling NaN then any - quiet NaN operand is treated as though it were a signaling NaN. - - .. method:: compare_total(other, context=None) - - Compare two operands using their abstract representation rather than their - numerical value. Similar to the :meth:`compare` method, but the result - gives a total ordering on :class:`Decimal` instances. Two - :class:`Decimal` instances with the same numeric value but different - representations compare unequal in this ordering: - - >>> Decimal('12.0').compare_total(Decimal('12')) - Decimal('-1') - - Quiet and signaling NaNs are also included in the total ordering. The - result of this function is ``Decimal('0')`` if both operands have the same - representation, ``Decimal('-1')`` if the first operand is lower in the - total order than the second, and ``Decimal('1')`` if the first operand is - higher in the total order than the second operand. See the specification - for details of the total order. - - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: compare_total_mag(other, context=None) - - Compare two operands using their abstract representation rather than their - value as in :meth:`compare_total`, but ignoring the sign of each operand. - ``x.compare_total_mag(y)`` is equivalent to - ``x.copy_abs().compare_total(y.copy_abs())``. - - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: conjugate() - - Just returns self, this method is only to comply with the Decimal - Specification. - - .. method:: copy_abs() - - Return the absolute value of the argument. This operation is unaffected - by the context and is quiet: no flags are changed and no rounding is - performed. - - .. method:: copy_negate() - - Return the negation of the argument. This operation is unaffected by the - context and is quiet: no flags are changed and no rounding is performed. - - .. method:: copy_sign(other, context=None) - - Return a copy of the first operand with the sign set to be the same as the - sign of the second operand. For example: - - >>> Decimal('2.3').copy_sign(Decimal('-1.5')) - Decimal('-2.3') - - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: exp(context=None) - - Return the value of the (natural) exponential function ``e**x`` at the - given number. The result is correctly rounded using the - :const:`ROUND_HALF_EVEN` rounding mode. - - >>> Decimal(1).exp() - Decimal('2.718281828459045235360287471') - >>> Decimal(321).exp() - Decimal('2.561702493119680037517373933E+139') - - .. classmethod:: from_float(f) - - Alternative constructor that only accepts instances of :class:`float` or - :class:`int`. - - Note ``Decimal.from_float(0.1)`` is not the same as ``Decimal('0.1')``. - Since 0.1 is not exactly representable in binary floating point, the - value is stored as the nearest representable value which is - ``0x1.999999999999ap-4``. That equivalent value in decimal is - ``0.1000000000000000055511151231257827021181583404541015625``. - - .. note:: From Python 3.2 onwards, a :class:`Decimal` instance - can also be constructed directly from a :class:`float`. - - .. doctest:: - - >>> Decimal.from_float(0.1) - Decimal('0.1000000000000000055511151231257827021181583404541015625') - >>> Decimal.from_float(float('nan')) - Decimal('NaN') - >>> Decimal.from_float(float('inf')) - Decimal('Infinity') - >>> Decimal.from_float(float('-inf')) - Decimal('-Infinity') - - .. versionadded:: 3.1 - - .. method:: fma(other, third, context=None) - - Fused multiply-add. Return self*other+third with no rounding of the - intermediate product self*other. - - >>> Decimal(2).fma(3, 5) - Decimal('11') - - .. method:: is_canonical() - - Return :const:`True` if the argument is canonical and :const:`False` - otherwise. Currently, a :class:`Decimal` instance is always canonical, so - this operation always returns :const:`True`. - - .. method:: is_finite() - - Return :const:`True` if the argument is a finite number, and - :const:`False` if the argument is an infinity or a NaN. - - .. method:: is_infinite() - - Return :const:`True` if the argument is either positive or negative - infinity and :const:`False` otherwise. - - .. method:: is_nan() - - Return :const:`True` if the argument is a (quiet or signaling) NaN and - :const:`False` otherwise. - - .. method:: is_normal(context=None) - - Return :const:`True` if the argument is a *normal* finite number. Return - :const:`False` if the argument is zero, subnormal, infinite or a NaN. - - .. method:: is_qnan() - - Return :const:`True` if the argument is a quiet NaN, and - :const:`False` otherwise. - - .. method:: is_signed() - - Return :const:`True` if the argument has a negative sign and - :const:`False` otherwise. Note that zeros and NaNs can both carry signs. - - .. method:: is_snan() - - Return :const:`True` if the argument is a signaling NaN and :const:`False` - otherwise. - - .. method:: is_subnormal(context=None) - - Return :const:`True` if the argument is subnormal, and :const:`False` - otherwise. - - .. method:: is_zero() - - Return :const:`True` if the argument is a (positive or negative) zero and - :const:`False` otherwise. - - .. method:: ln(context=None) - - Return the natural (base e) logarithm of the operand. The result is - correctly rounded using the :const:`ROUND_HALF_EVEN` rounding mode. - - .. method:: log10(context=None) - - Return the base ten logarithm of the operand. The result is correctly - rounded using the :const:`ROUND_HALF_EVEN` rounding mode. - - .. method:: logb(context=None) - - For a nonzero number, return the adjusted exponent of its operand as a - :class:`Decimal` instance. If the operand is a zero then - ``Decimal('-Infinity')`` is returned and the :const:`DivisionByZero` flag - is raised. If the operand is an infinity then ``Decimal('Infinity')`` is - returned. - - .. method:: logical_and(other, context=None) - - :meth:`logical_and` is a logical operation which takes two *logical - operands* (see :ref:`logical_operands_label`). The result is the - digit-wise ``and`` of the two operands. - - .. method:: logical_invert(context=None) - - :meth:`logical_invert` is a logical operation. The - result is the digit-wise inversion of the operand. - - .. method:: logical_or(other, context=None) - - :meth:`logical_or` is a logical operation which takes two *logical - operands* (see :ref:`logical_operands_label`). The result is the - digit-wise ``or`` of the two operands. - - .. method:: logical_xor(other, context=None) - - :meth:`logical_xor` is a logical operation which takes two *logical - operands* (see :ref:`logical_operands_label`). The result is the - digit-wise exclusive or of the two operands. - - .. method:: max(other, context=None) - - Like ``max(self, other)`` except that the context rounding rule is applied - before returning and that ``NaN`` values are either signaled or - ignored (depending on the context and whether they are signaling or - quiet). - - .. method:: max_mag(other, context=None) - - Similar to the :meth:`.max` method, but the comparison is done using the - absolute values of the operands. - - .. method:: min(other, context=None) - - Like ``min(self, other)`` except that the context rounding rule is applied - before returning and that ``NaN`` values are either signaled or - ignored (depending on the context and whether they are signaling or - quiet). - - .. method:: min_mag(other, context=None) - - Similar to the :meth:`.min` method, but the comparison is done using the - absolute values of the operands. - - .. method:: next_minus(context=None) - - Return the largest number representable in the given context (or in the - current thread's context if no context is given) that is smaller than the - given operand. - - .. method:: next_plus(context=None) - - Return the smallest number representable in the given context (or in the - current thread's context if no context is given) that is larger than the - given operand. - - .. method:: next_toward(other, context=None) - - If the two operands are unequal, return the number closest to the first - operand in the direction of the second operand. If both operands are - numerically equal, return a copy of the first operand with the sign set to - be the same as the sign of the second operand. - - .. method:: normalize(context=None) - - Used for producing canonical values of an equivalence - class within either the current context or the specified context. - - This has the same semantics as the unary plus operation, except that if - the final result is finite it is reduced to its simplest form, with all - trailing zeros removed and its sign preserved. That is, while the - coefficient is non-zero and a multiple of ten the coefficient is divided - by ten and the exponent is incremented by 1. Otherwise (the coefficient is - zero) the exponent is set to 0. In all cases the sign is unchanged. - - For example, ``Decimal('32.100')`` and ``Decimal('0.321000e+2')`` both - normalize to the equivalent value ``Decimal('32.1')``. - - Note that rounding is applied *before* reducing to simplest form. - - In the latest versions of the specification, this operation is also known - as ``reduce``. - - .. method:: number_class(context=None) - - Return a string describing the *class* of the operand. The returned value - is one of the following ten strings. - - * ``"-Infinity"``, indicating that the operand is negative infinity. - * ``"-Normal"``, indicating that the operand is a negative normal number. - * ``"-Subnormal"``, indicating that the operand is negative and subnormal. - * ``"-Zero"``, indicating that the operand is a negative zero. - * ``"+Zero"``, indicating that the operand is a positive zero. - * ``"+Subnormal"``, indicating that the operand is positive and subnormal. - * ``"+Normal"``, indicating that the operand is a positive normal number. - * ``"+Infinity"``, indicating that the operand is positive infinity. - * ``"NaN"``, indicating that the operand is a quiet NaN (Not a Number). - * ``"sNaN"``, indicating that the operand is a signaling NaN. - - .. method:: quantize(exp, rounding=None, context=None) - - Return a value equal to the first operand after rounding and having the - exponent of the second operand. - - >>> Decimal('1.41421356').quantize(Decimal('1.000')) - Decimal('1.414') - - Unlike other operations, if the length of the coefficient after the - quantize operation would be greater than precision, then an - :const:`InvalidOperation` is signaled. This guarantees that, unless there - is an error condition, the quantized exponent is always equal to that of - the right-hand operand. - - Also unlike other operations, quantize never signals Underflow, even if - the result is subnormal and inexact. - - If the exponent of the second operand is larger than that of the first - then rounding may be necessary. In this case, the rounding mode is - determined by the ``rounding`` argument if given, else by the given - ``context`` argument; if neither argument is given the rounding mode of - the current thread's context is used. - - An error is returned whenever the resulting exponent is greater than - :attr:`~Context.Emax` or less than :meth:`~Context.Etiny`. - - .. method:: radix() - - Return ``Decimal(10)``, the radix (base) in which the :class:`Decimal` - class does all its arithmetic. Included for compatibility with the - specification. - - .. method:: remainder_near(other, context=None) - - Return the remainder from dividing *self* by *other*. This differs from - ``self % other`` in that the sign of the remainder is chosen so as to - minimize its absolute value. More precisely, the return value is - ``self - n * other`` where ``n`` is the integer nearest to the exact - value of ``self / other``, and if two integers are equally near then the - even one is chosen. - - If the result is zero then its sign will be the sign of *self*. - - >>> Decimal(18).remainder_near(Decimal(10)) - Decimal('-2') - >>> Decimal(25).remainder_near(Decimal(10)) - Decimal('5') - >>> Decimal(35).remainder_near(Decimal(10)) - Decimal('-5') - - .. method:: rotate(other, context=None) - - Return the result of rotating the digits of the first operand by an amount - specified by the second operand. The second operand must be an integer in - the range -precision through precision. The absolute value of the second - operand gives the number of places to rotate. If the second operand is - positive then rotation is to the left; otherwise rotation is to the right. - The coefficient of the first operand is padded on the left with zeros to - length precision if necessary. The sign and exponent of the first operand - are unchanged. - - .. method:: same_quantum(other, context=None) - - Test whether self and other have the same exponent or whether both are - ``NaN``. - - This operation is unaffected by context and is quiet: no flags are changed - and no rounding is performed. As an exception, the C version may raise - InvalidOperation if the second operand cannot be converted exactly. - - .. method:: scaleb(other, context=None) - - Return the first operand with exponent adjusted by the second. - Equivalently, return the first operand multiplied by ``10**other``. The - second operand must be an integer. - - .. method:: shift(other, context=None) - - Return the result of shifting the digits of the first operand by an amount - specified by the second operand. The second operand must be an integer in - the range -precision through precision. The absolute value of the second - operand gives the number of places to shift. If the second operand is - positive then the shift is to the left; otherwise the shift is to the - right. Digits shifted into the coefficient are zeros. The sign and - exponent of the first operand are unchanged. - - .. method:: sqrt(context=None) - - Return the square root of the argument to full precision. - - - .. method:: to_eng_string(context=None) - - Convert to a string, using engineering notation if an exponent is needed. - - Engineering notation has an exponent which is a multiple of 3. This - can leave up to 3 digits to the left of the decimal place and may - require the addition of either one or two trailing zeros. - - For example, this converts ``Decimal('123E+1')`` to ``Decimal('1.23E+3')``. - - .. method:: to_integral(rounding=None, context=None) - - Identical to the :meth:`to_integral_value` method. The ``to_integral`` - name has been kept for compatibility with older versions. - - .. method:: to_integral_exact(rounding=None, context=None) - - Round to the nearest integer, signaling :const:`Inexact` or - :const:`Rounded` as appropriate if rounding occurs. The rounding mode is - determined by the ``rounding`` parameter if given, else by the given - ``context``. If neither parameter is given then the rounding mode of the - current context is used. - - .. method:: to_integral_value(rounding=None, context=None) - - Round to the nearest integer without signaling :const:`Inexact` or - :const:`Rounded`. If given, applies *rounding*; otherwise, uses the - rounding method in either the supplied *context* or the current context. - - -.. _logical_operands_label: - -Logical operands -^^^^^^^^^^^^^^^^ - -The :meth:`~Decimal.logical_and`, :meth:`~Decimal.logical_invert`, :meth:`~Decimal.logical_or`, -and :meth:`~Decimal.logical_xor` methods expect their arguments to be *logical -operands*. A *logical operand* is a :class:`Decimal` instance whose -exponent and sign are both zero, and whose digits are all either -``0`` or ``1``. - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-context: - -Context objects ---------------- - -Contexts are environments for arithmetic operations. They govern precision, set -rules for rounding, determine which signals are treated as exceptions, and limit -the range for exponents. - -Each thread has its own current context which is accessed or changed using the -:func:`getcontext` and :func:`setcontext` functions: - - -.. function:: getcontext() - - Return the current context for the active thread. - - -.. function:: setcontext(c) - - Set the current context for the active thread to *c*. - -You can also use the :keyword:`with` statement and the :func:`localcontext` -function to temporarily change the active context. - -.. function:: localcontext(ctx=None, \*\*kwargs) - - Return a context manager that will set the current context for the active thread - to a copy of *ctx* on entry to the with-statement and restore the previous context - when exiting the with-statement. If no context is specified, a copy of the - current context is used. The *kwargs* argument is used to set the attributes - of the new context. - - For example, the following code sets the current decimal precision to 42 places, - performs a calculation, and then automatically restores the previous context:: - - from decimal import localcontext - - with localcontext() as ctx: - ctx.prec = 42 # Perform a high precision calculation - s = calculate_something() - s = +s # Round the final result back to the default precision - - Using keyword arguments, the code would be the following:: - - from decimal import localcontext - - with localcontext(prec=42) as ctx: - s = calculate_something() - s = +s - - Raises :exc:`TypeError` if *kwargs* supplies an attribute that :class:`Context` doesn't - support. Raises either :exc:`TypeError` or :exc:`ValueError` if *kwargs* supplies an - invalid value for an attribute. - - .. versionchanged:: 3.11 - :meth:`localcontext` now supports setting context attributes through the use of keyword arguments. - -New contexts can also be created using the :class:`Context` constructor -described below. In addition, the module provides three pre-made contexts: - - -.. class:: BasicContext - - This is a standard context defined by the General Decimal Arithmetic - Specification. Precision is set to nine. Rounding is set to - :const:`ROUND_HALF_UP`. All flags are cleared. All traps are enabled (treated - as exceptions) except :const:`Inexact`, :const:`Rounded`, and - :const:`Subnormal`. - - Because many of the traps are enabled, this context is useful for debugging. - - -.. class:: ExtendedContext - - This is a standard context defined by the General Decimal Arithmetic - Specification. Precision is set to nine. Rounding is set to - :const:`ROUND_HALF_EVEN`. All flags are cleared. No traps are enabled (so that - exceptions are not raised during computations). - - Because the traps are disabled, this context is useful for applications that - prefer to have result value of ``NaN`` or ``Infinity`` instead of - raising exceptions. This allows an application to complete a run in the - presence of conditions that would otherwise halt the program. - - -.. class:: DefaultContext - - This context is used by the :class:`Context` constructor as a prototype for new - contexts. Changing a field (such a precision) has the effect of changing the - default for new contexts created by the :class:`Context` constructor. - - This context is most useful in multi-threaded environments. Changing one of the - fields before threads are started has the effect of setting system-wide - defaults. Changing the fields after threads have started is not recommended as - it would require thread synchronization to prevent race conditions. - - In single threaded environments, it is preferable to not use this context at - all. Instead, simply create contexts explicitly as described below. - - The default values are :attr:`Context.prec`\ =\ ``28``, - :attr:`Context.rounding`\ =\ :const:`ROUND_HALF_EVEN`, - and enabled traps for :class:`Overflow`, :class:`InvalidOperation`, and - :class:`DivisionByZero`. - -In addition to the three supplied contexts, new contexts can be created with the -:class:`Context` constructor. - - -.. class:: Context(prec=None, rounding=None, Emin=None, Emax=None, capitals=None, clamp=None, flags=None, traps=None) - - Creates a new context. If a field is not specified or is :const:`None`, the - default values are copied from the :const:`DefaultContext`. If the *flags* - field is not specified or is :const:`None`, all flags are cleared. - - *prec* is an integer in the range [``1``, :const:`MAX_PREC`] that sets - the precision for arithmetic operations in the context. - - The *rounding* option is one of the constants listed in the section - `Rounding Modes`_. - - The *traps* and *flags* fields list any signals to be set. Generally, new - contexts should only set traps and leave the flags clear. - - The *Emin* and *Emax* fields are integers specifying the outer limits allowable - for exponents. *Emin* must be in the range [:const:`MIN_EMIN`, ``0``], - *Emax* in the range [``0``, :const:`MAX_EMAX`]. - - The *capitals* field is either ``0`` or ``1`` (the default). If set to - ``1``, exponents are printed with a capital ``E``; otherwise, a - lowercase ``e`` is used: ``Decimal('6.02e+23')``. - - The *clamp* field is either ``0`` (the default) or ``1``. - If set to ``1``, the exponent ``e`` of a :class:`Decimal` - instance representable in this context is strictly limited to the - range ``Emin - prec + 1 <= e <= Emax - prec + 1``. If *clamp* is - ``0`` then a weaker condition holds: the adjusted exponent of - the :class:`Decimal` instance is at most :attr:`~Context.Emax`. When *clamp* is - ``1``, a large normal number will, where possible, have its - exponent reduced and a corresponding number of zeros added to its - coefficient, in order to fit the exponent constraints; this - preserves the value of the number but loses information about - significant trailing zeros. For example:: - - >>> Context(prec=6, Emax=999, clamp=1).create_decimal('1.23e999') - Decimal('1.23000E+999') - - A *clamp* value of ``1`` allows compatibility with the - fixed-width decimal interchange formats specified in IEEE 754. - - The :class:`Context` class defines several general purpose methods as well as - a large number of methods for doing arithmetic directly in a given context. - In addition, for each of the :class:`Decimal` methods described above (with - the exception of the :meth:`~Decimal.adjusted` and :meth:`~Decimal.as_tuple` methods) there is - a corresponding :class:`Context` method. For example, for a :class:`Context` - instance ``C`` and :class:`Decimal` instance ``x``, ``C.exp(x)`` is - equivalent to ``x.exp(context=C)``. Each :class:`Context` method accepts a - Python integer (an instance of :class:`int`) anywhere that a - Decimal instance is accepted. - - - .. method:: clear_flags() - - Resets all of the flags to ``0``. - - .. method:: clear_traps() - - Resets all of the traps to ``0``. - - .. versionadded:: 3.3 - - .. method:: copy() - - Return a duplicate of the context. - - .. method:: copy_decimal(num) - - Return a copy of the Decimal instance num. - - .. method:: create_decimal(num) - - Creates a new Decimal instance from *num* but using *self* as - context. Unlike the :class:`Decimal` constructor, the context precision, - rounding method, flags, and traps are applied to the conversion. - - This is useful because constants are often given to a greater precision - than is needed by the application. Another benefit is that rounding - immediately eliminates unintended effects from digits beyond the current - precision. In the following example, using unrounded inputs means that - adding zero to a sum can change the result: - - .. doctest:: newcontext - - >>> getcontext().prec = 3 - >>> Decimal('3.4445') + Decimal('1.0023') - Decimal('4.45') - >>> Decimal('3.4445') + Decimal(0) + Decimal('1.0023') - Decimal('4.44') - - This method implements the to-number operation of the IBM specification. - If the argument is a string, no leading or trailing whitespace or - underscores are permitted. - - .. method:: create_decimal_from_float(f) - - Creates a new Decimal instance from a float *f* but rounding using *self* - as the context. Unlike the :meth:`Decimal.from_float` class method, - the context precision, rounding method, flags, and traps are applied to - the conversion. - - .. doctest:: - - >>> context = Context(prec=5, rounding=ROUND_DOWN) - >>> context.create_decimal_from_float(math.pi) - Decimal('3.1415') - >>> context = Context(prec=5, traps=[Inexact]) - >>> context.create_decimal_from_float(math.pi) - Traceback (most recent call last): - ... - decimal.Inexact: None - - .. versionadded:: 3.1 - - .. method:: Etiny() - - Returns a value equal to ``Emin - prec + 1`` which is the minimum exponent - value for subnormal results. When underflow occurs, the exponent is set - to :const:`Etiny`. - - .. method:: Etop() - - Returns a value equal to ``Emax - prec + 1``. - - The usual approach to working with decimals is to create :class:`Decimal` - instances and then apply arithmetic operations which take place within the - current context for the active thread. An alternative approach is to use - context methods for calculating within a specific context. The methods are - similar to those for the :class:`Decimal` class and are only briefly - recounted here. - - - .. method:: abs(x) - - Returns the absolute value of *x*. - - - .. method:: add(x, y) - - Return the sum of *x* and *y*. - - - .. method:: canonical(x) - - Returns the same Decimal object *x*. - - - .. method:: compare(x, y) - - Compares *x* and *y* numerically. - - - .. method:: compare_signal(x, y) - - Compares the values of the two operands numerically. - - - .. method:: compare_total(x, y) - - Compares two operands using their abstract representation. - - - .. method:: compare_total_mag(x, y) - - Compares two operands using their abstract representation, ignoring sign. - - - .. method:: copy_abs(x) - - Returns a copy of *x* with the sign set to 0. - - - .. method:: copy_negate(x) - - Returns a copy of *x* with the sign inverted. - - - .. method:: copy_sign(x, y) - - Copies the sign from *y* to *x*. - - - .. method:: divide(x, y) - - Return *x* divided by *y*. - - - .. method:: divide_int(x, y) - - Return *x* divided by *y*, truncated to an integer. - - - .. method:: divmod(x, y) - - Divides two numbers and returns the integer part of the result. - - - .. method:: exp(x) - - Returns ``e ** x``. - - - .. method:: fma(x, y, z) - - Returns *x* multiplied by *y*, plus *z*. - - - .. method:: is_canonical(x) - - Returns ``True`` if *x* is canonical; otherwise returns ``False``. - - - .. method:: is_finite(x) - - Returns ``True`` if *x* is finite; otherwise returns ``False``. - - - .. method:: is_infinite(x) - - Returns ``True`` if *x* is infinite; otherwise returns ``False``. - - - .. method:: is_nan(x) - - Returns ``True`` if *x* is a qNaN or sNaN; otherwise returns ``False``. - - - .. method:: is_normal(x) - - Returns ``True`` if *x* is a normal number; otherwise returns ``False``. - - - .. method:: is_qnan(x) - - Returns ``True`` if *x* is a quiet NaN; otherwise returns ``False``. - - - .. method:: is_signed(x) - - Returns ``True`` if *x* is negative; otherwise returns ``False``. - - - .. method:: is_snan(x) - - Returns ``True`` if *x* is a signaling NaN; otherwise returns ``False``. - - - .. method:: is_subnormal(x) - - Returns ``True`` if *x* is subnormal; otherwise returns ``False``. - - - .. method:: is_zero(x) - - Returns ``True`` if *x* is a zero; otherwise returns ``False``. - - - .. method:: ln(x) - - Returns the natural (base e) logarithm of *x*. - - - .. method:: log10(x) - - Returns the base 10 logarithm of *x*. - - - .. method:: logb(x) - - Returns the exponent of the magnitude of the operand's MSD. - - - .. method:: logical_and(x, y) - - Applies the logical operation *and* between each operand's digits. - - - .. method:: logical_invert(x) - - Invert all the digits in *x*. - - - .. method:: logical_or(x, y) - - Applies the logical operation *or* between each operand's digits. - - - .. method:: logical_xor(x, y) - - Applies the logical operation *xor* between each operand's digits. - - - .. method:: max(x, y) - - Compares two values numerically and returns the maximum. - - - .. method:: max_mag(x, y) - - Compares the values numerically with their sign ignored. - - - .. method:: min(x, y) - - Compares two values numerically and returns the minimum. - - - .. method:: min_mag(x, y) - - Compares the values numerically with their sign ignored. - - - .. method:: minus(x) - - Minus corresponds to the unary prefix minus operator in Python. - - - .. method:: multiply(x, y) - - Return the product of *x* and *y*. - - - .. method:: next_minus(x) - - Returns the largest representable number smaller than *x*. - - - .. method:: next_plus(x) - - Returns the smallest representable number larger than *x*. - - - .. method:: next_toward(x, y) - - Returns the number closest to *x*, in direction towards *y*. - - - .. method:: normalize(x) - - Reduces *x* to its simplest form. - - - .. method:: number_class(x) - - Returns an indication of the class of *x*. - - - .. method:: plus(x) - - Plus corresponds to the unary prefix plus operator in Python. This - operation applies the context precision and rounding, so it is *not* an - identity operation. - - - .. method:: power(x, y, modulo=None) - - Return ``x`` to the power of ``y``, reduced modulo ``modulo`` if given. - - With two arguments, compute ``x**y``. If ``x`` is negative then ``y`` - must be integral. The result will be inexact unless ``y`` is integral and - the result is finite and can be expressed exactly in 'precision' digits. - The rounding mode of the context is used. Results are always correctly rounded - in the Python version. - - ``Decimal(0) ** Decimal(0)`` results in ``InvalidOperation``, and if ``InvalidOperation`` - is not trapped, then results in ``Decimal('NaN')``. - - .. versionchanged:: 3.3 - The C module computes :meth:`power` in terms of the correctly rounded - :meth:`exp` and :meth:`ln` functions. The result is well-defined but - only "almost always correctly rounded". - - With three arguments, compute ``(x**y) % modulo``. For the three argument - form, the following restrictions on the arguments hold: - - - all three arguments must be integral - - ``y`` must be nonnegative - - at least one of ``x`` or ``y`` must be nonzero - - ``modulo`` must be nonzero and have at most 'precision' digits - - The value resulting from ``Context.power(x, y, modulo)`` is - equal to the value that would be obtained by computing ``(x**y) - % modulo`` with unbounded precision, but is computed more - efficiently. The exponent of the result is zero, regardless of - the exponents of ``x``, ``y`` and ``modulo``. The result is - always exact. - - - .. method:: quantize(x, y) - - Returns a value equal to *x* (rounded), having the exponent of *y*. - - - .. method:: radix() - - Just returns 10, as this is Decimal, :) - - - .. method:: remainder(x, y) - - Returns the remainder from integer division. - - The sign of the result, if non-zero, is the same as that of the original - dividend. - - - .. method:: remainder_near(x, y) - - Returns ``x - y * n``, where *n* is the integer nearest the exact value - of ``x / y`` (if the result is 0 then its sign will be the sign of *x*). - - - .. method:: rotate(x, y) - - Returns a rotated copy of *x*, *y* times. - - - .. method:: same_quantum(x, y) - - Returns ``True`` if the two operands have the same exponent. - - - .. method:: scaleb (x, y) - - Returns the first operand after adding the second value its exp. - - - .. method:: shift(x, y) - - Returns a shifted copy of *x*, *y* times. - - - .. method:: sqrt(x) - - Square root of a non-negative number to context precision. - - - .. method:: subtract(x, y) - - Return the difference between *x* and *y*. - - - .. method:: to_eng_string(x) - - Convert to a string, using engineering notation if an exponent is needed. - - Engineering notation has an exponent which is a multiple of 3. This - can leave up to 3 digits to the left of the decimal place and may - require the addition of either one or two trailing zeros. - - - .. method:: to_integral_exact(x) - - Rounds to an integer. - - - .. method:: to_sci_string(x) - - Converts a number to a string using scientific notation. - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - -.. _decimal-rounding-modes: - -Constants ---------- - -The constants in this section are only relevant for the C module. They -are also included in the pure Python version for compatibility. - -+---------------------+---------------------+-------------------------------+ -| | 32-bit | 64-bit | -+=====================+=====================+===============================+ -| .. data:: MAX_PREC | ``425000000`` | ``999999999999999999`` | -+---------------------+---------------------+-------------------------------+ -| .. data:: MAX_EMAX | ``425000000`` | ``999999999999999999`` | -+---------------------+---------------------+-------------------------------+ -| .. data:: MIN_EMIN | ``-425000000`` | ``-999999999999999999`` | -+---------------------+---------------------+-------------------------------+ -| .. data:: MIN_ETINY | ``-849999999`` | ``-1999999999999999997`` | -+---------------------+---------------------+-------------------------------+ - - -.. data:: HAVE_THREADS - - The value is ``True``. Deprecated, because Python now always has threads. - -.. deprecated:: 3.9 - -.. data:: HAVE_CONTEXTVAR - - The default value is ``True``. If Python is :option:`configured using - the --without-decimal-contextvar option <--without-decimal-contextvar>`, - the C version uses a thread-local rather than a coroutine-local context and the value - is ``False``. This is slightly faster in some nested context scenarios. - -.. versionadded:: 3.9 backported to 3.7 and 3.8. - - -Rounding modes --------------- - -.. data:: ROUND_CEILING - - Round towards ``Infinity``. - -.. data:: ROUND_DOWN - - Round towards zero. - -.. data:: ROUND_FLOOR - - Round towards ``-Infinity``. - -.. data:: ROUND_HALF_DOWN - - Round to nearest with ties going towards zero. - -.. data:: ROUND_HALF_EVEN - - Round to nearest with ties going to nearest even integer. - -.. data:: ROUND_HALF_UP - - Round to nearest with ties going away from zero. - -.. data:: ROUND_UP - - Round away from zero. - -.. data:: ROUND_05UP - - Round away from zero if last digit after rounding towards zero would have - been 0 or 5; otherwise round towards zero. - - -.. _decimal-signals: - -Signals -------- - -Signals represent conditions that arise during computation. Each corresponds to -one context flag and one context trap enabler. - -The context flag is set whenever the condition is encountered. After the -computation, flags may be checked for informational purposes (for instance, to -determine whether a computation was exact). After checking the flags, be sure to -clear all flags before starting the next computation. - -If the context's trap enabler is set for the signal, then the condition causes a -Python exception to be raised. For example, if the :class:`DivisionByZero` trap -is set, then a :exc:`DivisionByZero` exception is raised upon encountering the -condition. - - -.. class:: Clamped - - Altered an exponent to fit representation constraints. - - Typically, clamping occurs when an exponent falls outside the context's - :attr:`~Context.Emin` and :attr:`~Context.Emax` limits. If possible, the exponent is reduced to - fit by adding zeros to the coefficient. - - -.. class:: DecimalException - - Base class for other signals and a subclass of :exc:`ArithmeticError`. - - -.. class:: DivisionByZero - - Signals the division of a non-infinite number by zero. - - Can occur with division, modulo division, or when raising a number to a negative - power. If this signal is not trapped, returns ``Infinity`` or - ``-Infinity`` with the sign determined by the inputs to the calculation. - - -.. class:: Inexact - - Indicates that rounding occurred and the result is not exact. - - Signals when non-zero digits were discarded during rounding. The rounded result - is returned. The signal flag or trap is used to detect when results are - inexact. - - -.. class:: InvalidOperation - - An invalid operation was performed. - - Indicates that an operation was requested that does not make sense. If not - trapped, returns ``NaN``. Possible causes include:: - - Infinity - Infinity - 0 * Infinity - Infinity / Infinity - x % 0 - Infinity % x - sqrt(-x) and x > 0 - 0 ** 0 - x ** (non-integer) - x ** Infinity - - -.. class:: Overflow - - Numerical overflow. - - Indicates the exponent is larger than :attr:`Context.Emax` after rounding has - occurred. If not trapped, the result depends on the rounding mode, either - pulling inward to the largest representable finite number or rounding outward - to ``Infinity``. In either case, :class:`Inexact` and :class:`Rounded` - are also signaled. - - -.. class:: Rounded - - Rounding occurred though possibly no information was lost. - - Signaled whenever rounding discards digits; even if those digits are zero - (such as rounding ``5.00`` to ``5.0``). If not trapped, returns - the result unchanged. This signal is used to detect loss of significant - digits. - - -.. class:: Subnormal - - Exponent was lower than :attr:`~Context.Emin` prior to rounding. - - Occurs when an operation result is subnormal (the exponent is too small). If - not trapped, returns the result unchanged. - - -.. class:: Underflow - - Numerical underflow with result rounded to zero. - - Occurs when a subnormal result is pushed to zero by rounding. :class:`Inexact` - and :class:`Subnormal` are also signaled. - - -.. class:: FloatOperation - - Enable stricter semantics for mixing floats and Decimals. - - If the signal is not trapped (default), mixing floats and Decimals is - permitted in the :class:`~decimal.Decimal` constructor, - :meth:`~decimal.Context.create_decimal` and all comparison operators. - Both conversion and comparisons are exact. Any occurrence of a mixed - operation is silently recorded by setting :exc:`FloatOperation` in the - context flags. Explicit conversions with :meth:`~decimal.Decimal.from_float` - or :meth:`~decimal.Context.create_decimal_from_float` do not set the flag. - - Otherwise (the signal is trapped), only equality comparisons and explicit - conversions are silent. All other mixed operations raise :exc:`FloatOperation`. - - -The following table summarizes the hierarchy of signals:: - - exceptions.ArithmeticError(exceptions.Exception) - DecimalException - Clamped - DivisionByZero(DecimalException, exceptions.ZeroDivisionError) - Inexact - Overflow(Inexact, Rounded) - Underflow(Inexact, Rounded, Subnormal) - InvalidOperation - Rounded - Subnormal - FloatOperation(DecimalException, exceptions.TypeError) - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - - -.. _decimal-notes: - -Floating Point Notes --------------------- - - -Mitigating round-off error with increased precision -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The use of decimal floating point eliminates decimal representation error -(making it possible to represent ``0.1`` exactly); however, some operations -can still incur round-off error when non-zero digits exceed the fixed precision. - -The effects of round-off error can be amplified by the addition or subtraction -of nearly offsetting quantities resulting in loss of significance. Knuth -provides two instructive examples where rounded floating point arithmetic with -insufficient precision causes the breakdown of the associative and distributive -properties of addition: - -.. doctest:: newcontext - - # Examples from Seminumerical Algorithms, Section 4.2.2. - >>> from decimal import Decimal, getcontext - >>> getcontext().prec = 8 - - >>> u, v, w = Decimal(11111113), Decimal(-11111111), Decimal('7.51111111') - >>> (u + v) + w - Decimal('9.5111111') - >>> u + (v + w) - Decimal('10') - - >>> u, v, w = Decimal(20000), Decimal(-6), Decimal('6.0000003') - >>> (u*v) + (u*w) - Decimal('0.01') - >>> u * (v+w) - Decimal('0.0060000') - -The :mod:`decimal` module makes it possible to restore the identities by -expanding the precision sufficiently to avoid loss of significance: - -.. doctest:: newcontext - - >>> getcontext().prec = 20 - >>> u, v, w = Decimal(11111113), Decimal(-11111111), Decimal('7.51111111') - >>> (u + v) + w - Decimal('9.51111111') - >>> u + (v + w) - Decimal('9.51111111') - >>> - >>> u, v, w = Decimal(20000), Decimal(-6), Decimal('6.0000003') - >>> (u*v) + (u*w) - Decimal('0.0060000') - >>> u * (v+w) - Decimal('0.0060000') - - -Special values -^^^^^^^^^^^^^^ - -The number system for the :mod:`decimal` module provides special values -including ``NaN``, ``sNaN``, ``-Infinity``, ``Infinity``, -and two zeros, ``+0`` and ``-0``. - -Infinities can be constructed directly with: ``Decimal('Infinity')``. Also, -they can arise from dividing by zero when the :exc:`DivisionByZero` signal is -not trapped. Likewise, when the :exc:`Overflow` signal is not trapped, infinity -can result from rounding beyond the limits of the largest representable number. - -The infinities are signed (affine) and can be used in arithmetic operations -where they get treated as very large, indeterminate numbers. For instance, -adding a constant to infinity gives another infinite result. - -Some operations are indeterminate and return ``NaN``, or if the -:exc:`InvalidOperation` signal is trapped, raise an exception. For example, -``0/0`` returns ``NaN`` which means "not a number". This variety of -``NaN`` is quiet and, once created, will flow through other computations -always resulting in another ``NaN``. This behavior can be useful for a -series of computations that occasionally have missing inputs --- it allows the -calculation to proceed while flagging specific results as invalid. - -A variant is ``sNaN`` which signals rather than remaining quiet after every -operation. This is a useful return value when an invalid result needs to -interrupt a calculation for special handling. - -The behavior of Python's comparison operators can be a little surprising where a -``NaN`` is involved. A test for equality where one of the operands is a -quiet or signaling ``NaN`` always returns :const:`False` (even when doing -``Decimal('NaN')==Decimal('NaN')``), while a test for inequality always returns -:const:`True`. An attempt to compare two Decimals using any of the ``<``, -``<=``, ``>`` or ``>=`` operators will raise the :exc:`InvalidOperation` signal -if either operand is a ``NaN``, and return :const:`False` if this signal is -not trapped. Note that the General Decimal Arithmetic specification does not -specify the behavior of direct comparisons; these rules for comparisons -involving a ``NaN`` were taken from the IEEE 854 standard (see Table 3 in -section 5.7). To ensure strict standards-compliance, use the :meth:`~Decimal.compare` -and :meth:`~Decimal.compare_signal` methods instead. - -The signed zeros can result from calculations that underflow. They keep the sign -that would have resulted if the calculation had been carried out to greater -precision. Since their magnitude is zero, both positive and negative zeros are -treated as equal and their sign is informational. - -In addition to the two signed zeros which are distinct yet equal, there are -various representations of zero with differing precisions yet equivalent in -value. This takes a bit of getting used to. For an eye accustomed to -normalized floating point representations, it is not immediately obvious that -the following calculation returns a value equal to zero: - - >>> 1 / Decimal('Infinity') - Decimal('0E-1000026') - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-threads: - -Working with threads --------------------- - -The :func:`getcontext` function accesses a different :class:`Context` object for -each thread. Having separate thread contexts means that threads may make -changes (such as ``getcontext().prec=10``) without interfering with other threads. - -Likewise, the :func:`setcontext` function automatically assigns its target to -the current thread. - -If :func:`setcontext` has not been called before :func:`getcontext`, then -:func:`getcontext` will automatically create a new context for use in the -current thread. - -The new context is copied from a prototype context called *DefaultContext*. To -control the defaults so that each thread will use the same values throughout the -application, directly modify the *DefaultContext* object. This should be done -*before* any threads are started so that there won't be a race condition between -threads calling :func:`getcontext`. For example:: - - # Set applicationwide defaults for all threads about to be launched - DefaultContext.prec = 12 - DefaultContext.rounding = ROUND_DOWN - DefaultContext.traps = ExtendedContext.traps.copy() - DefaultContext.traps[InvalidOperation] = 1 - setcontext(DefaultContext) - - # Afterwards, the threads can be started - t1.start() - t2.start() - t3.start() - . . . - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-recipes: - -Recipes -------- - -Here are a few recipes that serve as utility functions and that demonstrate ways -to work with the :class:`Decimal` class:: - - def moneyfmt(value, places=2, curr='', sep=',', dp='.', - pos='', neg='-', trailneg=''): - """Convert Decimal to a money formatted string. - - places: required number of places after the decimal point - curr: optional currency symbol before the sign (may be blank) - sep: optional grouping separator (comma, period, space, or blank) - dp: decimal point indicator (comma or period) - only specify as blank when places is zero - pos: optional sign for positive numbers: '+', space or blank - neg: optional sign for negative numbers: '-', '(', space or blank - trailneg:optional trailing minus indicator: '-', ')', space or blank - - >>> d = Decimal('-1234567.8901') - >>> moneyfmt(d, curr='$') - '-$1,234,567.89' - >>> moneyfmt(d, places=0, sep='.', dp='', neg='', trailneg='-') - '1.234.568-' - >>> moneyfmt(d, curr='$', neg='(', trailneg=')') - '($1,234,567.89)' - >>> moneyfmt(Decimal(123456789), sep=' ') - '123 456 789.00' - >>> moneyfmt(Decimal('-0.02'), neg='<', trailneg='>') - '<0.02>' - - """ - q = Decimal(10) ** -places # 2 places --> '0.01' - sign, digits, exp = value.quantize(q).as_tuple() - result = [] - digits = list(map(str, digits)) - build, next = result.append, digits.pop - if sign: - build(trailneg) - for i in range(places): - build(next() if digits else '0') - if places: - build(dp) - if not digits: - build('0') - i = 0 - while digits: - build(next()) - i += 1 - if i == 3 and digits: - i = 0 - build(sep) - build(curr) - build(neg if sign else pos) - return ''.join(reversed(result)) - - def pi(): - """Compute Pi to the current precision. - - >>> print(pi()) - 3.141592653589793238462643383 - - """ - getcontext().prec += 2 # extra digits for intermediate steps - three = Decimal(3) # substitute "three=3.0" for regular floats - lasts, t, s, n, na, d, da = 0, three, 3, 1, 0, 0, 24 - while s != lasts: - lasts = s - n, na = n+na, na+8 - d, da = d+da, da+32 - t = (t * n) / d - s += t - getcontext().prec -= 2 - return +s # unary plus applies the new precision - - def exp(x): - """Return e raised to the power of x. Result type matches input type. - - >>> print(exp(Decimal(1))) - 2.718281828459045235360287471 - >>> print(exp(Decimal(2))) - 7.389056098930650227230427461 - >>> print(exp(2.0)) - 7.38905609893 - >>> print(exp(2+0j)) - (7.38905609893+0j) - - """ - getcontext().prec += 2 - i, lasts, s, fact, num = 0, 0, 1, 1, 1 - while s != lasts: - lasts = s - i += 1 - fact *= i - num *= x - s += num / fact - getcontext().prec -= 2 - return +s - - def cos(x): - """Return the cosine of x as measured in radians. - - The Taylor series approximation works best for a small value of x. - For larger values, first compute x = x % (2 * pi). - - >>> print(cos(Decimal('0.5'))) - 0.8775825618903727161162815826 - >>> print(cos(0.5)) - 0.87758256189 - >>> print(cos(0.5+0j)) - (0.87758256189+0j) - - """ - getcontext().prec += 2 - i, lasts, s, fact, num, sign = 0, 0, 1, 1, 1, 1 - while s != lasts: - lasts = s - i += 2 - fact *= i * (i-1) - num *= x * x - sign *= -1 - s += num / fact * sign - getcontext().prec -= 2 - return +s - - def sin(x): - """Return the sine of x as measured in radians. - - The Taylor series approximation works best for a small value of x. - For larger values, first compute x = x % (2 * pi). - - >>> print(sin(Decimal('0.5'))) - 0.4794255386042030002732879352 - >>> print(sin(0.5)) - 0.479425538604 - >>> print(sin(0.5+0j)) - (0.479425538604+0j) - - """ - getcontext().prec += 2 - i, lasts, s, fact, num, sign = 1, 0, x, 1, x, 1 - while s != lasts: - lasts = s - i += 2 - fact *= i * (i-1) - num *= x * x - sign *= -1 - s += num / fact * sign - getcontext().prec -= 2 - return +s - - -.. %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% - - -.. _decimal-faq: - -Decimal FAQ ------------ - -Q. It is cumbersome to type ``decimal.Decimal('1234.5')``. Is there a way to -minimize typing when using the interactive interpreter? - -A. Some users abbreviate the constructor to just a single letter: - - >>> D = decimal.Decimal - >>> D('1.23') + D('3.45') - Decimal('4.68') - -Q. In a fixed-point application with two decimal places, some inputs have many -places and need to be rounded. Others are not supposed to have excess digits -and need to be validated. What methods should be used? - -A. The :meth:`~Decimal.quantize` method rounds to a fixed number of decimal places. If -the :const:`Inexact` trap is set, it is also useful for validation: - - >>> TWOPLACES = Decimal(10) ** -2 # same as Decimal('0.01') - - >>> # Round to two places - >>> Decimal('3.214').quantize(TWOPLACES) - Decimal('3.21') - - >>> # Validate that a number does not exceed two places - >>> Decimal('3.21').quantize(TWOPLACES, context=Context(traps=[Inexact])) - Decimal('3.21') - - >>> Decimal('3.214').quantize(TWOPLACES, context=Context(traps=[Inexact])) - Traceback (most recent call last): - ... - Inexact: None - -Q. Once I have valid two place inputs, how do I maintain that invariant -throughout an application? - -A. Some operations like addition, subtraction, and multiplication by an integer -will automatically preserve fixed point. Others operations, like division and -non-integer multiplication, will change the number of decimal places and need to -be followed-up with a :meth:`~Decimal.quantize` step: - - >>> a = Decimal('102.72') # Initial fixed-point values - >>> b = Decimal('3.17') - >>> a + b # Addition preserves fixed-point - Decimal('105.89') - >>> a - b - Decimal('99.55') - >>> a * 42 # So does integer multiplication - Decimal('4314.24') - >>> (a * b).quantize(TWOPLACES) # Must quantize non-integer multiplication - Decimal('325.62') - >>> (b / a).quantize(TWOPLACES) # And quantize division - Decimal('0.03') - -In developing fixed-point applications, it is convenient to define functions -to handle the :meth:`~Decimal.quantize` step: - - >>> def mul(x, y, fp=TWOPLACES): - ... return (x * y).quantize(fp) - >>> def div(x, y, fp=TWOPLACES): - ... return (x / y).quantize(fp) - - >>> mul(a, b) # Automatically preserve fixed-point - Decimal('325.62') - >>> div(b, a) - Decimal('0.03') - -Q. There are many ways to express the same value. The numbers ``200``, -``200.000``, ``2E2``, and ``.02E+4`` all have the same value at -various precisions. Is there a way to transform them to a single recognizable -canonical value? - -A. The :meth:`~Decimal.normalize` method maps all equivalent values to a single -representative: - - >>> values = map(Decimal, '200 200.000 2E2 .02E+4'.split()) - >>> [v.normalize() for v in values] - [Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2'), Decimal('2E+2')] - -Q. When does rounding occur in a computation? - -A. It occurs *after* the computation. The philosophy of the decimal -specification is that numbers are considered exact and are created -independent of the current context. They can even have greater -precision than current context. Computations process with those -exact inputs and then rounding (or other context operations) is -applied to the *result* of the computation:: - - >>> getcontext().prec = 5 - >>> pi = Decimal('3.1415926535') # More than 5 digits - >>> pi # All digits are retained - Decimal('3.1415926535') - >>> pi + 0 # Rounded after an addition - Decimal('3.1416') - >>> pi - Decimal('0.00005') # Subtract unrounded numbers, then round - Decimal('3.1415') - >>> pi + 0 - Decimal('0.00005'). # Intermediate values are rounded - Decimal('3.1416') - -Q. Some decimal values always print with exponential notation. Is there a way -to get a non-exponential representation? - -A. For some values, exponential notation is the only way to express the number -of significant places in the coefficient. For example, expressing -``5.0E+3`` as ``5000`` keeps the value constant but cannot show the -original's two-place significance. - -If an application does not care about tracking significance, it is easy to -remove the exponent and trailing zeroes, losing significance, but keeping the -value unchanged: - - >>> def remove_exponent(d): - ... return d.quantize(Decimal(1)) if d == d.to_integral() else d.normalize() - - >>> remove_exponent(Decimal('5E+3')) - Decimal('5000') - -Q. Is there a way to convert a regular float to a :class:`Decimal`? - -A. Yes, any binary floating point number can be exactly expressed as a -Decimal though an exact conversion may take more precision than intuition would -suggest: - -.. doctest:: - - >>> Decimal(math.pi) - Decimal('3.141592653589793115997963468544185161590576171875') - -Q. Within a complex calculation, how can I make sure that I haven't gotten a -spurious result because of insufficient precision or rounding anomalies. - -A. The decimal module makes it easy to test results. A best practice is to -re-run calculations using greater precision and with various rounding modes. -Widely differing results indicate insufficient precision, rounding mode issues, -ill-conditioned inputs, or a numerically unstable algorithm. - -Q. I noticed that context precision is applied to the results of operations but -not to the inputs. Is there anything to watch out for when mixing values of -different precisions? - -A. Yes. The principle is that all values are considered to be exact and so is -the arithmetic on those values. Only the results are rounded. The advantage -for inputs is that "what you type is what you get". A disadvantage is that the -results can look odd if you forget that the inputs haven't been rounded: - -.. doctest:: newcontext - - >>> getcontext().prec = 3 - >>> Decimal('3.104') + Decimal('2.104') - Decimal('5.21') - >>> Decimal('3.104') + Decimal('0.000') + Decimal('2.104') - Decimal('5.20') - -The solution is either to increase precision or to force rounding of inputs -using the unary plus operation: - -.. doctest:: newcontext - - >>> getcontext().prec = 3 - >>> +Decimal('1.23456789') # unary plus triggers rounding - Decimal('1.23') - -Alternatively, inputs can be rounded upon creation using the -:meth:`Context.create_decimal` method: - - >>> Context(prec=5, rounding=ROUND_DOWN).create_decimal('1.2345678') - Decimal('1.2345') - -Q. Is the CPython implementation fast for large numbers? - -A. Yes. In the CPython and PyPy3 implementations, the C/CFFI versions of -the decimal module integrate the high speed `libmpdec -`_ library for -arbitrary precision correctly rounded decimal floating point arithmetic [#]_. -``libmpdec`` uses `Karatsuba multiplication -`_ -for medium-sized numbers and the `Number Theoretic Transform -`_ -for very large numbers. - -The context must be adapted for exact arbitrary precision arithmetic. :attr:`~Context.Emin` -and :attr:`~Context.Emax` should always be set to the maximum values, :attr:`~Context.clamp` -should always be 0 (the default). Setting :attr:`~Context.prec` requires some care. - -The easiest approach for trying out bignum arithmetic is to use the maximum -value for :attr:`~Context.prec` as well [#]_:: - - >>> setcontext(Context(prec=MAX_PREC, Emax=MAX_EMAX, Emin=MIN_EMIN)) - >>> x = Decimal(2) ** 256 - >>> x / 128 - Decimal('904625697166532776746648320380374280103671755200316906558262375061821325312') - - -For inexact results, :attr:`MAX_PREC` is far too large on 64-bit platforms and -the available memory will be insufficient:: - - >>> Decimal(1) / 3 - Traceback (most recent call last): - File "", line 1, in - MemoryError - -On systems with overallocation (e.g. Linux), a more sophisticated approach is to -adjust :attr:`~Context.prec` to the amount of available RAM. Suppose that you have 8GB of -RAM and expect 10 simultaneous operands using a maximum of 500MB each:: - - >>> import sys - >>> - >>> # Maximum number of digits for a single operand using 500MB in 8-byte words - >>> # with 19 digits per word (4-byte and 9 digits for the 32-bit build): - >>> maxdigits = 19 * ((500 * 1024**2) // 8) - >>> - >>> # Check that this works: - >>> c = Context(prec=maxdigits, Emax=MAX_EMAX, Emin=MIN_EMIN) - >>> c.traps[Inexact] = True - >>> setcontext(c) - >>> - >>> # Fill the available precision with nines: - >>> x = Decimal(0).logical_invert() * 9 - >>> sys.getsizeof(x) - 524288112 - >>> x + 2 - Traceback (most recent call last): - File "", line 1, in - decimal.Inexact: [] - -In general (and especially on systems without overallocation), it is recommended -to estimate even tighter bounds and set the :attr:`Inexact` trap if all calculations -are expected to be exact. - - -.. [#] - .. versionadded:: 3.3 - -.. [#] - .. versionchanged:: 3.9 - This approach now works for all exact results except for non-integer powers. diff --git a/Doc/library/development.rst b/Doc/library/development.rst deleted file mode 100644 index 9edce758688e2d..00000000000000 --- a/Doc/library/development.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _development: - -***************** -Development Tools -***************** - -The modules described in this chapter help you write software. For example, the -:mod:`pydoc` module takes a module and generates documentation based on the -module's contents. The :mod:`doctest` and :mod:`unittest` modules contains -frameworks for writing unit tests that automatically exercise code and verify -that the expected output is produced. :program:`2to3` can translate Python 2.x -source code into valid Python 3.x code. - -The list of modules described in this chapter is: - - -.. toctree:: - - typing.rst - pydoc.rst - devmode.rst - doctest.rst - unittest.rst - unittest.mock.rst - unittest.mock-examples.rst - 2to3.rst - test.rst diff --git a/Doc/library/devmode.rst b/Doc/library/devmode.rst deleted file mode 100644 index b3a66bf5383fdb..00000000000000 --- a/Doc/library/devmode.rst +++ /dev/null @@ -1,220 +0,0 @@ -.. _devmode: - -Python Development Mode -======================= - -.. versionadded:: 3.7 - -The Python Development Mode introduces additional runtime checks that are too -expensive to be enabled by default. It should not be more verbose than the -default if the code is correct; new warnings are only emitted when an issue is -detected. - -It can be enabled using the :option:`-X dev <-X>` command line option or by -setting the :envvar:`PYTHONDEVMODE` environment variable to ``1``. - -See also :ref:`Python debug build `. - -Effects of the Python Development Mode --------------------------------------- - -Enabling the Python Development Mode is similar to the following command, but -with additional effects described below:: - - PYTHONMALLOC=debug PYTHONASYNCIODEBUG=1 python3 -W default -X faulthandler - -Effects of the Python Development Mode: - -* Add ``default`` :ref:`warning filter `. The - following warnings are shown: - - * :exc:`DeprecationWarning` - * :exc:`ImportWarning` - * :exc:`PendingDeprecationWarning` - * :exc:`ResourceWarning` - - Normally, the above warnings are filtered by the default :ref:`warning - filters `. - - It behaves as if the :option:`-W default <-W>` command line option is used. - - Use the :option:`-W error <-W>` command line option or set the - :envvar:`PYTHONWARNINGS` environment variable to ``error`` to treat warnings - as errors. - -* Install debug hooks on memory allocators to check for: - - * Buffer underflow - * Buffer overflow - * Memory allocator API violation - * Unsafe usage of the GIL - - See the :c:func:`PyMem_SetupDebugHooks` C function. - - It behaves as if the :envvar:`PYTHONMALLOC` environment variable is set to - ``debug``. - - To enable the Python Development Mode without installing debug hooks on - memory allocators, set the :envvar:`PYTHONMALLOC` environment variable to - ``default``. - -* Call :func:`faulthandler.enable` at Python startup to install handlers for - the :const:`~signal.SIGSEGV`, :const:`~signal.SIGFPE`, - :const:`~signal.SIGABRT`, :const:`~signal.SIGBUS` and - :const:`~signal.SIGILL` signals to dump the Python traceback on a crash. - - It behaves as if the :option:`-X faulthandler <-X>` command line option is - used or if the :envvar:`PYTHONFAULTHANDLER` environment variable is set to - ``1``. - -* Enable :ref:`asyncio debug mode `. For example, - :mod:`asyncio` checks for coroutines that were not awaited and logs them. - - It behaves as if the :envvar:`PYTHONASYNCIODEBUG` environment variable is set - to ``1``. - -* Check the *encoding* and *errors* arguments for string encoding and decoding - operations. Examples: :func:`open`, :meth:`str.encode` and - :meth:`bytes.decode`. - - By default, for best performance, the *errors* argument is only checked at - the first encoding/decoding error and the *encoding* argument is sometimes - ignored for empty strings. - -* The :class:`io.IOBase` destructor logs ``close()`` exceptions. -* Set the :attr:`~sys.flags.dev_mode` attribute of :data:`sys.flags` to - ``True``. - -The Python Development Mode does not enable the :mod:`tracemalloc` module by -default, because the overhead cost (to performance and memory) would be too -large. Enabling the :mod:`tracemalloc` module provides additional information -on the origin of some errors. For example, :exc:`ResourceWarning` logs the -traceback where the resource was allocated, and a buffer overflow error logs -the traceback where the memory block was allocated. - -The Python Development Mode does not prevent the :option:`-O` command line -option from removing :keyword:`assert` statements nor from setting -:const:`__debug__` to ``False``. - -The Python Development Mode can only be enabled at the Python startup. Its -value can be read from :data:`sys.flags.dev_mode `. - -.. versionchanged:: 3.8 - The :class:`io.IOBase` destructor now logs ``close()`` exceptions. - -.. versionchanged:: 3.9 - The *encoding* and *errors* arguments are now checked for string encoding - and decoding operations. - - -ResourceWarning Example ------------------------ - -Example of a script counting the number of lines of the text file specified in -the command line:: - - import sys - - def main(): - fp = open(sys.argv[1]) - nlines = len(fp.readlines()) - print(nlines) - # The file is closed implicitly - - if __name__ == "__main__": - main() - -The script does not close the file explicitly. By default, Python does not emit -any warning. Example using README.txt, which has 269 lines: - -.. code-block:: shell-session - - $ python3 script.py README.txt - 269 - -Enabling the Python Development Mode displays a :exc:`ResourceWarning` warning: - -.. code-block:: shell-session - - $ python3 -X dev script.py README.txt - 269 - script.py:10: ResourceWarning: unclosed file <_io.TextIOWrapper name='README.rst' mode='r' encoding='UTF-8'> - main() - ResourceWarning: Enable tracemalloc to get the object allocation traceback - -In addition, enabling :mod:`tracemalloc` shows the line where the file was -opened: - -.. code-block:: shell-session - - $ python3 -X dev -X tracemalloc=5 script.py README.rst - 269 - script.py:10: ResourceWarning: unclosed file <_io.TextIOWrapper name='README.rst' mode='r' encoding='UTF-8'> - main() - Object allocated at (most recent call last): - File "script.py", lineno 10 - main() - File "script.py", lineno 4 - fp = open(sys.argv[1]) - -The fix is to close explicitly the file. Example using a context manager:: - - def main(): - # Close the file explicitly when exiting the with block - with open(sys.argv[1]) as fp: - nlines = len(fp.readlines()) - print(nlines) - -Not closing a resource explicitly can leave a resource open for way longer than -expected; it can cause severe issues upon exiting Python. It is bad in -CPython, but it is even worse in PyPy. Closing resources explicitly makes an -application more deterministic and more reliable. - - -Bad file descriptor error example ---------------------------------- - -Script displaying the first line of itself:: - - import os - - def main(): - fp = open(__file__) - firstline = fp.readline() - print(firstline.rstrip()) - os.close(fp.fileno()) - # The file is closed implicitly - - main() - -By default, Python does not emit any warning: - -.. code-block:: shell-session - - $ python3 script.py - import os - -The Python Development Mode shows a :exc:`ResourceWarning` and logs a "Bad file -descriptor" error when finalizing the file object: - -.. code-block:: shell-session - - $ python3 -X dev script.py - import os - script.py:10: ResourceWarning: unclosed file <_io.TextIOWrapper name='script.py' mode='r' encoding='UTF-8'> - main() - ResourceWarning: Enable tracemalloc to get the object allocation traceback - Exception ignored in: <_io.TextIOWrapper name='script.py' mode='r' encoding='UTF-8'> - Traceback (most recent call last): - File "script.py", line 10, in - main() - OSError: [Errno 9] Bad file descriptor - -``os.close(fp.fileno())`` closes the file descriptor. When the file object -finalizer tries to close the file descriptor again, it fails with the ``Bad -file descriptor`` error. A file descriptor must be closed only once. In the -worst case scenario, closing it twice can lead to a crash (see :issue:`18748` -for an example). - -The fix is to remove the ``os.close(fp.fileno())`` line, or open the file with -``closefd=False``. diff --git a/Doc/library/dialog.rst b/Doc/library/dialog.rst deleted file mode 100644 index 191e0da12103fa..00000000000000 --- a/Doc/library/dialog.rst +++ /dev/null @@ -1,230 +0,0 @@ -Tkinter Dialogs -=============== - -:mod:`tkinter.simpledialog` --- Standard Tkinter input dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. module:: tkinter.simpledialog - :platform: Tk - :synopsis: Simple dialog windows - -**Source code:** :source:`Lib/tkinter/simpledialog.py` - --------------- - -The :mod:`tkinter.simpledialog` module contains convenience classes and -functions for creating simple modal dialogs to get a value from the user. - - -.. function:: askfloat(title, prompt, **kw) - askinteger(title, prompt, **kw) - askstring(title, prompt, **kw) - - The above three functions provide dialogs that prompt the user to enter a value - of the desired type. - -.. class:: Dialog(parent, title=None) - - The base class for custom dialogs. - - .. method:: body(master) - - Override to construct the dialog's interface and return the widget that - should have initial focus. - - .. method:: buttonbox() - - Default behaviour adds OK and Cancel buttons. Override for custom button - layouts. - - - -:mod:`tkinter.filedialog` --- File selection dialogs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. module:: tkinter.filedialog - :platform: Tk - :synopsis: Dialog classes for file selection - -**Source code:** :source:`Lib/tkinter/filedialog.py` - --------------- - -The :mod:`tkinter.filedialog` module provides classes and factory functions for -creating file/directory selection windows. - -Native Load/Save Dialogs ------------------------- - -The following classes and functions provide file dialog windows that combine a -native look-and-feel with configuration options to customize behaviour. -The following keyword arguments are applicable to the classes and functions -listed below: - - | *parent* - the window to place the dialog on top of - - | *title* - the title of the window - - | *initialdir* - the directory that the dialog starts in - - | *initialfile* - the file selected upon opening of the dialog - - | *filetypes* - a sequence of (label, pattern) tuples, '*' wildcard is allowed - - | *defaultextension* - default extension to append to file (save dialogs) - - | *multiple* - when true, selection of multiple items is allowed - - -**Static factory functions** - -The below functions when called create a modal, native look-and-feel dialog, -wait for the user's selection, then return the selected value(s) or ``None`` to the -caller. - -.. function:: askopenfile(mode="r", **options) - askopenfiles(mode="r", **options) - - The above two functions create an :class:`Open` dialog and return the opened - file object(s) in read-only mode. - -.. function:: asksaveasfile(mode="w", **options) - - Create a :class:`SaveAs` dialog and return a file object opened in write-only mode. - -.. function:: askopenfilename(**options) - askopenfilenames(**options) - - The above two functions create an :class:`Open` dialog and return the - selected filename(s) that correspond to existing file(s). - -.. function:: asksaveasfilename(**options) - - Create a :class:`SaveAs` dialog and return the selected filename. - -.. function:: askdirectory(**options) - - | Prompt user to select a directory. - | Additional keyword option: - | *mustexist* - determines if selection must be an existing directory. - -.. class:: Open(master=None, **options) - SaveAs(master=None, **options) - - The above two classes provide native dialog windows for saving and loading - files. - -**Convenience classes** - -The below classes are used for creating file/directory windows from scratch. -These do not emulate the native look-and-feel of the platform. - -.. class:: Directory(master=None, **options) - - Create a dialog prompting the user to select a directory. - -.. note:: The *FileDialog* class should be subclassed for custom event - handling and behaviour. - -.. class:: FileDialog(master, title=None) - - Create a basic file selection dialog. - - .. method:: cancel_command(event=None) - - Trigger the termination of the dialog window. - - .. method:: dirs_double_event(event) - - Event handler for double-click event on directory. - - .. method:: dirs_select_event(event) - - Event handler for click event on directory. - - .. method:: files_double_event(event) - - Event handler for double-click event on file. - - .. method:: files_select_event(event) - - Event handler for single-click event on file. - - .. method:: filter_command(event=None) - - Filter the files by directory. - - .. method:: get_filter() - - Retrieve the file filter currently in use. - - .. method:: get_selection() - - Retrieve the currently selected item. - - .. method:: go(dir_or_file=os.curdir, pattern="*", default="", key=None) - - Render dialog and start event loop. - - .. method:: ok_event(event) - - Exit dialog returning current selection. - - .. method:: quit(how=None) - - Exit dialog returning filename, if any. - - .. method:: set_filter(dir, pat) - - Set the file filter. - - .. method:: set_selection(file) - - Update the current file selection to *file*. - - -.. class:: LoadFileDialog(master, title=None) - - A subclass of FileDialog that creates a dialog window for selecting an - existing file. - - .. method:: ok_command() - - Test that a file is provided and that the selection indicates an - already existing file. - -.. class:: SaveFileDialog(master, title=None) - - A subclass of FileDialog that creates a dialog window for selecting a - destination file. - - .. method:: ok_command() - - Test whether or not the selection points to a valid file that is not a - directory. Confirmation is required if an already existing file is - selected. - -:mod:`tkinter.commondialog` --- Dialog window templates -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. module:: tkinter.commondialog - :platform: Tk - :synopsis: Tkinter base class for dialogs - -**Source code:** :source:`Lib/tkinter/commondialog.py` - --------------- - -The :mod:`tkinter.commondialog` module provides the :class:`Dialog` class that -is the base class for dialogs defined in other supporting modules. - -.. class:: Dialog(master=None, **options) - - .. method:: show(color=None, **options) - - Render the Dialog window. - - -.. seealso:: - - Modules :mod:`tkinter.messagebox`, :ref:`tut-files` diff --git a/Doc/library/difflib.rst b/Doc/library/difflib.rst deleted file mode 100644 index 83c669e50f1f85..00000000000000 --- a/Doc/library/difflib.rst +++ /dev/null @@ -1,767 +0,0 @@ -:mod:`difflib` --- Helpers for computing deltas -=============================================== - -.. module:: difflib - :synopsis: Helpers for computing differences between objects. - -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. Markup by Fred L. Drake, Jr. - -**Source code:** :source:`Lib/difflib.py` - -.. testsetup:: - - import sys - from difflib import * - --------------- - -This module provides classes and functions for comparing sequences. It -can be used for example, for comparing files, and can produce information -about file differences in various formats, including HTML and context and unified -diffs. For comparing directories and files, see also, the :mod:`filecmp` module. - - -.. class:: SequenceMatcher - :noindex: - - This is a flexible class for comparing pairs of sequences of any type, so long - as the sequence elements are :term:`hashable`. The basic algorithm predates, and is a - little fancier than, an algorithm published in the late 1980's by Ratcliff and - Obershelp under the hyperbolic name "gestalt pattern matching." The idea is to - find the longest contiguous matching subsequence that contains no "junk" - elements; these "junk" elements are ones that are uninteresting in some - sense, such as blank lines or whitespace. (Handling junk is an - extension to the Ratcliff and Obershelp algorithm.) The same - idea is then applied recursively to the pieces of the sequences to the left and - to the right of the matching subsequence. This does not yield minimal edit - sequences, but does tend to yield matches that "look right" to people. - - **Timing:** The basic Ratcliff-Obershelp algorithm is cubic time in the worst - case and quadratic time in the expected case. :class:`SequenceMatcher` is - quadratic time for the worst case and has expected-case behavior dependent in a - complicated way on how many elements the sequences have in common; best case - time is linear. - - **Automatic junk heuristic:** :class:`SequenceMatcher` supports a heuristic that - automatically treats certain sequence items as junk. The heuristic counts how many - times each individual item appears in the sequence. If an item's duplicates (after - the first one) account for more than 1% of the sequence and the sequence is at least - 200 items long, this item is marked as "popular" and is treated as junk for - the purpose of sequence matching. This heuristic can be turned off by setting - the ``autojunk`` argument to ``False`` when creating the :class:`SequenceMatcher`. - - .. versionadded:: 3.2 - The *autojunk* parameter. - - -.. class:: Differ - - This is a class for comparing sequences of lines of text, and producing - human-readable differences or deltas. Differ uses :class:`SequenceMatcher` - both to compare sequences of lines, and to compare sequences of characters - within similar (near-matching) lines. - - Each line of a :class:`Differ` delta begins with a two-letter code: - - +----------+-------------------------------------------+ - | Code | Meaning | - +==========+===========================================+ - | ``'- '`` | line unique to sequence 1 | - +----------+-------------------------------------------+ - | ``'+ '`` | line unique to sequence 2 | - +----------+-------------------------------------------+ - | ``' '`` | line common to both sequences | - +----------+-------------------------------------------+ - | ``'? '`` | line not present in either input sequence | - +----------+-------------------------------------------+ - - Lines beginning with '``?``' attempt to guide the eye to intraline differences, - and were not present in either input sequence. These lines can be confusing if - the sequences contain tab characters. - - -.. class:: HtmlDiff - - This class can be used to create an HTML table (or a complete HTML file - containing the table) showing a side by side, line by line comparison of text - with inter-line and intra-line change highlights. The table can be generated in - either full or contextual difference mode. - - The constructor for this class is: - - - .. method:: __init__(tabsize=8, wrapcolumn=None, linejunk=None, charjunk=IS_CHARACTER_JUNK) - - Initializes instance of :class:`HtmlDiff`. - - *tabsize* is an optional keyword argument to specify tab stop spacing and - defaults to ``8``. - - *wrapcolumn* is an optional keyword to specify column number where lines are - broken and wrapped, defaults to ``None`` where lines are not wrapped. - - *linejunk* and *charjunk* are optional keyword arguments passed into :func:`ndiff` - (used by :class:`HtmlDiff` to generate the side by side HTML differences). See - :func:`ndiff` documentation for argument default values and descriptions. - - The following methods are public: - - .. method:: make_file(fromlines, tolines, fromdesc='', todesc='', context=False, \ - numlines=5, *, charset='utf-8') - - Compares *fromlines* and *tolines* (lists of strings) and returns a string which - is a complete HTML file containing a table showing line by line differences with - inter-line and intra-line changes highlighted. - - *fromdesc* and *todesc* are optional keyword arguments to specify from/to file - column header strings (both default to an empty string). - - *context* and *numlines* are both optional keyword arguments. Set *context* to - ``True`` when contextual differences are to be shown, else the default is - ``False`` to show the full files. *numlines* defaults to ``5``. When *context* - is ``True`` *numlines* controls the number of context lines which surround the - difference highlights. When *context* is ``False`` *numlines* controls the - number of lines which are shown before a difference highlight when using the - "next" hyperlinks (setting to zero would cause the "next" hyperlinks to place - the next difference highlight at the top of the browser without any leading - context). - - .. note:: - *fromdesc* and *todesc* are interpreted as unescaped HTML and should be - properly escaped while receiving input from untrusted sources. - - .. versionchanged:: 3.5 - *charset* keyword-only argument was added. The default charset of - HTML document changed from ``'ISO-8859-1'`` to ``'utf-8'``. - - .. method:: make_table(fromlines, tolines, fromdesc='', todesc='', context=False, numlines=5) - - Compares *fromlines* and *tolines* (lists of strings) and returns a string which - is a complete HTML table showing line by line differences with inter-line and - intra-line changes highlighted. - - The arguments for this method are the same as those for the :meth:`make_file` - method. - - :file:`Tools/scripts/diff.py` is a command-line front-end to this class and - contains a good example of its use. - - -.. function:: context_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') - - Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` - generating the delta lines) in context diff format. - - Context diffs are a compact way of showing just the lines that have changed plus - a few lines of context. The changes are shown in a before/after style. The - number of context lines is set by *n* which defaults to three. - - By default, the diff control lines (those with ``***`` or ``---``) are created - with a trailing newline. This is helpful so that inputs created from - :func:`io.IOBase.readlines` result in diffs that are suitable for use with - :func:`io.IOBase.writelines` since both the inputs and outputs have trailing - newlines. - - For inputs that do not have trailing newlines, set the *lineterm* argument to - ``""`` so that the output will be uniformly newline free. - - The context diff format normally has a header for filenames and modification - times. Any or all of these may be specified using strings for *fromfile*, - *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally - expressed in the ISO 8601 format. If not specified, the - strings default to blanks. - - >>> import sys - >>> from difflib import * - >>> s1 = ['bacon\n', 'eggs\n', 'ham\n', 'guido\n'] - >>> s2 = ['python\n', 'eggy\n', 'hamster\n', 'guido\n'] - >>> sys.stdout.writelines(context_diff(s1, s2, fromfile='before.py', - ... tofile='after.py')) - *** before.py - --- after.py - *************** - *** 1,4 **** - ! bacon - ! eggs - ! ham - guido - --- 1,4 ---- - ! python - ! eggy - ! hamster - guido - - See :ref:`difflib-interface` for a more detailed example. - - -.. function:: get_close_matches(word, possibilities, n=3, cutoff=0.6) - - Return a list of the best "good enough" matches. *word* is a sequence for which - close matches are desired (typically a string), and *possibilities* is a list of - sequences against which to match *word* (typically a list of strings). - - Optional argument *n* (default ``3``) is the maximum number of close matches to - return; *n* must be greater than ``0``. - - Optional argument *cutoff* (default ``0.6``) is a float in the range [0, 1]. - Possibilities that don't score at least that similar to *word* are ignored. - - The best (no more than *n*) matches among the possibilities are returned in a - list, sorted by similarity score, most similar first. - - >>> get_close_matches('appel', ['ape', 'apple', 'peach', 'puppy']) - ['apple', 'ape'] - >>> import keyword - >>> get_close_matches('wheel', keyword.kwlist) - ['while'] - >>> get_close_matches('pineapple', keyword.kwlist) - [] - >>> get_close_matches('accept', keyword.kwlist) - ['except'] - - -.. function:: ndiff(a, b, linejunk=None, charjunk=IS_CHARACTER_JUNK) - - Compare *a* and *b* (lists of strings); return a :class:`Differ`\ -style - delta (a :term:`generator` generating the delta lines). - - Optional keyword parameters *linejunk* and *charjunk* are filtering functions - (or ``None``): - - *linejunk*: A function that accepts a single string argument, and returns - true if the string is junk, or false if not. The default is ``None``. There - is also a module-level function :func:`IS_LINE_JUNK`, which filters out lines - without visible characters, except for at most one pound character (``'#'``) - -- however the underlying :class:`SequenceMatcher` class does a dynamic - analysis of which lines are so frequent as to constitute noise, and this - usually works better than using this function. - - *charjunk*: A function that accepts a character (a string of length 1), and - returns if the character is junk, or false if not. The default is module-level - function :func:`IS_CHARACTER_JUNK`, which filters out whitespace characters (a - blank or tab; it's a bad idea to include newline in this!). - - :file:`Tools/scripts/ndiff.py` is a command-line front-end to this function. - - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), - ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) - >>> print(''.join(diff), end="") - - one - ? ^ - + ore - ? ^ - - two - - three - ? - - + tree - + emu - - -.. function:: restore(sequence, which) - - Return one of the two sequences that generated a delta. - - Given a *sequence* produced by :meth:`Differ.compare` or :func:`ndiff`, extract - lines originating from file 1 or 2 (parameter *which*), stripping off line - prefixes. - - Example: - - >>> diff = ndiff('one\ntwo\nthree\n'.splitlines(keepends=True), - ... 'ore\ntree\nemu\n'.splitlines(keepends=True)) - >>> diff = list(diff) # materialize the generated delta into a list - >>> print(''.join(restore(diff, 1)), end="") - one - two - three - >>> print(''.join(restore(diff, 2)), end="") - ore - tree - emu - - -.. function:: unified_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n') - - Compare *a* and *b* (lists of strings); return a delta (a :term:`generator` - generating the delta lines) in unified diff format. - - Unified diffs are a compact way of showing just the lines that have changed plus - a few lines of context. The changes are shown in an inline style (instead of - separate before/after blocks). The number of context lines is set by *n* which - defaults to three. - - By default, the diff control lines (those with ``---``, ``+++``, or ``@@``) are - created with a trailing newline. This is helpful so that inputs created from - :func:`io.IOBase.readlines` result in diffs that are suitable for use with - :func:`io.IOBase.writelines` since both the inputs and outputs have trailing - newlines. - - For inputs that do not have trailing newlines, set the *lineterm* argument to - ``""`` so that the output will be uniformly newline free. - - The unified diff format normally has a header for filenames and modification - times. Any or all of these may be specified using strings for *fromfile*, - *tofile*, *fromfiledate*, and *tofiledate*. The modification times are normally - expressed in the ISO 8601 format. If not specified, the - strings default to blanks. - - >>> s1 = ['bacon\n', 'eggs\n', 'ham\n', 'guido\n'] - >>> s2 = ['python\n', 'eggy\n', 'hamster\n', 'guido\n'] - >>> sys.stdout.writelines(unified_diff(s1, s2, fromfile='before.py', tofile='after.py')) - --- before.py - +++ after.py - @@ -1,4 +1,4 @@ - -bacon - -eggs - -ham - +python - +eggy - +hamster - guido - - See :ref:`difflib-interface` for a more detailed example. - -.. function:: diff_bytes(dfunc, a, b, fromfile=b'', tofile=b'', fromfiledate=b'', tofiledate=b'', n=3, lineterm=b'\n') - - Compare *a* and *b* (lists of bytes objects) using *dfunc*; yield a - sequence of delta lines (also bytes) in the format returned by *dfunc*. - *dfunc* must be a callable, typically either :func:`unified_diff` or - :func:`context_diff`. - - Allows you to compare data with unknown or inconsistent encoding. All - inputs except *n* must be bytes objects, not str. Works by losslessly - converting all inputs (except *n*) to str, and calling ``dfunc(a, b, - fromfile, tofile, fromfiledate, tofiledate, n, lineterm)``. The output of - *dfunc* is then converted back to bytes, so the delta lines that you - receive have the same unknown/inconsistent encodings as *a* and *b*. - - .. versionadded:: 3.5 - -.. function:: IS_LINE_JUNK(line) - - Return ``True`` for ignorable lines. The line *line* is ignorable if *line* is - blank or contains a single ``'#'``, otherwise it is not ignorable. Used as a - default for parameter *linejunk* in :func:`ndiff` in older versions. - - -.. function:: IS_CHARACTER_JUNK(ch) - - Return ``True`` for ignorable characters. The character *ch* is ignorable if *ch* - is a space or tab, otherwise it is not ignorable. Used as a default for - parameter *charjunk* in :func:`ndiff`. - - -.. seealso:: - - `Pattern Matching: The Gestalt Approach `_ - Discussion of a similar algorithm by John W. Ratcliff and D. E. Metzener. This - was published in `Dr. Dobb's Journal `_ in July, 1988. - - -.. _sequence-matcher: - -SequenceMatcher Objects ------------------------ - -The :class:`SequenceMatcher` class has this constructor: - - -.. class:: SequenceMatcher(isjunk=None, a='', b='', autojunk=True) - - Optional argument *isjunk* must be ``None`` (the default) or a one-argument - function that takes a sequence element and returns true if and only if the - element is "junk" and should be ignored. Passing ``None`` for *isjunk* is - equivalent to passing ``lambda x: False``; in other words, no elements are ignored. - For example, pass:: - - lambda x: x in " \t" - - if you're comparing lines as sequences of characters, and don't want to synch up - on blanks or hard tabs. - - The optional arguments *a* and *b* are sequences to be compared; both default to - empty strings. The elements of both sequences must be :term:`hashable`. - - The optional argument *autojunk* can be used to disable the automatic junk - heuristic. - - .. versionadded:: 3.2 - The *autojunk* parameter. - - SequenceMatcher objects get three data attributes: *bjunk* is the - set of elements of *b* for which *isjunk* is ``True``; *bpopular* is the set of - non-junk elements considered popular by the heuristic (if it is not - disabled); *b2j* is a dict mapping the remaining elements of *b* to a list - of positions where they occur. All three are reset whenever *b* is reset - with :meth:`set_seqs` or :meth:`set_seq2`. - - .. versionadded:: 3.2 - The *bjunk* and *bpopular* attributes. - - :class:`SequenceMatcher` objects have the following methods: - - .. method:: set_seqs(a, b) - - Set the two sequences to be compared. - - :class:`SequenceMatcher` computes and caches detailed information about the - second sequence, so if you want to compare one sequence against many - sequences, use :meth:`set_seq2` to set the commonly used sequence once and - call :meth:`set_seq1` repeatedly, once for each of the other sequences. - - - .. method:: set_seq1(a) - - Set the first sequence to be compared. The second sequence to be compared - is not changed. - - - .. method:: set_seq2(b) - - Set the second sequence to be compared. The first sequence to be compared - is not changed. - - - .. method:: find_longest_match(alo=0, ahi=None, blo=0, bhi=None) - - Find longest matching block in ``a[alo:ahi]`` and ``b[blo:bhi]``. - - If *isjunk* was omitted or ``None``, :meth:`find_longest_match` returns - ``(i, j, k)`` such that ``a[i:i+k]`` is equal to ``b[j:j+k]``, where ``alo - <= i <= i+k <= ahi`` and ``blo <= j <= j+k <= bhi``. For all ``(i', j', - k')`` meeting those conditions, the additional conditions ``k >= k'``, ``i - <= i'``, and if ``i == i'``, ``j <= j'`` are also met. In other words, of - all maximal matching blocks, return one that starts earliest in *a*, and - of all those maximal matching blocks that start earliest in *a*, return - the one that starts earliest in *b*. - - >>> s = SequenceMatcher(None, " abcd", "abcd abcd") - >>> s.find_longest_match(0, 5, 0, 9) - Match(a=0, b=4, size=5) - - If *isjunk* was provided, first the longest matching block is determined - as above, but with the additional restriction that no junk element appears - in the block. Then that block is extended as far as possible by matching - (only) junk elements on both sides. So the resulting block never matches - on junk except as identical junk happens to be adjacent to an interesting - match. - - Here's the same example as before, but considering blanks to be junk. That - prevents ``' abcd'`` from matching the ``' abcd'`` at the tail end of the - second sequence directly. Instead only the ``'abcd'`` can match, and - matches the leftmost ``'abcd'`` in the second sequence: - - >>> s = SequenceMatcher(lambda x: x==" ", " abcd", "abcd abcd") - >>> s.find_longest_match(0, 5, 0, 9) - Match(a=1, b=0, size=4) - - If no blocks match, this returns ``(alo, blo, 0)``. - - This method returns a :term:`named tuple` ``Match(a, b, size)``. - - .. versionchanged:: 3.9 - Added default arguments. - - - .. method:: get_matching_blocks() - - Return list of triples describing non-overlapping matching subsequences. - Each triple is of the form ``(i, j, n)``, - and means that ``a[i:i+n] == b[j:j+n]``. The - triples are monotonically increasing in *i* and *j*. - - The last triple is a dummy, and has the value ``(len(a), len(b), 0)``. It - is the only triple with ``n == 0``. If ``(i, j, n)`` and ``(i', j', n')`` - are adjacent triples in the list, and the second is not the last triple in - the list, then ``i+n < i'`` or ``j+n < j'``; in other words, adjacent - triples always describe non-adjacent equal blocks. - - .. XXX Explain why a dummy is used! - - .. doctest:: - - >>> s = SequenceMatcher(None, "abxcd", "abcd") - >>> s.get_matching_blocks() - [Match(a=0, b=0, size=2), Match(a=3, b=2, size=2), Match(a=5, b=4, size=0)] - - - .. method:: get_opcodes() - - Return list of 5-tuples describing how to turn *a* into *b*. Each tuple is - of the form ``(tag, i1, i2, j1, j2)``. The first tuple has ``i1 == j1 == - 0``, and remaining tuples have *i1* equal to the *i2* from the preceding - tuple, and, likewise, *j1* equal to the previous *j2*. - - The *tag* values are strings, with these meanings: - - +---------------+---------------------------------------------+ - | Value | Meaning | - +===============+=============================================+ - | ``'replace'`` | ``a[i1:i2]`` should be replaced by | - | | ``b[j1:j2]``. | - +---------------+---------------------------------------------+ - | ``'delete'`` | ``a[i1:i2]`` should be deleted. Note that | - | | ``j1 == j2`` in this case. | - +---------------+---------------------------------------------+ - | ``'insert'`` | ``b[j1:j2]`` should be inserted at | - | | ``a[i1:i1]``. Note that ``i1 == i2`` in | - | | this case. | - +---------------+---------------------------------------------+ - | ``'equal'`` | ``a[i1:i2] == b[j1:j2]`` (the sub-sequences | - | | are equal). | - +---------------+---------------------------------------------+ - - For example:: - - >>> a = "qabxcd" - >>> b = "abycdf" - >>> s = SequenceMatcher(None, a, b) - >>> for tag, i1, i2, j1, j2 in s.get_opcodes(): - ... print('{:7} a[{}:{}] --> b[{}:{}] {!r:>8} --> {!r}'.format( - ... tag, i1, i2, j1, j2, a[i1:i2], b[j1:j2])) - delete a[0:1] --> b[0:0] 'q' --> '' - equal a[1:3] --> b[0:2] 'ab' --> 'ab' - replace a[3:4] --> b[2:3] 'x' --> 'y' - equal a[4:6] --> b[3:5] 'cd' --> 'cd' - insert a[6:6] --> b[5:6] '' --> 'f' - - - .. method:: get_grouped_opcodes(n=3) - - Return a :term:`generator` of groups with up to *n* lines of context. - - Starting with the groups returned by :meth:`get_opcodes`, this method - splits out smaller change clusters and eliminates intervening ranges which - have no changes. - - The groups are returned in the same format as :meth:`get_opcodes`. - - - .. method:: ratio() - - Return a measure of the sequences' similarity as a float in the range [0, - 1]. - - Where T is the total number of elements in both sequences, and M is the - number of matches, this is 2.0\*M / T. Note that this is ``1.0`` if the - sequences are identical, and ``0.0`` if they have nothing in common. - - This is expensive to compute if :meth:`get_matching_blocks` or - :meth:`get_opcodes` hasn't already been called, in which case you may want - to try :meth:`quick_ratio` or :meth:`real_quick_ratio` first to get an - upper bound. - - .. note:: - - Caution: The result of a :meth:`ratio` call may depend on the order of - the arguments. For instance:: - - >>> SequenceMatcher(None, 'tide', 'diet').ratio() - 0.25 - >>> SequenceMatcher(None, 'diet', 'tide').ratio() - 0.5 - - - .. method:: quick_ratio() - - Return an upper bound on :meth:`ratio` relatively quickly. - - - .. method:: real_quick_ratio() - - Return an upper bound on :meth:`ratio` very quickly. - - -The three methods that return the ratio of matching to total characters can give -different results due to differing levels of approximation, although -:meth:`~SequenceMatcher.quick_ratio` and :meth:`~SequenceMatcher.real_quick_ratio` -are always at least as large as :meth:`~SequenceMatcher.ratio`: - - >>> s = SequenceMatcher(None, "abcd", "bcde") - >>> s.ratio() - 0.75 - >>> s.quick_ratio() - 0.75 - >>> s.real_quick_ratio() - 1.0 - - -.. _sequencematcher-examples: - -SequenceMatcher Examples ------------------------- - -This example compares two strings, considering blanks to be "junk": - - >>> s = SequenceMatcher(lambda x: x == " ", - ... "private Thread currentThread;", - ... "private volatile Thread currentThread;") - -:meth:`~SequenceMatcher.ratio` returns a float in [0, 1], measuring the similarity of the -sequences. As a rule of thumb, a :meth:`~SequenceMatcher.ratio` value over 0.6 means the -sequences are close matches: - - >>> print(round(s.ratio(), 3)) - 0.866 - -If you're only interested in where the sequences match, -:meth:`~SequenceMatcher.get_matching_blocks` is handy: - - >>> for block in s.get_matching_blocks(): - ... print("a[%d] and b[%d] match for %d elements" % block) - a[0] and b[0] match for 8 elements - a[8] and b[17] match for 21 elements - a[29] and b[38] match for 0 elements - -Note that the last tuple returned by :meth:`~SequenceMatcher.get_matching_blocks` -is always a dummy, ``(len(a), len(b), 0)``, and this is the only case in which the last -tuple element (number of elements matched) is ``0``. - -If you want to know how to change the first sequence into the second, use -:meth:`~SequenceMatcher.get_opcodes`: - - >>> for opcode in s.get_opcodes(): - ... print("%6s a[%d:%d] b[%d:%d]" % opcode) - equal a[0:8] b[0:8] - insert a[8:8] b[8:17] - equal a[8:29] b[17:38] - -.. seealso:: - - * The :func:`get_close_matches` function in this module which shows how - simple code building on :class:`SequenceMatcher` can be used to do useful - work. - - * `Simple version control recipe - `_ for a small application - built with :class:`SequenceMatcher`. - - -.. _differ-objects: - -Differ Objects --------------- - -Note that :class:`Differ`\ -generated deltas make no claim to be **minimal** -diffs. To the contrary, minimal diffs are often counter-intuitive, because they -synch up anywhere possible, sometimes accidental matches 100 pages apart. -Restricting synch points to contiguous matches preserves some notion of -locality, at the occasional cost of producing a longer diff. - -The :class:`Differ` class has this constructor: - - -.. class:: Differ(linejunk=None, charjunk=None) - :noindex: - - Optional keyword parameters *linejunk* and *charjunk* are for filter functions - (or ``None``): - - *linejunk*: A function that accepts a single string argument, and returns true - if the string is junk. The default is ``None``, meaning that no line is - considered junk. - - *charjunk*: A function that accepts a single character argument (a string of - length 1), and returns true if the character is junk. The default is ``None``, - meaning that no character is considered junk. - - These junk-filtering functions speed up matching to find - differences and do not cause any differing lines or characters to - be ignored. Read the description of the - :meth:`~SequenceMatcher.find_longest_match` method's *isjunk* - parameter for an explanation. - - :class:`Differ` objects are used (deltas generated) via a single method: - - - .. method:: Differ.compare(a, b) - - Compare two sequences of lines, and generate the delta (a sequence of lines). - - Each sequence must contain individual single-line strings ending with - newlines. Such sequences can be obtained from the - :meth:`~io.IOBase.readlines` method of file-like objects. The delta - generated also consists of newline-terminated strings, ready to be - printed as-is via the :meth:`~io.IOBase.writelines` method of a - file-like object. - - -.. _differ-examples: - -Differ Example --------------- - -This example compares two texts. First we set up the texts, sequences of -individual single-line strings ending with newlines (such sequences can also be -obtained from the :meth:`~io.IOBase.readlines` method of file-like objects): - - >>> text1 = ''' 1. Beautiful is better than ugly. - ... 2. Explicit is better than implicit. - ... 3. Simple is better than complex. - ... 4. Complex is better than complicated. - ... '''.splitlines(keepends=True) - >>> len(text1) - 4 - >>> text1[0][-1] - '\n' - >>> text2 = ''' 1. Beautiful is better than ugly. - ... 3. Simple is better than complex. - ... 4. Complicated is better than complex. - ... 5. Flat is better than nested. - ... '''.splitlines(keepends=True) - -Next we instantiate a Differ object: - - >>> d = Differ() - -Note that when instantiating a :class:`Differ` object we may pass functions to -filter out line and character "junk." See the :meth:`Differ` constructor for -details. - -Finally, we compare the two: - - >>> result = list(d.compare(text1, text2)) - -``result`` is a list of strings, so let's pretty-print it: - - >>> from pprint import pprint - >>> pprint(result) - [' 1. Beautiful is better than ugly.\n', - '- 2. Explicit is better than implicit.\n', - '- 3. Simple is better than complex.\n', - '+ 3. Simple is better than complex.\n', - '? ++\n', - '- 4. Complex is better than complicated.\n', - '? ^ ---- ^\n', - '+ 4. Complicated is better than complex.\n', - '? ++++ ^ ^\n', - '+ 5. Flat is better than nested.\n'] - -As a single multi-line string it looks like this: - - >>> import sys - >>> sys.stdout.writelines(result) - 1. Beautiful is better than ugly. - - 2. Explicit is better than implicit. - - 3. Simple is better than complex. - + 3. Simple is better than complex. - ? ++ - - 4. Complex is better than complicated. - ? ^ ---- ^ - + 4. Complicated is better than complex. - ? ++++ ^ ^ - + 5. Flat is better than nested. - - -.. _difflib-interface: - -A command-line interface to difflib ------------------------------------ - -This example shows how to use difflib to create a ``diff``-like utility. -It is also contained in the Python source distribution, as -:file:`Tools/scripts/diff.py`. - -.. literalinclude:: ../../Tools/scripts/diff.py diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst deleted file mode 100644 index a12f5937cd379e..00000000000000 --- a/Doc/library/dis.rst +++ /dev/null @@ -1,1461 +0,0 @@ -:mod:`dis` --- Disassembler for Python bytecode -=============================================== - -.. module:: dis - :synopsis: Disassembler for Python bytecode. - -**Source code:** :source:`Lib/dis.py` - -.. testsetup:: - - import dis - def myfunc(alist): - return len(alist) - --------------- - -The :mod:`dis` module supports the analysis of CPython :term:`bytecode` by -disassembling it. The CPython bytecode which this module takes as an input is -defined in the file :file:`Include/opcode.h` and used by the compiler and the -interpreter. - -.. impl-detail:: - - Bytecode is an implementation detail of the CPython interpreter. No - guarantees are made that bytecode will not be added, removed, or changed - between versions of Python. Use of this module should not be considered to - work across Python VMs or Python releases. - - .. versionchanged:: 3.6 - Use 2 bytes for each instruction. Previously the number of bytes varied - by instruction. - - .. versionchanged:: 3.10 - The argument of jump, exception handling and loop instructions is now - the instruction offset rather than the byte offset. - - .. versionchanged:: 3.11 - Some instructions are accompanied by one or more inline cache entries, - which take the form of :opcode:`CACHE` instructions. These instructions - are hidden by default, but can be shown by passing ``show_caches=True`` to - any :mod:`dis` utility. Furthermore, the interpreter now adapts the - bytecode to specialize it for different runtime conditions. The - adaptive bytecode can be shown by passing ``adaptive=True``. - - -Example: Given the function :func:`myfunc`:: - - def myfunc(alist): - return len(alist) - -the following command can be used to display the disassembly of -:func:`myfunc`: - -.. doctest:: - - >>> dis.dis(myfunc) - 2 0 RESUME 0 - - 3 2 LOAD_GLOBAL 1 (NULL + len) - 14 LOAD_FAST 0 (alist) - 16 PRECALL 1 - 20 CALL 1 - 30 RETURN_VALUE - -(The "2" is a line number). - -.. _dis-cli: - -Command-line interface ----------------------- - -The :mod:`dis` module can be invoked as a script from the command line: - -.. code-block:: sh - - python -m dis [-h] [-C] [infile] - -The following options are accepted: - -.. program:: dis - -.. cmdoption:: -h, --help - - Display usage and exit. - -.. cmdoption:: -C, --show-caches - - Show inline caches. - -If :file:`infile` is specified, its disassembled code will be written to stdout. -Otherwise, disassembly is performed on compiled source code recieved from stdin. - -Bytecode analysis ------------------ - -.. versionadded:: 3.4 - -The bytecode analysis API allows pieces of Python code to be wrapped in a -:class:`Bytecode` object that provides easy access to details of the compiled -code. - -.. class:: Bytecode(x, *, first_line=None, current_offset=None,\ - show_caches=False, adaptive=False) - - Analyse the bytecode corresponding to a function, generator, asynchronous - generator, coroutine, method, string of source code, or a code object (as - returned by :func:`compile`). - - This is a convenience wrapper around many of the functions listed below, most - notably :func:`get_instructions`, as iterating over a :class:`Bytecode` - instance yields the bytecode operations as :class:`Instruction` instances. - - If *first_line* is not ``None``, it indicates the line number that should be - reported for the first source line in the disassembled code. Otherwise, the - source line information (if any) is taken directly from the disassembled code - object. - - If *current_offset* is not ``None``, it refers to an instruction offset in the - disassembled code. Setting this means :meth:`.dis` will display a "current - instruction" marker against the specified opcode. - - If *show_caches* is ``True``, :meth:`.dis` will display inline cache - entries used by the interpreter to specialize the bytecode. - - If *adaptive* is ``True``, :meth:`.dis` will display specialized bytecode - that may be different from the original bytecode. - - .. classmethod:: from_traceback(tb, *, show_caches=False) - - Construct a :class:`Bytecode` instance from the given traceback, setting - *current_offset* to the instruction responsible for the exception. - - .. data:: codeobj - - The compiled code object. - - .. data:: first_line - - The first source line of the code object (if available) - - .. method:: dis() - - Return a formatted view of the bytecode operations (the same as printed by - :func:`dis.dis`, but returned as a multi-line string). - - .. method:: info() - - Return a formatted multi-line string with detailed information about the - code object, like :func:`code_info`. - - .. versionchanged:: 3.7 - This can now handle coroutine and asynchronous generator objects. - - .. versionchanged:: 3.11 - Added the *show_caches* and *adaptive* parameters. - -Example: - -.. doctest:: - - >>> bytecode = dis.Bytecode(myfunc) - >>> for instr in bytecode: - ... print(instr.opname) - ... - RESUME - LOAD_GLOBAL - LOAD_FAST - PRECALL - CALL - RETURN_VALUE - - -Analysis functions ------------------- - -The :mod:`dis` module also defines the following analysis functions that convert -the input directly to the desired output. They can be useful if only a single -operation is being performed, so the intermediate analysis object isn't useful: - -.. function:: code_info(x) - - Return a formatted multi-line string with detailed code object information - for the supplied function, generator, asynchronous generator, coroutine, - method, source code string or code object. - - Note that the exact contents of code info strings are highly implementation - dependent and they may change arbitrarily across Python VMs or Python - releases. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.7 - This can now handle coroutine and asynchronous generator objects. - - -.. function:: show_code(x, *, file=None) - - Print detailed code object information for the supplied function, method, - source code string or code object to *file* (or ``sys.stdout`` if *file* - is not specified). - - This is a convenient shorthand for ``print(code_info(x), file=file)``, - intended for interactive exploration at the interpreter prompt. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Added *file* parameter. - - -.. function:: dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False) - - Disassemble the *x* object. *x* can denote either a module, a class, a - method, a function, a generator, an asynchronous generator, a coroutine, - a code object, a string of source code or a byte sequence of raw bytecode. - For a module, it disassembles all functions. For a class, it disassembles - all methods (including class and static methods). For a code object or - sequence of raw bytecode, it prints one line per bytecode instruction. - It also recursively disassembles nested code objects (the code of - comprehensions, generator expressions and nested functions, and the code - used for building nested classes). - Strings are first compiled to code objects with the :func:`compile` - built-in function before being disassembled. If no object is provided, this - function disassembles the last traceback. - - The disassembly is written as text to the supplied *file* argument if - provided and to ``sys.stdout`` otherwise. - - The maximal depth of recursion is limited by *depth* unless it is ``None``. - ``depth=0`` means no recursion. - - If *show_caches* is ``True``, this function will display inline cache - entries used by the interpreter to specialize the bytecode. - - If *adaptive* is ``True``, this function will display specialized bytecode - that may be different from the original bytecode. - - .. versionchanged:: 3.4 - Added *file* parameter. - - .. versionchanged:: 3.7 - Implemented recursive disassembling and added *depth* parameter. - - .. versionchanged:: 3.7 - This can now handle coroutine and asynchronous generator objects. - - .. versionchanged:: 3.11 - Added the *show_caches* and *adaptive* parameters. - - -.. function:: distb(tb=None, *, file=None, show_caches=False, adaptive=False) - - Disassemble the top-of-stack function of a traceback, using the last - traceback if none was passed. The instruction causing the exception is - indicated. - - The disassembly is written as text to the supplied *file* argument if - provided and to ``sys.stdout`` otherwise. - - .. versionchanged:: 3.4 - Added *file* parameter. - - .. versionchanged:: 3.11 - Added the *show_caches* and *adaptive* parameters. - - -.. function:: disassemble(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) - disco(code, lasti=-1, *, file=None, show_caches=False, adaptive=False) - - Disassemble a code object, indicating the last instruction if *lasti* was - provided. The output is divided in the following columns: - - #. the line number, for the first instruction of each line - #. the current instruction, indicated as ``-->``, - #. a labelled instruction, indicated with ``>>``, - #. the address of the instruction, - #. the operation code name, - #. operation parameters, and - #. interpretation of the parameters in parentheses. - - The parameter interpretation recognizes local and global variable names, - constant values, branch targets, and compare operators. - - The disassembly is written as text to the supplied *file* argument if - provided and to ``sys.stdout`` otherwise. - - .. versionchanged:: 3.4 - Added *file* parameter. - - .. versionchanged:: 3.11 - Added the *show_caches* and *adaptive* parameters. - - -.. function:: get_instructions(x, *, first_line=None, show_caches=False, adaptive=False) - - Return an iterator over the instructions in the supplied function, method, - source code string or code object. - - The iterator generates a series of :class:`Instruction` named tuples giving - the details of each operation in the supplied code. - - If *first_line* is not ``None``, it indicates the line number that should be - reported for the first source line in the disassembled code. Otherwise, the - source line information (if any) is taken directly from the disassembled code - object. - - The *show_caches* and *adaptive* parameters work as they do in :func:`dis`. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.11 - Added the *show_caches* and *adaptive* parameters. - - -.. function:: findlinestarts(code) - - This generator function uses the ``co_lines`` method - of the code object *code* to find the offsets which are starts of - lines in the source code. They are generated as ``(offset, lineno)`` pairs. - - .. versionchanged:: 3.6 - Line numbers can be decreasing. Before, they were always increasing. - - .. versionchanged:: 3.10 - The :pep:`626` ``co_lines`` method is used instead of the ``co_firstlineno`` - and ``co_lnotab`` attributes of the code object. - - -.. function:: findlabels(code) - - Detect all offsets in the raw compiled bytecode string *code* which are jump targets, and - return a list of these offsets. - - -.. function:: stack_effect(opcode, oparg=None, *, jump=None) - - Compute the stack effect of *opcode* with argument *oparg*. - - If the code has a jump target and *jump* is ``True``, :func:`~stack_effect` - will return the stack effect of jumping. If *jump* is ``False``, - it will return the stack effect of not jumping. And if *jump* is - ``None`` (default), it will return the maximal stack effect of both cases. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.8 - Added *jump* parameter. - - -.. _bytecodes: - -Python Bytecode Instructions ----------------------------- - -The :func:`get_instructions` function and :class:`Bytecode` class provide -details of bytecode instructions as :class:`Instruction` instances: - -.. class:: Instruction - - Details for a bytecode operation - - .. data:: opcode - - numeric code for operation, corresponding to the opcode values listed - below and the bytecode values in the :ref:`opcode_collections`. - - - .. data:: opname - - human readable name for operation - - - .. data:: arg - - numeric argument to operation (if any), otherwise ``None`` - - - .. data:: argval - - resolved arg value (if any), otherwise ``None`` - - - .. data:: argrepr - - human readable description of operation argument (if any), - otherwise an empty string. - - - .. data:: offset - - start index of operation within bytecode sequence - - - .. data:: starts_line - - line started by this opcode (if any), otherwise ``None`` - - - .. data:: is_jump_target - - ``True`` if other code jumps to here, otherwise ``False`` - - - .. data:: positions - - :class:`dis.Positions` object holding the - start and end locations that are covered by this instruction. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.11 - - Field ``positions`` is added. - - -.. class:: Positions - - In case the information is not available, some fields might be ``None``. - - .. data:: lineno - .. data:: end_lineno - .. data:: col_offset - .. data:: end_col_offset - - .. versionadded:: 3.11 - - -The Python compiler currently generates the following bytecode instructions. - - -**General instructions** - -.. opcode:: NOP - - Do nothing code. Used as a placeholder by the bytecode optimizer, and to - generate line tracing events. - - -.. opcode:: POP_TOP - - Removes the top-of-stack (TOS) item. - - -.. opcode:: COPY (i) - - Push the *i*-th item to the top of the stack. The item is not removed from its - original location. - - .. versionadded:: 3.11 - - -.. opcode:: SWAP (i) - - Swap TOS with the item at position *i*. - - .. versionadded:: 3.11 - - -.. opcode:: CACHE - - Rather than being an actual instruction, this opcode is used to mark extra - space for the interpreter to cache useful data directly in the bytecode - itself. It is automatically hidden by all ``dis`` utilities, but can be - viewed with ``show_caches=True``. - - Logically, this space is part of the preceding instruction. Many opcodes - expect to be followed by an exact number of caches, and will instruct the - interpreter to skip over them at runtime. - - Populated caches can look like arbitrary instructions, so great care should - be taken when reading or modifying raw, adaptive bytecode containing - quickened data. - - .. versionadded:: 3.11 - - -**Unary operations** - -Unary operations take the top of the stack, apply the operation, and push the -result back on the stack. - -.. opcode:: UNARY_POSITIVE - - Implements ``TOS = +TOS``. - - -.. opcode:: UNARY_NEGATIVE - - Implements ``TOS = -TOS``. - - -.. opcode:: UNARY_NOT - - Implements ``TOS = not TOS``. - - -.. opcode:: UNARY_INVERT - - Implements ``TOS = ~TOS``. - - -.. opcode:: GET_ITER - - Implements ``TOS = iter(TOS)``. - - -.. opcode:: GET_YIELD_FROM_ITER - - If ``TOS`` is a :term:`generator iterator` or :term:`coroutine` object - it is left as is. Otherwise, implements ``TOS = iter(TOS)``. - - .. versionadded:: 3.5 - - -**Binary and in-place operations** - -Binary operations remove the top of the stack (TOS) and the second top-most -stack item (TOS1) from the stack. They perform the operation, and put the -result back on the stack. - -In-place operations are like binary operations, in that they remove TOS and -TOS1, and push the result back on the stack, but the operation is done in-place -when TOS1 supports it, and the resulting TOS may be (but does not have to be) -the original TOS1. - - -.. opcode:: BINARY_OP (op) - - Implements the binary and in-place operators (depending on the value of - *op*). - - .. versionadded:: 3.11 - - -.. opcode:: BINARY_SUBSCR - - Implements ``TOS = TOS1[TOS]``. - - -.. opcode:: STORE_SUBSCR - - Implements ``TOS1[TOS] = TOS2``. - - -.. opcode:: DELETE_SUBSCR - - Implements ``del TOS1[TOS]``. - - -**Coroutine opcodes** - -.. opcode:: GET_AWAITABLE (where) - - Implements ``TOS = get_awaitable(TOS)``, where ``get_awaitable(o)`` - returns ``o`` if ``o`` is a coroutine object or a generator object with - the CO_ITERABLE_COROUTINE flag, or resolves - ``o.__await__``. - - If the ``where`` operand is nonzero, it indicates where the instruction - occurs: - - * ``1`` After a call to ``__aenter__`` - * ``2`` After a call to ``__aexit__`` - - .. versionadded:: 3.5 - - .. versionchanged:: 3.11 - Previously, this instruction did not have an oparg. - - -.. opcode:: GET_AITER - - Implements ``TOS = TOS.__aiter__()``. - - .. versionadded:: 3.5 - .. versionchanged:: 3.7 - Returning awaitable objects from ``__aiter__`` is no longer - supported. - - -.. opcode:: GET_ANEXT - - Pushes ``get_awaitable(TOS.__anext__())`` to the stack. See - ``GET_AWAITABLE`` for details about ``get_awaitable``. - - .. versionadded:: 3.5 - - -.. opcode:: END_ASYNC_FOR - - Terminates an :keyword:`async for` loop. Handles an exception raised - when awaiting a next item. The stack contains the async iterable in - TOS1 and the raised exception in TOS. Both are popped. - If the exception is not :exc:`StopAsyncIteration`, it is re-raised. - - .. versionadded:: 3.8 - - .. versionchanged:: 3.11 - Exception representation on the stack now consist of one, not three, items. - -.. opcode:: BEFORE_ASYNC_WITH - - Resolves ``__aenter__`` and ``__aexit__`` from the object on top of the - stack. Pushes ``__aexit__`` and result of ``__aenter__()`` to the stack. - - .. versionadded:: 3.5 - - - -**Miscellaneous opcodes** - -.. opcode:: PRINT_EXPR - - Implements the expression statement for the interactive mode. TOS is removed - from the stack and printed. In non-interactive mode, an expression statement - is terminated with :opcode:`POP_TOP`. - - -.. opcode:: SET_ADD (i) - - Calls ``set.add(TOS1[-i], TOS)``. Used to implement set comprehensions. - - -.. opcode:: LIST_APPEND (i) - - Calls ``list.append(TOS1[-i], TOS)``. Used to implement list comprehensions. - - -.. opcode:: MAP_ADD (i) - - Calls ``dict.__setitem__(TOS1[-i], TOS1, TOS)``. Used to implement dict - comprehensions. - - .. versionadded:: 3.1 - .. versionchanged:: 3.8 - Map value is TOS and map key is TOS1. Before, those were reversed. - -For all of the :opcode:`SET_ADD`, :opcode:`LIST_APPEND` and :opcode:`MAP_ADD` -instructions, while the added value or key/value pair is popped off, the -container object remains on the stack so that it is available for further -iterations of the loop. - - -.. opcode:: RETURN_VALUE - - Returns with TOS to the caller of the function. - - -.. opcode:: YIELD_VALUE - - Pops TOS and yields it from a :term:`generator`. - - - -.. opcode:: SETUP_ANNOTATIONS - - Checks whether ``__annotations__`` is defined in ``locals()``, if not it is - set up to an empty ``dict``. This opcode is only emitted if a class - or module body contains :term:`variable annotations ` - statically. - - .. versionadded:: 3.6 - - -.. opcode:: IMPORT_STAR - - Loads all symbols not starting with ``'_'`` directly from the module TOS to - the local namespace. The module is popped after loading all names. This - opcode implements ``from module import *``. - - -.. opcode:: POP_EXCEPT - - Pops a value from the stack, which is used to restore the exception state. - - .. versionchanged:: 3.11 - Exception representation on the stack now consist of one, not three, items. - -.. opcode:: RERAISE - - Re-raises the exception currently on top of the stack. If oparg is non-zero, - pops an additional value from the stack which is used to set ``f_lasti`` - of the current frame. - - .. versionadded:: 3.9 - - .. versionchanged:: 3.11 - Exception representation on the stack now consist of one, not three, items. - -.. opcode:: PUSH_EXC_INFO - - Pops a value from the stack. Pushes the current exception to the top of the stack. - Pushes the value originally popped back to the stack. - Used in exception handlers. - - .. versionadded:: 3.11 - -.. opcode:: CHECK_EXC_MATCH - - Performs exception matching for ``except``. Tests whether the TOS1 is an exception - matching TOS. Pops TOS and pushes the boolean result of the test. - - .. versionadded:: 3.11 - -.. opcode:: CHECK_EG_MATCH - - Performs exception matching for ``except*``. Applies ``split(TOS)`` on - the exception group representing TOS1. - - In case of a match, pops two items from the stack and pushes the - non-matching subgroup (``None`` in case of full match) followed by the - matching subgroup. When there is no match, pops one item (the match - type) and pushes ``None``. - - .. versionadded:: 3.11 - -.. opcode:: PREP_RERAISE_STAR - - Combines the raised and reraised exceptions list from TOS, into an exception - group to propagate from a try-except* block. Uses the original exception - group from TOS1 to reconstruct the structure of reraised exceptions. Pops - two items from the stack and pushes the exception to reraise or ``None`` - if there isn't one. - - .. versionadded:: 3.11 - -.. opcode:: WITH_EXCEPT_START - - Calls the function in position 4 on the stack with arguments (type, val, tb) - representing the exception at the top of the stack. - Used to implement the call ``context_manager.__exit__(*exc_info())`` when an exception - has occurred in a :keyword:`with` statement. - - .. versionadded:: 3.9 - - .. versionchanged:: 3.11 - The ``__exit__`` function is in position 4 of the stack rather than 7. - Exception representation on the stack now consist of one, not three, items. - - -.. opcode:: LOAD_ASSERTION_ERROR - - Pushes :exc:`AssertionError` onto the stack. Used by the :keyword:`assert` - statement. - - .. versionadded:: 3.9 - - -.. opcode:: LOAD_BUILD_CLASS - - Pushes :func:`builtins.__build_class__` onto the stack. It is later called - to construct a class. - - -.. opcode:: BEFORE_WITH (delta) - - This opcode performs several operations before a with block starts. First, - it loads :meth:`~object.__exit__` from the context manager and pushes it onto - the stack for later use by :opcode:`WITH_EXCEPT_START`. Then, - :meth:`~object.__enter__` is called. Finally, the result of calling the - ``__enter__()`` method is pushed onto the stack. - - .. versionadded:: 3.11 - - -.. opcode:: GET_LEN - - Push ``len(TOS)`` onto the stack. - - .. versionadded:: 3.10 - - -.. opcode:: MATCH_MAPPING - - If TOS is an instance of :class:`collections.abc.Mapping` (or, more technically: if - it has the :c:macro:`Py_TPFLAGS_MAPPING` flag set in its - :c:member:`~PyTypeObject.tp_flags`), push ``True`` onto the stack. Otherwise, push - ``False``. - - .. versionadded:: 3.10 - - -.. opcode:: MATCH_SEQUENCE - - If TOS is an instance of :class:`collections.abc.Sequence` and is *not* an instance - of :class:`str`/:class:`bytes`/:class:`bytearray` (or, more technically: if it has - the :c:macro:`Py_TPFLAGS_SEQUENCE` flag set in its :c:member:`~PyTypeObject.tp_flags`), - push ``True`` onto the stack. Otherwise, push ``False``. - - .. versionadded:: 3.10 - - -.. opcode:: MATCH_KEYS - - TOS is a tuple of mapping keys, and TOS1 is the match subject. If TOS1 - contains all of the keys in TOS, push a :class:`tuple` containing the - corresponding values. Otherwise, push ``None``. - - .. versionadded:: 3.10 - - .. versionchanged:: 3.11 - Previously, this instruction also pushed a boolean value indicating - success (``True``) or failure (``False``). - - -.. opcode:: STORE_NAME (namei) - - Implements ``name = TOS``. *namei* is the index of *name* in the attribute - :attr:`co_names` of the code object. The compiler tries to use - :opcode:`STORE_FAST` or :opcode:`STORE_GLOBAL` if possible. - - -.. opcode:: DELETE_NAME (namei) - - Implements ``del name``, where *namei* is the index into :attr:`co_names` - attribute of the code object. - - -.. opcode:: UNPACK_SEQUENCE (count) - - Unpacks TOS into *count* individual values, which are put onto the stack - right-to-left. - - -.. opcode:: UNPACK_EX (counts) - - Implements assignment with a starred target: Unpacks an iterable in TOS into - individual values, where the total number of values can be smaller than the - number of items in the iterable: one of the new values will be a list of all - leftover items. - - The low byte of *counts* is the number of values before the list value, the - high byte of *counts* the number of values after it. The resulting values - are put onto the stack right-to-left. - - -.. opcode:: STORE_ATTR (namei) - - Implements ``TOS.name = TOS1``, where *namei* is the index of name in - :attr:`co_names`. - - -.. opcode:: DELETE_ATTR (namei) - - Implements ``del TOS.name``, using *namei* as index into :attr:`co_names`. - - -.. opcode:: STORE_GLOBAL (namei) - - Works as :opcode:`STORE_NAME`, but stores the name as a global. - - -.. opcode:: DELETE_GLOBAL (namei) - - Works as :opcode:`DELETE_NAME`, but deletes a global name. - - -.. opcode:: LOAD_CONST (consti) - - Pushes ``co_consts[consti]`` onto the stack. - - -.. opcode:: LOAD_NAME (namei) - - Pushes the value associated with ``co_names[namei]`` onto the stack. - - -.. opcode:: BUILD_TUPLE (count) - - Creates a tuple consuming *count* items from the stack, and pushes the - resulting tuple onto the stack. - - -.. opcode:: BUILD_LIST (count) - - Works as :opcode:`BUILD_TUPLE`, but creates a list. - - -.. opcode:: BUILD_SET (count) - - Works as :opcode:`BUILD_TUPLE`, but creates a set. - - -.. opcode:: BUILD_MAP (count) - - Pushes a new dictionary object onto the stack. Pops ``2 * count`` items - so that the dictionary holds *count* entries: - ``{..., TOS3: TOS2, TOS1: TOS}``. - - .. versionchanged:: 3.5 - The dictionary is created from stack items instead of creating an - empty dictionary pre-sized to hold *count* items. - - -.. opcode:: BUILD_CONST_KEY_MAP (count) - - The version of :opcode:`BUILD_MAP` specialized for constant keys. Pops the - top element on the stack which contains a tuple of keys, then starting from - ``TOS1``, pops *count* values to form values in the built dictionary. - - .. versionadded:: 3.6 - - -.. opcode:: BUILD_STRING (count) - - Concatenates *count* strings from the stack and pushes the resulting string - onto the stack. - - .. versionadded:: 3.6 - - -.. opcode:: LIST_TO_TUPLE - - Pops a list from the stack and pushes a tuple containing the same values. - - .. versionadded:: 3.9 - - -.. opcode:: LIST_EXTEND (i) - - Calls ``list.extend(TOS1[-i], TOS)``. Used to build lists. - - .. versionadded:: 3.9 - - -.. opcode:: SET_UPDATE (i) - - Calls ``set.update(TOS1[-i], TOS)``. Used to build sets. - - .. versionadded:: 3.9 - - -.. opcode:: DICT_UPDATE (i) - - Calls ``dict.update(TOS1[-i], TOS)``. Used to build dicts. - - .. versionadded:: 3.9 - - -.. opcode:: DICT_MERGE (i) - - Like :opcode:`DICT_UPDATE` but raises an exception for duplicate keys. - - .. versionadded:: 3.9 - - -.. opcode:: LOAD_ATTR (namei) - - Replaces TOS with ``getattr(TOS, co_names[namei])``. - - -.. opcode:: COMPARE_OP (opname) - - Performs a Boolean operation. The operation name can be found in - ``cmp_op[opname]``. - - -.. opcode:: IS_OP (invert) - - Performs ``is`` comparison, or ``is not`` if ``invert`` is 1. - - .. versionadded:: 3.9 - - -.. opcode:: CONTAINS_OP (invert) - - Performs ``in`` comparison, or ``not in`` if ``invert`` is 1. - - .. versionadded:: 3.9 - - -.. opcode:: IMPORT_NAME (namei) - - Imports the module ``co_names[namei]``. TOS and TOS1 are popped and provide - the *fromlist* and *level* arguments of :func:`__import__`. The module - object is pushed onto the stack. The current namespace is not affected: for - a proper import statement, a subsequent :opcode:`STORE_FAST` instruction - modifies the namespace. - - -.. opcode:: IMPORT_FROM (namei) - - Loads the attribute ``co_names[namei]`` from the module found in TOS. The - resulting object is pushed onto the stack, to be subsequently stored by a - :opcode:`STORE_FAST` instruction. - - -.. opcode:: JUMP_FORWARD (delta) - - Increments bytecode counter by *delta*. - - -.. opcode:: JUMP_BACKWARD (delta) - - Decrements bytecode counter by *delta*. Checks for interrupts. - - .. versionadded:: 3.11 - - -.. opcode:: JUMP_BACKWARD_NO_INTERRUPT (delta) - - Decrements bytecode counter by *delta*. Does not check for interrupts. - - .. versionadded:: 3.11 - - -.. opcode:: POP_JUMP_FORWARD_IF_TRUE (delta) - - If TOS is true, increments the bytecode counter by *delta*. TOS is popped. - - .. versionadded:: 3.11 - - -.. opcode:: POP_JUMP_BACKWARD_IF_TRUE (delta) - - If TOS is true, decrements the bytecode counter by *delta*. TOS is popped. - - .. versionadded:: 3.11 - - -.. opcode:: POP_JUMP_FORWARD_IF_FALSE (delta) - - If TOS is false, increments the bytecode counter by *delta*. TOS is popped. - - .. versionadded:: 3.11 - - -.. opcode:: POP_JUMP_BACKWARD_IF_FALSE (delta) - - If TOS is false, decrements the bytecode counter by *delta*. TOS is popped. - - .. versionadded:: 3.11 - - -.. opcode:: POP_JUMP_FORWARD_IF_NOT_NONE (delta) - - If TOS is not ``None``, increments the bytecode counter by *delta*. TOS is popped. - - .. versionadded:: 3.11 - - -.. opcode:: POP_JUMP_BACKWARD_IF_NOT_NONE (delta) - - If TOS is not ``None``, decrements the bytecode counter by *delta*. TOS is popped. - - .. versionadded:: 3.11 - - -.. opcode:: POP_JUMP_FORWARD_IF_NONE (delta) - - If TOS is ``None``, increments the bytecode counter by *delta*. TOS is popped. - - .. versionadded:: 3.11 - - -.. opcode:: POP_JUMP_BACKWARD_IF_NONE (delta) - - If TOS is ``None``, decrements the bytecode counter by *delta*. TOS is popped. - - .. versionadded:: 3.11 - - -.. opcode:: JUMP_IF_TRUE_OR_POP (delta) - - If TOS is true, increments the bytecode counter by *delta* and leaves TOS on the - stack. Otherwise (TOS is false), TOS is popped. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.11 - The oparg is now a relative delta rather than an absolute target. - -.. opcode:: JUMP_IF_FALSE_OR_POP (delta) - - If TOS is false, increments the bytecode counter by *delta* and leaves TOS on the - stack. Otherwise (TOS is true), TOS is popped. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.11 - The oparg is now a relative delta rather than an absolute target. - - -.. opcode:: FOR_ITER (delta) - - TOS is an :term:`iterator`. Call its :meth:`~iterator.__next__` method. If - this yields a new value, push it on the stack (leaving the iterator below - it). If the iterator indicates it is exhausted, TOS is popped, and the byte - code counter is incremented by *delta*. - - -.. opcode:: LOAD_GLOBAL (namei) - - Loads the global named ``co_names[namei>>1]`` onto the stack. - - .. versionchanged:: 3.11 - If the low bit of ``namei`` is set, then a ``NULL`` is pushed to the - stack before the global variable. - -.. opcode:: LOAD_FAST (var_num) - - Pushes a reference to the local ``co_varnames[var_num]`` onto the stack. - - -.. opcode:: STORE_FAST (var_num) - - Stores TOS into the local ``co_varnames[var_num]``. - - -.. opcode:: DELETE_FAST (var_num) - - Deletes local ``co_varnames[var_num]``. - - -.. opcode:: MAKE_CELL (i) - - Creates a new cell in slot ``i``. If that slot is nonempty then - that value is stored into the new cell. - - .. versionadded:: 3.11 - - -.. opcode:: LOAD_CLOSURE (i) - - Pushes a reference to the cell contained in slot ``i`` of the "fast locals" - storage. The name of the variable is ``co_fastlocalnames[i]``. - - Note that ``LOAD_CLOSURE`` is effectively an alias for ``LOAD_FAST``. - It exists to keep bytecode a little more readable. - - .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. - - -.. opcode:: LOAD_DEREF (i) - - Loads the cell contained in slot ``i`` of the "fast locals" storage. - Pushes a reference to the object the cell contains on the stack. - - .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. - - -.. opcode:: LOAD_CLASSDEREF (i) - - Much like :opcode:`LOAD_DEREF` but first checks the locals dictionary before - consulting the cell. This is used for loading free variables in class - bodies. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. - - -.. opcode:: STORE_DEREF (i) - - Stores TOS into the cell contained in slot ``i`` of the "fast locals" - storage. - - .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. - - -.. opcode:: DELETE_DEREF (i) - - Empties the cell contained in slot ``i`` of the "fast locals" storage. - Used by the :keyword:`del` statement. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.11 - ``i`` is no longer offset by the length of ``co_varnames``. - - -.. opcode:: COPY_FREE_VARS (n) - - Copies the ``n`` free variables from the closure into the frame. - Removes the need for special code on the caller's side when calling - closures. - - .. versionadded:: 3.11 - - -.. opcode:: RAISE_VARARGS (argc) - - Raises an exception using one of the 3 forms of the ``raise`` statement, - depending on the value of *argc*: - - * 0: ``raise`` (re-raise previous exception) - * 1: ``raise TOS`` (raise exception instance or type at ``TOS``) - * 2: ``raise TOS1 from TOS`` (raise exception instance or type at ``TOS1`` - with ``__cause__`` set to ``TOS``) - - -.. opcode:: CALL (argc) - - Calls a callable object with the number of arguments specified by ``argc``, - including the named arguments specified by the preceding - :opcode:`KW_NAMES`, if any. - On the stack are (in ascending order), either: - - * NULL - * The callable - * The positional arguments - * The named arguments - - or: - - * The callable - * ``self`` - * The remaining positional arguments - * The named arguments - - ``argc`` is the total of the positional and named arguments, excluding - ``self`` when a ``NULL`` is not present. - - ``CALL`` pops all arguments and the callable object off the stack, - calls the callable object with those arguments, and pushes the return value - returned by the callable object. - - .. versionadded:: 3.11 - - -.. opcode:: CALL_FUNCTION_EX (flags) - - Calls a callable object with variable set of positional and keyword - arguments. If the lowest bit of *flags* is set, the top of the stack - contains a mapping object containing additional keyword arguments. - Before the callable is called, the mapping object and iterable object - are each "unpacked" and their contents passed in as keyword and - positional arguments respectively. - ``CALL_FUNCTION_EX`` pops all arguments and the callable object off the stack, - calls the callable object with those arguments, and pushes the return value - returned by the callable object. - - .. versionadded:: 3.6 - - -.. opcode:: LOAD_METHOD (namei) - - Loads a method named ``co_names[namei]`` from the TOS object. TOS is popped. - This bytecode distinguishes two cases: if TOS has a method with the correct - name, the bytecode pushes the unbound method and TOS. TOS will be used as - the first argument (``self``) by :opcode:`CALL` when calling the - unbound method. Otherwise, ``NULL`` and the object return by the attribute - lookup are pushed. - - .. versionadded:: 3.7 - - -.. opcode:: PRECALL (argc) - - Prefixes :opcode:`CALL`. Logically this is a no op. - It exists to enable effective specialization of calls. - ``argc`` is the number of arguments as described in :opcode:`CALL`. - - .. versionadded:: 3.11 - - -.. opcode:: PUSH_NULL - - Pushes a ``NULL`` to the stack. - Used in the call sequence to match the ``NULL`` pushed by - :opcode:`LOAD_METHOD` for non-method calls. - - .. versionadded:: 3.11 - - -.. opcode:: KW_NAMES (i) - - Prefixes :opcode:`PRECALL`. - Stores a reference to ``co_consts[consti]`` into an internal variable - for use by :opcode:`CALL`. ``co_consts[consti]`` must be a tuple of strings. - - .. versionadded:: 3.11 - - -.. opcode:: MAKE_FUNCTION (flags) - - Pushes a new function object on the stack. From bottom to top, the consumed - stack must consist of values if the argument carries a specified flag value - - * ``0x01`` a tuple of default values for positional-only and - positional-or-keyword parameters in positional order - * ``0x02`` a dictionary of keyword-only parameters' default values - * ``0x04`` a tuple of strings containing parameters' annotations - * ``0x08`` a tuple containing cells for free variables, making a closure - * the code associated with the function (at TOS) - - .. versionchanged:: 3.10 - Flag value ``0x04`` is a tuple of strings instead of dictionary - - .. versionchanged:: 3.11 - Qualified name at TOS was removed. - - -.. opcode:: BUILD_SLICE (argc) - - .. index:: pair: built-in function; slice - - Pushes a slice object on the stack. *argc* must be 2 or 3. If it is 2, - ``slice(TOS1, TOS)`` is pushed; if it is 3, ``slice(TOS2, TOS1, TOS)`` is - pushed. See the :func:`slice` built-in function for more information. - - -.. opcode:: EXTENDED_ARG (ext) - - Prefixes any opcode which has an argument too big to fit into the default one - byte. *ext* holds an additional byte which act as higher bits in the argument. - For each opcode, at most three prefixal ``EXTENDED_ARG`` are allowed, forming - an argument from two-byte to four-byte. - - -.. opcode:: FORMAT_VALUE (flags) - - Used for implementing formatted literal strings (f-strings). Pops - an optional *fmt_spec* from the stack, then a required *value*. - *flags* is interpreted as follows: - - * ``(flags & 0x03) == 0x00``: *value* is formatted as-is. - * ``(flags & 0x03) == 0x01``: call :func:`str` on *value* before - formatting it. - * ``(flags & 0x03) == 0x02``: call :func:`repr` on *value* before - formatting it. - * ``(flags & 0x03) == 0x03``: call :func:`ascii` on *value* before - formatting it. - * ``(flags & 0x04) == 0x04``: pop *fmt_spec* from the stack and use - it, else use an empty *fmt_spec*. - - Formatting is performed using :c:func:`PyObject_Format`. The - result is pushed on the stack. - - .. versionadded:: 3.6 - - -.. opcode:: MATCH_CLASS (count) - - TOS is a tuple of keyword attribute names, TOS1 is the class being matched - against, and TOS2 is the match subject. *count* is the number of positional - sub-patterns. - - Pop TOS, TOS1, and TOS2. If TOS2 is an instance of TOS1 and has the - positional and keyword attributes required by *count* and TOS, push a tuple - of extracted attributes. Otherwise, push ``None``. - - .. versionadded:: 3.10 - - .. versionchanged:: 3.11 - Previously, this instruction also pushed a boolean value indicating - success (``True``) or failure (``False``). - - -.. opcode:: RESUME (where) - - A no-op. Performs internal tracing, debugging and optimization checks. - - The ``where`` operand marks where the ``RESUME`` occurs: - - * ``0`` The start of a function - * ``1`` After a ``yield`` expression - * ``2`` After a ``yield from`` expression - * ``3`` After an ``await`` expression - - .. versionadded:: 3.11 - - -.. opcode:: RETURN_GENERATOR - - Create a generator, coroutine, or async generator from the current frame. - Clear the current frame and return the newly created generator. - - .. versionadded:: 3.11 - - -.. opcode:: SEND - - Sends ``None`` to the sub-generator of this generator. - Used in ``yield from`` and ``await`` statements. - - .. versionadded:: 3.11 - - -.. opcode:: ASYNC_GEN_WRAP - - Wraps the value on top of the stack in an ``async_generator_wrapped_value``. - Used to yield in async generators. - - .. versionadded:: 3.11 - - -.. opcode:: HAVE_ARGUMENT - - This is not really an opcode. It identifies the dividing line between - opcodes which don't use their argument and those that do - (``< HAVE_ARGUMENT`` and ``>= HAVE_ARGUMENT``, respectively). - - .. versionchanged:: 3.6 - Now every instruction has an argument, but opcodes ``< HAVE_ARGUMENT`` - ignore it. Before, only opcodes ``>= HAVE_ARGUMENT`` had an argument. - - -.. _opcode_collections: - -Opcode collections ------------------- - -These collections are provided for automatic introspection of bytecode -instructions: - -.. data:: opname - - Sequence of operation names, indexable using the bytecode. - - -.. data:: opmap - - Dictionary mapping operation names to bytecodes. - - -.. data:: cmp_op - - Sequence of all compare operation names. - - -.. data:: hasconst - - Sequence of bytecodes that access a constant. - - -.. data:: hasfree - - Sequence of bytecodes that access a free variable (note that 'free' in this - context refers to names in the current scope that are referenced by inner - scopes or names in outer scopes that are referenced from this scope. It does - *not* include references to global or builtin scopes). - - -.. data:: hasname - - Sequence of bytecodes that access an attribute by name. - - -.. data:: hasjrel - - Sequence of bytecodes that have a relative jump target. - - -.. data:: hasjabs - - Sequence of bytecodes that have an absolute jump target. - - -.. data:: haslocal - - Sequence of bytecodes that access a local variable. - - -.. data:: hascompare - - Sequence of bytecodes of Boolean operations. diff --git a/Doc/library/distribution.rst b/Doc/library/distribution.rst deleted file mode 100644 index 8d4befe41b329c..00000000000000 --- a/Doc/library/distribution.rst +++ /dev/null @@ -1,15 +0,0 @@ -*********************************** -Software Packaging and Distribution -*********************************** - -These libraries help you with publishing and installing Python software. -While these modules are designed to work in conjunction with the -`Python Package Index `__, they can also be used -with a local index server, or without any index server at all. - -.. toctree:: - - distutils.rst - ensurepip.rst - venv.rst - zipapp.rst diff --git a/Doc/library/distutils.rst b/Doc/library/distutils.rst deleted file mode 100644 index 31c4ae5b23906b..00000000000000 --- a/Doc/library/distutils.rst +++ /dev/null @@ -1,49 +0,0 @@ -:mod:`distutils` --- Building and installing Python modules -=========================================================== - -.. module:: distutils - :synopsis: Support for building and installing Python modules into an - existing Python installation. - -.. sectionauthor:: Fred L. Drake, Jr. - --------------- - -:mod:`distutils` is deprecated with removal planned for Python 3.12. -See the :ref:`What's New ` entry for more information. - --------------- - -The :mod:`distutils` package provides support for building and installing -additional modules into a Python installation. The new modules may be either -100%-pure Python, or may be extension modules written in C, or may be -collections of Python packages which include modules coded in both Python and C. - -Most Python users will *not* want to use this module directly, but instead -use the cross-version tools maintained by the Python Packaging Authority. In -particular, -`setuptools `__ is an -enhanced alternative to :mod:`distutils` that provides: - -* support for declaring project dependencies -* additional mechanisms for configuring which files to include in source - releases (including plugins for integration with version control systems) -* the ability to declare project "entry points", which can be used as the - basis for application plugin systems -* the ability to automatically generate Windows command line executables at - installation time rather than needing to prebuild them -* consistent behaviour across all supported Python versions - -The recommended `pip `__ installer runs all -``setup.py`` scripts with ``setuptools``, even if the script itself only -imports ``distutils``. Refer to the -`Python Packaging User Guide `_ for more -information. - -For the benefits of packaging tool authors and users seeking a deeper -understanding of the details of the current packaging and distribution -system, the legacy :mod:`distutils` based user documentation and API -reference remain available: - -* :ref:`install-index` -* :ref:`distutils-index` diff --git a/Doc/library/doctest.rst b/Doc/library/doctest.rst deleted file mode 100644 index d00c4c88d57176..00000000000000 --- a/Doc/library/doctest.rst +++ /dev/null @@ -1,1891 +0,0 @@ -:mod:`doctest` --- Test interactive Python examples -=================================================== - -.. module:: doctest - :synopsis: Test pieces of code within docstrings. - -.. moduleauthor:: Tim Peters -.. sectionauthor:: Tim Peters -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Edward Loper - -**Source code:** :source:`Lib/doctest.py` - --------------- - -The :mod:`doctest` module searches for pieces of text that look like interactive -Python sessions, and then executes those sessions to verify that they work -exactly as shown. There are several common ways to use doctest: - -* To check that a module's docstrings are up-to-date by verifying that all - interactive examples still work as documented. - -* To perform regression testing by verifying that interactive examples from a - test file or a test object work as expected. - -* To write tutorial documentation for a package, liberally illustrated with - input-output examples. Depending on whether the examples or the expository text - are emphasized, this has the flavor of "literate testing" or "executable - documentation". - -Here's a complete but small example module:: - - """ - This is the "example" module. - - The example module supplies one function, factorial(). For example, - - >>> factorial(5) - 120 - """ - - def factorial(n): - """Return the factorial of n, an exact integer >= 0. - - >>> [factorial(n) for n in range(6)] - [1, 1, 2, 6, 24, 120] - >>> factorial(30) - 265252859812191058636308480000000 - >>> factorial(-1) - Traceback (most recent call last): - ... - ValueError: n must be >= 0 - - Factorials of floats are OK, but the float must be an exact integer: - >>> factorial(30.1) - Traceback (most recent call last): - ... - ValueError: n must be exact integer - >>> factorial(30.0) - 265252859812191058636308480000000 - - It must also not be ridiculously large: - >>> factorial(1e100) - Traceback (most recent call last): - ... - OverflowError: n too large - """ - - import math - if not n >= 0: - raise ValueError("n must be >= 0") - if math.floor(n) != n: - raise ValueError("n must be exact integer") - if n+1 == n: # catch a value like 1e300 - raise OverflowError("n too large") - result = 1 - factor = 2 - while factor <= n: - result *= factor - factor += 1 - return result - - - if __name__ == "__main__": - import doctest - doctest.testmod() - -If you run :file:`example.py` directly from the command line, :mod:`doctest` -works its magic: - -.. code-block:: shell-session - - $ python example.py - $ - -There's no output! That's normal, and it means all the examples worked. Pass -``-v`` to the script, and :mod:`doctest` prints a detailed log of what -it's trying, and prints a summary at the end: - -.. code-block:: shell-session - - $ python example.py -v - Trying: - factorial(5) - Expecting: - 120 - ok - Trying: - [factorial(n) for n in range(6)] - Expecting: - [1, 1, 2, 6, 24, 120] - ok - -And so on, eventually ending with: - -.. code-block:: none - - Trying: - factorial(1e100) - Expecting: - Traceback (most recent call last): - ... - OverflowError: n too large - ok - 2 items passed all tests: - 1 tests in __main__ - 8 tests in __main__.factorial - 9 tests in 2 items. - 9 passed and 0 failed. - Test passed. - $ - -That's all you need to know to start making productive use of :mod:`doctest`! -Jump in. The following sections provide full details. Note that there are many -examples of doctests in the standard Python test suite and libraries. -Especially useful examples can be found in the standard test file -:file:`Lib/test/test_doctest.py`. - - -.. _doctest-simple-testmod: - -Simple Usage: Checking Examples in Docstrings ---------------------------------------------- - -The simplest way to start using doctest (but not necessarily the way you'll -continue to do it) is to end each module :mod:`M` with:: - - if __name__ == "__main__": - import doctest - doctest.testmod() - -:mod:`doctest` then examines docstrings in module :mod:`M`. - -Running the module as a script causes the examples in the docstrings to get -executed and verified:: - - python M.py - -This won't display anything unless an example fails, in which case the failing -example(s) and the cause(s) of the failure(s) are printed to stdout, and the -final line of output is ``***Test Failed*** N failures.``, where *N* is the -number of examples that failed. - -Run it with the ``-v`` switch instead:: - - python M.py -v - -and a detailed report of all examples tried is printed to standard output, along -with assorted summaries at the end. - -You can force verbose mode by passing ``verbose=True`` to :func:`testmod`, or -prohibit it by passing ``verbose=False``. In either of those cases, -``sys.argv`` is not examined by :func:`testmod` (so passing ``-v`` or not -has no effect). - -There is also a command line shortcut for running :func:`testmod`. You can -instruct the Python interpreter to run the doctest module directly from the -standard library and pass the module name(s) on the command line:: - - python -m doctest -v example.py - -This will import :file:`example.py` as a standalone module and run -:func:`testmod` on it. Note that this may not work correctly if the file is -part of a package and imports other submodules from that package. - -For more information on :func:`testmod`, see section :ref:`doctest-basic-api`. - - -.. _doctest-simple-testfile: - -Simple Usage: Checking Examples in a Text File ----------------------------------------------- - -Another simple application of doctest is testing interactive examples in a text -file. This can be done with the :func:`testfile` function:: - - import doctest - doctest.testfile("example.txt") - -That short script executes and verifies any interactive Python examples -contained in the file :file:`example.txt`. The file content is treated as if it -were a single giant docstring; the file doesn't need to contain a Python -program! For example, perhaps :file:`example.txt` contains this: - -.. code-block:: none - - The ``example`` module - ====================== - - Using ``factorial`` - ------------------- - - This is an example text file in reStructuredText format. First import - ``factorial`` from the ``example`` module: - - >>> from example import factorial - - Now use it: - - >>> factorial(6) - 120 - -Running ``doctest.testfile("example.txt")`` then finds the error in this -documentation:: - - File "./example.txt", line 14, in example.txt - Failed example: - factorial(6) - Expected: - 120 - Got: - 720 - -As with :func:`testmod`, :func:`testfile` won't display anything unless an -example fails. If an example does fail, then the failing example(s) and the -cause(s) of the failure(s) are printed to stdout, using the same format as -:func:`testmod`. - -By default, :func:`testfile` looks for files in the calling module's directory. -See section :ref:`doctest-basic-api` for a description of the optional arguments -that can be used to tell it to look for files in other locations. - -Like :func:`testmod`, :func:`testfile`'s verbosity can be set with the -``-v`` command-line switch or with the optional keyword argument -*verbose*. - -There is also a command line shortcut for running :func:`testfile`. You can -instruct the Python interpreter to run the doctest module directly from the -standard library and pass the file name(s) on the command line:: - - python -m doctest -v example.txt - -Because the file name does not end with :file:`.py`, :mod:`doctest` infers that -it must be run with :func:`testfile`, not :func:`testmod`. - -For more information on :func:`testfile`, see section :ref:`doctest-basic-api`. - - -.. _doctest-how-it-works: - -How It Works ------------- - -This section examines in detail how doctest works: which docstrings it looks at, -how it finds interactive examples, what execution context it uses, how it -handles exceptions, and how option flags can be used to control its behavior. -This is the information that you need to know to write doctest examples; for -information about actually running doctest on these examples, see the following -sections. - - -.. _doctest-which-docstrings: - -Which Docstrings Are Examined? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The module docstring, and all function, class and method docstrings are -searched. Objects imported into the module are not searched. - -In addition, there are cases when you want tests to be part of a module but not part -of the help text, which requires that the tests not be included in the docstring. -Doctest looks for a module-level variable called ``__test__`` and uses it to locate other -tests. If ``M.__test__`` exists and is truthy, it must be a dict, and each -entry maps a (string) name to a function object, class object, or string. -Function and class object docstrings found from ``M.__test__`` are searched, and -strings are treated as if they were docstrings. In output, a key ``K`` in -``M.__test__`` appears with name ``M.__test__.K``. - -For example, place this block of code at the top of :file:`example.py`: - -.. code-block:: python - - __test__ = { - 'numbers': """ - >>> factorial(6) - 720 - - >>> [factorial(n) for n in range(6)] - [1, 1, 2, 6, 24, 120] - """ - } - -The value of ``example.__test__["numbers"]`` will be treated as a -docstring and all the tests inside it will be run. It is -important to note that the value can be mapped to a function, -class object, or module; if so, :mod:`!doctest` -searches them recursively for docstrings, which are then scanned for tests. - -Any classes found are recursively searched similarly, to test docstrings in -their contained methods and nested classes. - - -.. _doctest-finding-examples: - -How are Docstring Examples Recognized? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -In most cases a copy-and-paste of an interactive console session works fine, -but doctest isn't trying to do an exact emulation of any specific Python shell. - -:: - - >>> # comments are ignored - >>> x = 12 - >>> x - 12 - >>> if x == 13: - ... print("yes") - ... else: - ... print("no") - ... print("NO") - ... print("NO!!!") - ... - no - NO - NO!!! - >>> - -.. index:: - single: >>>; interpreter prompt - single: ...; interpreter prompt - -Any expected output must immediately follow the final ``'>>> '`` or ``'... '`` -line containing the code, and the expected output (if any) extends to the next -``'>>> '`` or all-whitespace line. - -The fine print: - -* Expected output cannot contain an all-whitespace line, since such a line is - taken to signal the end of expected output. If expected output does contain a - blank line, put ```` in your doctest example each place a blank line - is expected. - -* All hard tab characters are expanded to spaces, using 8-column tab stops. - Tabs in output generated by the tested code are not modified. Because any - hard tabs in the sample output *are* expanded, this means that if the code - output includes hard tabs, the only way the doctest can pass is if the - :const:`NORMALIZE_WHITESPACE` option or :ref:`directive ` - is in effect. - Alternatively, the test can be rewritten to capture the output and compare it - to an expected value as part of the test. This handling of tabs in the - source was arrived at through trial and error, and has proven to be the least - error prone way of handling them. It is possible to use a different - algorithm for handling tabs by writing a custom :class:`DocTestParser` class. - -* Output to stdout is captured, but not output to stderr (exception tracebacks - are captured via a different means). - -* If you continue a line via backslashing in an interactive session, or for any - other reason use a backslash, you should use a raw docstring, which will - preserve your backslashes exactly as you type them:: - - >>> def f(x): - ... r'''Backslashes in a raw docstring: m\n''' - >>> print(f.__doc__) - Backslashes in a raw docstring: m\n - - Otherwise, the backslash will be interpreted as part of the string. For example, - the ``\n`` above would be interpreted as a newline character. Alternatively, you - can double each backslash in the doctest version (and not use a raw string):: - - >>> def f(x): - ... '''Backslashes in a raw docstring: m\\n''' - >>> print(f.__doc__) - Backslashes in a raw docstring: m\n - -* The starting column doesn't matter:: - - >>> assert "Easy!" - >>> import math - >>> math.floor(1.9) - 1 - - and as many leading whitespace characters are stripped from the expected output - as appeared in the initial ``'>>> '`` line that started the example. - - -.. _doctest-execution-context: - -What's the Execution Context? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, each time :mod:`doctest` finds a docstring to test, it uses a -*shallow copy* of :mod:`M`'s globals, so that running tests doesn't change the -module's real globals, and so that one test in :mod:`M` can't leave behind -crumbs that accidentally allow another test to work. This means examples can -freely use any names defined at top-level in :mod:`M`, and names defined earlier -in the docstring being run. Examples cannot see names defined in other -docstrings. - -You can force use of your own dict as the execution context by passing -``globs=your_dict`` to :func:`testmod` or :func:`testfile` instead. - - -.. _doctest-exceptions: - -What About Exceptions? -^^^^^^^^^^^^^^^^^^^^^^ - -No problem, provided that the traceback is the only output produced by the -example: just paste in the traceback. [#]_ Since tracebacks contain details -that are likely to change rapidly (for example, exact file paths and line -numbers), this is one case where doctest works hard to be flexible in what it -accepts. - -Simple example:: - - >>> [1, 2, 3].remove(42) - Traceback (most recent call last): - File "", line 1, in - ValueError: list.remove(x): x not in list - -That doctest succeeds if :exc:`ValueError` is raised, with the ``list.remove(x): -x not in list`` detail as shown. - -The expected output for an exception must start with a traceback header, which -may be either of the following two lines, indented the same as the first line of -the example:: - - Traceback (most recent call last): - Traceback (innermost last): - -The traceback header is followed by an optional traceback stack, whose contents -are ignored by doctest. The traceback stack is typically omitted, or copied -verbatim from an interactive session. - -The traceback stack is followed by the most interesting part: the line(s) -containing the exception type and detail. This is usually the last line of a -traceback, but can extend across multiple lines if the exception has a -multi-line detail:: - - >>> raise ValueError('multi\n line\ndetail') - Traceback (most recent call last): - File "", line 1, in - ValueError: multi - line - detail - -The last three lines (starting with :exc:`ValueError`) are compared against the -exception's type and detail, and the rest are ignored. - -Best practice is to omit the traceback stack, unless it adds significant -documentation value to the example. So the last example is probably better as:: - - >>> raise ValueError('multi\n line\ndetail') - Traceback (most recent call last): - ... - ValueError: multi - line - detail - -Note that tracebacks are treated very specially. In particular, in the -rewritten example, the use of ``...`` is independent of doctest's -:const:`ELLIPSIS` option. The ellipsis in that example could be left out, or -could just as well be three (or three hundred) commas or digits, or an indented -transcript of a Monty Python skit. - -Some details you should read once, but won't need to remember: - -* Doctest can't guess whether your expected output came from an exception - traceback or from ordinary printing. So, e.g., an example that expects - ``ValueError: 42 is prime`` will pass whether :exc:`ValueError` is actually - raised or if the example merely prints that traceback text. In practice, - ordinary output rarely begins with a traceback header line, so this doesn't - create real problems. - -* Each line of the traceback stack (if present) must be indented further than - the first line of the example, *or* start with a non-alphanumeric character. - The first line following the traceback header indented the same and starting - with an alphanumeric is taken to be the start of the exception detail. Of - course this does the right thing for genuine tracebacks. - -* When the :const:`IGNORE_EXCEPTION_DETAIL` doctest option is specified, - everything following the leftmost colon and any module information in the - exception name is ignored. - -* The interactive shell omits the traceback header line for some - :exc:`SyntaxError`\ s. But doctest uses the traceback header line to - distinguish exceptions from non-exceptions. So in the rare case where you need - to test a :exc:`SyntaxError` that omits the traceback header, you will need to - manually add the traceback header line to your test example. - -.. index:: single: ^ (caret); marker - -* For some exceptions, Python displays the position of the error using ``^`` - markers and tildes:: - - >>> 1 + None - File "", line 1 - 1 + None - ~~^~~~~~ - TypeError: unsupported operand type(s) for +: 'int' and 'NoneType' - - Since the lines showing the position of the error come before the exception type - and detail, they are not checked by doctest. For example, the following test - would pass, even though it puts the ``^`` marker in the wrong location:: - - >>> 1 + None - File "", line 1 - 1 + None - ^~~~~~~~ - TypeError: unsupported operand type(s) for +: 'int' and 'NoneType' - - -.. _option-flags-and-directives: -.. _doctest-options: - -Option Flags -^^^^^^^^^^^^ - -A number of option flags control various aspects of doctest's behavior. -Symbolic names for the flags are supplied as module constants, which can be -:ref:`bitwise ORed ` together and passed to various functions. -The names can also be used in :ref:`doctest directives `, -and may be passed to the doctest command line interface via the ``-o`` option. - -.. versionadded:: 3.4 - The ``-o`` command line option. - -The first group of options define test semantics, controlling aspects of how -doctest decides whether actual output matches an example's expected output: - - -.. data:: DONT_ACCEPT_TRUE_FOR_1 - - By default, if an expected output block contains just ``1``, an actual output - block containing just ``1`` or just ``True`` is considered to be a match, and - similarly for ``0`` versus ``False``. When :const:`DONT_ACCEPT_TRUE_FOR_1` is - specified, neither substitution is allowed. The default behavior caters to that - Python changed the return type of many functions from integer to boolean; - doctests expecting "little integer" output still work in these cases. This - option will probably go away, but not for several years. - - -.. index:: single: -.. data:: DONT_ACCEPT_BLANKLINE - - By default, if an expected output block contains a line containing only the - string ````, then that line will match a blank line in the actual - output. Because a genuinely blank line delimits the expected output, this is - the only way to communicate that a blank line is expected. When - :const:`DONT_ACCEPT_BLANKLINE` is specified, this substitution is not allowed. - - -.. data:: NORMALIZE_WHITESPACE - - When specified, all sequences of whitespace (blanks and newlines) are treated as - equal. Any sequence of whitespace within the expected output will match any - sequence of whitespace within the actual output. By default, whitespace must - match exactly. :const:`NORMALIZE_WHITESPACE` is especially useful when a line of - expected output is very long, and you want to wrap it across multiple lines in - your source. - - -.. index:: single: ...; in doctests -.. data:: ELLIPSIS - - When specified, an ellipsis marker (``...``) in the expected output can match - any substring in the actual output. This includes substrings that span line - boundaries, and empty substrings, so it's best to keep usage of this simple. - Complicated uses can lead to the same kinds of "oops, it matched too much!" - surprises that ``.*`` is prone to in regular expressions. - - -.. data:: IGNORE_EXCEPTION_DETAIL - - When specified, doctests expecting exceptions pass so long as an exception - of the expected type is raised, even if the details - (message and fully qualified exception name) don't match. - - For example, an example expecting ``ValueError: 42`` will pass if the actual - exception raised is ``ValueError: 3*14``, but will fail if, say, a - :exc:`TypeError` is raised instead. - It will also ignore any fully qualified name included before the - exception class, which can vary between implementations and versions - of Python and the code/libraries in use. - Hence, all three of these variations will work with the flag specified: - - .. code-block:: pycon - - >>> raise Exception('message') - Traceback (most recent call last): - Exception: message - - >>> raise Exception('message') - Traceback (most recent call last): - builtins.Exception: message - - >>> raise Exception('message') - Traceback (most recent call last): - __main__.Exception: message - - Note that :const:`ELLIPSIS` can also be used to ignore the - details of the exception message, but such a test may still fail based - on whether the module name is present or matches exactly. - - .. versionchanged:: 3.2 - :const:`IGNORE_EXCEPTION_DETAIL` now also ignores any information relating - to the module containing the exception under test. - - -.. data:: SKIP - - When specified, do not run the example at all. This can be useful in contexts - where doctest examples serve as both documentation and test cases, and an - example should be included for documentation purposes, but should not be - checked. E.g., the example's output might be random; or the example might - depend on resources which would be unavailable to the test driver. - - The SKIP flag can also be used for temporarily "commenting out" examples. - - -.. data:: COMPARISON_FLAGS - - A bitmask or'ing together all the comparison flags above. - -The second group of options controls how test failures are reported: - - -.. data:: REPORT_UDIFF - - When specified, failures that involve multi-line expected and actual outputs are - displayed using a unified diff. - - -.. data:: REPORT_CDIFF - - When specified, failures that involve multi-line expected and actual outputs - will be displayed using a context diff. - - -.. data:: REPORT_NDIFF - - When specified, differences are computed by ``difflib.Differ``, using the same - algorithm as the popular :file:`ndiff.py` utility. This is the only method that - marks differences within lines as well as across lines. For example, if a line - of expected output contains digit ``1`` where actual output contains letter - ``l``, a line is inserted with a caret marking the mismatching column positions. - - -.. data:: REPORT_ONLY_FIRST_FAILURE - - When specified, display the first failing example in each doctest, but suppress - output for all remaining examples. This will prevent doctest from reporting - correct examples that break because of earlier failures; but it might also hide - incorrect examples that fail independently of the first failure. When - :const:`REPORT_ONLY_FIRST_FAILURE` is specified, the remaining examples are - still run, and still count towards the total number of failures reported; only - the output is suppressed. - - -.. data:: FAIL_FAST - - When specified, exit after the first failing example and don't attempt to run - the remaining examples. Thus, the number of failures reported will be at most - 1. This flag may be useful during debugging, since examples after the first - failure won't even produce debugging output. - - The doctest command line accepts the option ``-f`` as a shorthand for ``-o - FAIL_FAST``. - - .. versionadded:: 3.4 - - -.. data:: REPORTING_FLAGS - - A bitmask or'ing together all the reporting flags above. - - -There is also a way to register new option flag names, though this isn't -useful unless you intend to extend :mod:`doctest` internals via subclassing: - - -.. function:: register_optionflag(name) - - Create a new option flag with a given name, and return the new flag's integer - value. :func:`register_optionflag` can be used when subclassing - :class:`OutputChecker` or :class:`DocTestRunner` to create new options that are - supported by your subclasses. :func:`register_optionflag` should always be - called using the following idiom:: - - MY_FLAG = register_optionflag('MY_FLAG') - - -.. index:: - single: # (hash); in doctests - single: + (plus); in doctests - single: - (minus); in doctests -.. _doctest-directives: - -Directives -^^^^^^^^^^ - -Doctest directives may be used to modify the :ref:`option flags -` for an individual example. Doctest directives are -special Python comments following an example's source code: - -.. productionlist:: doctest - directive: "#" "doctest:" `directive_options` - directive_options: `directive_option` ("," `directive_option`)* - directive_option: `on_or_off` `directive_option_name` - on_or_off: "+" | "-" - directive_option_name: "DONT_ACCEPT_BLANKLINE" | "NORMALIZE_WHITESPACE" | ... - -Whitespace is not allowed between the ``+`` or ``-`` and the directive option -name. The directive option name can be any of the option flag names explained -above. - -An example's doctest directives modify doctest's behavior for that single -example. Use ``+`` to enable the named behavior, or ``-`` to disable it. - -For example, this test passes: - -.. doctest:: - :no-trim-doctest-flags: - - >>> print(list(range(20))) # doctest: +NORMALIZE_WHITESPACE - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, - 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] - -Without the directive it would fail, both because the actual output doesn't have -two blanks before the single-digit list elements, and because the actual output -is on a single line. This test also passes, and also requires a directive to do -so: - -.. doctest:: - :no-trim-doctest-flags: - - >>> print(list(range(20))) # doctest: +ELLIPSIS - [0, 1, ..., 18, 19] - -Multiple directives can be used on a single physical line, separated by -commas: - -.. doctest:: - :no-trim-doctest-flags: - - >>> print(list(range(20))) # doctest: +ELLIPSIS, +NORMALIZE_WHITESPACE - [0, 1, ..., 18, 19] - -If multiple directive comments are used for a single example, then they are -combined: - -.. doctest:: - :no-trim-doctest-flags: - - >>> print(list(range(20))) # doctest: +ELLIPSIS - ... # doctest: +NORMALIZE_WHITESPACE - [0, 1, ..., 18, 19] - -As the previous example shows, you can add ``...`` lines to your example -containing only directives. This can be useful when an example is too long for -a directive to comfortably fit on the same line: - -.. doctest:: - :no-trim-doctest-flags: - - >>> print(list(range(5)) + list(range(10, 20)) + list(range(30, 40))) - ... # doctest: +ELLIPSIS - [0, ..., 4, 10, ..., 19, 30, ..., 39] - -Note that since all options are disabled by default, and directives apply only -to the example they appear in, enabling options (via ``+`` in a directive) is -usually the only meaningful choice. However, option flags can also be passed to -functions that run doctests, establishing different defaults. In such cases, -disabling an option via ``-`` in a directive can be useful. - - -.. _doctest-warnings: - -Warnings -^^^^^^^^ - -:mod:`doctest` is serious about requiring exact matches in expected output. If -even a single character doesn't match, the test fails. This will probably -surprise you a few times, as you learn exactly what Python does and doesn't -guarantee about output. For example, when printing a set, Python doesn't -guarantee that the element is printed in any particular order, so a test like :: - - >>> foo() - {"Hermione", "Harry"} - -is vulnerable! One workaround is to do :: - - >>> foo() == {"Hermione", "Harry"} - True - -instead. Another is to do :: - - >>> d = sorted(foo()) - >>> d - ['Harry', 'Hermione'] - -There are others, but you get the idea. - -Another bad idea is to print things that embed an object address, like - -.. doctest:: - - >>> id(1.0) # certain to fail some of the time # doctest: +SKIP - 7948648 - >>> class C: pass - >>> C() # the default repr() for instances embeds an address # doctest: +SKIP - - -The :const:`ELLIPSIS` directive gives a nice approach for the last example: - -.. doctest:: - :no-trim-doctest-flags: - - >>> C() # doctest: +ELLIPSIS - - -Floating-point numbers are also subject to small output variations across -platforms, because Python defers to the platform C library for float formatting, -and C libraries vary widely in quality here. :: - - >>> 1./7 # risky - 0.14285714285714285 - >>> print(1./7) # safer - 0.142857142857 - >>> print(round(1./7, 6)) # much safer - 0.142857 - -Numbers of the form ``I/2.**J`` are safe across all platforms, and I often -contrive doctest examples to produce numbers of that form:: - - >>> 3./4 # utterly safe - 0.75 - -Simple fractions are also easier for people to understand, and that makes for -better documentation. - - -.. _doctest-basic-api: - -Basic API ---------- - -The functions :func:`testmod` and :func:`testfile` provide a simple interface to -doctest that should be sufficient for most basic uses. For a less formal -introduction to these two functions, see sections :ref:`doctest-simple-testmod` -and :ref:`doctest-simple-testfile`. - - -.. function:: testfile(filename, module_relative=True, name=None, package=None, globs=None, verbose=None, report=True, optionflags=0, extraglobs=None, raise_on_error=False, parser=DocTestParser(), encoding=None) - - All arguments except *filename* are optional, and should be specified in keyword - form. - - Test examples in the file named *filename*. Return ``(failure_count, - test_count)``. - - Optional argument *module_relative* specifies how the filename should be - interpreted: - - * If *module_relative* is ``True`` (the default), then *filename* specifies an - OS-independent module-relative path. By default, this path is relative to the - calling module's directory; but if the *package* argument is specified, then it - is relative to that package. To ensure OS-independence, *filename* should use - ``/`` characters to separate path segments, and may not be an absolute path - (i.e., it may not begin with ``/``). - - * If *module_relative* is ``False``, then *filename* specifies an OS-specific - path. The path may be absolute or relative; relative paths are resolved with - respect to the current working directory. - - Optional argument *name* gives the name of the test; by default, or if ``None``, - ``os.path.basename(filename)`` is used. - - Optional argument *package* is a Python package or the name of a Python package - whose directory should be used as the base directory for a module-relative - filename. If no package is specified, then the calling module's directory is - used as the base directory for module-relative filenames. It is an error to - specify *package* if *module_relative* is ``False``. - - Optional argument *globs* gives a dict to be used as the globals when executing - examples. A new shallow copy of this dict is created for the doctest, so its - examples start with a clean slate. By default, or if ``None``, a new empty dict - is used. - - Optional argument *extraglobs* gives a dict merged into the globals used to - execute examples. This works like :meth:`dict.update`: if *globs* and - *extraglobs* have a common key, the associated value in *extraglobs* appears in - the combined dict. By default, or if ``None``, no extra globals are used. This - is an advanced feature that allows parameterization of doctests. For example, a - doctest can be written for a base class, using a generic name for the class, - then reused to test any number of subclasses by passing an *extraglobs* dict - mapping the generic name to the subclass to be tested. - - Optional argument *verbose* prints lots of stuff if true, and prints only - failures if false; by default, or if ``None``, it's true if and only if ``'-v'`` - is in ``sys.argv``. - - Optional argument *report* prints a summary at the end when true, else prints - nothing at the end. In verbose mode, the summary is detailed, else the summary - is very brief (in fact, empty if all tests passed). - - Optional argument *optionflags* (default value 0) takes the - :ref:`bitwise OR ` of option flags. - See section :ref:`doctest-options`. - - Optional argument *raise_on_error* defaults to false. If true, an exception is - raised upon the first failure or unexpected exception in an example. This - allows failures to be post-mortem debugged. Default behavior is to continue - running examples. - - Optional argument *parser* specifies a :class:`DocTestParser` (or subclass) that - should be used to extract tests from the files. It defaults to a normal parser - (i.e., ``DocTestParser()``). - - Optional argument *encoding* specifies an encoding that should be used to - convert the file to unicode. - - -.. function:: testmod(m=None, name=None, globs=None, verbose=None, report=True, optionflags=0, extraglobs=None, raise_on_error=False, exclude_empty=False) - - All arguments are optional, and all except for *m* should be specified in - keyword form. - - Test examples in docstrings in functions and classes reachable from module *m* - (or module :mod:`__main__` if *m* is not supplied or is ``None``), starting with - ``m.__doc__``. - - Also test examples reachable from dict ``m.__test__``, if it exists and is not - ``None``. ``m.__test__`` maps names (strings) to functions, classes and - strings; function and class docstrings are searched for examples; strings are - searched directly, as if they were docstrings. - - Only docstrings attached to objects belonging to module *m* are searched. - - Return ``(failure_count, test_count)``. - - Optional argument *name* gives the name of the module; by default, or if - ``None``, ``m.__name__`` is used. - - Optional argument *exclude_empty* defaults to false. If true, objects for which - no doctests are found are excluded from consideration. The default is a backward - compatibility hack, so that code still using :meth:`doctest.master.summarize` in - conjunction with :func:`testmod` continues to get output for objects with no - tests. The *exclude_empty* argument to the newer :class:`DocTestFinder` - constructor defaults to true. - - Optional arguments *extraglobs*, *verbose*, *report*, *optionflags*, - *raise_on_error*, and *globs* are the same as for function :func:`testfile` - above, except that *globs* defaults to ``m.__dict__``. - - -.. function:: run_docstring_examples(f, globs, verbose=False, name="NoName", compileflags=None, optionflags=0) - - Test examples associated with object *f*; for example, *f* may be a string, - a module, a function, or a class object. - - A shallow copy of dictionary argument *globs* is used for the execution context. - - Optional argument *name* is used in failure messages, and defaults to - ``"NoName"``. - - If optional argument *verbose* is true, output is generated even if there are no - failures. By default, output is generated only in case of an example failure. - - Optional argument *compileflags* gives the set of flags that should be used by - the Python compiler when running the examples. By default, or if ``None``, - flags are deduced corresponding to the set of future features found in *globs*. - - Optional argument *optionflags* works as for function :func:`testfile` above. - - -.. _doctest-unittest-api: - -Unittest API ------------- - -As your collection of doctest'ed modules grows, you'll want a way to run all -their doctests systematically. :mod:`doctest` provides two functions that can -be used to create :mod:`unittest` test suites from modules and text files -containing doctests. To integrate with :mod:`unittest` test discovery, include -a :func:`load_tests` function in your test module:: - - import unittest - import doctest - import my_module_with_doctests - - def load_tests(loader, tests, ignore): - tests.addTests(doctest.DocTestSuite(my_module_with_doctests)) - return tests - -There are two main functions for creating :class:`unittest.TestSuite` instances -from text files and modules with doctests: - - -.. function:: DocFileSuite(*paths, module_relative=True, package=None, setUp=None, tearDown=None, globs=None, optionflags=0, parser=DocTestParser(), encoding=None) - - Convert doctest tests from one or more text files to a - :class:`unittest.TestSuite`. - - The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs the interactive examples in each file. If an example in any file - fails, then the synthesized unit test fails, and a :exc:`failureException` - exception is raised showing the name of the file containing the test and a - (sometimes approximate) line number. - - Pass one or more paths (as strings) to text files to be examined. - - Options may be provided as keyword arguments: - - Optional argument *module_relative* specifies how the filenames in *paths* - should be interpreted: - - * If *module_relative* is ``True`` (the default), then each filename in - *paths* specifies an OS-independent module-relative path. By default, this - path is relative to the calling module's directory; but if the *package* - argument is specified, then it is relative to that package. To ensure - OS-independence, each filename should use ``/`` characters to separate path - segments, and may not be an absolute path (i.e., it may not begin with - ``/``). - - * If *module_relative* is ``False``, then each filename in *paths* specifies - an OS-specific path. The path may be absolute or relative; relative paths - are resolved with respect to the current working directory. - - Optional argument *package* is a Python package or the name of a Python - package whose directory should be used as the base directory for - module-relative filenames in *paths*. If no package is specified, then the - calling module's directory is used as the base directory for module-relative - filenames. It is an error to specify *package* if *module_relative* is - ``False``. - - Optional argument *setUp* specifies a set-up function for the test suite. - This is called before running the tests in each file. The *setUp* function - will be passed a :class:`DocTest` object. The setUp function can access the - test globals as the *globs* attribute of the test passed. - - Optional argument *tearDown* specifies a tear-down function for the test - suite. This is called after running the tests in each file. The *tearDown* - function will be passed a :class:`DocTest` object. The setUp function can - access the test globals as the *globs* attribute of the test passed. - - Optional argument *globs* is a dictionary containing the initial global - variables for the tests. A new copy of this dictionary is created for each - test. By default, *globs* is a new empty dictionary. - - Optional argument *optionflags* specifies the default doctest options for the - tests, created by or-ing together individual option flags. See section - :ref:`doctest-options`. See function :func:`set_unittest_reportflags` below - for a better way to set reporting options. - - Optional argument *parser* specifies a :class:`DocTestParser` (or subclass) - that should be used to extract tests from the files. It defaults to a normal - parser (i.e., ``DocTestParser()``). - - Optional argument *encoding* specifies an encoding that should be used to - convert the file to unicode. - - The global ``__file__`` is added to the globals provided to doctests loaded - from a text file using :func:`DocFileSuite`. - - -.. function:: DocTestSuite(module=None, globs=None, extraglobs=None, test_finder=None, setUp=None, tearDown=None, checker=None) - - Convert doctest tests for a module to a :class:`unittest.TestSuite`. - - The returned :class:`unittest.TestSuite` is to be run by the unittest framework - and runs each doctest in the module. If any of the doctests fail, then the - synthesized unit test fails, and a :exc:`failureException` exception is raised - showing the name of the file containing the test and a (sometimes approximate) - line number. - - Optional argument *module* provides the module to be tested. It can be a module - object or a (possibly dotted) module name. If not specified, the module calling - this function is used. - - Optional argument *globs* is a dictionary containing the initial global - variables for the tests. A new copy of this dictionary is created for each - test. By default, *globs* is a new empty dictionary. - - Optional argument *extraglobs* specifies an extra set of global variables, which - is merged into *globs*. By default, no extra globals are used. - - Optional argument *test_finder* is the :class:`DocTestFinder` object (or a - drop-in replacement) that is used to extract doctests from the module. - - Optional arguments *setUp*, *tearDown*, and *optionflags* are the same as for - function :func:`DocFileSuite` above. - - This function uses the same search technique as :func:`testmod`. - - .. versionchanged:: 3.5 - :func:`DocTestSuite` returns an empty :class:`unittest.TestSuite` if *module* - contains no docstrings instead of raising :exc:`ValueError`. - - -Under the covers, :func:`DocTestSuite` creates a :class:`unittest.TestSuite` out -of :class:`doctest.DocTestCase` instances, and :class:`DocTestCase` is a -subclass of :class:`unittest.TestCase`. :class:`DocTestCase` isn't documented -here (it's an internal detail), but studying its code can answer questions about -the exact details of :mod:`unittest` integration. - -Similarly, :func:`DocFileSuite` creates a :class:`unittest.TestSuite` out of -:class:`doctest.DocFileCase` instances, and :class:`DocFileCase` is a subclass -of :class:`DocTestCase`. - -So both ways of creating a :class:`unittest.TestSuite` run instances of -:class:`DocTestCase`. This is important for a subtle reason: when you run -:mod:`doctest` functions yourself, you can control the :mod:`doctest` options in -use directly, by passing option flags to :mod:`doctest` functions. However, if -you're writing a :mod:`unittest` framework, :mod:`unittest` ultimately controls -when and how tests get run. The framework author typically wants to control -:mod:`doctest` reporting options (perhaps, e.g., specified by command line -options), but there's no way to pass options through :mod:`unittest` to -:mod:`doctest` test runners. - -For this reason, :mod:`doctest` also supports a notion of :mod:`doctest` -reporting flags specific to :mod:`unittest` support, via this function: - - -.. function:: set_unittest_reportflags(flags) - - Set the :mod:`doctest` reporting flags to use. - - Argument *flags* takes the :ref:`bitwise OR ` of option flags. See - section :ref:`doctest-options`. Only "reporting flags" can be used. - - This is a module-global setting, and affects all future doctests run by module - :mod:`unittest`: the :meth:`runTest` method of :class:`DocTestCase` looks at - the option flags specified for the test case when the :class:`DocTestCase` - instance was constructed. If no reporting flags were specified (which is the - typical and expected case), :mod:`doctest`'s :mod:`unittest` reporting flags are - :ref:`bitwise ORed ` into the option flags, and the option flags - so augmented are passed to the :class:`DocTestRunner` instance created to - run the doctest. If any reporting flags were specified when the - :class:`DocTestCase` instance was constructed, :mod:`doctest`'s - :mod:`unittest` reporting flags are ignored. - - The value of the :mod:`unittest` reporting flags in effect before the function - was called is returned by the function. - - -.. _doctest-advanced-api: - -Advanced API ------------- - -The basic API is a simple wrapper that's intended to make doctest easy to use. -It is fairly flexible, and should meet most users' needs; however, if you -require more fine-grained control over testing, or wish to extend doctest's -capabilities, then you should use the advanced API. - -The advanced API revolves around two container classes, which are used to store -the interactive examples extracted from doctest cases: - -* :class:`Example`: A single Python :term:`statement`, paired with its expected - output. - -* :class:`DocTest`: A collection of :class:`Example`\ s, typically extracted - from a single docstring or text file. - -Additional processing classes are defined to find, parse, and run, and check -doctest examples: - -* :class:`DocTestFinder`: Finds all docstrings in a given module, and uses a - :class:`DocTestParser` to create a :class:`DocTest` from every docstring that - contains interactive examples. - -* :class:`DocTestParser`: Creates a :class:`DocTest` object from a string (such - as an object's docstring). - -* :class:`DocTestRunner`: Executes the examples in a :class:`DocTest`, and uses - an :class:`OutputChecker` to verify their output. - -* :class:`OutputChecker`: Compares the actual output from a doctest example with - the expected output, and decides whether they match. - -The relationships among these processing classes are summarized in the following -diagram:: - - list of: - +------+ +---------+ - |module| --DocTestFinder-> | DocTest | --DocTestRunner-> results - +------+ | ^ +---------+ | ^ (printed) - | | | Example | | | - v | | ... | v | - DocTestParser | Example | OutputChecker - +---------+ - - -.. _doctest-doctest: - -DocTest Objects -^^^^^^^^^^^^^^^ - - -.. class:: DocTest(examples, globs, name, filename, lineno, docstring) - - A collection of doctest examples that should be run in a single namespace. The - constructor arguments are used to initialize the attributes of the same names. - - - :class:`DocTest` defines the following attributes. They are initialized by - the constructor, and should not be modified directly. - - - .. attribute:: examples - - A list of :class:`Example` objects encoding the individual interactive Python - examples that should be run by this test. - - - .. attribute:: globs - - The namespace (aka globals) that the examples should be run in. This is a - dictionary mapping names to values. Any changes to the namespace made by the - examples (such as binding new variables) will be reflected in :attr:`globs` - after the test is run. - - - .. attribute:: name - - A string name identifying the :class:`DocTest`. Typically, this is the name - of the object or file that the test was extracted from. - - - .. attribute:: filename - - The name of the file that this :class:`DocTest` was extracted from; or - ``None`` if the filename is unknown, or if the :class:`DocTest` was not - extracted from a file. - - - .. attribute:: lineno - - The line number within :attr:`filename` where this :class:`DocTest` begins, or - ``None`` if the line number is unavailable. This line number is zero-based - with respect to the beginning of the file. - - - .. attribute:: docstring - - The string that the test was extracted from, or ``None`` if the string is - unavailable, or if the test was not extracted from a string. - - -.. _doctest-example: - -Example Objects -^^^^^^^^^^^^^^^ - - -.. class:: Example(source, want, exc_msg=None, lineno=0, indent=0, options=None) - - A single interactive example, consisting of a Python statement and its expected - output. The constructor arguments are used to initialize the attributes of - the same names. - - - :class:`Example` defines the following attributes. They are initialized by - the constructor, and should not be modified directly. - - - .. attribute:: source - - A string containing the example's source code. This source code consists of a - single Python statement, and always ends with a newline; the constructor adds - a newline when necessary. - - - .. attribute:: want - - The expected output from running the example's source code (either from - stdout, or a traceback in case of exception). :attr:`want` ends with a - newline unless no output is expected, in which case it's an empty string. The - constructor adds a newline when necessary. - - - .. attribute:: exc_msg - - The exception message generated by the example, if the example is expected to - generate an exception; or ``None`` if it is not expected to generate an - exception. This exception message is compared against the return value of - :func:`traceback.format_exception_only`. :attr:`exc_msg` ends with a newline - unless it's ``None``. The constructor adds a newline if needed. - - - .. attribute:: lineno - - The line number within the string containing this example where the example - begins. This line number is zero-based with respect to the beginning of the - containing string. - - - .. attribute:: indent - - The example's indentation in the containing string, i.e., the number of space - characters that precede the example's first prompt. - - - .. attribute:: options - - A dictionary mapping from option flags to ``True`` or ``False``, which is used - to override default options for this example. Any option flags not contained - in this dictionary are left at their default value (as specified by the - :class:`DocTestRunner`'s :attr:`optionflags`). By default, no options are set. - - -.. _doctest-doctestfinder: - -DocTestFinder objects -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: DocTestFinder(verbose=False, parser=DocTestParser(), recurse=True, exclude_empty=True) - - A processing class used to extract the :class:`DocTest`\ s that are relevant to - a given object, from its docstring and the docstrings of its contained objects. - :class:`DocTest`\ s can be extracted from modules, classes, functions, - methods, staticmethods, classmethods, and properties. - - The optional argument *verbose* can be used to display the objects searched by - the finder. It defaults to ``False`` (no output). - - The optional argument *parser* specifies the :class:`DocTestParser` object (or a - drop-in replacement) that is used to extract doctests from docstrings. - - If the optional argument *recurse* is false, then :meth:`DocTestFinder.find` - will only examine the given object, and not any contained objects. - - If the optional argument *exclude_empty* is false, then - :meth:`DocTestFinder.find` will include tests for objects with empty docstrings. - - - :class:`DocTestFinder` defines the following method: - - - .. method:: find(obj[, name][, module][, globs][, extraglobs]) - - Return a list of the :class:`DocTest`\ s that are defined by *obj*'s - docstring, or by any of its contained objects' docstrings. - - The optional argument *name* specifies the object's name; this name will be - used to construct names for the returned :class:`DocTest`\ s. If *name* is - not specified, then ``obj.__name__`` is used. - - The optional parameter *module* is the module that contains the given object. - If the module is not specified or is ``None``, then the test finder will attempt - to automatically determine the correct module. The object's module is used: - - * As a default namespace, if *globs* is not specified. - - * To prevent the DocTestFinder from extracting DocTests from objects that are - imported from other modules. (Contained objects with modules other than - *module* are ignored.) - - * To find the name of the file containing the object. - - * To help find the line number of the object within its file. - - If *module* is ``False``, no attempt to find the module will be made. This is - obscure, of use mostly in testing doctest itself: if *module* is ``False``, or - is ``None`` but cannot be found automatically, then all objects are considered - to belong to the (non-existent) module, so all contained objects will - (recursively) be searched for doctests. - - The globals for each :class:`DocTest` is formed by combining *globs* and - *extraglobs* (bindings in *extraglobs* override bindings in *globs*). A new - shallow copy of the globals dictionary is created for each :class:`DocTest`. - If *globs* is not specified, then it defaults to the module's *__dict__*, if - specified, or ``{}`` otherwise. If *extraglobs* is not specified, then it - defaults to ``{}``. - - -.. _doctest-doctestparser: - -DocTestParser objects -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: DocTestParser() - - A processing class used to extract interactive examples from a string, and use - them to create a :class:`DocTest` object. - - - :class:`DocTestParser` defines the following methods: - - - .. method:: get_doctest(string, globs, name, filename, lineno) - - Extract all doctest examples from the given string, and collect them into a - :class:`DocTest` object. - - *globs*, *name*, *filename*, and *lineno* are attributes for the new - :class:`DocTest` object. See the documentation for :class:`DocTest` for more - information. - - - .. method:: get_examples(string, name='') - - Extract all doctest examples from the given string, and return them as a list - of :class:`Example` objects. Line numbers are 0-based. The optional argument - *name* is a name identifying this string, and is only used for error messages. - - - .. method:: parse(string, name='') - - Divide the given string into examples and intervening text, and return them as - a list of alternating :class:`Example`\ s and strings. Line numbers for the - :class:`Example`\ s are 0-based. The optional argument *name* is a name - identifying this string, and is only used for error messages. - - -.. _doctest-doctestrunner: - -DocTestRunner objects -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: DocTestRunner(checker=None, verbose=None, optionflags=0) - - A processing class used to execute and verify the interactive examples in a - :class:`DocTest`. - - The comparison between expected outputs and actual outputs is done by an - :class:`OutputChecker`. This comparison may be customized with a number of - option flags; see section :ref:`doctest-options` for more information. If the - option flags are insufficient, then the comparison may also be customized by - passing a subclass of :class:`OutputChecker` to the constructor. - - The test runner's display output can be controlled in two ways. First, an output - function can be passed to :meth:`TestRunner.run`; this function will be called - with strings that should be displayed. It defaults to ``sys.stdout.write``. If - capturing the output is not sufficient, then the display output can be also - customized by subclassing DocTestRunner, and overriding the methods - :meth:`report_start`, :meth:`report_success`, - :meth:`report_unexpected_exception`, and :meth:`report_failure`. - - The optional keyword argument *checker* specifies the :class:`OutputChecker` - object (or drop-in replacement) that should be used to compare the expected - outputs to the actual outputs of doctest examples. - - The optional keyword argument *verbose* controls the :class:`DocTestRunner`'s - verbosity. If *verbose* is ``True``, then information is printed about each - example, as it is run. If *verbose* is ``False``, then only failures are - printed. If *verbose* is unspecified, or ``None``, then verbose output is used - iff the command-line switch ``-v`` is used. - - The optional keyword argument *optionflags* can be used to control how the test - runner compares expected output to actual output, and how it displays failures. - For more information, see section :ref:`doctest-options`. - - - :class:`DocTestParser` defines the following methods: - - - .. method:: report_start(out, test, example) - - Report that the test runner is about to process the given example. This method - is provided to allow subclasses of :class:`DocTestRunner` to customize their - output; it should not be called directly. - - *example* is the example about to be processed. *test* is the test - *containing example*. *out* is the output function that was passed to - :meth:`DocTestRunner.run`. - - - .. method:: report_success(out, test, example, got) - - Report that the given example ran successfully. This method is provided to - allow subclasses of :class:`DocTestRunner` to customize their output; it - should not be called directly. - - *example* is the example about to be processed. *got* is the actual output - from the example. *test* is the test containing *example*. *out* is the - output function that was passed to :meth:`DocTestRunner.run`. - - - .. method:: report_failure(out, test, example, got) - - Report that the given example failed. This method is provided to allow - subclasses of :class:`DocTestRunner` to customize their output; it should not - be called directly. - - *example* is the example about to be processed. *got* is the actual output - from the example. *test* is the test containing *example*. *out* is the - output function that was passed to :meth:`DocTestRunner.run`. - - - .. method:: report_unexpected_exception(out, test, example, exc_info) - - Report that the given example raised an unexpected exception. This method is - provided to allow subclasses of :class:`DocTestRunner` to customize their - output; it should not be called directly. - - *example* is the example about to be processed. *exc_info* is a tuple - containing information about the unexpected exception (as returned by - :func:`sys.exc_info`). *test* is the test containing *example*. *out* is the - output function that was passed to :meth:`DocTestRunner.run`. - - - .. method:: run(test, compileflags=None, out=None, clear_globs=True) - - Run the examples in *test* (a :class:`DocTest` object), and display the - results using the writer function *out*. - - The examples are run in the namespace ``test.globs``. If *clear_globs* is - true (the default), then this namespace will be cleared after the test runs, - to help with garbage collection. If you would like to examine the namespace - after the test completes, then use *clear_globs=False*. - - *compileflags* gives the set of flags that should be used by the Python - compiler when running the examples. If not specified, then it will default to - the set of future-import flags that apply to *globs*. - - The output of each example is checked using the :class:`DocTestRunner`'s - output checker, and the results are formatted by the - :meth:`DocTestRunner.report_\*` methods. - - - .. method:: summarize(verbose=None) - - Print a summary of all the test cases that have been run by this DocTestRunner, - and return a :term:`named tuple` ``TestResults(failed, attempted)``. - - The optional *verbose* argument controls how detailed the summary is. If the - verbosity is not specified, then the :class:`DocTestRunner`'s verbosity is - used. - -.. _doctest-outputchecker: - -OutputChecker objects -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: OutputChecker() - - A class used to check the whether the actual output from a doctest example - matches the expected output. :class:`OutputChecker` defines two methods: - :meth:`check_output`, which compares a given pair of outputs, and returns ``True`` - if they match; and :meth:`output_difference`, which returns a string describing - the differences between two outputs. - - - :class:`OutputChecker` defines the following methods: - - .. method:: check_output(want, got, optionflags) - - Return ``True`` iff the actual output from an example (*got*) matches the - expected output (*want*). These strings are always considered to match if - they are identical; but depending on what option flags the test runner is - using, several non-exact match types are also possible. See section - :ref:`doctest-options` for more information about option flags. - - - .. method:: output_difference(example, got, optionflags) - - Return a string describing the differences between the expected output for a - given example (*example*) and the actual output (*got*). *optionflags* is the - set of option flags used to compare *want* and *got*. - - -.. _doctest-debugging: - -Debugging ---------- - -Doctest provides several mechanisms for debugging doctest examples: - -* Several functions convert doctests to executable Python programs, which can be - run under the Python debugger, :mod:`pdb`. - -* The :class:`DebugRunner` class is a subclass of :class:`DocTestRunner` that - raises an exception for the first failing example, containing information about - that example. This information can be used to perform post-mortem debugging on - the example. - -* The :mod:`unittest` cases generated by :func:`DocTestSuite` support the - :meth:`debug` method defined by :class:`unittest.TestCase`. - -* You can add a call to :func:`pdb.set_trace` in a doctest example, and you'll - drop into the Python debugger when that line is executed. Then you can inspect - current values of variables, and so on. For example, suppose :file:`a.py` - contains just this module docstring:: - - """ - >>> def f(x): - ... g(x*2) - >>> def g(x): - ... print(x+3) - ... import pdb; pdb.set_trace() - >>> f(3) - 9 - """ - - Then an interactive Python session may look like this:: - - >>> import a, doctest - >>> doctest.testmod(a) - --Return-- - > (3)g()->None - -> import pdb; pdb.set_trace() - (Pdb) list - 1 def g(x): - 2 print(x+3) - 3 -> import pdb; pdb.set_trace() - [EOF] - (Pdb) p x - 6 - (Pdb) step - --Return-- - > (2)f()->None - -> g(x*2) - (Pdb) list - 1 def f(x): - 2 -> g(x*2) - [EOF] - (Pdb) p x - 3 - (Pdb) step - --Return-- - > (1)?()->None - -> f(3) - (Pdb) cont - (0, 3) - >>> - - -Functions that convert doctests to Python code, and possibly run the synthesized -code under the debugger: - - -.. function:: script_from_examples(s) - - Convert text with examples to a script. - - Argument *s* is a string containing doctest examples. The string is converted - to a Python script, where doctest examples in *s* are converted to regular code, - and everything else is converted to Python comments. The generated script is - returned as a string. For example, :: - - import doctest - print(doctest.script_from_examples(r""" - Set x and y to 1 and 2. - >>> x, y = 1, 2 - - Print their sum: - >>> print(x+y) - 3 - """)) - - displays:: - - # Set x and y to 1 and 2. - x, y = 1, 2 - # - # Print their sum: - print(x+y) - # Expected: - ## 3 - - This function is used internally by other functions (see below), but can also be - useful when you want to transform an interactive Python session into a Python - script. - - -.. function:: testsource(module, name) - - Convert the doctest for an object to a script. - - Argument *module* is a module object, or dotted name of a module, containing the - object whose doctests are of interest. Argument *name* is the name (within the - module) of the object with the doctests of interest. The result is a string, - containing the object's docstring converted to a Python script, as described for - :func:`script_from_examples` above. For example, if module :file:`a.py` - contains a top-level function :func:`f`, then :: - - import a, doctest - print(doctest.testsource(a, "a.f")) - - prints a script version of function :func:`f`'s docstring, with doctests - converted to code, and the rest placed in comments. - - -.. function:: debug(module, name, pm=False) - - Debug the doctests for an object. - - The *module* and *name* arguments are the same as for function - :func:`testsource` above. The synthesized Python script for the named object's - docstring is written to a temporary file, and then that file is run under the - control of the Python debugger, :mod:`pdb`. - - A shallow copy of ``module.__dict__`` is used for both local and global - execution context. - - Optional argument *pm* controls whether post-mortem debugging is used. If *pm* - has a true value, the script file is run directly, and the debugger gets - involved only if the script terminates via raising an unhandled exception. If - it does, then post-mortem debugging is invoked, via :func:`pdb.post_mortem`, - passing the traceback object from the unhandled exception. If *pm* is not - specified, or is false, the script is run under the debugger from the start, via - passing an appropriate :func:`exec` call to :func:`pdb.run`. - - -.. function:: debug_src(src, pm=False, globs=None) - - Debug the doctests in a string. - - This is like function :func:`debug` above, except that a string containing - doctest examples is specified directly, via the *src* argument. - - Optional argument *pm* has the same meaning as in function :func:`debug` above. - - Optional argument *globs* gives a dictionary to use as both local and global - execution context. If not specified, or ``None``, an empty dictionary is used. - If specified, a shallow copy of the dictionary is used. - - -The :class:`DebugRunner` class, and the special exceptions it may raise, are of -most interest to testing framework authors, and will only be sketched here. See -the source code, and especially :class:`DebugRunner`'s docstring (which is a -doctest!) for more details: - - -.. class:: DebugRunner(checker=None, verbose=None, optionflags=0) - - A subclass of :class:`DocTestRunner` that raises an exception as soon as a - failure is encountered. If an unexpected exception occurs, an - :exc:`UnexpectedException` exception is raised, containing the test, the - example, and the original exception. If the output doesn't match, then a - :exc:`DocTestFailure` exception is raised, containing the test, the example, and - the actual output. - - For information about the constructor parameters and methods, see the - documentation for :class:`DocTestRunner` in section :ref:`doctest-advanced-api`. - -There are two exceptions that may be raised by :class:`DebugRunner` instances: - - -.. exception:: DocTestFailure(test, example, got) - - An exception raised by :class:`DocTestRunner` to signal that a doctest example's - actual output did not match its expected output. The constructor arguments are - used to initialize the attributes of the same names. - -:exc:`DocTestFailure` defines the following attributes: - - -.. attribute:: DocTestFailure.test - - The :class:`DocTest` object that was being run when the example failed. - - -.. attribute:: DocTestFailure.example - - The :class:`Example` that failed. - - -.. attribute:: DocTestFailure.got - - The example's actual output. - - -.. exception:: UnexpectedException(test, example, exc_info) - - An exception raised by :class:`DocTestRunner` to signal that a doctest - example raised an unexpected exception. The constructor arguments are used - to initialize the attributes of the same names. - -:exc:`UnexpectedException` defines the following attributes: - - -.. attribute:: UnexpectedException.test - - The :class:`DocTest` object that was being run when the example failed. - - -.. attribute:: UnexpectedException.example - - The :class:`Example` that failed. - - -.. attribute:: UnexpectedException.exc_info - - A tuple containing information about the unexpected exception, as returned by - :func:`sys.exc_info`. - - -.. _doctest-soapbox: - -Soapbox -------- - -As mentioned in the introduction, :mod:`doctest` has grown to have three primary -uses: - -#. Checking examples in docstrings. - -#. Regression testing. - -#. Executable documentation / literate testing. - -These uses have different requirements, and it is important to distinguish them. -In particular, filling your docstrings with obscure test cases makes for bad -documentation. - -When writing a docstring, choose docstring examples with care. There's an art to -this that needs to be learned---it may not be natural at first. Examples should -add genuine value to the documentation. A good example can often be worth many -words. If done with care, the examples will be invaluable for your users, and -will pay back the time it takes to collect them many times over as the years go -by and things change. I'm still amazed at how often one of my :mod:`doctest` -examples stops working after a "harmless" change. - -Doctest also makes an excellent tool for regression testing, especially if you -don't skimp on explanatory text. By interleaving prose and examples, it becomes -much easier to keep track of what's actually being tested, and why. When a test -fails, good prose can make it much easier to figure out what the problem is, and -how it should be fixed. It's true that you could write extensive comments in -code-based testing, but few programmers do. Many have found that using doctest -approaches instead leads to much clearer tests. Perhaps this is simply because -doctest makes writing prose a little easier than writing code, while writing -comments in code is a little harder. I think it goes deeper than just that: -the natural attitude when writing a doctest-based test is that you want to -explain the fine points of your software, and illustrate them with examples. -This in turn naturally leads to test files that start with the simplest -features, and logically progress to complications and edge cases. A coherent -narrative is the result, instead of a collection of isolated functions that test -isolated bits of functionality seemingly at random. It's a different attitude, -and produces different results, blurring the distinction between testing and -explaining. - -Regression testing is best confined to dedicated objects or files. There are -several options for organizing tests: - -* Write text files containing test cases as interactive examples, and test the - files using :func:`testfile` or :func:`DocFileSuite`. This is recommended, - although is easiest to do for new projects, designed from the start to use - doctest. - -* Define functions named ``_regrtest_topic`` that consist of single docstrings, - containing test cases for the named topics. These functions can be included in - the same file as the module, or separated out into a separate test file. - -* Define a ``__test__`` dictionary mapping from regression test topics to - docstrings containing test cases. - -When you have placed your tests in a module, the module can itself be the test -runner. When a test fails, you can arrange for your test runner to re-run only -the failing doctest while you debug the problem. Here is a minimal example of -such a test runner:: - - if __name__ == '__main__': - import doctest - flags = doctest.REPORT_NDIFF|doctest.FAIL_FAST - if len(sys.argv) > 1: - name = sys.argv[1] - if name in globals(): - obj = globals()[name] - else: - obj = __test__[name] - doctest.run_docstring_examples(obj, globals(), name=name, - optionflags=flags) - else: - fail, total = doctest.testmod(optionflags=flags) - print("{} failures out of {} tests".format(fail, total)) - - -.. rubric:: Footnotes - -.. [#] Examples containing both expected output and an exception are not supported. - Trying to guess where one ends and the other begins is too error-prone, and that - also makes for a confusing test. diff --git a/Doc/library/email.charset.rst b/Doc/library/email.charset.rst deleted file mode 100644 index 1b1cd17bc81ffe..00000000000000 --- a/Doc/library/email.charset.rst +++ /dev/null @@ -1,214 +0,0 @@ -:mod:`email.charset`: Representing character sets -------------------------------------------------- - -.. module:: email.charset - :synopsis: Character Sets - -**Source code:** :source:`Lib/email/charset.py` - --------------- - -This module is part of the legacy (``Compat32``) email API. In the new -API only the aliases table is used. - -The remaining text in this section is the original documentation of the module. - -This module provides a class :class:`Charset` for representing character sets -and character set conversions in email messages, as well as a character set -registry and several convenience methods for manipulating this registry. -Instances of :class:`Charset` are used in several other modules within the -:mod:`email` package. - -Import this class from the :mod:`email.charset` module. - - -.. class:: Charset(input_charset=DEFAULT_CHARSET) - - Map character sets to their email properties. - - This class provides information about the requirements imposed on email for a - specific character set. It also provides convenience routines for converting - between character sets, given the availability of the applicable codecs. Given - a character set, it will do its best to provide information on how to use that - character set in an email message in an RFC-compliant way. - - Certain character sets must be encoded with quoted-printable or base64 when used - in email headers or bodies. Certain character sets must be converted outright, - and are not allowed in email. - - Optional *input_charset* is as described below; it is always coerced to lower - case. After being alias normalized it is also used as a lookup into the - registry of character sets to find out the header encoding, body encoding, and - output conversion codec to be used for the character set. For example, if - *input_charset* is ``iso-8859-1``, then headers and bodies will be encoded using - quoted-printable and no output conversion codec is necessary. If - *input_charset* is ``euc-jp``, then headers will be encoded with base64, bodies - will not be encoded, but output text will be converted from the ``euc-jp`` - character set to the ``iso-2022-jp`` character set. - - :class:`Charset` instances have the following data attributes: - - .. attribute:: input_charset - - The initial character set specified. Common aliases are converted to - their *official* email names (e.g. ``latin_1`` is converted to - ``iso-8859-1``). Defaults to 7-bit ``us-ascii``. - - - .. attribute:: header_encoding - - If the character set must be encoded before it can be used in an email - header, this attribute will be set to ``Charset.QP`` (for - quoted-printable), ``Charset.BASE64`` (for base64 encoding), or - ``Charset.SHORTEST`` for the shortest of QP or BASE64 encoding. Otherwise, - it will be ``None``. - - - .. attribute:: body_encoding - - Same as *header_encoding*, but describes the encoding for the mail - message's body, which indeed may be different than the header encoding. - ``Charset.SHORTEST`` is not allowed for *body_encoding*. - - - .. attribute:: output_charset - - Some character sets must be converted before they can be used in email - headers or bodies. If the *input_charset* is one of them, this attribute - will contain the name of the character set output will be converted to. - Otherwise, it will be ``None``. - - - .. attribute:: input_codec - - The name of the Python codec used to convert the *input_charset* to - Unicode. If no conversion codec is necessary, this attribute will be - ``None``. - - - .. attribute:: output_codec - - The name of the Python codec used to convert Unicode to the - *output_charset*. If no conversion codec is necessary, this attribute - will have the same value as the *input_codec*. - - - :class:`Charset` instances also have the following methods: - - .. method:: get_body_encoding() - - Return the content transfer encoding used for body encoding. - - This is either the string ``quoted-printable`` or ``base64`` depending on - the encoding used, or it is a function, in which case you should call the - function with a single argument, the Message object being encoded. The - function should then set the :mailheader:`Content-Transfer-Encoding` - header itself to whatever is appropriate. - - Returns the string ``quoted-printable`` if *body_encoding* is ``QP``, - returns the string ``base64`` if *body_encoding* is ``BASE64``, and - returns the string ``7bit`` otherwise. - - - .. method:: get_output_charset() - - Return the output character set. - - This is the *output_charset* attribute if that is not ``None``, otherwise - it is *input_charset*. - - - .. method:: header_encode(string) - - Header-encode the string *string*. - - The type of encoding (base64 or quoted-printable) will be based on the - *header_encoding* attribute. - - - .. method:: header_encode_lines(string, maxlengths) - - Header-encode a *string* by converting it first to bytes. - - This is similar to :meth:`header_encode` except that the string is fit - into maximum line lengths as given by the argument *maxlengths*, which - must be an iterator: each element returned from this iterator will provide - the next maximum line length. - - - .. method:: body_encode(string) - - Body-encode the string *string*. - - The type of encoding (base64 or quoted-printable) will be based on the - *body_encoding* attribute. - - The :class:`Charset` class also provides a number of methods to support - standard operations and built-in functions. - - - .. method:: __str__() - - Returns *input_charset* as a string coerced to lower - case. :meth:`!__repr__` is an alias for :meth:`!__str__`. - - - .. method:: __eq__(other) - - This method allows you to compare two :class:`Charset` instances for - equality. - - - .. method:: __ne__(other) - - This method allows you to compare two :class:`Charset` instances for - inequality. - -The :mod:`email.charset` module also provides the following functions for adding -new entries to the global character set, alias, and codec registries: - - -.. function:: add_charset(charset, header_enc=None, body_enc=None, output_charset=None) - - Add character properties to the global registry. - - *charset* is the input character set, and must be the canonical name of a - character set. - - Optional *header_enc* and *body_enc* is either ``Charset.QP`` for - quoted-printable, ``Charset.BASE64`` for base64 encoding, - ``Charset.SHORTEST`` for the shortest of quoted-printable or base64 encoding, - or ``None`` for no encoding. ``SHORTEST`` is only valid for - *header_enc*. The default is ``None`` for no encoding. - - Optional *output_charset* is the character set that the output should be in. - Conversions will proceed from input charset, to Unicode, to the output charset - when the method :meth:`Charset.convert` is called. The default is to output in - the same character set as the input. - - Both *input_charset* and *output_charset* must have Unicode codec entries in the - module's character set-to-codec mapping; use :func:`add_codec` to add codecs the - module does not know about. See the :mod:`codecs` module's documentation for - more information. - - The global character set registry is kept in the module global dictionary - ``CHARSETS``. - - -.. function:: add_alias(alias, canonical) - - Add a character set alias. *alias* is the alias name, e.g. ``latin-1``. - *canonical* is the character set's canonical name, e.g. ``iso-8859-1``. - - The global charset alias registry is kept in the module global dictionary - ``ALIASES``. - - -.. function:: add_codec(charset, codecname) - - Add a codec that map characters in the given character set to and from Unicode. - - *charset* is the canonical name of a character set. *codecname* is the name of a - Python codec, as appropriate for the second argument to the :class:`str`'s - :meth:`~str.encode` method. - diff --git a/Doc/library/email.compat32-message.rst b/Doc/library/email.compat32-message.rst deleted file mode 100644 index c4c322a82e1f44..00000000000000 --- a/Doc/library/email.compat32-message.rst +++ /dev/null @@ -1,759 +0,0 @@ -.. _compat32_message: - -:mod:`email.message.Message`: Representing an email message using the :data:`~email.policy.compat32` API --------------------------------------------------------------------------------------------------------- - -.. module:: email.message - :synopsis: The base class representing email messages in a fashion - backward compatible with Python 3.2 - :noindex: - - -The :class:`Message` class is very similar to the -:class:`~email.message.EmailMessage` class, without the methods added by that -class, and with the default behavior of certain other methods being slightly -different. We also document here some methods that, while supported by the -:class:`~email.message.EmailMessage` class, are not recommended unless you are -dealing with legacy code. - -The philosophy and structure of the two classes is otherwise the same. - -This document describes the behavior under the default (for :class:`Message`) -policy :attr:`~email.policy.Compat32`. If you are going to use another policy, -you should be using the :class:`~email.message.EmailMessage` class instead. - -An email message consists of *headers* and a *payload*. Headers must be -:rfc:`5322` style names and values, where the field name and value are -separated by a colon. The colon is not part of either the field name or the -field value. The payload may be a simple text message, or a binary object, or -a structured sequence of sub-messages each with their own set of headers and -their own payload. The latter type of payload is indicated by the message -having a MIME type such as :mimetype:`multipart/\*` or -:mimetype:`message/rfc822`. - -The conceptual model provided by a :class:`Message` object is that of an -ordered dictionary of headers with additional methods for accessing both -specialized information from the headers, for accessing the payload, for -generating a serialized version of the message, and for recursively walking -over the object tree. Note that duplicate headers are supported but special -methods must be used to access them. - -The :class:`Message` pseudo-dictionary is indexed by the header names, which -must be ASCII values. The values of the dictionary are strings that are -supposed to contain only ASCII characters; there is some special handling for -non-ASCII input, but it doesn't always produce the correct results. Headers -are stored and returned in case-preserving form, but field names are matched -case-insensitively. There may also be a single envelope header, also known as -the *Unix-From* header or the ``From_`` header. The *payload* is either a -string or bytes, in the case of simple message objects, or a list of -:class:`Message` objects, for MIME container documents (e.g. -:mimetype:`multipart/\*` and :mimetype:`message/rfc822`). - -Here are the methods of the :class:`Message` class: - - -.. class:: Message(policy=compat32) - - If *policy* is specified (it must be an instance of a :mod:`~email.policy` - class) use the rules it specifies to update and serialize the representation - of the message. If *policy* is not set, use the :class:`compat32 - ` policy, which maintains backward compatibility with - the Python 3.2 version of the email package. For more information see the - :mod:`~email.policy` documentation. - - .. versionchanged:: 3.3 The *policy* keyword argument was added. - - - .. method:: as_string(unixfrom=False, maxheaderlen=0, policy=None) - - Return the entire message flattened as a string. When optional *unixfrom* - is true, the envelope header is included in the returned string. - *unixfrom* defaults to ``False``. For backward compatibility reasons, - *maxheaderlen* defaults to ``0``, so if you want a different value you - must override it explicitly (the value specified for *max_line_length* in - the policy will be ignored by this method). The *policy* argument may be - used to override the default policy obtained from the message instance. - This can be used to control some of the formatting produced by the - method, since the specified *policy* will be passed to the ``Generator``. - - Flattening the message may trigger changes to the :class:`Message` if - defaults need to be filled in to complete the transformation to a string - (for example, MIME boundaries may be generated or modified). - - Note that this method is provided as a convenience and may not always - format the message the way you want. For example, by default it does - not do the mangling of lines that begin with ``From`` that is - required by the Unix mbox format. For more flexibility, instantiate a - :class:`~email.generator.Generator` instance and use its - :meth:`~email.generator.Generator.flatten` method directly. For example:: - - from io import StringIO - from email.generator import Generator - fp = StringIO() - g = Generator(fp, mangle_from_=True, maxheaderlen=60) - g.flatten(msg) - text = fp.getvalue() - - If the message object contains binary data that is not encoded according - to RFC standards, the non-compliant data will be replaced by unicode - "unknown character" code points. (See also :meth:`.as_bytes` and - :class:`~email.generator.BytesGenerator`.) - - .. versionchanged:: 3.4 the *policy* keyword argument was added. - - - .. method:: __str__() - - Equivalent to :meth:`.as_string()`. Allows ``str(msg)`` to produce a - string containing the formatted message. - - - .. method:: as_bytes(unixfrom=False, policy=None) - - Return the entire message flattened as a bytes object. When optional - *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. The *policy* argument may be - used to override the default policy obtained from the message instance. - This can be used to control some of the formatting produced by the - method, since the specified *policy* will be passed to the - ``BytesGenerator``. - - Flattening the message may trigger changes to the :class:`Message` if - defaults need to be filled in to complete the transformation to a string - (for example, MIME boundaries may be generated or modified). - - Note that this method is provided as a convenience and may not always - format the message the way you want. For example, by default it does - not do the mangling of lines that begin with ``From`` that is - required by the Unix mbox format. For more flexibility, instantiate a - :class:`~email.generator.BytesGenerator` instance and use its - :meth:`~email.generator.BytesGenerator.flatten` method directly. - For example:: - - from io import BytesIO - from email.generator import BytesGenerator - fp = BytesIO() - g = BytesGenerator(fp, mangle_from_=True, maxheaderlen=60) - g.flatten(msg) - text = fp.getvalue() - - .. versionadded:: 3.4 - - - .. method:: __bytes__() - - Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a - bytes object containing the formatted message. - - .. versionadded:: 3.4 - - - .. method:: is_multipart() - - Return ``True`` if the message's payload is a list of - sub-\ :class:`Message` objects, otherwise return ``False``. When - :meth:`is_multipart` returns ``False``, the payload should be a string - object (which might be a CTE encoded binary payload). (Note that - :meth:`is_multipart` returning ``True`` does not necessarily mean that - "msg.get_content_maintype() == 'multipart'" will return the ``True``. - For example, ``is_multipart`` will return ``True`` when the - :class:`Message` is of type ``message/rfc822``.) - - - .. method:: set_unixfrom(unixfrom) - - Set the message's envelope header to *unixfrom*, which should be a string. - - - .. method:: get_unixfrom() - - Return the message's envelope header. Defaults to ``None`` if the - envelope header was never set. - - - .. method:: attach(payload) - - Add the given *payload* to the current payload, which must be ``None`` or - a list of :class:`Message` objects before the call. After the call, the - payload will always be a list of :class:`Message` objects. If you want to - set the payload to a scalar object (e.g. a string), use - :meth:`set_payload` instead. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by :meth:`~email.message.EmailMessage.set_content` and the - related ``make`` and ``add`` methods. - - - .. method:: get_payload(i=None, decode=False) - - Return the current payload, which will be a list of - :class:`Message` objects when :meth:`is_multipart` is ``True``, or a - string when :meth:`is_multipart` is ``False``. If the payload is a list - and you mutate the list object, you modify the message's payload in place. - - With optional argument *i*, :meth:`get_payload` will return the *i*-th - element of the payload, counting from zero, if :meth:`is_multipart` is - ``True``. An :exc:`IndexError` will be raised if *i* is less than 0 or - greater than or equal to the number of items in the payload. If the - payload is a string (i.e. :meth:`is_multipart` is ``False``) and *i* is - given, a :exc:`TypeError` is raised. - - Optional *decode* is a flag indicating whether the payload should be - decoded or not, according to the :mailheader:`Content-Transfer-Encoding` - header. When ``True`` and the message is not a multipart, the payload will - be decoded if this header's value is ``quoted-printable`` or ``base64``. - If some other encoding is used, or :mailheader:`Content-Transfer-Encoding` - header is missing, the payload is - returned as-is (undecoded). In all cases the returned value is binary - data. If the message is a multipart and the *decode* flag is ``True``, - then ``None`` is returned. If the payload is base64 and it was not - perfectly formed (missing padding, characters outside the base64 - alphabet), then an appropriate defect will be added to the message's - defect property (:class:`~email.errors.InvalidBase64PaddingDefect` or - :class:`~email.errors.InvalidBase64CharactersDefect`, respectively). - - When *decode* is ``False`` (the default) the body is returned as a string - without decoding the :mailheader:`Content-Transfer-Encoding`. However, - for a :mailheader:`Content-Transfer-Encoding` of 8bit, an attempt is made - to decode the original bytes using the ``charset`` specified by the - :mailheader:`Content-Type` header, using the ``replace`` error handler. - If no ``charset`` is specified, or if the ``charset`` given is not - recognized by the email package, the body is decoded using the default - ASCII charset. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by :meth:`~email.message.EmailMessage.get_content` and - :meth:`~email.message.EmailMessage.iter_parts`. - - - .. method:: set_payload(payload, charset=None) - - Set the entire message object's payload to *payload*. It is the client's - responsibility to ensure the payload invariants. Optional *charset* sets - the message's default character set; see :meth:`set_charset` for details. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by :meth:`~email.message.EmailMessage.set_content`. - - - .. method:: set_charset(charset) - - Set the character set of the payload to *charset*, which can either be a - :class:`~email.charset.Charset` instance (see :mod:`email.charset`), a - string naming a character set, or ``None``. If it is a string, it will - be converted to a :class:`~email.charset.Charset` instance. If *charset* - is ``None``, the ``charset`` parameter will be removed from the - :mailheader:`Content-Type` header (the message will not be otherwise - modified). Anything else will generate a :exc:`TypeError`. - - If there is no existing :mailheader:`MIME-Version` header one will be - added. If there is no existing :mailheader:`Content-Type` header, one - will be added with a value of :mimetype:`text/plain`. Whether the - :mailheader:`Content-Type` header already exists or not, its ``charset`` - parameter will be set to *charset.output_charset*. If - *charset.input_charset* and *charset.output_charset* differ, the payload - will be re-encoded to the *output_charset*. If there is no existing - :mailheader:`Content-Transfer-Encoding` header, then the payload will be - transfer-encoded, if needed, using the specified - :class:`~email.charset.Charset`, and a header with the appropriate value - will be added. If a :mailheader:`Content-Transfer-Encoding` header - already exists, the payload is assumed to already be correctly encoded - using that :mailheader:`Content-Transfer-Encoding` and is not modified. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by the *charset* parameter of the - :meth:`email.emailmessage.EmailMessage.set_content` method. - - - .. method:: get_charset() - - Return the :class:`~email.charset.Charset` instance associated with the - message's payload. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class it always returns - ``None``. - - - The following methods implement a mapping-like interface for accessing the - message's :rfc:`2822` headers. Note that there are some semantic differences - between these methods and a normal mapping (i.e. dictionary) interface. For - example, in a dictionary there are no duplicate keys, but here there may be - duplicate message headers. Also, in dictionaries there is no guaranteed - order to the keys returned by :meth:`keys`, but in a :class:`Message` object, - headers are always returned in the order they appeared in the original - message, or were added to the message later. Any header deleted and then - re-added are always appended to the end of the header list. - - These semantic differences are intentional and are biased toward maximal - convenience. - - Note that in all cases, any envelope header present in the message is not - included in the mapping interface. - - In a model generated from bytes, any header values that (in contravention of - the RFCs) contain non-ASCII bytes will, when retrieved through this - interface, be represented as :class:`~email.header.Header` objects with - a charset of ``unknown-8bit``. - - - .. method:: __len__() - - Return the total number of headers, including duplicates. - - - .. method:: __contains__(name) - - Return ``True`` if the message object has a field named *name*. Matching is - done case-insensitively and *name* should not include the trailing colon. - Used for the ``in`` operator, e.g.:: - - if 'message-id' in myMessage: - print('Message-ID:', myMessage['message-id']) - - - .. method:: __getitem__(name) - - Return the value of the named header field. *name* should not include the - colon field separator. If the header is missing, ``None`` is returned; a - :exc:`KeyError` is never raised. - - Note that if the named field appears more than once in the message's - headers, exactly which of those field values will be returned is - undefined. Use the :meth:`get_all` method to get the values of all the - extant named headers. - - - .. method:: __setitem__(name, val) - - Add a header to the message with field name *name* and value *val*. The - field is appended to the end of the message's existing fields. - - Note that this does *not* overwrite or delete any existing header with the same - name. If you want to ensure that the new header is the only one present in the - message with field name *name*, delete the field first, e.g.:: - - del msg['subject'] - msg['subject'] = 'Python roolz!' - - - .. method:: __delitem__(name) - - Delete all occurrences of the field with name *name* from the message's - headers. No exception is raised if the named field isn't present in the - headers. - - - .. method:: keys() - - Return a list of all the message's header field names. - - - .. method:: values() - - Return a list of all the message's field values. - - - .. method:: items() - - Return a list of 2-tuples containing all the message's field headers and - values. - - - .. method:: get(name, failobj=None) - - Return the value of the named header field. This is identical to - :meth:`~object.__getitem__` except that optional *failobj* is returned if the - named header is missing (defaults to ``None``). - - Here are some additional useful methods: - - - .. method:: get_all(name, failobj=None) - - Return a list of all the values for the field named *name*. If there are - no such named headers in the message, *failobj* is returned (defaults to - ``None``). - - - .. method:: add_header(_name, _value, **_params) - - Extended header setting. This method is similar to :meth:`__setitem__` - except that additional header parameters can be provided as keyword - arguments. *_name* is the header field to add and *_value* is the - *primary* value for the header. - - For each item in the keyword argument dictionary *_params*, the key is - taken as the parameter name, with underscores converted to dashes (since - dashes are illegal in Python identifiers). Normally, the parameter will - be added as ``key="value"`` unless the value is ``None``, in which case - only the key will be added. If the value contains non-ASCII characters, - it can be specified as a three tuple in the format - ``(CHARSET, LANGUAGE, VALUE)``, where ``CHARSET`` is a string naming the - charset to be used to encode the value, ``LANGUAGE`` can usually be set - to ``None`` or the empty string (see :rfc:`2231` for other possibilities), - and ``VALUE`` is the string value containing non-ASCII code points. If - a three tuple is not passed and the value contains non-ASCII characters, - it is automatically encoded in :rfc:`2231` format using a ``CHARSET`` - of ``utf-8`` and a ``LANGUAGE`` of ``None``. - - Here's an example:: - - msg.add_header('Content-Disposition', 'attachment', filename='bud.gif') - - This will add a header that looks like :: - - Content-Disposition: attachment; filename="bud.gif" - - An example with non-ASCII characters:: - - msg.add_header('Content-Disposition', 'attachment', - filename=('iso-8859-1', '', 'Fußballer.ppt')) - - Which produces :: - - Content-Disposition: attachment; filename*="iso-8859-1''Fu%DFballer.ppt" - - - .. method:: replace_header(_name, _value) - - Replace a header. Replace the first header found in the message that - matches *_name*, retaining header order and field name case. If no - matching header was found, a :exc:`KeyError` is raised. - - - .. method:: get_content_type() - - Return the message's content type. The returned string is coerced to - lower case of the form :mimetype:`maintype/subtype`. If there was no - :mailheader:`Content-Type` header in the message the default type as given - by :meth:`get_default_type` will be returned. Since according to - :rfc:`2045`, messages always have a default type, :meth:`get_content_type` - will always return a value. - - :rfc:`2045` defines a message's default type to be :mimetype:`text/plain` - unless it appears inside a :mimetype:`multipart/digest` container, in - which case it would be :mimetype:`message/rfc822`. If the - :mailheader:`Content-Type` header has an invalid type specification, - :rfc:`2045` mandates that the default type be :mimetype:`text/plain`. - - - .. method:: get_content_maintype() - - Return the message's main content type. This is the :mimetype:`maintype` - part of the string returned by :meth:`get_content_type`. - - - .. method:: get_content_subtype() - - Return the message's sub-content type. This is the :mimetype:`subtype` - part of the string returned by :meth:`get_content_type`. - - - .. method:: get_default_type() - - Return the default content type. Most messages have a default content - type of :mimetype:`text/plain`, except for messages that are subparts of - :mimetype:`multipart/digest` containers. Such subparts have a default - content type of :mimetype:`message/rfc822`. - - - .. method:: set_default_type(ctype) - - Set the default content type. *ctype* should either be - :mimetype:`text/plain` or :mimetype:`message/rfc822`, although this is not - enforced. The default content type is not stored in the - :mailheader:`Content-Type` header. - - - .. method:: get_params(failobj=None, header='content-type', unquote=True) - - Return the message's :mailheader:`Content-Type` parameters, as a list. - The elements of the returned list are 2-tuples of key/value pairs, as - split on the ``'='`` sign. The left hand side of the ``'='`` is the key, - while the right hand side is the value. If there is no ``'='`` sign in - the parameter the value is the empty string, otherwise the value is as - described in :meth:`get_param` and is unquoted if optional *unquote* is - ``True`` (the default). - - Optional *failobj* is the object to return if there is no - :mailheader:`Content-Type` header. Optional *header* is the header to - search instead of :mailheader:`Content-Type`. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by the *params* property of the individual header objects - returned by the header access methods. - - - .. method:: get_param(param, failobj=None, header='content-type', unquote=True) - - Return the value of the :mailheader:`Content-Type` header's parameter - *param* as a string. If the message has no :mailheader:`Content-Type` - header or if there is no such parameter, then *failobj* is returned - (defaults to ``None``). - - Optional *header* if given, specifies the message header to use instead of - :mailheader:`Content-Type`. - - Parameter keys are always compared case insensitively. The return value - can either be a string, or a 3-tuple if the parameter was :rfc:`2231` - encoded. When it's a 3-tuple, the elements of the value are of the form - ``(CHARSET, LANGUAGE, VALUE)``. Note that both ``CHARSET`` and - ``LANGUAGE`` can be ``None``, in which case you should consider ``VALUE`` - to be encoded in the ``us-ascii`` charset. You can usually ignore - ``LANGUAGE``. - - If your application doesn't care whether the parameter was encoded as in - :rfc:`2231`, you can collapse the parameter value by calling - :func:`email.utils.collapse_rfc2231_value`, passing in the return value - from :meth:`get_param`. This will return a suitably decoded Unicode - string when the value is a tuple, or the original string unquoted if it - isn't. For example:: - - rawparam = msg.get_param('foo') - param = email.utils.collapse_rfc2231_value(rawparam) - - In any case, the parameter value (either the returned string, or the - ``VALUE`` item in the 3-tuple) is always unquoted, unless *unquote* is set - to ``False``. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by the *params* property of the individual header objects - returned by the header access methods. - - - .. method:: set_param(param, value, header='Content-Type', requote=True, \ - charset=None, language='', replace=False) - - Set a parameter in the :mailheader:`Content-Type` header. If the - parameter already exists in the header, its value will be replaced with - *value*. If the :mailheader:`Content-Type` header as not yet been defined - for this message, it will be set to :mimetype:`text/plain` and the new - parameter value will be appended as per :rfc:`2045`. - - Optional *header* specifies an alternative header to - :mailheader:`Content-Type`, and all parameters will be quoted as necessary - unless optional *requote* is ``False`` (the default is ``True``). - - If optional *charset* is specified, the parameter will be encoded - according to :rfc:`2231`. Optional *language* specifies the RFC 2231 - language, defaulting to the empty string. Both *charset* and *language* - should be strings. - - If *replace* is ``False`` (the default) the header is moved to the - end of the list of headers. If *replace* is ``True``, the header - will be updated in place. - - .. versionchanged:: 3.4 ``replace`` keyword was added. - - - .. method:: del_param(param, header='content-type', requote=True) - - Remove the given parameter completely from the :mailheader:`Content-Type` - header. The header will be re-written in place without the parameter or - its value. All values will be quoted as necessary unless *requote* is - ``False`` (the default is ``True``). Optional *header* specifies an - alternative to :mailheader:`Content-Type`. - - - .. method:: set_type(type, header='Content-Type', requote=True) - - Set the main type and subtype for the :mailheader:`Content-Type` - header. *type* must be a string in the form :mimetype:`maintype/subtype`, - otherwise a :exc:`ValueError` is raised. - - This method replaces the :mailheader:`Content-Type` header, keeping all - the parameters in place. If *requote* is ``False``, this leaves the - existing header's quoting as is, otherwise the parameters will be quoted - (the default). - - An alternative header can be specified in the *header* argument. When the - :mailheader:`Content-Type` header is set a :mailheader:`MIME-Version` - header is also added. - - This is a legacy method. On the - :class:`~email.emailmessage.EmailMessage` class its functionality is - replaced by the ``make_`` and ``add_`` methods. - - - .. method:: get_filename(failobj=None) - - Return the value of the ``filename`` parameter of the - :mailheader:`Content-Disposition` header of the message. If the header - does not have a ``filename`` parameter, this method falls back to looking - for the ``name`` parameter on the :mailheader:`Content-Type` header. If - neither is found, or the header is missing, then *failobj* is returned. - The returned string will always be unquoted as per - :func:`email.utils.unquote`. - - - .. method:: get_boundary(failobj=None) - - Return the value of the ``boundary`` parameter of the - :mailheader:`Content-Type` header of the message, or *failobj* if either - the header is missing, or has no ``boundary`` parameter. The returned - string will always be unquoted as per :func:`email.utils.unquote`. - - - .. method:: set_boundary(boundary) - - Set the ``boundary`` parameter of the :mailheader:`Content-Type` header to - *boundary*. :meth:`set_boundary` will always quote *boundary* if - necessary. A :exc:`~email.errors.HeaderParseError` is raised if the - message object has no :mailheader:`Content-Type` header. - - Note that using this method is subtly different than deleting the old - :mailheader:`Content-Type` header and adding a new one with the new - boundary via :meth:`add_header`, because :meth:`set_boundary` preserves - the order of the :mailheader:`Content-Type` header in the list of - headers. However, it does *not* preserve any continuation lines which may - have been present in the original :mailheader:`Content-Type` header. - - - .. method:: get_content_charset(failobj=None) - - Return the ``charset`` parameter of the :mailheader:`Content-Type` header, - coerced to lower case. If there is no :mailheader:`Content-Type` header, or if - that header has no ``charset`` parameter, *failobj* is returned. - - Note that this method differs from :meth:`get_charset` which returns the - :class:`~email.charset.Charset` instance for the default encoding of the message body. - - - .. method:: get_charsets(failobj=None) - - Return a list containing the character set names in the message. If the - message is a :mimetype:`multipart`, then the list will contain one element - for each subpart in the payload, otherwise, it will be a list of length 1. - - Each item in the list will be a string which is the value of the - ``charset`` parameter in the :mailheader:`Content-Type` header for the - represented subpart. However, if the subpart has no - :mailheader:`Content-Type` header, no ``charset`` parameter, or is not of - the :mimetype:`text` main MIME type, then that item in the returned list - will be *failobj*. - - - .. method:: get_content_disposition() - - Return the lowercased value (without parameters) of the message's - :mailheader:`Content-Disposition` header if it has one, or ``None``. The - possible values for this method are *inline*, *attachment* or ``None`` - if the message follows :rfc:`2183`. - - .. versionadded:: 3.5 - - .. method:: walk() - - The :meth:`walk` method is an all-purpose generator which can be used to - iterate over all the parts and subparts of a message object tree, in - depth-first traversal order. You will typically use :meth:`walk` as the - iterator in a ``for`` loop; each iteration returns the next subpart. - - Here's an example that prints the MIME type of every part of a multipart - message structure: - - .. testsetup:: - - import email - from email import message_from_binary_file - from os.path import join, dirname - lib_dir = dirname(dirname(email.__file__)) - file_path = join(lib_dir, 'test/test_email/data/msg_16.txt') - with open(file_path, 'rb') as f: - msg = message_from_binary_file(f) - from email.iterators import _structure - - .. doctest:: - - >>> for part in msg.walk(): - ... print(part.get_content_type()) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - ``walk`` iterates over the subparts of any part where - :meth:`is_multipart` returns ``True``, even though - ``msg.get_content_maintype() == 'multipart'`` may return ``False``. We - can see this in our example by making use of the ``_structure`` debug - helper function: - - .. doctest:: - - >>> for part in msg.walk(): - ... print(part.get_content_maintype() == 'multipart', - ... part.is_multipart()) - True True - False False - False True - False False - False False - False True - False False - >>> _structure(msg) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - Here the ``message`` parts are not ``multiparts``, but they do contain - subparts. ``is_multipart()`` returns ``True`` and ``walk`` descends - into the subparts. - - - :class:`Message` objects can also optionally contain two instance attributes, - which can be used when generating the plain text of a MIME message. - - - .. attribute:: preamble - - The format of a MIME document allows for some text between the blank line - following the headers, and the first multipart boundary string. Normally, - this text is never visible in a MIME-aware mail reader because it falls - outside the standard MIME armor. However, when viewing the raw text of - the message, or when viewing the message in a non-MIME aware reader, this - text can become visible. - - The *preamble* attribute contains this leading extra-armor text for MIME - documents. When the :class:`~email.parser.Parser` discovers some text - after the headers but before the first boundary string, it assigns this - text to the message's *preamble* attribute. When the - :class:`~email.generator.Generator` is writing out the plain text - representation of a MIME message, and it finds the - message has a *preamble* attribute, it will write this text in the area - between the headers and the first boundary. See :mod:`email.parser` and - :mod:`email.generator` for details. - - Note that if the message object has no preamble, the *preamble* attribute - will be ``None``. - - - .. attribute:: epilogue - - The *epilogue* attribute acts the same way as the *preamble* attribute, - except that it contains text that appears between the last boundary and - the end of the message. - - You do not need to set the epilogue to the empty string in order for the - :class:`~email.generator.Generator` to print a newline at the end of the - file. - - - .. attribute:: defects - - The *defects* attribute contains a list of all the problems found when - parsing this message. See :mod:`email.errors` for a detailed description - of the possible parsing defects. diff --git a/Doc/library/email.contentmanager.rst b/Doc/library/email.contentmanager.rst deleted file mode 100644 index 5b49339650f0e9..00000000000000 --- a/Doc/library/email.contentmanager.rst +++ /dev/null @@ -1,198 +0,0 @@ -:mod:`email.contentmanager`: Managing MIME Content --------------------------------------------------- - -.. module:: email.contentmanager - :synopsis: Storing and Retrieving Content from MIME Parts - -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - -**Source code:** :source:`Lib/email/contentmanager.py` - ------------- - -.. versionadded:: 3.6 [1]_ - - -.. class:: ContentManager() - - Base class for content managers. Provides the standard registry mechanisms - to register converters between MIME content and other representations, as - well as the ``get_content`` and ``set_content`` dispatch methods. - - - .. method:: get_content(msg, *args, **kw) - - Look up a handler function based on the ``mimetype`` of *msg* (see next - paragraph), call it, passing through all arguments, and return the result - of the call. The expectation is that the handler will extract the - payload from *msg* and return an object that encodes information about - the extracted data. - - To find the handler, look for the following keys in the registry, - stopping with the first one found: - - * the string representing the full MIME type (``maintype/subtype``) - * the string representing the ``maintype`` - * the empty string - - If none of these keys produce a handler, raise a :exc:`KeyError` for the - full MIME type. - - - .. method:: set_content(msg, obj, *args, **kw) - - If the ``maintype`` is ``multipart``, raise a :exc:`TypeError`; otherwise - look up a handler function based on the type of *obj* (see next - paragraph), call :meth:`~email.message.EmailMessage.clear_content` on the - *msg*, and call the handler function, passing through all arguments. The - expectation is that the handler will transform and store *obj* into - *msg*, possibly making other changes to *msg* as well, such as adding - various MIME headers to encode information needed to interpret the stored - data. - - To find the handler, obtain the type of *obj* (``typ = type(obj)``), and - look for the following keys in the registry, stopping with the first one - found: - - * the type itself (``typ``) - * the type's fully qualified name (``typ.__module__ + '.' + - typ.__qualname__``). - * the type's qualname (``typ.__qualname__``) - * the type's name (``typ.__name__``). - - If none of the above match, repeat all of the checks above for each of - the types in the :term:`MRO` (``typ.__mro__``). Finally, if no other key - yields a handler, check for a handler for the key ``None``. If there is - no handler for ``None``, raise a :exc:`KeyError` for the fully - qualified name of the type. - - Also add a :mailheader:`MIME-Version` header if one is not present (see - also :class:`.MIMEPart`). - - - .. method:: add_get_handler(key, handler) - - Record the function *handler* as the handler for *key*. For the possible - values of *key*, see :meth:`get_content`. - - - .. method:: add_set_handler(typekey, handler) - - Record *handler* as the function to call when an object of a type - matching *typekey* is passed to :meth:`set_content`. For the possible - values of *typekey*, see :meth:`set_content`. - - -Content Manager Instances -~~~~~~~~~~~~~~~~~~~~~~~~~ - -Currently the email package provides only one concrete content manager, -:data:`raw_data_manager`, although more may be added in the future. -:data:`raw_data_manager` is the -:attr:`~email.policy.EmailPolicy.content_manager` provided by -:attr:`~email.policy.EmailPolicy` and its derivatives. - - -.. data:: raw_data_manager - - This content manager provides only a minimum interface beyond that provided - by :class:`~email.message.Message` itself: it deals only with text, raw - byte strings, and :class:`~email.message.Message` objects. Nevertheless, it - provides significant advantages compared to the base API: ``get_content`` on - a text part will return a unicode string without the application needing to - manually decode it, ``set_content`` provides a rich set of options for - controlling the headers added to a part and controlling the content transfer - encoding, and it enables the use of the various ``add_`` methods, thereby - simplifying the creation of multipart messages. - - .. method:: get_content(msg, errors='replace') - - Return the payload of the part as either a string (for ``text`` parts), an - :class:`~email.message.EmailMessage` object (for ``message/rfc822`` - parts), or a ``bytes`` object (for all other non-multipart types). Raise - a :exc:`KeyError` if called on a ``multipart``. If the part is a - ``text`` part and *errors* is specified, use it as the error handler when - decoding the payload to unicode. The default error handler is - ``replace``. - - .. method:: set_content(msg, <'str'>, subtype="plain", charset='utf-8', \ - cte=None, \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - set_content(msg, <'bytes'>, maintype, subtype, cte="base64", \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - set_content(msg, <'EmailMessage'>, cte=None, \ - disposition=None, filename=None, cid=None, \ - params=None, headers=None) - - Add headers and payload to *msg*: - - Add a :mailheader:`Content-Type` header with a ``maintype/subtype`` - value. - - * For ``str``, set the MIME ``maintype`` to ``text``, and set the - subtype to *subtype* if it is specified, or ``plain`` if it is not. - * For ``bytes``, use the specified *maintype* and *subtype*, or - raise a :exc:`TypeError` if they are not specified. - * For :class:`~email.message.EmailMessage` objects, set the maintype - to ``message``, and set the subtype to *subtype* if it is - specified or ``rfc822`` if it is not. If *subtype* is - ``partial``, raise an error (``bytes`` objects must be used to - construct ``message/partial`` parts). - - If *charset* is provided (which is valid only for ``str``), encode the - string to bytes using the specified character set. The default is - ``utf-8``. If the specified *charset* is a known alias for a standard - MIME charset name, use the standard charset instead. - - If *cte* is set, encode the payload using the specified content transfer - encoding, and set the :mailheader:`Content-Transfer-Encoding` header to - that value. Possible values for *cte* are ``quoted-printable``, - ``base64``, ``7bit``, ``8bit``, and ``binary``. If the input cannot be - encoded in the specified encoding (for example, specifying a *cte* of - ``7bit`` for an input that contains non-ASCII values), raise a - :exc:`ValueError`. - - * For ``str`` objects, if *cte* is not set use heuristics to - determine the most compact encoding. - * For :class:`~email.message.EmailMessage`, per :rfc:`2046`, raise - an error if a *cte* of ``quoted-printable`` or ``base64`` is - requested for *subtype* ``rfc822``, and for any *cte* other than - ``7bit`` for *subtype* ``external-body``. For - ``message/rfc822``, use ``8bit`` if *cte* is not specified. For - all other values of *subtype*, use ``7bit``. - - .. note:: A *cte* of ``binary`` does not actually work correctly yet. - The ``EmailMessage`` object as modified by ``set_content`` is - correct, but :class:`~email.generator.BytesGenerator` does not - serialize it correctly. - - If *disposition* is set, use it as the value of the - :mailheader:`Content-Disposition` header. If not specified, and - *filename* is specified, add the header with the value ``attachment``. - If *disposition* is not specified and *filename* is also not specified, - do not add the header. The only valid values for *disposition* are - ``attachment`` and ``inline``. - - If *filename* is specified, use it as the value of the ``filename`` - parameter of the :mailheader:`Content-Disposition` header. - - If *cid* is specified, add a :mailheader:`Content-ID` header with - *cid* as its value. - - If *params* is specified, iterate its ``items`` method and use the - resulting ``(key, value)`` pairs to set additional parameters on the - :mailheader:`Content-Type` header. - - If *headers* is specified and is a list of strings of the form - ``headername: headervalue`` or a list of ``header`` objects - (distinguished from strings by having a ``name`` attribute), add the - headers to *msg*. - - -.. rubric:: Footnotes - -.. [1] Originally added in 3.4 as a :term:`provisional module ` diff --git a/Doc/library/email.encoders.rst b/Doc/library/email.encoders.rst deleted file mode 100644 index 3bd377e33f6c15..00000000000000 --- a/Doc/library/email.encoders.rst +++ /dev/null @@ -1,75 +0,0 @@ -:mod:`email.encoders`: Encoders -------------------------------- - -.. module:: email.encoders - :synopsis: Encoders for email message payloads. - -**Source code:** :source:`Lib/email/encoders.py` - --------------- - -This module is part of the legacy (``Compat32``) email API. In the -new API the functionality is provided by the *cte* parameter of -the :meth:`~email.message.EmailMessage.set_content` method. - -This module is deprecated in Python 3. The functions provided here -should not be called explicitly since the :class:`~email.mime.text.MIMEText` -class sets the content type and CTE header using the *_subtype* and *_charset* -values passed during the instantiation of that class. - -The remaining text in this section is the original documentation of the module. - -When creating :class:`~email.message.Message` objects from scratch, you often -need to encode the payloads for transport through compliant mail servers. This -is especially true for :mimetype:`image/\*` and :mimetype:`text/\*` type messages -containing binary data. - -The :mod:`email` package provides some convenient encoders in its -:mod:`~email.encoders` module. These encoders are actually used by the -:class:`~email.mime.audio.MIMEAudio` and :class:`~email.mime.image.MIMEImage` -class constructors to provide default encodings. All encoder functions take -exactly one argument, the message object to encode. They usually extract the -payload, encode it, and reset the payload to this newly encoded value. They -should also set the :mailheader:`Content-Transfer-Encoding` header as appropriate. - -Note that these functions are not meaningful for a multipart message. They -must be applied to individual subparts instead, and will raise a -:exc:`TypeError` if passed a message whose type is multipart. - -Here are the encoding functions provided: - - -.. function:: encode_quopri(msg) - - Encodes the payload into quoted-printable form and sets the - :mailheader:`Content-Transfer-Encoding` header to ``quoted-printable`` [#]_. - This is a good encoding to use when most of your payload is normal printable - data, but contains a few unprintable characters. - - -.. function:: encode_base64(msg) - - Encodes the payload into base64 form and sets the - :mailheader:`Content-Transfer-Encoding` header to ``base64``. This is a good - encoding to use when most of your payload is unprintable data since it is a more - compact form than quoted-printable. The drawback of base64 encoding is that it - renders the text non-human readable. - - -.. function:: encode_7or8bit(msg) - - This doesn't actually modify the message's payload, but it does set the - :mailheader:`Content-Transfer-Encoding` header to either ``7bit`` or ``8bit`` as - appropriate, based on the payload data. - - -.. function:: encode_noop(msg) - - This does nothing; it doesn't even set the - :mailheader:`Content-Transfer-Encoding` header. - -.. rubric:: Footnotes - -.. [#] Note that encoding with :meth:`encode_quopri` also encodes all tabs and space - characters in the data. - diff --git a/Doc/library/email.errors.rst b/Doc/library/email.errors.rst deleted file mode 100644 index 194a98696f437d..00000000000000 --- a/Doc/library/email.errors.rst +++ /dev/null @@ -1,117 +0,0 @@ -:mod:`email.errors`: Exception and Defect classes -------------------------------------------------- - -.. module:: email.errors - :synopsis: The exception classes used by the email package. - -**Source code:** :source:`Lib/email/errors.py` - --------------- - -The following exception classes are defined in the :mod:`email.errors` module: - - -.. exception:: MessageError() - - This is the base class for all exceptions that the :mod:`email` package can - raise. It is derived from the standard :exc:`Exception` class and defines no - additional methods. - - -.. exception:: MessageParseError() - - This is the base class for exceptions raised by the - :class:`~email.parser.Parser` class. It is derived from - :exc:`MessageError`. This class is also used internally by the parser used - by :mod:`~email.headerregistry`. - - -.. exception:: HeaderParseError() - - Raised under some error conditions when parsing the :rfc:`5322` headers of a - message, this class is derived from :exc:`MessageParseError`. The - :meth:`~email.message.EmailMessage.set_boundary` method will raise this - error if the content type is unknown when the method is called. - :class:`~email.header.Header` may raise this error for certain base64 - decoding errors, and when an attempt is made to create a header that appears - to contain an embedded header (that is, there is what is supposed to be a - continuation line that has no leading whitespace and looks like a header). - - -.. exception:: BoundaryError() - - Deprecated and no longer used. - - -.. exception:: MultipartConversionError() - - Raised when a payload is added to a :class:`~email.message.Message` object - using :meth:`add_payload`, but the payload is already a scalar and the - message's :mailheader:`Content-Type` main type is not either - :mimetype:`multipart` or missing. :exc:`MultipartConversionError` multiply - inherits from :exc:`MessageError` and the built-in :exc:`TypeError`. - - Since :meth:`Message.add_payload` is deprecated, this exception is rarely - raised in practice. However the exception may also be raised if the - :meth:`~email.message.Message.attach` - method is called on an instance of a class derived from - :class:`~email.mime.nonmultipart.MIMENonMultipart` (e.g. - :class:`~email.mime.image.MIMEImage`). - - -Here is the list of the defects that the :class:`~email.parser.FeedParser` -can find while parsing messages. Note that the defects are added to the message -where the problem was found, so for example, if a message nested inside a -:mimetype:`multipart/alternative` had a malformed header, that nested message -object would have a defect, but the containing messages would not. - -All defect classes are subclassed from :class:`email.errors.MessageDefect`. - -* :class:`NoBoundaryInMultipartDefect` -- A message claimed to be a multipart, - but had no :mimetype:`boundary` parameter. - -* :class:`StartBoundaryNotFoundDefect` -- The start boundary claimed in the - :mailheader:`Content-Type` header was never found. - -* :class:`CloseBoundaryNotFoundDefect` -- A start boundary was found, but - no corresponding close boundary was ever found. - - .. versionadded:: 3.3 - -* :class:`FirstHeaderLineIsContinuationDefect` -- The message had a continuation - line as its first header line. - -* :class:`MisplacedEnvelopeHeaderDefect` - A "Unix From" header was found in the - middle of a header block. - -* :class:`MissingHeaderBodySeparatorDefect` - A line was found while parsing - headers that had no leading white space but contained no ':'. Parsing - continues assuming that the line represents the first line of the body. - - .. versionadded:: 3.3 - -* :class:`MalformedHeaderDefect` -- A header was found that was missing a colon, - or was otherwise malformed. - - .. deprecated:: 3.3 - This defect has not been used for several Python versions. - -* :class:`MultipartInvariantViolationDefect` -- A message claimed to be a - :mimetype:`multipart`, but no subparts were found. Note that when a message - has this defect, its :meth:`~email.message.Message.is_multipart` method may - return ``False`` even though its content type claims to be :mimetype:`multipart`. - -* :class:`InvalidBase64PaddingDefect` -- When decoding a block of base64 - encoded bytes, the padding was not correct. Enough padding is added to - perform the decode, but the resulting decoded bytes may be invalid. - -* :class:`InvalidBase64CharactersDefect` -- When decoding a block of base64 - encoded bytes, characters outside the base64 alphabet were encountered. - The characters are ignored, but the resulting decoded bytes may be invalid. - -* :class:`InvalidBase64LengthDefect` -- When decoding a block of base64 encoded - bytes, the number of non-padding base64 characters was invalid (1 more than - a multiple of 4). The encoded block was kept as-is. - -* :class:`InvalidDateDefect` -- When decoding an invalid or unparsable date field. - The original value is kept as-is. diff --git a/Doc/library/email.examples.rst b/Doc/library/email.examples.rst deleted file mode 100644 index fc964622809d0e..00000000000000 --- a/Doc/library/email.examples.rst +++ /dev/null @@ -1,67 +0,0 @@ -.. _email-examples: - -:mod:`email`: Examples ----------------------- - -Here are a few examples of how to use the :mod:`email` package to read, write, -and send simple email messages, as well as more complex MIME messages. - -First, let's see how to create and send a simple text message (both the -text content and the addresses may contain unicode characters): - -.. literalinclude:: ../includes/email-simple.py - - -Parsing :rfc:`822` headers can easily be done by the using the classes -from the :mod:`~email.parser` module: - -.. literalinclude:: ../includes/email-headers.py - - -Here's an example of how to send a MIME message containing a bunch of family -pictures that may be residing in a directory: - -.. literalinclude:: ../includes/email-mime.py - - -Here's an example of how to send the entire contents of a directory as an email -message: [1]_ - -.. literalinclude:: ../includes/email-dir.py - - -Here's an example of how to unpack a MIME message like the one -above, into a directory of files: - -.. literalinclude:: ../includes/email-unpack.py - - -Here's an example of how to create an HTML message with an alternative plain -text version. To make things a bit more interesting, we include a related -image in the html part, and we save a copy of what we are going to send to -disk, as well as sending it. - -.. literalinclude:: ../includes/email-alternative.py - - -If we were sent the message from the last example, here is one way we could -process it: - -.. literalinclude:: ../includes/email-read-alternative.py - -Up to the prompt, the output from the above is: - -.. code-block:: none - - To: Penelope Pussycat , Fabrette Pussycat - From: Pepé Le Pew - Subject: Ayons asperges pour le déjeuner - - Salut! - - Cela ressemble à un excellent recipie[1] déjeuner. - - -.. rubric:: Footnotes - -.. [1] Thanks to Matthew Dixon Cowles for the original inspiration and examples. diff --git a/Doc/library/email.generator.rst b/Doc/library/email.generator.rst deleted file mode 100644 index 91d9d69a63d73f..00000000000000 --- a/Doc/library/email.generator.rst +++ /dev/null @@ -1,283 +0,0 @@ -:mod:`email.generator`: Generating MIME documents -------------------------------------------------- - -.. module:: email.generator - :synopsis: Generate flat text email messages from a message structure. - -**Source code:** :source:`Lib/email/generator.py` - --------------- - -One of the most common tasks is to generate the flat (serialized) version of -the email message represented by a message object structure. You will need to -do this if you want to send your message via :meth:`smtplib.SMTP.sendmail` or -the :mod:`nntplib` module, or print the message on the console. Taking a -message object structure and producing a serialized representation is the job -of the generator classes. - -As with the :mod:`email.parser` module, you aren't limited to the functionality -of the bundled generator; you could write one from scratch yourself. However -the bundled generator knows how to generate most email in a standards-compliant -way, should handle MIME and non-MIME email messages just fine, and is designed -so that the bytes-oriented parsing and generation operations are inverses, -assuming the same non-transforming :mod:`~email.policy` is used for both. That -is, parsing the serialized byte stream via the -:class:`~email.parser.BytesParser` class and then regenerating the serialized -byte stream using :class:`BytesGenerator` should produce output identical to -the input [#]_. (On the other hand, using the generator on an -:class:`~email.message.EmailMessage` constructed by program may result in -changes to the :class:`~email.message.EmailMessage` object as defaults are -filled in.) - -The :class:`Generator` class can be used to flatten a message into a text (as -opposed to binary) serialized representation, but since Unicode cannot -represent binary data directly, the message is of necessity transformed into -something that contains only ASCII characters, using the standard email RFC -Content Transfer Encoding techniques for encoding email messages for transport -over channels that are not "8 bit clean". - -To accommodate reproducible processing of SMIME-signed messages -:class:`Generator` disables header folding for message parts of type -``multipart/signed`` and all subparts. - - -.. class:: BytesGenerator(outfp, mangle_from_=None, maxheaderlen=None, *, \ - policy=None) - - Return a :class:`BytesGenerator` object that will write any message provided - to the :meth:`flatten` method, or any surrogateescape encoded text provided - to the :meth:`write` method, to the :term:`file-like object` *outfp*. - *outfp* must support a ``write`` method that accepts binary data. - - If optional *mangle_from_* is ``True``, put a ``>`` character in front of - any line in the body that starts with the exact string ``"From "``, that is - ``From`` followed by a space at the beginning of a line. *mangle_from_* - defaults to the value of the :attr:`~email.policy.Policy.mangle_from_` - setting of the *policy* (which is ``True`` for the - :data:`~email.policy.compat32` policy and ``False`` for all others). - *mangle_from_* is intended for use when messages are stored in Unix mbox - format (see :mod:`mailbox` and `WHY THE CONTENT-LENGTH FORMAT IS BAD - `_). - - If *maxheaderlen* is not ``None``, refold any header lines that are longer - than *maxheaderlen*, or if ``0``, do not rewrap any headers. If - *manheaderlen* is ``None`` (the default), wrap headers and other message - lines according to the *policy* settings. - - If *policy* is specified, use that policy to control message generation. If - *policy* is ``None`` (the default), use the policy associated with the - :class:`~email.message.Message` or :class:`~email.message.EmailMessage` - object passed to ``flatten`` to control the message generation. See - :mod:`email.policy` for details on what *policy* controls. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 Added the *policy* keyword. - - .. versionchanged:: 3.6 The default behavior of the *mangle_from_* - and *maxheaderlen* parameters is to follow the policy. - - - .. method:: flatten(msg, unixfrom=False, linesep=None) - - Print the textual representation of the message object structure rooted - at *msg* to the output file specified when the :class:`BytesGenerator` - instance was created. - - If the :mod:`~email.policy` option :attr:`~email.policy.Policy.cte_type` - is ``8bit`` (the default), copy any headers in the original parsed - message that have not been modified to the output with any bytes with the - high bit set reproduced as in the original, and preserve the non-ASCII - :mailheader:`Content-Transfer-Encoding` of any body parts that have them. - If ``cte_type`` is ``7bit``, convert the bytes with the high bit set as - needed using an ASCII-compatible :mailheader:`Content-Transfer-Encoding`. - That is, transform parts with non-ASCII - :mailheader:`Content-Transfer-Encoding` - (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible - :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII - bytes in headers using the MIME ``unknown-8bit`` character set, thus - rendering them RFC-compliant. - - .. XXX: There should be an option that just does the RFC - compliance transformation on headers but leaves CTE 8bit parts alone. - - If *unixfrom* is ``True``, print the envelope header delimiter used by - the Unix mailbox format (see :mod:`mailbox`) before the first of the - :rfc:`5322` headers of the root message object. If the root object has - no envelope header, craft a standard one. The default is ``False``. - Note that for subparts, no envelope header is ever printed. - - If *linesep* is not ``None``, use it as the separator character between - all the lines of the flattened message. If *linesep* is ``None`` (the - default), use the value specified in the *policy*. - - .. XXX: flatten should take a *policy* keyword. - - - .. method:: clone(fp) - - Return an independent clone of this :class:`BytesGenerator` instance with - the exact same option settings, and *fp* as the new *outfp*. - - - .. method:: write(s) - - Encode *s* using the ``ASCII`` codec and the ``surrogateescape`` error - handler, and pass it to the *write* method of the *outfp* passed to the - :class:`BytesGenerator`'s constructor. - - -As a convenience, :class:`~email.message.EmailMessage` provides the methods -:meth:`~email.message.EmailMessage.as_bytes` and ``bytes(aMessage)`` (a.k.a. -:meth:`~email.message.EmailMessage.__bytes__`), which simplify the generation of -a serialized binary representation of a message object. For more detail, see -:mod:`email.message`. - - -Because strings cannot represent binary data, the :class:`Generator` class must -convert any binary data in any message it flattens to an ASCII compatible -format, by converting them to an ASCII compatible -:mailheader:`Content-Transfer_Encoding`. Using the terminology of the email -RFCs, you can think of this as :class:`Generator` serializing to an I/O stream -that is not "8 bit clean". In other words, most applications will want -to be using :class:`BytesGenerator`, and not :class:`Generator`. - -.. class:: Generator(outfp, mangle_from_=None, maxheaderlen=None, *, \ - policy=None) - - Return a :class:`Generator` object that will write any message provided - to the :meth:`flatten` method, or any text provided to the :meth:`write` - method, to the :term:`file-like object` *outfp*. *outfp* must support a - ``write`` method that accepts string data. - - If optional *mangle_from_* is ``True``, put a ``>`` character in front of - any line in the body that starts with the exact string ``"From "``, that is - ``From`` followed by a space at the beginning of a line. *mangle_from_* - defaults to the value of the :attr:`~email.policy.Policy.mangle_from_` - setting of the *policy* (which is ``True`` for the - :data:`~email.policy.compat32` policy and ``False`` for all others). - *mangle_from_* is intended for use when messages are stored in Unix mbox - format (see :mod:`mailbox` and `WHY THE CONTENT-LENGTH FORMAT IS BAD - `_). - - If *maxheaderlen* is not ``None``, refold any header lines that are longer - than *maxheaderlen*, or if ``0``, do not rewrap any headers. If - *manheaderlen* is ``None`` (the default), wrap headers and other message - lines according to the *policy* settings. - - If *policy* is specified, use that policy to control message generation. If - *policy* is ``None`` (the default), use the policy associated with the - :class:`~email.message.Message` or :class:`~email.message.EmailMessage` - object passed to ``flatten`` to control the message generation. See - :mod:`email.policy` for details on what *policy* controls. - - .. versionchanged:: 3.3 Added the *policy* keyword. - - .. versionchanged:: 3.6 The default behavior of the *mangle_from_* - and *maxheaderlen* parameters is to follow the policy. - - - .. method:: flatten(msg, unixfrom=False, linesep=None) - - Print the textual representation of the message object structure rooted - at *msg* to the output file specified when the :class:`Generator` - instance was created. - - If the :mod:`~email.policy` option :attr:`~email.policy.Policy.cte_type` - is ``8bit``, generate the message as if the option were set to ``7bit``. - (This is required because strings cannot represent non-ASCII bytes.) - Convert any bytes with the high bit set as needed using an - ASCII-compatible :mailheader:`Content-Transfer-Encoding`. That is, - transform parts with non-ASCII :mailheader:`Content-Transfer-Encoding` - (:mailheader:`Content-Transfer-Encoding: 8bit`) to an ASCII compatible - :mailheader:`Content-Transfer-Encoding`, and encode RFC-invalid non-ASCII - bytes in headers using the MIME ``unknown-8bit`` character set, thus - rendering them RFC-compliant. - - If *unixfrom* is ``True``, print the envelope header delimiter used by - the Unix mailbox format (see :mod:`mailbox`) before the first of the - :rfc:`5322` headers of the root message object. If the root object has - no envelope header, craft a standard one. The default is ``False``. - Note that for subparts, no envelope header is ever printed. - - If *linesep* is not ``None``, use it as the separator character between - all the lines of the flattened message. If *linesep* is ``None`` (the - default), use the value specified in the *policy*. - - .. XXX: flatten should take a *policy* keyword. - - .. versionchanged:: 3.2 - Added support for re-encoding ``8bit`` message bodies, and the - *linesep* argument. - - - .. method:: clone(fp) - - Return an independent clone of this :class:`Generator` instance with the - exact same options, and *fp* as the new *outfp*. - - - .. method:: write(s) - - Write *s* to the *write* method of the *outfp* passed to the - :class:`Generator`'s constructor. This provides just enough file-like - API for :class:`Generator` instances to be used in the :func:`print` - function. - - -As a convenience, :class:`~email.message.EmailMessage` provides the methods -:meth:`~email.message.EmailMessage.as_string` and ``str(aMessage)`` (a.k.a. -:meth:`~email.message.EmailMessage.__str__`), which simplify the generation of -a formatted string representation of a message object. For more detail, see -:mod:`email.message`. - - -The :mod:`email.generator` module also provides a derived class, -:class:`DecodedGenerator`, which is like the :class:`Generator` base class, -except that non-\ :mimetype:`text` parts are not serialized, but are instead -represented in the output stream by a string derived from a template filled -in with information about the part. - -.. class:: DecodedGenerator(outfp, mangle_from_=None, maxheaderlen=None, \ - fmt=None, *, policy=None) - - Act like :class:`Generator`, except that for any subpart of the message - passed to :meth:`Generator.flatten`, if the subpart is of main type - :mimetype:`text`, print the decoded payload of the subpart, and if the main - type is not :mimetype:`text`, instead of printing it fill in the string - *fmt* using information from the part and print the resulting - filled-in string. - - To fill in *fmt*, execute ``fmt % part_info``, where ``part_info`` - is a dictionary composed of the following keys and values: - - * ``type`` -- Full MIME type of the non-\ :mimetype:`text` part - - * ``maintype`` -- Main MIME type of the non-\ :mimetype:`text` part - - * ``subtype`` -- Sub-MIME type of the non-\ :mimetype:`text` part - - * ``filename`` -- Filename of the non-\ :mimetype:`text` part - - * ``description`` -- Description associated with the non-\ :mimetype:`text` part - - * ``encoding`` -- Content transfer encoding of the non-\ :mimetype:`text` part - - If *fmt* is ``None``, use the following default *fmt*: - - "[Non-text (%(type)s) part of message omitted, filename %(filename)s]" - - Optional *_mangle_from_* and *maxheaderlen* are as with the - :class:`Generator` base class. - - -.. rubric:: Footnotes - -.. [#] This statement assumes that you use the appropriate setting for - ``unixfrom``, and that there are no :mod:`email.policy` settings calling for - automatic adjustments (for example, - :attr:`~email.policy.EmailPolicy.refold_source` must be ``none``, which is - *not* the default). It is also not 100% true, since if the message - does not conform to the RFC standards occasionally information about the - exact original text is lost during parsing error recovery. It is a goal - to fix these latter edge cases when possible. diff --git a/Doc/library/email.header.rst b/Doc/library/email.header.rst deleted file mode 100644 index e093f138936b36..00000000000000 --- a/Doc/library/email.header.rst +++ /dev/null @@ -1,205 +0,0 @@ -:mod:`email.header`: Internationalized headers ----------------------------------------------- - -.. module:: email.header - :synopsis: Representing non-ASCII headers - -**Source code:** :source:`Lib/email/header.py` - --------------- - -This module is part of the legacy (``Compat32``) email API. In the current API -encoding and decoding of headers is handled transparently by the -dictionary-like API of the :class:`~email.message.EmailMessage` class. In -addition to uses in legacy code, this module can be useful in applications that -need to completely control the character sets used when encoding headers. - -The remaining text in this section is the original documentation of the module. - -:rfc:`2822` is the base standard that describes the format of email messages. -It derives from the older :rfc:`822` standard which came into widespread use at -a time when most email was composed of ASCII characters only. :rfc:`2822` is a -specification written assuming email contains only 7-bit ASCII characters. - -Of course, as email has been deployed worldwide, it has become -internationalized, such that language specific character sets can now be used in -email messages. The base standard still requires email messages to be -transferred using only 7-bit ASCII characters, so a slew of RFCs have been -written describing how to encode email containing non-ASCII characters into -:rfc:`2822`\ -compliant format. These RFCs include :rfc:`2045`, :rfc:`2046`, -:rfc:`2047`, and :rfc:`2231`. The :mod:`email` package supports these standards -in its :mod:`email.header` and :mod:`email.charset` modules. - -If you want to include non-ASCII characters in your email headers, say in the -:mailheader:`Subject` or :mailheader:`To` fields, you should use the -:class:`Header` class and assign the field in the :class:`~email.message.Message` -object to an instance of :class:`Header` instead of using a string for the header -value. Import the :class:`Header` class from the :mod:`email.header` module. -For example:: - - >>> from email.message import Message - >>> from email.header import Header - >>> msg = Message() - >>> h = Header('p\xf6stal', 'iso-8859-1') - >>> msg['Subject'] = h - >>> msg.as_string() - 'Subject: =?iso-8859-1?q?p=F6stal?=\n\n' - - - -Notice here how we wanted the :mailheader:`Subject` field to contain a non-ASCII -character? We did this by creating a :class:`Header` instance and passing in -the character set that the byte string was encoded in. When the subsequent -:class:`~email.message.Message` instance was flattened, the :mailheader:`Subject` -field was properly :rfc:`2047` encoded. MIME-aware mail readers would show this -header using the embedded ISO-8859-1 character. - -Here is the :class:`Header` class description: - - -.. class:: Header(s=None, charset=None, maxlinelen=None, header_name=None, continuation_ws=' ', errors='strict') - - Create a MIME-compliant header that can contain strings in different character - sets. - - Optional *s* is the initial header value. If ``None`` (the default), the - initial header value is not set. You can later append to the header with - :meth:`append` method calls. *s* may be an instance of :class:`bytes` or - :class:`str`, but see the :meth:`append` documentation for semantics. - - Optional *charset* serves two purposes: it has the same meaning as the *charset* - argument to the :meth:`append` method. It also sets the default character set - for all subsequent :meth:`append` calls that omit the *charset* argument. If - *charset* is not provided in the constructor (the default), the ``us-ascii`` - character set is used both as *s*'s initial charset and as the default for - subsequent :meth:`append` calls. - - The maximum line length can be specified explicitly via *maxlinelen*. For - splitting the first line to a shorter value (to account for the field header - which isn't included in *s*, e.g. :mailheader:`Subject`) pass in the name of the - field in *header_name*. The default *maxlinelen* is 76, and the default value - for *header_name* is ``None``, meaning it is not taken into account for the - first line of a long, split header. - - Optional *continuation_ws* must be :rfc:`2822`\ -compliant folding - whitespace, and is usually either a space or a hard tab character. This - character will be prepended to continuation lines. *continuation_ws* - defaults to a single space character. - - Optional *errors* is passed straight through to the :meth:`append` method. - - - .. method:: append(s, charset=None, errors='strict') - - Append the string *s* to the MIME header. - - Optional *charset*, if given, should be a :class:`~email.charset.Charset` - instance (see :mod:`email.charset`) or the name of a character set, which - will be converted to a :class:`~email.charset.Charset` instance. A value - of ``None`` (the default) means that the *charset* given in the constructor - is used. - - *s* may be an instance of :class:`bytes` or :class:`str`. If it is an - instance of :class:`bytes`, then *charset* is the encoding of that byte - string, and a :exc:`UnicodeError` will be raised if the string cannot be - decoded with that character set. - - If *s* is an instance of :class:`str`, then *charset* is a hint specifying - the character set of the characters in the string. - - In either case, when producing an :rfc:`2822`\ -compliant header using - :rfc:`2047` rules, the string will be encoded using the output codec of - the charset. If the string cannot be encoded using the output codec, a - UnicodeError will be raised. - - Optional *errors* is passed as the errors argument to the decode call - if *s* is a byte string. - - - .. method:: encode(splitchars=';, \t', maxlinelen=None, linesep='\n') - - Encode a message header into an RFC-compliant format, possibly wrapping - long lines and encapsulating non-ASCII parts in base64 or quoted-printable - encodings. - - Optional *splitchars* is a string containing characters which should be - given extra weight by the splitting algorithm during normal header - wrapping. This is in very rough support of :RFC:`2822`\'s 'higher level - syntactic breaks': split points preceded by a splitchar are preferred - during line splitting, with the characters preferred in the order in - which they appear in the string. Space and tab may be included in the - string to indicate whether preference should be given to one over the - other as a split point when other split chars do not appear in the line - being split. Splitchars does not affect :RFC:`2047` encoded lines. - - *maxlinelen*, if given, overrides the instance's value for the maximum - line length. - - *linesep* specifies the characters used to separate the lines of the - folded header. It defaults to the most useful value for Python - application code (``\n``), but ``\r\n`` can be specified in order - to produce headers with RFC-compliant line separators. - - .. versionchanged:: 3.2 - Added the *linesep* argument. - - - The :class:`Header` class also provides a number of methods to support - standard operators and built-in functions. - - .. method:: __str__() - - Returns an approximation of the :class:`Header` as a string, using an - unlimited line length. All pieces are converted to unicode using the - specified encoding and joined together appropriately. Any pieces with a - charset of ``'unknown-8bit'`` are decoded as ASCII using the ``'replace'`` - error handler. - - .. versionchanged:: 3.2 - Added handling for the ``'unknown-8bit'`` charset. - - - .. method:: __eq__(other) - - This method allows you to compare two :class:`Header` instances for - equality. - - - .. method:: __ne__(other) - - This method allows you to compare two :class:`Header` instances for - inequality. - -The :mod:`email.header` module also provides the following convenient functions. - - -.. function:: decode_header(header) - - Decode a message header value without converting the character set. The header - value is in *header*. - - This function returns a list of ``(decoded_string, charset)`` pairs containing - each of the decoded parts of the header. *charset* is ``None`` for non-encoded - parts of the header, otherwise a lower case string containing the name of the - character set specified in the encoded string. - - Here's an example:: - - >>> from email.header import decode_header - >>> decode_header('=?iso-8859-1?q?p=F6stal?=') - [(b'p\xf6stal', 'iso-8859-1')] - - -.. function:: make_header(decoded_seq, maxlinelen=None, header_name=None, continuation_ws=' ') - - Create a :class:`Header` instance from a sequence of pairs as returned by - :func:`decode_header`. - - :func:`decode_header` takes a header value string and returns a sequence of - pairs of the format ``(decoded_string, charset)`` where *charset* is the name of - the character set. - - This function takes one of those sequence of pairs and returns a - :class:`Header` instance. Optional *maxlinelen*, *header_name*, and - *continuation_ws* are as in the :class:`Header` constructor. - diff --git a/Doc/library/email.headerregistry.rst b/Doc/library/email.headerregistry.rst deleted file mode 100644 index 00a954e0307ea6..00000000000000 --- a/Doc/library/email.headerregistry.rst +++ /dev/null @@ -1,462 +0,0 @@ -:mod:`email.headerregistry`: Custom Header Objects --------------------------------------------------- - -.. module:: email.headerregistry - :synopsis: Automatic Parsing of headers based on the field name - -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - -**Source code:** :source:`Lib/email/headerregistry.py` - --------------- - -.. versionadded:: 3.6 [1]_ - -Headers are represented by customized subclasses of :class:`str`. The -particular class used to represent a given header is determined by the -:attr:`~email.policy.EmailPolicy.header_factory` of the :mod:`~email.policy` in -effect when the headers are created. This section documents the particular -``header_factory`` implemented by the email package for handling :RFC:`5322` -compliant email messages, which not only provides customized header objects for -various header types, but also provides an extension mechanism for applications -to add their own custom header types. - -When using any of the policy objects derived from -:data:`~email.policy.EmailPolicy`, all headers are produced by -:class:`.HeaderRegistry` and have :class:`.BaseHeader` as their last base -class. Each header class has an additional base class that is determined by -the type of the header. For example, many headers have the class -:class:`.UnstructuredHeader` as their other base class. The specialized second -class for a header is determined by the name of the header, using a lookup -table stored in the :class:`.HeaderRegistry`. All of this is managed -transparently for the typical application program, but interfaces are provided -for modifying the default behavior for use by more complex applications. - -The sections below first document the header base classes and their attributes, -followed by the API for modifying the behavior of :class:`.HeaderRegistry`, and -finally the support classes used to represent the data parsed from structured -headers. - - -.. class:: BaseHeader(name, value) - - *name* and *value* are passed to ``BaseHeader`` from the - :attr:`~email.policy.EmailPolicy.header_factory` call. The string value of - any header object is the *value* fully decoded to unicode. - - This base class defines the following read-only properties: - - - .. attribute:: name - - The name of the header (the portion of the field before the ':'). This - is exactly the value passed in the - :attr:`~email.policy.EmailPolicy.header_factory` call for *name*; that - is, case is preserved. - - - .. attribute:: defects - - A tuple of :exc:`~email.errors.HeaderDefect` instances reporting any - RFC compliance problems found during parsing. The email package tries to - be complete about detecting compliance issues. See the :mod:`~email.errors` - module for a discussion of the types of defects that may be reported. - - - .. attribute:: max_count - - The maximum number of headers of this type that can have the same - ``name``. A value of ``None`` means unlimited. The ``BaseHeader`` value - for this attribute is ``None``; it is expected that specialized header - classes will override this value as needed. - - ``BaseHeader`` also provides the following method, which is called by the - email library code and should not in general be called by application - programs: - - .. method:: fold(*, policy) - - Return a string containing :attr:`~email.policy.Policy.linesep` - characters as required to correctly fold the header according to - *policy*. A :attr:`~email.policy.Policy.cte_type` of ``8bit`` will be - treated as if it were ``7bit``, since headers may not contain arbitrary - binary data. If :attr:`~email.policy.EmailPolicy.utf8` is ``False``, - non-ASCII data will be :rfc:`2047` encoded. - - - ``BaseHeader`` by itself cannot be used to create a header object. It - defines a protocol that each specialized header cooperates with in order to - produce the header object. Specifically, ``BaseHeader`` requires that - the specialized class provide a :func:`classmethod` named ``parse``. This - method is called as follows:: - - parse(string, kwds) - - ``kwds`` is a dictionary containing one pre-initialized key, ``defects``. - ``defects`` is an empty list. The parse method should append any detected - defects to this list. On return, the ``kwds`` dictionary *must* contain - values for at least the keys ``decoded`` and ``defects``. ``decoded`` - should be the string value for the header (that is, the header value fully - decoded to unicode). The parse method should assume that *string* may - contain content-transfer-encoded parts, but should correctly handle all valid - unicode characters as well so that it can parse un-encoded header values. - - ``BaseHeader``'s ``__new__`` then creates the header instance, and calls its - ``init`` method. The specialized class only needs to provide an ``init`` - method if it wishes to set additional attributes beyond those provided by - ``BaseHeader`` itself. Such an ``init`` method should look like this:: - - def init(self, /, *args, **kw): - self._myattr = kw.pop('myattr') - super().init(*args, **kw) - - That is, anything extra that the specialized class puts in to the ``kwds`` - dictionary should be removed and handled, and the remaining contents of - ``kw`` (and ``args``) passed to the ``BaseHeader`` ``init`` method. - - -.. class:: UnstructuredHeader - - An "unstructured" header is the default type of header in :rfc:`5322`. - Any header that does not have a specified syntax is treated as - unstructured. The classic example of an unstructured header is the - :mailheader:`Subject` header. - - In :rfc:`5322`, an unstructured header is a run of arbitrary text in the - ASCII character set. :rfc:`2047`, however, has an :rfc:`5322` compatible - mechanism for encoding non-ASCII text as ASCII characters within a header - value. When a *value* containing encoded words is passed to the - constructor, the ``UnstructuredHeader`` parser converts such encoded words - into unicode, following the :rfc:`2047` rules for unstructured text. The - parser uses heuristics to attempt to decode certain non-compliant encoded - words. Defects are registered in such cases, as well as defects for issues - such as invalid characters within the encoded words or the non-encoded text. - - This header type provides no additional attributes. - - -.. class:: DateHeader - - :rfc:`5322` specifies a very specific format for dates within email headers. - The ``DateHeader`` parser recognizes that date format, as well as - recognizing a number of variant forms that are sometimes found "in the - wild". - - This header type provides the following additional attributes: - - .. attribute:: datetime - - If the header value can be recognized as a valid date of one form or - another, this attribute will contain a :class:`~datetime.datetime` - instance representing that date. If the timezone of the input date is - specified as ``-0000`` (indicating it is in UTC but contains no - information about the source timezone), then :attr:`.datetime` will be a - naive :class:`~datetime.datetime`. If a specific timezone offset is - found (including ``+0000``), then :attr:`.datetime` will contain an aware - ``datetime`` that uses :class:`datetime.timezone` to record the timezone - offset. - - The ``decoded`` value of the header is determined by formatting the - ``datetime`` according to the :rfc:`5322` rules; that is, it is set to:: - - email.utils.format_datetime(self.datetime) - - When creating a ``DateHeader``, *value* may be - :class:`~datetime.datetime` instance. This means, for example, that - the following code is valid and does what one would expect:: - - msg['Date'] = datetime(2011, 7, 15, 21) - - Because this is a naive ``datetime`` it will be interpreted as a UTC - timestamp, and the resulting value will have a timezone of ``-0000``. Much - more useful is to use the :func:`~email.utils.localtime` function from the - :mod:`~email.utils` module:: - - msg['Date'] = utils.localtime() - - This example sets the date header to the current time and date using - the current timezone offset. - - -.. class:: AddressHeader - - Address headers are one of the most complex structured header types. - The ``AddressHeader`` class provides a generic interface to any address - header. - - This header type provides the following additional attributes: - - - .. attribute:: groups - - A tuple of :class:`.Group` objects encoding the - addresses and groups found in the header value. Addresses that are - not part of a group are represented in this list as single-address - ``Groups`` whose :attr:`~.Group.display_name` is ``None``. - - - .. attribute:: addresses - - A tuple of :class:`.Address` objects encoding all - of the individual addresses from the header value. If the header value - contains any groups, the individual addresses from the group are included - in the list at the point where the group occurs in the value (that is, - the list of addresses is "flattened" into a one dimensional list). - - The ``decoded`` value of the header will have all encoded words decoded to - unicode. :class:`~encodings.idna` encoded domain names are also decoded to - unicode. The ``decoded`` value is set by :ref:`joining ` the - :class:`str` value of the elements of the ``groups`` attribute with ``', - '``. - - A list of :class:`.Address` and :class:`.Group` objects in any combination - may be used to set the value of an address header. ``Group`` objects whose - ``display_name`` is ``None`` will be interpreted as single addresses, which - allows an address list to be copied with groups intact by using the list - obtained from the ``groups`` attribute of the source header. - - -.. class:: SingleAddressHeader - - A subclass of :class:`.AddressHeader` that adds one - additional attribute: - - - .. attribute:: address - - The single address encoded by the header value. If the header value - actually contains more than one address (which would be a violation of - the RFC under the default :mod:`~email.policy`), accessing this attribute - will result in a :exc:`ValueError`. - - -Many of the above classes also have a ``Unique`` variant (for example, -``UniqueUnstructuredHeader``). The only difference is that in the ``Unique`` -variant, :attr:`~.BaseHeader.max_count` is set to 1. - - -.. class:: MIMEVersionHeader - - There is really only one valid value for the :mailheader:`MIME-Version` - header, and that is ``1.0``. For future proofing, this header class - supports other valid version numbers. If a version number has a valid value - per :rfc:`2045`, then the header object will have non-``None`` values for - the following attributes: - - .. attribute:: version - - The version number as a string, with any whitespace and/or comments - removed. - - .. attribute:: major - - The major version number as an integer - - .. attribute:: minor - - The minor version number as an integer - - -.. class:: ParameterizedMIMEHeader - - MIME headers all start with the prefix 'Content-'. Each specific header has - a certain value, described under the class for that header. Some can - also take a list of supplemental parameters, which have a common format. - This class serves as a base for all the MIME headers that take parameters. - - .. attribute:: params - - A dictionary mapping parameter names to parameter values. - - -.. class:: ContentTypeHeader - - A :class:`ParameterizedMIMEHeader` class that handles the - :mailheader:`Content-Type` header. - - .. attribute:: content_type - - The content type string, in the form ``maintype/subtype``. - - .. attribute:: maintype - - .. attribute:: subtype - - -.. class:: ContentDispositionHeader - - A :class:`ParameterizedMIMEHeader` class that handles the - :mailheader:`Content-Disposition` header. - - .. attribute:: content_disposition - - ``inline`` and ``attachment`` are the only valid values in common use. - - -.. class:: ContentTransferEncoding - - Handles the :mailheader:`Content-Transfer-Encoding` header. - - .. attribute:: cte - - Valid values are ``7bit``, ``8bit``, ``base64``, and - ``quoted-printable``. See :rfc:`2045` for more information. - - - -.. class:: HeaderRegistry(base_class=BaseHeader, \ - default_class=UnstructuredHeader, \ - use_default_map=True) - - This is the factory used by :class:`~email.policy.EmailPolicy` by default. - ``HeaderRegistry`` builds the class used to create a header instance - dynamically, using *base_class* and a specialized class retrieved from a - registry that it holds. When a given header name does not appear in the - registry, the class specified by *default_class* is used as the specialized - class. When *use_default_map* is ``True`` (the default), the standard - mapping of header names to classes is copied in to the registry during - initialization. *base_class* is always the last class in the generated - class's ``__bases__`` list. - - The default mappings are: - - :subject: UniqueUnstructuredHeader - :date: UniqueDateHeader - :resent-date: DateHeader - :orig-date: UniqueDateHeader - :sender: UniqueSingleAddressHeader - :resent-sender: SingleAddressHeader - :to: UniqueAddressHeader - :resent-to: AddressHeader - :cc: UniqueAddressHeader - :resent-cc: AddressHeader - :bcc: UniqueAddressHeader - :resent-bcc: AddressHeader - :from: UniqueAddressHeader - :resent-from: AddressHeader - :reply-to: UniqueAddressHeader - :mime-version: MIMEVersionHeader - :content-type: ContentTypeHeader - :content-disposition: ContentDispositionHeader - :content-transfer-encoding: ContentTransferEncodingHeader - :message-id: MessageIDHeader - - ``HeaderRegistry`` has the following methods: - - - .. method:: map_to_type(self, name, cls) - - *name* is the name of the header to be mapped. It will be converted to - lower case in the registry. *cls* is the specialized class to be used, - along with *base_class*, to create the class used to instantiate headers - that match *name*. - - - .. method:: __getitem__(name) - - Construct and return a class to handle creating a *name* header. - - - .. method:: __call__(name, value) - - Retrieves the specialized header associated with *name* from the - registry (using *default_class* if *name* does not appear in the - registry) and composes it with *base_class* to produce a class, - calls the constructed class's constructor, passing it the same - argument list, and finally returns the class instance created thereby. - - -The following classes are the classes used to represent data parsed from -structured headers and can, in general, be used by an application program to -construct structured values to assign to specific headers. - - -.. class:: Address(display_name='', username='', domain='', addr_spec=None) - - The class used to represent an email address. The general form of an - address is:: - - [display_name] - - or:: - - username@domain - - where each part must conform to specific syntax rules spelled out in - :rfc:`5322`. - - As a convenience *addr_spec* can be specified instead of *username* and - *domain*, in which case *username* and *domain* will be parsed from the - *addr_spec*. An *addr_spec* must be a properly RFC quoted string; if it is - not ``Address`` will raise an error. Unicode characters are allowed and - will be property encoded when serialized. However, per the RFCs, unicode is - *not* allowed in the username portion of the address. - - .. attribute:: display_name - - The display name portion of the address, if any, with all quoting - removed. If the address does not have a display name, this attribute - will be an empty string. - - .. attribute:: username - - The ``username`` portion of the address, with all quoting removed. - - .. attribute:: domain - - The ``domain`` portion of the address. - - .. attribute:: addr_spec - - The ``username@domain`` portion of the address, correctly quoted - for use as a bare address (the second form shown above). This - attribute is not mutable. - - .. method:: __str__() - - The ``str`` value of the object is the address quoted according to - :rfc:`5322` rules, but with no Content Transfer Encoding of any non-ASCII - characters. - - To support SMTP (:rfc:`5321`), ``Address`` handles one special case: if - ``username`` and ``domain`` are both the empty string (or ``None``), then - the string value of the ``Address`` is ``<>``. - - -.. class:: Group(display_name=None, addresses=None) - - The class used to represent an address group. The general form of an - address group is:: - - display_name: [address-list]; - - As a convenience for processing lists of addresses that consist of a mixture - of groups and single addresses, a ``Group`` may also be used to represent - single addresses that are not part of a group by setting *display_name* to - ``None`` and providing a list of the single address as *addresses*. - - .. attribute:: display_name - - The ``display_name`` of the group. If it is ``None`` and there is - exactly one ``Address`` in ``addresses``, then the ``Group`` represents a - single address that is not in a group. - - .. attribute:: addresses - - A possibly empty tuple of :class:`.Address` objects representing the - addresses in the group. - - .. method:: __str__() - - The ``str`` value of a ``Group`` is formatted according to :rfc:`5322`, - but with no Content Transfer Encoding of any non-ASCII characters. If - ``display_name`` is none and there is a single ``Address`` in the - ``addresses`` list, the ``str`` value will be the same as the ``str`` of - that single ``Address``. - - -.. rubric:: Footnotes - -.. [1] Originally added in 3.3 as a :term:`provisional module ` diff --git a/Doc/library/email.iterators.rst b/Doc/library/email.iterators.rst deleted file mode 100644 index d53ab33b8904a7..00000000000000 --- a/Doc/library/email.iterators.rst +++ /dev/null @@ -1,83 +0,0 @@ -:mod:`email.iterators`: Iterators ---------------------------------- - -.. module:: email.iterators - :synopsis: Iterate over a message object tree. - -**Source code:** :source:`Lib/email/iterators.py` - --------------- - -Iterating over a message object tree is fairly easy with the -:meth:`Message.walk ` method. The -:mod:`email.iterators` module provides some useful higher level iterations over -message object trees. - - -.. function:: body_line_iterator(msg, decode=False) - - This iterates over all the payloads in all the subparts of *msg*, returning the - string payloads line-by-line. It skips over all the subpart headers, and it - skips over any subpart with a payload that isn't a Python string. This is - somewhat equivalent to reading the flat text representation of the message from - a file using :meth:`~io.TextIOBase.readline`, skipping over all the - intervening headers. - - Optional *decode* is passed through to :meth:`Message.get_payload - `. - - -.. function:: typed_subpart_iterator(msg, maintype='text', subtype=None) - - This iterates over all the subparts of *msg*, returning only those subparts that - match the MIME type specified by *maintype* and *subtype*. - - Note that *subtype* is optional; if omitted, then subpart MIME type matching is - done only with the main type. *maintype* is optional too; it defaults to - :mimetype:`text`. - - Thus, by default :func:`typed_subpart_iterator` returns each subpart that has a - MIME type of :mimetype:`text/\*`. - - -The following function has been added as a useful debugging tool. It should -*not* be considered part of the supported public interface for the package. - -.. function:: _structure(msg, fp=None, level=0, include_default=False) - - Prints an indented representation of the content types of the message object - structure. For example: - - .. testsetup:: - - import email - from email.iterators import _structure - somefile = open('../Lib/test/test_email/data/msg_02.txt') - - .. doctest:: - - >>> msg = email.message_from_file(somefile) - >>> _structure(msg) - multipart/mixed - text/plain - text/plain - multipart/digest - message/rfc822 - text/plain - message/rfc822 - text/plain - message/rfc822 - text/plain - message/rfc822 - text/plain - message/rfc822 - text/plain - text/plain - - .. testcleanup:: - - somefile.close() - - Optional *fp* is a file-like object to print the output to. It must be - suitable for Python's :func:`print` function. *level* is used internally. - *include_default*, if true, prints the default type as well. diff --git a/Doc/library/email.message.rst b/Doc/library/email.message.rst deleted file mode 100644 index f58d93da6ed687..00000000000000 --- a/Doc/library/email.message.rst +++ /dev/null @@ -1,751 +0,0 @@ -:mod:`email.message`: Representing an email message ---------------------------------------------------- - -.. module:: email.message - :synopsis: The base class representing email messages. -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray , - Barry A. Warsaw - -**Source code:** :source:`Lib/email/message.py` - --------------- - -.. versionadded:: 3.6 [1]_ - -The central class in the :mod:`email` package is the :class:`EmailMessage` -class, imported from the :mod:`email.message` module. It is the base class for -the :mod:`email` object model. :class:`EmailMessage` provides the core -functionality for setting and querying header fields, for accessing message -bodies, and for creating or modifying structured messages. - -An email message consists of *headers* and a *payload* (which is also referred -to as the *content*). Headers are :rfc:`5322` or :rfc:`6532` style field names -and values, where the field name and value are separated by a colon. The colon -is not part of either the field name or the field value. The payload may be a -simple text message, or a binary object, or a structured sequence of -sub-messages each with their own set of headers and their own payload. The -latter type of payload is indicated by the message having a MIME type such as -:mimetype:`multipart/\*` or :mimetype:`message/rfc822`. - -The conceptual model provided by an :class:`EmailMessage` object is that of an -ordered dictionary of headers coupled with a *payload* that represents the -:rfc:`5322` body of the message, which might be a list of sub-``EmailMessage`` -objects. In addition to the normal dictionary methods for accessing the header -names and values, there are methods for accessing specialized information from -the headers (for example the MIME content type), for operating on the payload, -for generating a serialized version of the message, and for recursively walking -over the object tree. - -The :class:`EmailMessage` dictionary-like interface is indexed by the header -names, which must be ASCII values. The values of the dictionary are strings -with some extra methods. Headers are stored and returned in case-preserving -form, but field names are matched case-insensitively. Unlike a real dict, -there is an ordering to the keys, and there can be duplicate keys. Additional -methods are provided for working with headers that have duplicate keys. - -The *payload* is either a string or bytes object, in the case of simple message -objects, or a list of :class:`EmailMessage` objects, for MIME container -documents such as :mimetype:`multipart/\*` and :mimetype:`message/rfc822` -message objects. - - -.. class:: EmailMessage(policy=default) - - If *policy* is specified use the rules it specifies to update and serialize - the representation of the message. If *policy* is not set, use the - :class:`~email.policy.default` policy, which follows the rules of the email - RFCs except for line endings (instead of the RFC mandated ``\r\n``, it uses - the Python standard ``\n`` line endings). For more information see the - :mod:`~email.policy` documentation. - - .. method:: as_string(unixfrom=False, maxheaderlen=None, policy=None) - - Return the entire message flattened as a string. When optional - *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. For backward compatibility - with the base :class:`~email.message.Message` class *maxheaderlen* is - accepted, but defaults to ``None``, which means that by default the line - length is controlled by the - :attr:`~email.policy.Policy.max_line_length` of the policy. The - *policy* argument may be used to override the default policy obtained - from the message instance. This can be used to control some of the - formatting produced by the method, since the specified *policy* will be - passed to the :class:`~email.generator.Generator`. - - Flattening the message may trigger changes to the :class:`EmailMessage` - if defaults need to be filled in to complete the transformation to a - string (for example, MIME boundaries may be generated or modified). - - Note that this method is provided as a convenience and may not be the - most useful way to serialize messages in your application, especially if - you are dealing with multiple messages. See - :class:`email.generator.Generator` for a more flexible API for - serializing messages. Note also that this method is restricted to - producing messages serialized as "7 bit clean" when - :attr:`~email.policy.EmailPolicy.utf8` is ``False``, which is the default. - - .. versionchanged:: 3.6 the default behavior when *maxheaderlen* - is not specified was changed from defaulting to 0 to defaulting - to the value of *max_line_length* from the policy. - - - .. method:: __str__() - - Equivalent to ``as_string(policy=self.policy.clone(utf8=True))``. Allows - ``str(msg)`` to produce a string containing the serialized message in a - readable format. - - .. versionchanged:: 3.4 the method was changed to use ``utf8=True``, - thus producing an :rfc:`6531`-like message representation, instead of - being a direct alias for :meth:`as_string`. - - - .. method:: as_bytes(unixfrom=False, policy=None) - - Return the entire message flattened as a bytes object. When optional - *unixfrom* is true, the envelope header is included in the returned - string. *unixfrom* defaults to ``False``. The *policy* argument may be - used to override the default policy obtained from the message instance. - This can be used to control some of the formatting produced by the - method, since the specified *policy* will be passed to the - :class:`~email.generator.BytesGenerator`. - - Flattening the message may trigger changes to the :class:`EmailMessage` - if defaults need to be filled in to complete the transformation to a - string (for example, MIME boundaries may be generated or modified). - - Note that this method is provided as a convenience and may not be the - most useful way to serialize messages in your application, especially if - you are dealing with multiple messages. See - :class:`email.generator.BytesGenerator` for a more flexible API for - serializing messages. - - - .. method:: __bytes__() - - Equivalent to :meth:`.as_bytes()`. Allows ``bytes(msg)`` to produce a - bytes object containing the serialized message. - - - .. method:: is_multipart() - - Return ``True`` if the message's payload is a list of - sub-\ :class:`EmailMessage` objects, otherwise return ``False``. When - :meth:`is_multipart` returns ``False``, the payload should be a string - object (which might be a CTE encoded binary payload). Note that - :meth:`is_multipart` returning ``True`` does not necessarily mean that - "msg.get_content_maintype() == 'multipart'" will return the ``True``. - For example, ``is_multipart`` will return ``True`` when the - :class:`EmailMessage` is of type ``message/rfc822``. - - - .. method:: set_unixfrom(unixfrom) - - Set the message's envelope header to *unixfrom*, which should be a - string. (See :class:`~mailbox.mboxMessage` for a brief description of - this header.) - - - .. method:: get_unixfrom() - - Return the message's envelope header. Defaults to ``None`` if the - envelope header was never set. - - - The following methods implement the mapping-like interface for accessing the - message's headers. Note that there are some semantic differences - between these methods and a normal mapping (i.e. dictionary) interface. For - example, in a dictionary there are no duplicate keys, but here there may be - duplicate message headers. Also, in dictionaries there is no guaranteed - order to the keys returned by :meth:`keys`, but in an :class:`EmailMessage` - object, headers are always returned in the order they appeared in the - original message, or in which they were added to the message later. Any - header deleted and then re-added is always appended to the end of the - header list. - - These semantic differences are intentional and are biased toward - convenience in the most common use cases. - - Note that in all cases, any envelope header present in the message is not - included in the mapping interface. - - - .. method:: __len__() - - Return the total number of headers, including duplicates. - - - .. method:: __contains__(name) - - Return ``True`` if the message object has a field named *name*. Matching is - done without regard to case and *name* does not include the trailing - colon. Used for the ``in`` operator. For example:: - - if 'message-id' in myMessage: - print('Message-ID:', myMessage['message-id']) - - - .. method:: __getitem__(name) - - Return the value of the named header field. *name* does not include the - colon field separator. If the header is missing, ``None`` is returned; a - :exc:`KeyError` is never raised. - - Note that if the named field appears more than once in the message's - headers, exactly which of those field values will be returned is - undefined. Use the :meth:`get_all` method to get the values of all the - extant headers named *name*. - - Using the standard (non-``compat32``) policies, the returned value is an - instance of a subclass of :class:`email.headerregistry.BaseHeader`. - - - .. method:: __setitem__(name, val) - - Add a header to the message with field name *name* and value *val*. The - field is appended to the end of the message's existing headers. - - Note that this does *not* overwrite or delete any existing header with the same - name. If you want to ensure that the new header is the only one present in the - message with field name *name*, delete the field first, e.g.:: - - del msg['subject'] - msg['subject'] = 'Python roolz!' - - If the :mod:`policy ` defines certain headers to be unique (as the standard - policies do), this method may raise a :exc:`ValueError` when an attempt - is made to assign a value to such a header when one already exists. This - behavior is intentional for consistency's sake, but do not depend on it - as we may choose to make such assignments do an automatic deletion of the - existing header in the future. - - - .. method:: __delitem__(name) - - Delete all occurrences of the field with name *name* from the message's - headers. No exception is raised if the named field isn't present in the - headers. - - - .. method:: keys() - - Return a list of all the message's header field names. - - - .. method:: values() - - Return a list of all the message's field values. - - - .. method:: items() - - Return a list of 2-tuples containing all the message's field headers and - values. - - - .. method:: get(name, failobj=None) - - Return the value of the named header field. This is identical to - :meth:`~object.__getitem__` except that optional *failobj* is returned if the - named header is missing (*failobj* defaults to ``None``). - - - Here are some additional useful header related methods: - - - .. method:: get_all(name, failobj=None) - - Return a list of all the values for the field named *name*. If there are - no such named headers in the message, *failobj* is returned (defaults to - ``None``). - - - .. method:: add_header(_name, _value, **_params) - - Extended header setting. This method is similar to :meth:`__setitem__` - except that additional header parameters can be provided as keyword - arguments. *_name* is the header field to add and *_value* is the - *primary* value for the header. - - For each item in the keyword argument dictionary *_params*, the key is - taken as the parameter name, with underscores converted to dashes (since - dashes are illegal in Python identifiers). Normally, the parameter will - be added as ``key="value"`` unless the value is ``None``, in which case - only the key will be added. - - If the value contains non-ASCII characters, the charset and language may - be explicitly controlled by specifying the value as a three tuple in the - format ``(CHARSET, LANGUAGE, VALUE)``, where ``CHARSET`` is a string - naming the charset to be used to encode the value, ``LANGUAGE`` can - usually be set to ``None`` or the empty string (see :rfc:`2231` for other - possibilities), and ``VALUE`` is the string value containing non-ASCII - code points. If a three tuple is not passed and the value contains - non-ASCII characters, it is automatically encoded in :rfc:`2231` format - using a ``CHARSET`` of ``utf-8`` and a ``LANGUAGE`` of ``None``. - - Here is an example:: - - msg.add_header('Content-Disposition', 'attachment', filename='bud.gif') - - This will add a header that looks like :: - - Content-Disposition: attachment; filename="bud.gif" - - An example of the extended interface with non-ASCII characters:: - - msg.add_header('Content-Disposition', 'attachment', - filename=('iso-8859-1', '', 'Fußballer.ppt')) - - - .. method:: replace_header(_name, _value) - - Replace a header. Replace the first header found in the message that - matches *_name*, retaining header order and field name case of the - original header. If no matching header is found, raise a - :exc:`KeyError`. - - - .. method:: get_content_type() - - Return the message's content type, coerced to lower case of the form - :mimetype:`maintype/subtype`. If there is no :mailheader:`Content-Type` - header in the message return the value returned by - :meth:`get_default_type`. If the :mailheader:`Content-Type` header is - invalid, return ``text/plain``. - - (According to :rfc:`2045`, messages always have a default type, - :meth:`get_content_type` will always return a value. :rfc:`2045` defines - a message's default type to be :mimetype:`text/plain` unless it appears - inside a :mimetype:`multipart/digest` container, in which case it would - be :mimetype:`message/rfc822`. If the :mailheader:`Content-Type` header - has an invalid type specification, :rfc:`2045` mandates that the default - type be :mimetype:`text/plain`.) - - - .. method:: get_content_maintype() - - Return the message's main content type. This is the :mimetype:`maintype` - part of the string returned by :meth:`get_content_type`. - - - .. method:: get_content_subtype() - - Return the message's sub-content type. This is the :mimetype:`subtype` - part of the string returned by :meth:`get_content_type`. - - - .. method:: get_default_type() - - Return the default content type. Most messages have a default content - type of :mimetype:`text/plain`, except for messages that are subparts of - :mimetype:`multipart/digest` containers. Such subparts have a default - content type of :mimetype:`message/rfc822`. - - - .. method:: set_default_type(ctype) - - Set the default content type. *ctype* should either be - :mimetype:`text/plain` or :mimetype:`message/rfc822`, although this is - not enforced. The default content type is not stored in the - :mailheader:`Content-Type` header, so it only affects the return value of - the ``get_content_type`` methods when no :mailheader:`Content-Type` - header is present in the message. - - - .. method:: set_param(param, value, header='Content-Type', requote=True, \ - charset=None, language='', replace=False) - - Set a parameter in the :mailheader:`Content-Type` header. If the - parameter already exists in the header, replace its value with *value*. - When *header* is ``Content-Type`` (the default) and the header does not - yet exist in the message, add it, set its value to - :mimetype:`text/plain`, and append the new parameter value. Optional - *header* specifies an alternative header to :mailheader:`Content-Type`. - - If the value contains non-ASCII characters, the charset and language may - be explicitly specified using the optional *charset* and *language* - parameters. Optional *language* specifies the :rfc:`2231` language, - defaulting to the empty string. Both *charset* and *language* should be - strings. The default is to use the ``utf8`` *charset* and ``None`` for - the *language*. - - If *replace* is ``False`` (the default) the header is moved to the - end of the list of headers. If *replace* is ``True``, the header - will be updated in place. - - Use of the *requote* parameter with :class:`EmailMessage` objects is - deprecated. - - Note that existing parameter values of headers may be accessed through - the :attr:`~email.headerregistry.ParameterizedMIMEHeader.params` attribute of the - header value (for example, ``msg['Content-Type'].params['charset']``). - - .. versionchanged:: 3.4 ``replace`` keyword was added. - - - .. method:: del_param(param, header='content-type', requote=True) - - Remove the given parameter completely from the :mailheader:`Content-Type` - header. The header will be re-written in place without the parameter or - its value. Optional *header* specifies an alternative to - :mailheader:`Content-Type`. - - Use of the *requote* parameter with :class:`EmailMessage` objects is - deprecated. - - - .. method:: get_filename(failobj=None) - - Return the value of the ``filename`` parameter of the - :mailheader:`Content-Disposition` header of the message. If the header - does not have a ``filename`` parameter, this method falls back to looking - for the ``name`` parameter on the :mailheader:`Content-Type` header. If - neither is found, or the header is missing, then *failobj* is returned. - The returned string will always be unquoted as per - :func:`email.utils.unquote`. - - - .. method:: get_boundary(failobj=None) - - Return the value of the ``boundary`` parameter of the - :mailheader:`Content-Type` header of the message, or *failobj* if either - the header is missing, or has no ``boundary`` parameter. The returned - string will always be unquoted as per :func:`email.utils.unquote`. - - - .. method:: set_boundary(boundary) - - Set the ``boundary`` parameter of the :mailheader:`Content-Type` header to - *boundary*. :meth:`set_boundary` will always quote *boundary* if - necessary. A :exc:`~email.errors.HeaderParseError` is raised if the - message object has no :mailheader:`Content-Type` header. - - Note that using this method is subtly different from deleting the old - :mailheader:`Content-Type` header and adding a new one with the new - boundary via :meth:`add_header`, because :meth:`set_boundary` preserves - the order of the :mailheader:`Content-Type` header in the list of - headers. - - - .. method:: get_content_charset(failobj=None) - - Return the ``charset`` parameter of the :mailheader:`Content-Type` header, - coerced to lower case. If there is no :mailheader:`Content-Type` header, or if - that header has no ``charset`` parameter, *failobj* is returned. - - - .. method:: get_charsets(failobj=None) - - Return a list containing the character set names in the message. If the - message is a :mimetype:`multipart`, then the list will contain one element - for each subpart in the payload, otherwise, it will be a list of length 1. - - Each item in the list will be a string which is the value of the - ``charset`` parameter in the :mailheader:`Content-Type` header for the - represented subpart. If the subpart has no :mailheader:`Content-Type` - header, no ``charset`` parameter, or is not of the :mimetype:`text` main - MIME type, then that item in the returned list will be *failobj*. - - - .. method:: is_attachment - - Return ``True`` if there is a :mailheader:`Content-Disposition` header - and its (case insensitive) value is ``attachment``, ``False`` otherwise. - - .. versionchanged:: 3.4.2 - is_attachment is now a method instead of a property, for consistency - with :meth:`~email.message.Message.is_multipart`. - - - .. method:: get_content_disposition() - - Return the lowercased value (without parameters) of the message's - :mailheader:`Content-Disposition` header if it has one, or ``None``. The - possible values for this method are *inline*, *attachment* or ``None`` - if the message follows :rfc:`2183`. - - .. versionadded:: 3.5 - - - The following methods relate to interrogating and manipulating the content - (payload) of the message. - - - .. method:: walk() - - The :meth:`walk` method is an all-purpose generator which can be used to - iterate over all the parts and subparts of a message object tree, in - depth-first traversal order. You will typically use :meth:`walk` as the - iterator in a ``for`` loop; each iteration returns the next subpart. - - Here's an example that prints the MIME type of every part of a multipart - message structure: - - .. testsetup:: - - from email import message_from_binary_file - with open('../Lib/test/test_email/data/msg_16.txt', 'rb') as f: - msg = message_from_binary_file(f) - - .. doctest:: - - >>> for part in msg.walk(): - ... print(part.get_content_type()) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - ``walk`` iterates over the subparts of any part where - :meth:`is_multipart` returns ``True``, even though - ``msg.get_content_maintype() == 'multipart'`` may return ``False``. We - can see this in our example by making use of the ``_structure`` debug - helper function: - - .. doctest:: - - >>> from email.iterators import _structure - >>> for part in msg.walk(): - ... print(part.get_content_maintype() == 'multipart', - ... part.is_multipart()) - True True - False False - False True - False False - False False - False True - False False - >>> _structure(msg) - multipart/report - text/plain - message/delivery-status - text/plain - text/plain - message/rfc822 - text/plain - - Here the ``message`` parts are not ``multiparts``, but they do contain - subparts. ``is_multipart()`` returns ``True`` and ``walk`` descends - into the subparts. - - - .. method:: get_body(preferencelist=('related', 'html', 'plain')) - - Return the MIME part that is the best candidate to be the "body" of the - message. - - *preferencelist* must be a sequence of strings from the set ``related``, - ``html``, and ``plain``, and indicates the order of preference for the - content type of the part returned. - - Start looking for candidate matches with the object on which the - ``get_body`` method is called. - - If ``related`` is not included in *preferencelist*, consider the root - part (or subpart of the root part) of any related encountered as a - candidate if the (sub-)part matches a preference. - - When encountering a ``multipart/related``, check the ``start`` parameter - and if a part with a matching :mailheader:`Content-ID` is found, consider - only it when looking for candidate matches. Otherwise consider only the - first (default root) part of the ``multipart/related``. - - If a part has a :mailheader:`Content-Disposition` header, only consider - the part a candidate match if the value of the header is ``inline``. - - If none of the candidates matches any of the preferences in - *preferencelist*, return ``None``. - - Notes: (1) For most applications the only *preferencelist* combinations - that really make sense are ``('plain',)``, ``('html', 'plain')``, and the - default ``('related', 'html', 'plain')``. (2) Because matching starts - with the object on which ``get_body`` is called, calling ``get_body`` on - a ``multipart/related`` will return the object itself unless - *preferencelist* has a non-default value. (3) Messages (or message parts) - that do not specify a :mailheader:`Content-Type` or whose - :mailheader:`Content-Type` header is invalid will be treated as if they - are of type ``text/plain``, which may occasionally cause ``get_body`` to - return unexpected results. - - - .. method:: iter_attachments() - - Return an iterator over all of the immediate sub-parts of the message - that are not candidate "body" parts. That is, skip the first occurrence - of each of ``text/plain``, ``text/html``, ``multipart/related``, or - ``multipart/alternative`` (unless they are explicitly marked as - attachments via :mailheader:`Content-Disposition: attachment`), and - return all remaining parts. When applied directly to a - ``multipart/related``, return an iterator over the all the related parts - except the root part (ie: the part pointed to by the ``start`` parameter, - or the first part if there is no ``start`` parameter or the ``start`` - parameter doesn't match the :mailheader:`Content-ID` of any of the - parts). When applied directly to a ``multipart/alternative`` or a - non-``multipart``, return an empty iterator. - - - .. method:: iter_parts() - - Return an iterator over all of the immediate sub-parts of the message, - which will be empty for a non-``multipart``. (See also - :meth:`~email.message.EmailMessage.walk`.) - - - .. method:: get_content(*args, content_manager=None, **kw) - - Call the :meth:`~email.contentmanager.ContentManager.get_content` method - of the *content_manager*, passing self as the message object, and passing - along any other arguments or keywords as additional arguments. If - *content_manager* is not specified, use the ``content_manager`` specified - by the current :mod:`~email.policy`. - - - .. method:: set_content(*args, content_manager=None, **kw) - - Call the :meth:`~email.contentmanager.ContentManager.set_content` method - of the *content_manager*, passing self as the message object, and passing - along any other arguments or keywords as additional arguments. If - *content_manager* is not specified, use the ``content_manager`` specified - by the current :mod:`~email.policy`. - - - .. method:: make_related(boundary=None) - - Convert a non-``multipart`` message into a ``multipart/related`` message, - moving any existing :mailheader:`Content-` headers and payload into a - (new) first part of the ``multipart``. If *boundary* is specified, use - it as the boundary string in the multipart, otherwise leave the boundary - to be automatically created when it is needed (for example, when the - message is serialized). - - - .. method:: make_alternative(boundary=None) - - Convert a non-``multipart`` or a ``multipart/related`` into a - ``multipart/alternative``, moving any existing :mailheader:`Content-` - headers and payload into a (new) first part of the ``multipart``. If - *boundary* is specified, use it as the boundary string in the multipart, - otherwise leave the boundary to be automatically created when it is - needed (for example, when the message is serialized). - - - .. method:: make_mixed(boundary=None) - - Convert a non-``multipart``, a ``multipart/related``, or a - ``multipart-alternative`` into a ``multipart/mixed``, moving any existing - :mailheader:`Content-` headers and payload into a (new) first part of the - ``multipart``. If *boundary* is specified, use it as the boundary string - in the multipart, otherwise leave the boundary to be automatically - created when it is needed (for example, when the message is serialized). - - - .. method:: add_related(*args, content_manager=None, **kw) - - If the message is a ``multipart/related``, create a new message - object, pass all of the arguments to its :meth:`set_content` method, - and :meth:`~email.message.Message.attach` it to the ``multipart``. If - the message is a non-``multipart``, call :meth:`make_related` and then - proceed as above. If the message is any other type of ``multipart``, - raise a :exc:`TypeError`. If *content_manager* is not specified, use - the ``content_manager`` specified by the current :mod:`~email.policy`. - If the added part has no :mailheader:`Content-Disposition` header, - add one with the value ``inline``. - - - .. method:: add_alternative(*args, content_manager=None, **kw) - - If the message is a ``multipart/alternative``, create a new message - object, pass all of the arguments to its :meth:`set_content` method, and - :meth:`~email.message.Message.attach` it to the ``multipart``. If the - message is a non-``multipart`` or ``multipart/related``, call - :meth:`make_alternative` and then proceed as above. If the message is - any other type of ``multipart``, raise a :exc:`TypeError`. If - *content_manager* is not specified, use the ``content_manager`` specified - by the current :mod:`~email.policy`. - - - .. method:: add_attachment(*args, content_manager=None, **kw) - - If the message is a ``multipart/mixed``, create a new message object, - pass all of the arguments to its :meth:`set_content` method, and - :meth:`~email.message.Message.attach` it to the ``multipart``. If the - message is a non-``multipart``, ``multipart/related``, or - ``multipart/alternative``, call :meth:`make_mixed` and then proceed as - above. If *content_manager* is not specified, use the ``content_manager`` - specified by the current :mod:`~email.policy`. If the added part - has no :mailheader:`Content-Disposition` header, add one with the value - ``attachment``. This method can be used both for explicit attachments - (:mailheader:`Content-Disposition: attachment`) and ``inline`` attachments - (:mailheader:`Content-Disposition: inline`), by passing appropriate - options to the ``content_manager``. - - - .. method:: clear() - - Remove the payload and all of the headers. - - - .. method:: clear_content() - - Remove the payload and all of the :mailheader:`!Content-` headers, leaving - all other headers intact and in their original order. - - - :class:`EmailMessage` objects have the following instance attributes: - - - .. attribute:: preamble - - The format of a MIME document allows for some text between the blank line - following the headers, and the first multipart boundary string. Normally, - this text is never visible in a MIME-aware mail reader because it falls - outside the standard MIME armor. However, when viewing the raw text of - the message, or when viewing the message in a non-MIME aware reader, this - text can become visible. - - The *preamble* attribute contains this leading extra-armor text for MIME - documents. When the :class:`~email.parser.Parser` discovers some text - after the headers but before the first boundary string, it assigns this - text to the message's *preamble* attribute. When the - :class:`~email.generator.Generator` is writing out the plain text - representation of a MIME message, and it finds the - message has a *preamble* attribute, it will write this text in the area - between the headers and the first boundary. See :mod:`email.parser` and - :mod:`email.generator` for details. - - Note that if the message object has no preamble, the *preamble* attribute - will be ``None``. - - - .. attribute:: epilogue - - The *epilogue* attribute acts the same way as the *preamble* attribute, - except that it contains text that appears between the last boundary and - the end of the message. As with the :attr:`~EmailMessage.preamble`, - if there is no epilog text this attribute will be ``None``. - - - .. attribute:: defects - - The *defects* attribute contains a list of all the problems found when - parsing this message. See :mod:`email.errors` for a detailed description - of the possible parsing defects. - - -.. class:: MIMEPart(policy=default) - - This class represents a subpart of a MIME message. It is identical to - :class:`EmailMessage`, except that no :mailheader:`MIME-Version` headers are - added when :meth:`~EmailMessage.set_content` is called, since sub-parts do - not need their own :mailheader:`MIME-Version` headers. - - -.. rubric:: Footnotes - -.. [1] Originally added in 3.4 as a :term:`provisional module `. Docs for legacy message class moved to - :ref:`compat32_message`. diff --git a/Doc/library/email.mime.rst b/Doc/library/email.mime.rst deleted file mode 100644 index d7c0d203d191f8..00000000000000 --- a/Doc/library/email.mime.rst +++ /dev/null @@ -1,260 +0,0 @@ -:mod:`email.mime`: Creating email and MIME objects from scratch ---------------------------------------------------------------- - -.. module:: email.mime - :synopsis: Build MIME messages. - -**Source code:** :source:`Lib/email/mime/` - --------------- - -This module is part of the legacy (``Compat32``) email API. Its functionality -is partially replaced by the :mod:`~email.contentmanager` in the new API, but -in certain applications these classes may still be useful, even in non-legacy -code. - -Ordinarily, you get a message object structure by passing a file or some text to -a parser, which parses the text and returns the root message object. However -you can also build a complete message structure from scratch, or even individual -:class:`~email.message.Message` objects by hand. In fact, you can also take an -existing structure and add new :class:`~email.message.Message` objects, move them -around, etc. This makes a very convenient interface for slicing-and-dicing MIME -messages. - -You can create a new object structure by creating :class:`~email.message.Message` -instances, adding attachments and all the appropriate headers manually. For MIME -messages though, the :mod:`email` package provides some convenient subclasses to -make things easier. - -Here are the classes: - -.. currentmodule:: email.mime.base - -.. class:: MIMEBase(_maintype, _subtype, *, policy=compat32, **_params) - - Module: :mod:`email.mime.base` - - This is the base class for all the MIME-specific subclasses of - :class:`~email.message.Message`. Ordinarily you won't create instances - specifically of :class:`MIMEBase`, although you could. :class:`MIMEBase` - is provided primarily as a convenient base class for more specific - MIME-aware subclasses. - - *_maintype* is the :mailheader:`Content-Type` major type (e.g. :mimetype:`text` - or :mimetype:`image`), and *_subtype* is the :mailheader:`Content-Type` minor - type (e.g. :mimetype:`plain` or :mimetype:`gif`). *_params* is a parameter - key/value dictionary and is passed directly to :meth:`Message.add_header - `. - - If *policy* is specified, (defaults to the - :class:`compat32 ` policy) it will be passed to - :class:`~email.message.Message`. - - The :class:`MIMEBase` class always adds a :mailheader:`Content-Type` header - (based on *_maintype*, *_subtype*, and *_params*), and a - :mailheader:`MIME-Version` header (always set to ``1.0``). - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - - -.. currentmodule:: email.mime.nonmultipart - -.. class:: MIMENonMultipart() - - Module: :mod:`email.mime.nonmultipart` - - A subclass of :class:`~email.mime.base.MIMEBase`, this is an intermediate base - class for MIME messages that are not :mimetype:`multipart`. The primary - purpose of this class is to prevent the use of the - :meth:`~email.message.Message.attach` method, which only makes sense for - :mimetype:`multipart` messages. If :meth:`~email.message.Message.attach` - is called, a :exc:`~email.errors.MultipartConversionError` exception is raised. - - -.. currentmodule:: email.mime.multipart - -.. class:: MIMEMultipart(_subtype='mixed', boundary=None, _subparts=None, \ - *, policy=compat32, **_params) - - Module: :mod:`email.mime.multipart` - - A subclass of :class:`~email.mime.base.MIMEBase`, this is an intermediate base - class for MIME messages that are :mimetype:`multipart`. Optional *_subtype* - defaults to :mimetype:`mixed`, but can be used to specify the subtype of the - message. A :mailheader:`Content-Type` header of :mimetype:`multipart/_subtype` - will be added to the message object. A :mailheader:`MIME-Version` header will - also be added. - - Optional *boundary* is the multipart boundary string. When ``None`` (the - default), the boundary is calculated when needed (for example, when the - message is serialized). - - *_subparts* is a sequence of initial subparts for the payload. It must be - possible to convert this sequence to a list. You can always attach new subparts - to the message by using the :meth:`Message.attach - ` method. - - Optional *policy* argument defaults to :class:`compat32 `. - - Additional parameters for the :mailheader:`Content-Type` header are taken from - the keyword arguments, or passed into the *_params* argument, which is a keyword - dictionary. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.application - -.. class:: MIMEApplication(_data, _subtype='octet-stream', \ - _encoder=email.encoders.encode_base64, \ - *, policy=compat32, **_params) - - Module: :mod:`email.mime.application` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEApplication` class is used to represent MIME message objects of - major type :mimetype:`application`. *_data* contains the bytes for the raw - application data. Optional *_subtype* specifies the MIME subtype and defaults - to :mimetype:`octet-stream`. - - Optional *_encoder* is a callable (i.e. function) which will perform the actual - encoding of the data for transport. This callable takes one argument, which is - the :class:`MIMEApplication` instance. It should use - :meth:`~email.message.Message.get_payload` and - :meth:`~email.message.Message.set_payload` to change the payload to encoded - form. It should also add - any :mailheader:`Content-Transfer-Encoding` or other headers to the message - object as necessary. The default encoding is base64. See the - :mod:`email.encoders` module for a list of the built-in encoders. - - Optional *policy* argument defaults to :class:`compat32 `. - - *_params* are passed straight through to the base class constructor. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.audio - -.. class:: MIMEAudio(_audiodata, _subtype=None, \ - _encoder=email.encoders.encode_base64, \ - *, policy=compat32, **_params) - - Module: :mod:`email.mime.audio` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEAudio` class is used to create MIME message objects of major type - :mimetype:`audio`. *_audiodata* contains the bytes for the raw audio data. If - this data can be decoded as au, wav, aiff, or aifc, then the - subtype will be automatically included in the :mailheader:`Content-Type` header. - Otherwise you can explicitly specify the audio subtype via the *_subtype* - argument. If the minor type could not be guessed and *_subtype* was not given, - then :exc:`TypeError` is raised. - - Optional *_encoder* is a callable (i.e. function) which will perform the actual - encoding of the audio data for transport. This callable takes one argument, - which is the :class:`MIMEAudio` instance. It should use - :meth:`~email.message.Message.get_payload` and - :meth:`~email.message.Message.set_payload` to change the payload to encoded - form. It should also add - any :mailheader:`Content-Transfer-Encoding` or other headers to the message - object as necessary. The default encoding is base64. See the - :mod:`email.encoders` module for a list of the built-in encoders. - - Optional *policy* argument defaults to :class:`compat32 `. - - *_params* are passed straight through to the base class constructor. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.image - -.. class:: MIMEImage(_imagedata, _subtype=None, \ - _encoder=email.encoders.encode_base64, \ - *, policy=compat32, **_params) - - Module: :mod:`email.mime.image` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEImage` class is used to create MIME message objects of major type - :mimetype:`image`. *_imagedata* contains the bytes for the raw image data. If - this data type can be detected (jpeg, png, gif, tiff, rgb, pbm, pgm, ppm, - rast, xbm, bmp, webp, and exr attempted), then the subtype will be - automatically included in the :mailheader:`Content-Type` header. Otherwise - you can explicitly specify the image subtype via the *_subtype* argument. - If the minor type could not be guessed and *_subtype* was not given, then - :exc:`TypeError` is raised. - - Optional *_encoder* is a callable (i.e. function) which will perform the actual - encoding of the image data for transport. This callable takes one argument, - which is the :class:`MIMEImage` instance. It should use - :meth:`~email.message.Message.get_payload` and - :meth:`~email.message.Message.set_payload` to change the payload to encoded - form. It should also add - any :mailheader:`Content-Transfer-Encoding` or other headers to the message - object as necessary. The default encoding is base64. See the - :mod:`email.encoders` module for a list of the built-in encoders. - - Optional *policy* argument defaults to :class:`compat32 `. - - *_params* are passed straight through to the :class:`~email.mime.base.MIMEBase` - constructor. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.message - -.. class:: MIMEMessage(_msg, _subtype='rfc822', *, policy=compat32) - - Module: :mod:`email.mime.message` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEMessage` class is used to create MIME objects of main type - :mimetype:`message`. *_msg* is used as the payload, and must be an instance - of class :class:`~email.message.Message` (or a subclass thereof), otherwise - a :exc:`TypeError` is raised. - - Optional *_subtype* sets the subtype of the message; it defaults to - :mimetype:`rfc822`. - - Optional *policy* argument defaults to :class:`compat32 `. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. - -.. currentmodule:: email.mime.text - -.. class:: MIMEText(_text, _subtype='plain', _charset=None, *, policy=compat32) - - Module: :mod:`email.mime.text` - - A subclass of :class:`~email.mime.nonmultipart.MIMENonMultipart`, the - :class:`MIMEText` class is used to create MIME objects of major type - :mimetype:`text`. *_text* is the string for the payload. *_subtype* is the - minor type and defaults to :mimetype:`plain`. *_charset* is the character - set of the text and is passed as an argument to the - :class:`~email.mime.nonmultipart.MIMENonMultipart` constructor; it defaults - to ``us-ascii`` if the string contains only ``ascii`` code points, and - ``utf-8`` otherwise. The *_charset* parameter accepts either a string or a - :class:`~email.charset.Charset` instance. - - Unless the *_charset* argument is explicitly set to ``None``, the - MIMEText object created will have both a :mailheader:`Content-Type` header - with a ``charset`` parameter, and a :mailheader:`Content-Transfer-Encoding` - header. This means that a subsequent ``set_payload`` call will not result - in an encoded payload, even if a charset is passed in the ``set_payload`` - command. You can "reset" this behavior by deleting the - ``Content-Transfer-Encoding`` header, after which a ``set_payload`` call - will automatically encode the new payload (and add a new - :mailheader:`Content-Transfer-Encoding` header). - - Optional *policy* argument defaults to :class:`compat32 `. - - .. versionchanged:: 3.5 - *_charset* also accepts :class:`~email.charset.Charset` instances. - - .. versionchanged:: 3.6 - Added *policy* keyword-only parameter. diff --git a/Doc/library/email.parser.rst b/Doc/library/email.parser.rst deleted file mode 100644 index dda0466a6afa7d..00000000000000 --- a/Doc/library/email.parser.rst +++ /dev/null @@ -1,320 +0,0 @@ -:mod:`email.parser`: Parsing email messages -------------------------------------------- - -.. module:: email.parser - :synopsis: Parse flat text email messages to produce a message object structure. - -**Source code:** :source:`Lib/email/parser.py` - --------------- - -Message object structures can be created in one of two ways: they can be -created from whole cloth by creating an :class:`~email.message.EmailMessage` -object, adding headers using the dictionary interface, and adding payload(s) -using :meth:`~email.message.EmailMessage.set_content` and related methods, or -they can be created by parsing a serialized representation of the email -message. - -The :mod:`email` package provides a standard parser that understands most email -document structures, including MIME documents. You can pass the parser a -bytes, string or file object, and the parser will return to you the root -:class:`~email.message.EmailMessage` instance of the object structure. For -simple, non-MIME messages the payload of this root object will likely be a -string containing the text of the message. For MIME messages, the root object -will return ``True`` from its :meth:`~email.message.EmailMessage.is_multipart` -method, and the subparts can be accessed via the payload manipulation methods, -such as :meth:`~email.message.EmailMessage.get_body`, -:meth:`~email.message.EmailMessage.iter_parts`, and -:meth:`~email.message.EmailMessage.walk`. - -There are actually two parser interfaces available for use, the :class:`Parser` -API and the incremental :class:`FeedParser` API. The :class:`Parser` API is -most useful if you have the entire text of the message in memory, or if the -entire message lives in a file on the file system. :class:`FeedParser` is more -appropriate when you are reading the message from a stream which might block -waiting for more input (such as reading an email message from a socket). The -:class:`FeedParser` can consume and parse the message incrementally, and only -returns the root object when you close the parser. - -Note that the parser can be extended in limited ways, and of course you can -implement your own parser completely from scratch. All of the logic that -connects the :mod:`email` package's bundled parser and the -:class:`~email.message.EmailMessage` class is embodied in the :class:`~email.policy.Policy` -class, so a custom parser can create message object trees any way it finds -necessary by implementing custom versions of the appropriate :class:`!Policy` -methods. - - -FeedParser API -^^^^^^^^^^^^^^ - -The :class:`BytesFeedParser`, imported from the :mod:`email.feedparser` module, -provides an API that is conducive to incremental parsing of email messages, -such as would be necessary when reading the text of an email message from a -source that can block (such as a socket). The :class:`BytesFeedParser` can of -course be used to parse an email message fully contained in a :term:`bytes-like -object`, string, or file, but the :class:`BytesParser` API may be more -convenient for such use cases. The semantics and results of the two parser -APIs are identical. - -The :class:`BytesFeedParser`'s API is simple; you create an instance, feed it a -bunch of bytes until there's no more to feed it, then close the parser to -retrieve the root message object. The :class:`BytesFeedParser` is extremely -accurate when parsing standards-compliant messages, and it does a very good job -of parsing non-compliant messages, providing information about how a message -was deemed broken. It will populate a message object's -:attr:`~email.message.EmailMessage.defects` attribute with a list of any -problems it found in a message. See the :mod:`email.errors` module for the -list of defects that it can find. - -Here is the API for the :class:`BytesFeedParser`: - - -.. class:: BytesFeedParser(_factory=None, *, policy=policy.compat32) - - Create a :class:`BytesFeedParser` instance. Optional *_factory* is a - no-argument callable; if not specified use the - :attr:`~email.policy.Policy.message_factory` from the *policy*. Call - *_factory* whenever a new message object is needed. - - If *policy* is specified use the rules it specifies to update the - representation of the message. If *policy* is not set, use the - :class:`compat32 ` policy, which maintains backward - compatibility with the Python 3.2 version of the email package and provides - :class:`~email.message.Message` as the default factory. All other policies - provide :class:`~email.message.EmailMessage` as the default *_factory*. For - more information on what else *policy* controls, see the - :mod:`~email.policy` documentation. - - Note: **The policy keyword should always be specified**; The default will - change to :data:`email.policy.default` in a future version of Python. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 Added the *policy* keyword. - .. versionchanged:: 3.6 *_factory* defaults to the policy ``message_factory``. - - - .. method:: feed(data) - - Feed the parser some more data. *data* should be a :term:`bytes-like - object` containing one or more lines. The lines can be partial and the - parser will stitch such partial lines together properly. The lines can - have any of the three common line endings: carriage return, newline, or - carriage return and newline (they can even be mixed). - - - .. method:: close() - - Complete the parsing of all previously fed data and return the root - message object. It is undefined what happens if :meth:`~feed` is called - after this method has been called. - - -.. class:: FeedParser(_factory=None, *, policy=policy.compat32) - - Works like :class:`BytesFeedParser` except that the input to the - :meth:`~BytesFeedParser.feed` method must be a string. This is of limited - utility, since the only way for such a message to be valid is for it to - contain only ASCII text or, if :attr:`~email.policy.Policy.utf8` is - ``True``, no binary attachments. - - .. versionchanged:: 3.3 Added the *policy* keyword. - - -Parser API -^^^^^^^^^^ - -The :class:`BytesParser` class, imported from the :mod:`email.parser` module, -provides an API that can be used to parse a message when the complete contents -of the message are available in a :term:`bytes-like object` or file. The -:mod:`email.parser` module also provides :class:`Parser` for parsing strings, -and header-only parsers, :class:`BytesHeaderParser` and -:class:`HeaderParser`, which can be used if you're only interested in the -headers of the message. :class:`BytesHeaderParser` and :class:`HeaderParser` -can be much faster in these situations, since they do not attempt to parse the -message body, instead setting the payload to the raw body. - - -.. class:: BytesParser(_class=None, *, policy=policy.compat32) - - Create a :class:`BytesParser` instance. The *_class* and *policy* - arguments have the same meaning and semantics as the *_factory* - and *policy* arguments of :class:`BytesFeedParser`. - - Note: **The policy keyword should always be specified**; The default will - change to :data:`email.policy.default` in a future version of Python. - - .. versionchanged:: 3.3 - Removed the *strict* argument that was deprecated in 2.4. Added the - *policy* keyword. - .. versionchanged:: 3.6 *_class* defaults to the policy ``message_factory``. - - - .. method:: parse(fp, headersonly=False) - - Read all the data from the binary file-like object *fp*, parse the - resulting bytes, and return the message object. *fp* must support - both the :meth:`~io.IOBase.readline` and the :meth:`~io.IOBase.read` - methods. - - The bytes contained in *fp* must be formatted as a block of :rfc:`5322` - (or, if :attr:`~email.policy.Policy.utf8` is ``True``, :rfc:`6532`) - style headers and header continuation lines, optionally preceded by an - envelope header. The header block is terminated either by the end of the - data or by a blank line. Following the header block is the body of the - message (which may contain MIME-encoded subparts, including subparts - with a :mailheader:`Content-Transfer-Encoding` of ``8bit``). - - Optional *headersonly* is a flag specifying whether to stop parsing after - reading the headers or not. The default is ``False``, meaning it parses - the entire contents of the file. - - - .. method:: parsebytes(bytes, headersonly=False) - - Similar to the :meth:`parse` method, except it takes a :term:`bytes-like - object` instead of a file-like object. Calling this method on a - :term:`bytes-like object` is equivalent to wrapping *bytes* in a - :class:`~io.BytesIO` instance first and calling :meth:`parse`. - - Optional *headersonly* is as with the :meth:`parse` method. - - .. versionadded:: 3.2 - - -.. class:: BytesHeaderParser(_class=None, *, policy=policy.compat32) - - Exactly like :class:`BytesParser`, except that *headersonly* - defaults to ``True``. - - .. versionadded:: 3.3 - - -.. class:: Parser(_class=None, *, policy=policy.compat32) - - This class is parallel to :class:`BytesParser`, but handles string input. - - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - .. versionchanged:: 3.6 *_class* defaults to the policy ``message_factory``. - - - .. method:: parse(fp, headersonly=False) - - Read all the data from the text-mode file-like object *fp*, parse the - resulting text, and return the root message object. *fp* must support - both the :meth:`~io.TextIOBase.readline` and the - :meth:`~io.TextIOBase.read` methods on file-like objects. - - Other than the text mode requirement, this method operates like - :meth:`BytesParser.parse`. - - - .. method:: parsestr(text, headersonly=False) - - Similar to the :meth:`parse` method, except it takes a string object - instead of a file-like object. Calling this method on a string is - equivalent to wrapping *text* in a :class:`~io.StringIO` instance first - and calling :meth:`parse`. - - Optional *headersonly* is as with the :meth:`parse` method. - - -.. class:: HeaderParser(_class=None, *, policy=policy.compat32) - - Exactly like :class:`Parser`, except that *headersonly* - defaults to ``True``. - - -Since creating a message object structure from a string or a file object is such -a common task, four functions are provided as a convenience. They are available -in the top-level :mod:`email` package namespace. - -.. currentmodule:: email - - -.. function:: message_from_bytes(s, _class=None, *, policy=policy.compat32) - - Return a message object structure from a :term:`bytes-like object`. This is - equivalent to ``BytesParser().parsebytes(s)``. Optional *_class* and - *policy* are interpreted as with the :class:`~email.parser.BytesParser` class - constructor. - - .. versionadded:: 3.2 - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - - -.. function:: message_from_binary_file(fp, _class=None, *, \ - policy=policy.compat32) - - Return a message object structure tree from an open binary :term:`file - object`. This is equivalent to ``BytesParser().parse(fp)``. *_class* and - *policy* are interpreted as with the :class:`~email.parser.BytesParser` class - constructor. - - .. versionadded:: 3.2 - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - - -.. function:: message_from_string(s, _class=None, *, policy=policy.compat32) - - Return a message object structure from a string. This is equivalent to - ``Parser().parsestr(s)``. *_class* and *policy* are interpreted as - with the :class:`~email.parser.Parser` class constructor. - - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - - -.. function:: message_from_file(fp, _class=None, *, policy=policy.compat32) - - Return a message object structure tree from an open :term:`file object`. - This is equivalent to ``Parser().parse(fp)``. *_class* and *policy* are - interpreted as with the :class:`~email.parser.Parser` class constructor. - - .. versionchanged:: 3.3 - Removed the *strict* argument. Added the *policy* keyword. - .. versionchanged:: 3.6 *_class* defaults to the policy ``message_factory``. - - -Here's an example of how you might use :func:`message_from_bytes` at an -interactive Python prompt:: - - >>> import email - >>> msg = email.message_from_bytes(myBytes) # doctest: +SKIP - - -Additional notes -^^^^^^^^^^^^^^^^ - -Here are some notes on the parsing semantics: - -* Most non-\ :mimetype:`multipart` type messages are parsed as a single message - object with a string payload. These objects will return ``False`` for - :meth:`~email.message.EmailMessage.is_multipart`, and - :meth:`~email.message.EmailMessage.iter_parts` will yield an empty list. - -* All :mimetype:`multipart` type messages will be parsed as a container message - object with a list of sub-message objects for their payload. The outer - container message will return ``True`` for - :meth:`~email.message.EmailMessage.is_multipart`, and - :meth:`~email.message.EmailMessage.iter_parts` will yield a list of subparts. - -* Most messages with a content type of :mimetype:`message/\*` (such as - :mimetype:`message/delivery-status` and :mimetype:`message/rfc822`) will also - be parsed as container object containing a list payload of length 1. Their - :meth:`~email.message.EmailMessage.is_multipart` method will return ``True``. - The single element yielded by :meth:`~email.message.EmailMessage.iter_parts` - will be a sub-message object. - -* Some non-standards-compliant messages may not be internally consistent about - their :mimetype:`multipart`\ -edness. Such messages may have a - :mailheader:`Content-Type` header of type :mimetype:`multipart`, but their - :meth:`~email.message.EmailMessage.is_multipart` method may return ``False``. - If such messages were parsed with the :class:`~email.parser.FeedParser`, - they will have an instance of the - :class:`~email.errors.MultipartInvariantViolationDefect` class in their - *defects* attribute list. See :mod:`email.errors` for details. diff --git a/Doc/library/email.policy.rst b/Doc/library/email.policy.rst deleted file mode 100644 index 9b462fab119561..00000000000000 --- a/Doc/library/email.policy.rst +++ /dev/null @@ -1,651 +0,0 @@ -:mod:`email.policy`: Policy Objects ------------------------------------ - -.. module:: email.policy - :synopsis: Controlling the parsing and generating of messages - -.. moduleauthor:: R. David Murray -.. sectionauthor:: R. David Murray - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/email/policy.py` - --------------- - -The :mod:`email` package's prime focus is the handling of email messages as -described by the various email and MIME RFCs. However, the general format of -email messages (a block of header fields each consisting of a name followed by -a colon followed by a value, the whole block followed by a blank line and an -arbitrary 'body'), is a format that has found utility outside of the realm of -email. Some of these uses conform fairly closely to the main email RFCs, some -do not. Even when working with email, there are times when it is desirable to -break strict compliance with the RFCs, such as generating emails that -interoperate with email servers that do not themselves follow the standards, or -that implement extensions you want to use in ways that violate the -standards. - -Policy objects give the email package the flexibility to handle all these -disparate use cases. - -A :class:`Policy` object encapsulates a set of attributes and methods that -control the behavior of various components of the email package during use. -:class:`Policy` instances can be passed to various classes and methods in the -email package to alter the default behavior. The settable values and their -defaults are described below. - -There is a default policy used by all classes in the email package. For all of -the :mod:`~email.parser` classes and the related convenience functions, and for -the :class:`~email.message.Message` class, this is the :class:`Compat32` -policy, via its corresponding pre-defined instance :const:`compat32`. This -policy provides for complete backward compatibility (in some cases, including -bug compatibility) with the pre-Python3.3 version of the email package. - -This default value for the *policy* keyword to -:class:`~email.message.EmailMessage` is the :class:`EmailPolicy` policy, via -its pre-defined instance :data:`~default`. - -When a :class:`~email.message.Message` or :class:`~email.message.EmailMessage` -object is created, it acquires a policy. If the message is created by a -:mod:`~email.parser`, a policy passed to the parser will be the policy used by -the message it creates. If the message is created by the program, then the -policy can be specified when it is created. When a message is passed to a -:mod:`~email.generator`, the generator uses the policy from the message by -default, but you can also pass a specific policy to the generator that will -override the one stored on the message object. - -The default value for the *policy* keyword for the :mod:`email.parser` classes -and the parser convenience functions **will be changing** in a future version of -Python. Therefore you should **always specify explicitly which policy you want -to use** when calling any of the classes and functions described in the -:mod:`~email.parser` module. - -The first part of this documentation covers the features of :class:`Policy`, an -:term:`abstract base class` that defines the features that are common to all -policy objects, including :const:`compat32`. This includes certain hook -methods that are called internally by the email package, which a custom policy -could override to obtain different behavior. The second part describes the -concrete classes :class:`EmailPolicy` and :class:`Compat32`, which implement -the hooks that provide the standard behavior and the backward compatible -behavior and features, respectively. - -:class:`Policy` instances are immutable, but they can be cloned, accepting the -same keyword arguments as the class constructor and returning a new -:class:`Policy` instance that is a copy of the original but with the specified -attributes values changed. - -As an example, the following code could be used to read an email message from a -file on disk and pass it to the system ``sendmail`` program on a Unix system: - -.. testsetup:: - - from unittest import mock - mocker = mock.patch('subprocess.Popen') - m = mocker.start() - proc = mock.MagicMock() - m.return_value = proc - proc.stdin.close.return_value = None - mymsg = open('mymsg.txt', 'w') - mymsg.write('To: abc@xyz.com\n\n') - mymsg.flush() - -.. doctest:: - - >>> from email import message_from_binary_file - >>> from email.generator import BytesGenerator - >>> from email import policy - >>> from subprocess import Popen, PIPE - >>> with open('mymsg.txt', 'rb') as f: - ... msg = message_from_binary_file(f, policy=policy.default) - >>> p = Popen(['sendmail', msg['To'].addresses[0]], stdin=PIPE) - >>> g = BytesGenerator(p.stdin, policy=msg.policy.clone(linesep='\r\n')) - >>> g.flatten(msg) - >>> p.stdin.close() - >>> rc = p.wait() - -.. testcleanup:: - - mymsg.close() - mocker.stop() - import os - os.remove('mymsg.txt') - -Here we are telling :class:`~email.generator.BytesGenerator` to use the RFC -correct line separator characters when creating the binary string to feed into -``sendmail's`` ``stdin``, where the default policy would use ``\n`` line -separators. - -Some email package methods accept a *policy* keyword argument, allowing the -policy to be overridden for that method. For example, the following code uses -the :meth:`~email.message.Message.as_bytes` method of the *msg* object from -the previous example and writes the message to a file using the native line -separators for the platform on which it is running:: - - >>> import os - >>> with open('converted.txt', 'wb') as f: - ... f.write(msg.as_bytes(policy=msg.policy.clone(linesep=os.linesep))) - 17 - -Policy objects can also be combined using the addition operator, producing a -policy object whose settings are a combination of the non-default values of the -summed objects:: - - >>> compat_SMTP = policy.compat32.clone(linesep='\r\n') - >>> compat_strict = policy.compat32.clone(raise_on_defect=True) - >>> compat_strict_SMTP = compat_SMTP + compat_strict - -This operation is not commutative; that is, the order in which the objects are -added matters. To illustrate:: - - >>> policy100 = policy.compat32.clone(max_line_length=100) - >>> policy80 = policy.compat32.clone(max_line_length=80) - >>> apolicy = policy100 + policy80 - >>> apolicy.max_line_length - 80 - >>> apolicy = policy80 + policy100 - >>> apolicy.max_line_length - 100 - - -.. class:: Policy(**kw) - - This is the :term:`abstract base class` for all policy classes. It provides - default implementations for a couple of trivial methods, as well as the - implementation of the immutability property, the :meth:`clone` method, and - the constructor semantics. - - The constructor of a policy class can be passed various keyword arguments. - The arguments that may be specified are any non-method properties on this - class, plus any additional non-method properties on the concrete class. A - value specified in the constructor will override the default value for the - corresponding attribute. - - This class defines the following properties, and thus values for the - following may be passed in the constructor of any policy class: - - - .. attribute:: max_line_length - - The maximum length of any line in the serialized output, not counting the - end of line character(s). Default is 78, per :rfc:`5322`. A value of - ``0`` or :const:`None` indicates that no line wrapping should be - done at all. - - - .. attribute:: linesep - - The string to be used to terminate lines in serialized output. The - default is ``\n`` because that's the internal end-of-line discipline used - by Python, though ``\r\n`` is required by the RFCs. - - - .. attribute:: cte_type - - Controls the type of Content Transfer Encodings that may be or are - required to be used. The possible values are: - - .. tabularcolumns:: |l|L| - - ======== =============================================================== - ``7bit`` all data must be "7 bit clean" (ASCII-only). This means that - where necessary data will be encoded using either - quoted-printable or base64 encoding. - - ``8bit`` data is not constrained to be 7 bit clean. Data in headers is - still required to be ASCII-only and so will be encoded (see - :meth:`fold_binary` and :attr:`~EmailPolicy.utf8` below for - exceptions), but body parts may use the ``8bit`` CTE. - ======== =============================================================== - - A ``cte_type`` value of ``8bit`` only works with ``BytesGenerator``, not - ``Generator``, because strings cannot contain binary data. If a - ``Generator`` is operating under a policy that specifies - ``cte_type=8bit``, it will act as if ``cte_type`` is ``7bit``. - - - .. attribute:: raise_on_defect - - If :const:`True`, any defects encountered will be raised as errors. If - :const:`False` (the default), defects will be passed to the - :meth:`register_defect` method. - - - .. attribute:: mangle_from_ - - If :const:`True`, lines starting with *"From "* in the body are - escaped by putting a ``>`` in front of them. This parameter is used when - the message is being serialized by a generator. - Default: :const:`False`. - - .. versionadded:: 3.5 - The *mangle_from_* parameter. - - - .. attribute:: message_factory - - A factory function for constructing a new empty message object. Used - by the parser when building messages. Defaults to ``None``, in - which case :class:`~email.message.Message` is used. - - .. versionadded:: 3.6 - - The following :class:`Policy` method is intended to be called by code using - the email library to create policy instances with custom settings: - - - .. method:: clone(**kw) - - Return a new :class:`Policy` instance whose attributes have the same - values as the current instance, except where those attributes are - given new values by the keyword arguments. - - - The remaining :class:`Policy` methods are called by the email package code, - and are not intended to be called by an application using the email package. - A custom policy must implement all of these methods. - - - .. method:: handle_defect(obj, defect) - - Handle a *defect* found on *obj*. When the email package calls this - method, *defect* will always be a subclass of - :class:`~email.errors.Defect`. - - The default implementation checks the :attr:`raise_on_defect` flag. If - it is ``True``, *defect* is raised as an exception. If it is ``False`` - (the default), *obj* and *defect* are passed to :meth:`register_defect`. - - - .. method:: register_defect(obj, defect) - - Register a *defect* on *obj*. In the email package, *defect* will always - be a subclass of :class:`~email.errors.Defect`. - - The default implementation calls the ``append`` method of the ``defects`` - attribute of *obj*. When the email package calls :attr:`handle_defect`, - *obj* will normally have a ``defects`` attribute that has an ``append`` - method. Custom object types used with the email package (for example, - custom ``Message`` objects) should also provide such an attribute, - otherwise defects in parsed messages will raise unexpected errors. - - - .. method:: header_max_count(name) - - Return the maximum allowed number of headers named *name*. - - Called when a header is added to an :class:`~email.message.EmailMessage` - or :class:`~email.message.Message` object. If the returned value is not - ``0`` or ``None``, and there are already a number of headers with the - name *name* greater than or equal to the value returned, a - :exc:`ValueError` is raised. - - Because the default behavior of ``Message.__setitem__`` is to append the - value to the list of headers, it is easy to create duplicate headers - without realizing it. This method allows certain headers to be limited - in the number of instances of that header that may be added to a - ``Message`` programmatically. (The limit is not observed by the parser, - which will faithfully produce as many headers as exist in the message - being parsed.) - - The default implementation returns ``None`` for all header names. - - - .. method:: header_source_parse(sourcelines) - - The email package calls this method with a list of strings, each string - ending with the line separation characters found in the source being - parsed. The first line includes the field header name and separator. - All whitespace in the source is preserved. The method should return the - ``(name, value)`` tuple that is to be stored in the ``Message`` to - represent the parsed header. - - If an implementation wishes to retain compatibility with the existing - email package policies, *name* should be the case preserved name (all - characters up to the '``:``' separator), while *value* should be the - unfolded value (all line separator characters removed, but whitespace - kept intact), stripped of leading whitespace. - - *sourcelines* may contain surrogateescaped binary data. - - There is no default implementation - - - .. method:: header_store_parse(name, value) - - The email package calls this method with the name and value provided by - the application program when the application program is modifying a - ``Message`` programmatically (as opposed to a ``Message`` created by a - parser). The method should return the ``(name, value)`` tuple that is to - be stored in the ``Message`` to represent the header. - - If an implementation wishes to retain compatibility with the existing - email package policies, the *name* and *value* should be strings or - string subclasses that do not change the content of the passed in - arguments. - - There is no default implementation - - - .. method:: header_fetch_parse(name, value) - - The email package calls this method with the *name* and *value* currently - stored in the ``Message`` when that header is requested by the - application program, and whatever the method returns is what is passed - back to the application as the value of the header being retrieved. - Note that there may be more than one header with the same name stored in - the ``Message``; the method is passed the specific name and value of the - header destined to be returned to the application. - - *value* may contain surrogateescaped binary data. There should be no - surrogateescaped binary data in the value returned by the method. - - There is no default implementation - - - .. method:: fold(name, value) - - The email package calls this method with the *name* and *value* currently - stored in the ``Message`` for a given header. The method should return a - string that represents that header "folded" correctly (according to the - policy settings) by composing the *name* with the *value* and inserting - :attr:`linesep` characters at the appropriate places. See :rfc:`5322` - for a discussion of the rules for folding email headers. - - *value* may contain surrogateescaped binary data. There should be no - surrogateescaped binary data in the string returned by the method. - - - .. method:: fold_binary(name, value) - - The same as :meth:`fold`, except that the returned value should be a - bytes object rather than a string. - - *value* may contain surrogateescaped binary data. These could be - converted back into binary data in the returned bytes object. - - - -.. class:: EmailPolicy(**kw) - - This concrete :class:`Policy` provides behavior that is intended to be fully - compliant with the current email RFCs. These include (but are not limited - to) :rfc:`5322`, :rfc:`2047`, and the current MIME RFCs. - - This policy adds new header parsing and folding algorithms. Instead of - simple strings, headers are ``str`` subclasses with attributes that depend - on the type of the field. The parsing and folding algorithm fully implement - :rfc:`2047` and :rfc:`5322`. - - The default value for the :attr:`~email.policy.Policy.message_factory` - attribute is :class:`~email.message.EmailMessage`. - - In addition to the settable attributes listed above that apply to all - policies, this policy adds the following additional attributes: - - .. versionadded:: 3.6 [1]_ - - - .. attribute:: utf8 - - If ``False``, follow :rfc:`5322`, supporting non-ASCII characters in - headers by encoding them as "encoded words". If ``True``, follow - :rfc:`6532` and use ``utf-8`` encoding for headers. Messages - formatted in this way may be passed to SMTP servers that support - the ``SMTPUTF8`` extension (:rfc:`6531`). - - - .. attribute:: refold_source - - If the value for a header in the ``Message`` object originated from a - :mod:`~email.parser` (as opposed to being set by a program), this - attribute indicates whether or not a generator should refold that value - when transforming the message back into serialized form. The possible - values are: - - ======== =============================================================== - ``none`` all source values use original folding - - ``long`` source values that have any line that is longer than - ``max_line_length`` will be refolded - - ``all`` all values are refolded. - ======== =============================================================== - - The default is ``long``. - - - .. attribute:: header_factory - - A callable that takes two arguments, ``name`` and ``value``, where - ``name`` is a header field name and ``value`` is an unfolded header field - value, and returns a string subclass that represents that header. A - default ``header_factory`` (see :mod:`~email.headerregistry`) is provided - that supports custom parsing for the various address and date :RFC:`5322` - header field types, and the major MIME header field stypes. Support for - additional custom parsing will be added in the future. - - - .. attribute:: content_manager - - An object with at least two methods: get_content and set_content. When - the :meth:`~email.message.EmailMessage.get_content` or - :meth:`~email.message.EmailMessage.set_content` method of an - :class:`~email.message.EmailMessage` object is called, it calls the - corresponding method of this object, passing it the message object as its - first argument, and any arguments or keywords that were passed to it as - additional arguments. By default ``content_manager`` is set to - :data:`~email.contentmanager.raw_data_manager`. - - .. versionadded:: 3.4 - - - The class provides the following concrete implementations of the abstract - methods of :class:`Policy`: - - - .. method:: header_max_count(name) - - Returns the value of the - :attr:`~email.headerregistry.BaseHeader.max_count` attribute of the - specialized class used to represent the header with the given name. - - - .. method:: header_source_parse(sourcelines) - - - The name is parsed as everything up to the '``:``' and returned - unmodified. The value is determined by stripping leading whitespace off - the remainder of the first line, joining all subsequent lines together, - and stripping any trailing carriage return or linefeed characters. - - - .. method:: header_store_parse(name, value) - - The name is returned unchanged. If the input value has a ``name`` - attribute and it matches *name* ignoring case, the value is returned - unchanged. Otherwise the *name* and *value* are passed to - ``header_factory``, and the resulting header object is returned as - the value. In this case a ``ValueError`` is raised if the input value - contains CR or LF characters. - - - .. method:: header_fetch_parse(name, value) - - If the value has a ``name`` attribute, it is returned to unmodified. - Otherwise the *name*, and the *value* with any CR or LF characters - removed, are passed to the ``header_factory``, and the resulting - header object is returned. Any surrogateescaped bytes get turned into - the unicode unknown-character glyph. - - - .. method:: fold(name, value) - - Header folding is controlled by the :attr:`refold_source` policy setting. - A value is considered to be a 'source value' if and only if it does not - have a ``name`` attribute (having a ``name`` attribute means it is a - header object of some sort). If a source value needs to be refolded - according to the policy, it is converted into a header object by - passing the *name* and the *value* with any CR and LF characters removed - to the ``header_factory``. Folding of a header object is done by - calling its ``fold`` method with the current policy. - - Source values are split into lines using :meth:`~str.splitlines`. If - the value is not to be refolded, the lines are rejoined using the - ``linesep`` from the policy and returned. The exception is lines - containing non-ascii binary data. In that case the value is refolded - regardless of the ``refold_source`` setting, which causes the binary data - to be CTE encoded using the ``unknown-8bit`` charset. - - - .. method:: fold_binary(name, value) - - The same as :meth:`fold` if :attr:`~Policy.cte_type` is ``7bit``, except - that the returned value is bytes. - - If :attr:`~Policy.cte_type` is ``8bit``, non-ASCII binary data is - converted back - into bytes. Headers with binary data are not refolded, regardless of the - ``refold_header`` setting, since there is no way to know whether the - binary data consists of single byte characters or multibyte characters. - - -The following instances of :class:`EmailPolicy` provide defaults suitable for -specific application domains. Note that in the future the behavior of these -instances (in particular the ``HTTP`` instance) may be adjusted to conform even -more closely to the RFCs relevant to their domains. - - -.. data:: default - - An instance of ``EmailPolicy`` with all defaults unchanged. This policy - uses the standard Python ``\n`` line endings rather than the RFC-correct - ``\r\n``. - - -.. data:: SMTP - - Suitable for serializing messages in conformance with the email RFCs. - Like ``default``, but with ``linesep`` set to ``\r\n``, which is RFC - compliant. - - -.. data:: SMTPUTF8 - - The same as ``SMTP`` except that :attr:`~EmailPolicy.utf8` is ``True``. - Useful for serializing messages to a message store without using encoded - words in the headers. Should only be used for SMTP transmission if the - sender or recipient addresses have non-ASCII characters (the - :meth:`smtplib.SMTP.send_message` method handles this automatically). - - -.. data:: HTTP - - Suitable for serializing headers with for use in HTTP traffic. Like - ``SMTP`` except that ``max_line_length`` is set to ``None`` (unlimited). - - -.. data:: strict - - Convenience instance. The same as ``default`` except that - ``raise_on_defect`` is set to ``True``. This allows any policy to be made - strict by writing:: - - somepolicy + policy.strict - - -With all of these :class:`EmailPolicies <.EmailPolicy>`, the effective API of -the email package is changed from the Python 3.2 API in the following ways: - -* Setting a header on a :class:`~email.message.Message` results in that - header being parsed and a header object created. - -* Fetching a header value from a :class:`~email.message.Message` results - in that header being parsed and a header object created and - returned. - -* Any header object, or any header that is refolded due to the - policy settings, is folded using an algorithm that fully implements the - RFC folding algorithms, including knowing where encoded words are required - and allowed. - -From the application view, this means that any header obtained through the -:class:`~email.message.EmailMessage` is a header object with extra -attributes, whose string value is the fully decoded unicode value of the -header. Likewise, a header may be assigned a new value, or a new header -created, using a unicode string, and the policy will take care of converting -the unicode string into the correct RFC encoded form. - -The header objects and their attributes are described in -:mod:`~email.headerregistry`. - - - -.. class:: Compat32(**kw) - - This concrete :class:`Policy` is the backward compatibility policy. It - replicates the behavior of the email package in Python 3.2. The - :mod:`~email.policy` module also defines an instance of this class, - :const:`compat32`, that is used as the default policy. Thus the default - behavior of the email package is to maintain compatibility with Python 3.2. - - The following attributes have values that are different from the - :class:`Policy` default: - - - .. attribute:: mangle_from_ - - The default is ``True``. - - - The class provides the following concrete implementations of the - abstract methods of :class:`Policy`: - - - .. method:: header_source_parse(sourcelines) - - The name is parsed as everything up to the '``:``' and returned - unmodified. The value is determined by stripping leading whitespace off - the remainder of the first line, joining all subsequent lines together, - and stripping any trailing carriage return or linefeed characters. - - - .. method:: header_store_parse(name, value) - - The name and value are returned unmodified. - - - .. method:: header_fetch_parse(name, value) - - If the value contains binary data, it is converted into a - :class:`~email.header.Header` object using the ``unknown-8bit`` charset. - Otherwise it is returned unmodified. - - - .. method:: fold(name, value) - - Headers are folded using the :class:`~email.header.Header` folding - algorithm, which preserves existing line breaks in the value, and wraps - each resulting line to the ``max_line_length``. Non-ASCII binary data are - CTE encoded using the ``unknown-8bit`` charset. - - - .. method:: fold_binary(name, value) - - Headers are folded using the :class:`~email.header.Header` folding - algorithm, which preserves existing line breaks in the value, and wraps - each resulting line to the ``max_line_length``. If ``cte_type`` is - ``7bit``, non-ascii binary data is CTE encoded using the ``unknown-8bit`` - charset. Otherwise the original source header is used, with its existing - line breaks and any (RFC invalid) binary data it may contain. - - -.. data:: compat32 - - An instance of :class:`Compat32`, providing backward compatibility with the - behavior of the email package in Python 3.2. - - -.. rubric:: Footnotes - -.. [1] Originally added in 3.3 as a :term:`provisional feature `. diff --git a/Doc/library/email.rst b/Doc/library/email.rst deleted file mode 100644 index 5eebcd9e896d93..00000000000000 --- a/Doc/library/email.rst +++ /dev/null @@ -1,152 +0,0 @@ -:mod:`email` --- An email and MIME handling package -=================================================== - -.. module:: email - :synopsis: Package supporting the parsing, manipulating, and generating - email messages. -.. moduleauthor:: Barry A. Warsaw , - R. David Murray -.. sectionauthor:: R. David Murray - -**Source code:** :source:`Lib/email/__init__.py` - --------------- - -The :mod:`email` package is a library for managing email messages. It is -specifically *not* designed to do any sending of email messages to SMTP -(:rfc:`2821`), NNTP, or other servers; those are functions of modules such as -:mod:`smtplib` and :mod:`nntplib`. The :mod:`email` package attempts to be as -RFC-compliant as possible, supporting :rfc:`5322` and :rfc:`6532`, as well as -such MIME-related RFCs as :rfc:`2045`, :rfc:`2046`, :rfc:`2047`, :rfc:`2183`, -and :rfc:`2231`. - -The overall structure of the email package can be divided into three major -components, plus a fourth component that controls the behavior of the other -components. - -The central component of the package is an "object model" that represents email -messages. An application interacts with the package primarily through the -object model interface defined in the :mod:`~email.message` sub-module. The -application can use this API to ask questions about an existing email, to -construct a new email, or to add or remove email subcomponents that themselves -use the same object model interface. That is, following the nature of email -messages and their MIME subcomponents, the email object model is a tree -structure of objects that all provide the :class:`~email.message.EmailMessage` -API. - -The other two major components of the package are the :mod:`~email.parser` and -the :mod:`~email.generator`. The parser takes the serialized version of an -email message (a stream of bytes) and converts it into a tree of -:class:`~email.message.EmailMessage` objects. The generator takes an -:class:`~email.message.EmailMessage` and turns it back into a serialized byte -stream. (The parser and generator also handle streams of text characters, but -this usage is discouraged as it is too easy to end up with messages that are -not valid in one way or another.) - -The control component is the :mod:`~email.policy` module. Every -:class:`~email.message.EmailMessage`, every :mod:`~email.generator`, and every -:mod:`~email.parser` has an associated :mod:`~email.policy` object that -controls its behavior. Usually an application only needs to specify the policy -when an :class:`~email.message.EmailMessage` is created, either by directly -instantiating an :class:`~email.message.EmailMessage` to create a new email, -or by parsing an input stream using a :mod:`~email.parser`. But the policy can -be changed when the message is serialized using a :mod:`~email.generator`. -This allows, for example, a generic email message to be parsed from disk, but -to serialize it using standard SMTP settings when sending it to an email -server. - -The email package does its best to hide the details of the various governing -RFCs from the application. Conceptually the application should be able to -treat the email message as a structured tree of unicode text and binary -attachments, without having to worry about how these are represented when -serialized. In practice, however, it is often necessary to be aware of at -least some of the rules governing MIME messages and their structure, -specifically the names and nature of the MIME "content types" and how they -identify multipart documents. For the most part this knowledge should only be -required for more complex applications, and even then it should only be the -high level structure in question, and not the details of how those structures -are represented. Since MIME content types are used widely in modern internet -software (not just email), this will be a familiar concept to many programmers. - -The following sections describe the functionality of the :mod:`email` package. -We start with the :mod:`~email.message` object model, which is the primary -interface an application will use, and follow that with the -:mod:`~email.parser` and :mod:`~email.generator` components. Then we cover the -:mod:`~email.policy` controls, which completes the treatment of the main -components of the library. - -The next three sections cover the exceptions the package may raise and the -defects (non-compliance with the RFCs) that the :mod:`~email.parser` may -detect. Then we cover the :mod:`~email.headerregistry` and the -:mod:`~email.contentmanager` sub-components, which provide tools for doing more -detailed manipulation of headers and payloads, respectively. Both of these -components contain features relevant to consuming and producing non-trivial -messages, but also document their extensibility APIs, which will be of interest -to advanced applications. - -Following those is a set of examples of using the fundamental parts of the APIs -covered in the preceding sections. - -The foregoing represent the modern (unicode friendly) API of the email package. -The remaining sections, starting with the :class:`~email.message.Message` -class, cover the legacy :data:`~email.policy.compat32` API that deals much more -directly with the details of how email messages are represented. The -:data:`~email.policy.compat32` API does *not* hide the details of the RFCs from -the application, but for applications that need to operate at that level, they -can be useful tools. This documentation is also relevant for applications that -are still using the :mod:`~email.policy.compat32` API for backward -compatibility reasons. - -.. versionchanged:: 3.6 - Docs reorganized and rewritten to promote the new - :class:`~email.message.EmailMessage`/:class:`~email.policy.EmailPolicy` - API. - -Contents of the :mod:`email` package documentation: - -.. toctree:: - - email.message.rst - email.parser.rst - email.generator.rst - email.policy.rst - - email.errors.rst - email.headerregistry.rst - email.contentmanager.rst - - email.examples.rst - -Legacy API: - -.. toctree:: - - email.compat32-message.rst - email.mime.rst - email.header.rst - email.charset.rst - email.encoders.rst - email.utils.rst - email.iterators.rst - - -.. seealso:: - - Module :mod:`smtplib` - SMTP (Simple Mail Transport Protocol) client - - Module :mod:`poplib` - POP (Post Office Protocol) client - - Module :mod:`imaplib` - IMAP (Internet Message Access Protocol) client - - Module :mod:`nntplib` - NNTP (Net News Transport Protocol) client - - Module :mod:`mailbox` - Tools for creating, reading, and managing collections of messages on disk - using a variety standard formats. - - Module :mod:`smtpd` - SMTP server framework (primarily useful for testing) diff --git a/Doc/library/email.utils.rst b/Doc/library/email.utils.rst deleted file mode 100644 index 0e266b6a45782a..00000000000000 --- a/Doc/library/email.utils.rst +++ /dev/null @@ -1,220 +0,0 @@ -:mod:`email.utils`: Miscellaneous utilities -------------------------------------------- - -.. module:: email.utils - :synopsis: Miscellaneous email package utilities. - -**Source code:** :source:`Lib/email/utils.py` - --------------- - -There are a couple of useful utilities provided in the :mod:`email.utils` -module: - -.. function:: localtime(dt=None) - - Return local time as an aware datetime object. If called without - arguments, return current time. Otherwise *dt* argument should be a - :class:`~datetime.datetime` instance, and it is converted to the local time - zone according to the system time zone database. If *dt* is naive (that - is, ``dt.tzinfo`` is ``None``), it is assumed to be in local time. In this - case, a positive or zero value for *isdst* causes ``localtime`` to presume - initially that summer time (for example, Daylight Saving Time) is or is not - (respectively) in effect for the specified time. A negative value for - *isdst* causes the ``localtime`` to attempt to divine whether summer time - is in effect for the specified time. - - .. versionadded:: 3.3 - - -.. function:: make_msgid(idstring=None, domain=None) - - Returns a string suitable for an :rfc:`2822`\ -compliant - :mailheader:`Message-ID` header. Optional *idstring* if given, is a string - used to strengthen the uniqueness of the message id. Optional *domain* if - given provides the portion of the msgid after the '@'. The default is the - local hostname. It is not normally necessary to override this default, but - may be useful certain cases, such as a constructing distributed system that - uses a consistent domain name across multiple hosts. - - .. versionchanged:: 3.2 - Added the *domain* keyword. - - -The remaining functions are part of the legacy (``Compat32``) email API. There -is no need to directly use these with the new API, since the parsing and -formatting they provide is done automatically by the header parsing machinery -of the new API. - - -.. function:: quote(str) - - Return a new string with backslashes in *str* replaced by two backslashes, and - double quotes replaced by backslash-double quote. - - -.. function:: unquote(str) - - Return a new string which is an *unquoted* version of *str*. If *str* ends and - begins with double quotes, they are stripped off. Likewise if *str* ends and - begins with angle brackets, they are stripped off. - - -.. function:: parseaddr(address) - - Parse address -- which should be the value of some address-containing field such - as :mailheader:`To` or :mailheader:`Cc` -- into its constituent *realname* and - *email address* parts. Returns a tuple of that information, unless the parse - fails, in which case a 2-tuple of ``('', '')`` is returned. - - -.. function:: formataddr(pair, charset='utf-8') - - The inverse of :meth:`parseaddr`, this takes a 2-tuple of the form ``(realname, - email_address)`` and returns the string value suitable for a :mailheader:`To` or - :mailheader:`Cc` header. If the first element of *pair* is false, then the - second element is returned unmodified. - - Optional *charset* is the character set that will be used in the :rfc:`2047` - encoding of the ``realname`` if the ``realname`` contains non-ASCII - characters. Can be an instance of :class:`str` or a - :class:`~email.charset.Charset`. Defaults to ``utf-8``. - - .. versionchanged:: 3.3 - Added the *charset* option. - - -.. function:: getaddresses(fieldvalues) - - This method returns a list of 2-tuples of the form returned by ``parseaddr()``. - *fieldvalues* is a sequence of header field values as might be returned by - :meth:`Message.get_all `. Here's a simple - example that gets all the recipients of a message:: - - from email.utils import getaddresses - - tos = msg.get_all('to', []) - ccs = msg.get_all('cc', []) - resent_tos = msg.get_all('resent-to', []) - resent_ccs = msg.get_all('resent-cc', []) - all_recipients = getaddresses(tos + ccs + resent_tos + resent_ccs) - - -.. function:: parsedate(date) - - Attempts to parse a date according to the rules in :rfc:`2822`. however, some - mailers don't follow that format as specified, so :func:`parsedate` tries to - guess correctly in such cases. *date* is a string containing an :rfc:`2822` - date, such as ``"Mon, 20 Nov 1995 19:12:08 -0500"``. If it succeeds in parsing - the date, :func:`parsedate` returns a 9-tuple that can be passed directly to - :func:`time.mktime`; otherwise ``None`` will be returned. Note that indexes 6, - 7, and 8 of the result tuple are not usable. - - -.. function:: parsedate_tz(date) - - Performs the same function as :func:`parsedate`, but returns either ``None`` or - a 10-tuple; the first 9 elements make up a tuple that can be passed directly to - :func:`time.mktime`, and the tenth is the offset of the date's timezone from UTC - (which is the official term for Greenwich Mean Time) [#]_. If the input string - has no timezone, the last element of the tuple returned is ``0``, which represents - UTC. Note that indexes 6, 7, and 8 of the result tuple are not usable. - - -.. function:: parsedate_to_datetime(date) - - The inverse of :func:`format_datetime`. Performs the same function as - :func:`parsedate`, but on success returns a :mod:`~datetime.datetime`; - otherwise ``ValueError`` is raised if *date* contains an invalid value such - as an hour greater than 23 or a timezone offset not between -24 and 24 hours. - If the input date has a timezone of ``-0000``, the ``datetime`` will be a naive - ``datetime``, and if the date is conforming to the RFCs it will represent a - time in UTC but with no indication of the actual source timezone of the - message the date comes from. If the input date has any other valid timezone - offset, the ``datetime`` will be an aware ``datetime`` with the - corresponding a :class:`~datetime.timezone` :class:`~datetime.tzinfo`. - - .. versionadded:: 3.3 - - -.. function:: mktime_tz(tuple) - - Turn a 10-tuple as returned by :func:`parsedate_tz` into a UTC - timestamp (seconds since the Epoch). If the timezone item in the - tuple is ``None``, assume local time. - - -.. function:: formatdate(timeval=None, localtime=False, usegmt=False) - - Returns a date string as per :rfc:`2822`, e.g.:: - - Fri, 09 Nov 2001 01:08:47 -0000 - - Optional *timeval* if given is a floating point time value as accepted by - :func:`time.gmtime` and :func:`time.localtime`, otherwise the current time is - used. - - Optional *localtime* is a flag that when ``True``, interprets *timeval*, and - returns a date relative to the local timezone instead of UTC, properly taking - daylight savings time into account. The default is ``False`` meaning UTC is - used. - - Optional *usegmt* is a flag that when ``True``, outputs a date string with the - timezone as an ascii string ``GMT``, rather than a numeric ``-0000``. This is - needed for some protocols (such as HTTP). This only applies when *localtime* is - ``False``. The default is ``False``. - - -.. function:: format_datetime(dt, usegmt=False) - - Like ``formatdate``, but the input is a :mod:`datetime` instance. If it is - a naive datetime, it is assumed to be "UTC with no information about the - source timezone", and the conventional ``-0000`` is used for the timezone. - If it is an aware ``datetime``, then the numeric timezone offset is used. - If it is an aware timezone with offset zero, then *usegmt* may be set to - ``True``, in which case the string ``GMT`` is used instead of the numeric - timezone offset. This provides a way to generate standards conformant HTTP - date headers. - - .. versionadded:: 3.3 - - -.. function:: decode_rfc2231(s) - - Decode the string *s* according to :rfc:`2231`. - - -.. function:: encode_rfc2231(s, charset=None, language=None) - - Encode the string *s* according to :rfc:`2231`. Optional *charset* and - *language*, if given is the character set name and language name to use. If - neither is given, *s* is returned as-is. If *charset* is given but *language* - is not, the string is encoded using the empty string for *language*. - - -.. function:: collapse_rfc2231_value(value, errors='replace', fallback_charset='us-ascii') - - When a header parameter is encoded in :rfc:`2231` format, - :meth:`Message.get_param ` may return a - 3-tuple containing the character set, - language, and value. :func:`collapse_rfc2231_value` turns this into a unicode - string. Optional *errors* is passed to the *errors* argument of :class:`str`'s - :func:`~str.encode` method; it defaults to ``'replace'``. Optional - *fallback_charset* specifies the character set to use if the one in the - :rfc:`2231` header is not known by Python; it defaults to ``'us-ascii'``. - - For convenience, if the *value* passed to :func:`collapse_rfc2231_value` is not - a tuple, it should be a string and it is returned unquoted. - - -.. function:: decode_params(params) - - Decode parameters list according to :rfc:`2231`. *params* is a sequence of - 2-tuples containing elements of the form ``(content-type, string-value)``. - - -.. rubric:: Footnotes - -.. [#] Note that the sign of the timezone offset is the opposite of the sign of the - ``time.timezone`` variable for the same timezone; the latter variable follows - the POSIX standard while this module follows :rfc:`2822`. diff --git a/Doc/library/ensurepip.rst b/Doc/library/ensurepip.rst deleted file mode 100644 index de3b93f5e61073..00000000000000 --- a/Doc/library/ensurepip.rst +++ /dev/null @@ -1,138 +0,0 @@ -:mod:`ensurepip` --- Bootstrapping the ``pip`` installer -======================================================== - -.. module:: ensurepip - :synopsis: Bootstrapping the "pip" installer into an existing Python - installation or virtual environment. - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/ensurepip` - --------------- - -The :mod:`ensurepip` package provides support for bootstrapping the ``pip`` -installer into an existing Python installation or virtual environment. This -bootstrapping approach reflects the fact that ``pip`` is an independent -project with its own release cycle, and the latest available stable version -is bundled with maintenance and feature releases of the CPython reference -interpreter. - -In most cases, end users of Python shouldn't need to invoke this module -directly (as ``pip`` should be bootstrapped by default), but it may be -needed if installing ``pip`` was skipped when installing Python (or -when creating a virtual environment) or after explicitly uninstalling -``pip``. - -.. note:: - - This module *does not* access the internet. All of the components - needed to bootstrap ``pip`` are included as internal parts of the - package. - -.. seealso:: - - :ref:`installing-index` - The end user guide for installing Python packages - - :pep:`453`: Explicit bootstrapping of pip in Python installations - The original rationale and specification for this module. - -.. include:: ../includes/wasm-notavail.rst - -Command line interface ----------------------- - -The command line interface is invoked using the interpreter's ``-m`` switch. - -The simplest possible invocation is:: - - python -m ensurepip - -This invocation will install ``pip`` if it is not already installed, -but otherwise does nothing. To ensure the installed version of ``pip`` -is at least as recent as the one available in ``ensurepip``, pass the -``--upgrade`` option:: - - python -m ensurepip --upgrade - -By default, ``pip`` is installed into the current virtual environment -(if one is active) or into the system site packages (if there is no -active virtual environment). The installation location can be controlled -through two additional command line options: - -* :samp:`--root {dir}`: Installs ``pip`` relative to the given root directory - rather than the root of the currently active virtual environment (if any) - or the default root for the current Python installation. -* ``--user``: Installs ``pip`` into the user site packages directory rather - than globally for the current Python installation (this option is not - permitted inside an active virtual environment). - -By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where -X.Y stands for the version of Python used to invoke ``ensurepip``). The -scripts installed can be controlled through two additional command line -options: - -* ``--altinstall``: if an alternate installation is requested, the ``pipX`` - script will *not* be installed. - -* ``--default-pip``: if a "default pip" installation is requested, the - ``pip`` script will be installed in addition to the two regular scripts. - -Providing both of the script selection options will trigger an exception. - - -Module API ----------- - -:mod:`ensurepip` exposes two functions for programmatic use: - -.. function:: version() - - Returns a string specifying the available version of pip that will be - installed when bootstrapping an environment. - -.. function:: bootstrap(root=None, upgrade=False, user=False, \ - altinstall=False, default_pip=False, \ - verbosity=0) - - Bootstraps ``pip`` into the current or designated environment. - - *root* specifies an alternative root directory to install relative to. - If *root* is ``None``, then installation uses the default install location - for the current environment. - - *upgrade* indicates whether or not to upgrade an existing installation - of an earlier version of ``pip`` to the available version. - - *user* indicates whether to use the user scheme rather than installing - globally. - - By default, the scripts ``pipX`` and ``pipX.Y`` will be installed (where - X.Y stands for the current version of Python). - - If *altinstall* is set, then ``pipX`` will *not* be installed. - - If *default_pip* is set, then ``pip`` will be installed in addition to - the two regular scripts. - - Setting both *altinstall* and *default_pip* will trigger - :exc:`ValueError`. - - *verbosity* controls the level of output to :data:`sys.stdout` from the - bootstrapping operation. - - .. audit-event:: ensurepip.bootstrap root ensurepip.bootstrap - - .. note:: - - The bootstrapping process has side effects on both ``sys.path`` and - ``os.environ``. Invoking the command line interface in a subprocess - instead allows these side effects to be avoided. - - .. note:: - - The bootstrapping process may install additional modules required by - ``pip``, but other software should not assume those dependencies will - always be present by default (as the dependencies may be removed in a - future version of ``pip``). diff --git a/Doc/library/enum.rst b/Doc/library/enum.rst deleted file mode 100644 index 8fb97ea9ed1c84..00000000000000 --- a/Doc/library/enum.rst +++ /dev/null @@ -1,923 +0,0 @@ -:mod:`enum` --- Support for enumerations -======================================== - -.. module:: enum - :synopsis: Implementation of an enumeration class. - -.. moduleauthor:: Ethan Furman -.. sectionauthor:: Barry Warsaw -.. sectionauthor:: Eli Bendersky -.. sectionauthor:: Ethan Furman - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/enum.py` - -.. sidebar:: Important - - This page contains the API reference information. For tutorial - information and discussion of more advanced topics, see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Enum Cookbook ` - ---------------- - -An enumeration: - -* is a set of symbolic names (members) bound to unique values -* can be iterated over to return its canonical (i.e. non-alias) members in - definition order -* uses *call* syntax to return members by value -* uses *index* syntax to return members by name - -Enumerations are created either by using :keyword:`class` syntax, or by -using function-call syntax:: - - >>> from enum import Enum - - >>> # class syntax - >>> class Color(Enum): - ... RED = 1 - ... GREEN = 2 - ... BLUE = 3 - - >>> # functional syntax - >>> Color = Enum('Color', ['RED', 'GREEN', 'BLUE']) - -Even though we can use :keyword:`class` syntax to create Enums, Enums -are not normal Python classes. See -:ref:`How are Enums different? ` for more details. - -.. note:: Nomenclature - - - The class :class:`!Color` is an *enumeration* (or *enum*) - - The attributes :attr:`!Color.RED`, :attr:`!Color.GREEN`, etc., are - *enumeration members* (or *members*) and are functionally constants. - - The enum members have *names* and *values* (the name of - :attr:`!Color.RED` is ``RED``, the value of :attr:`!Color.BLUE` is - ``3``, etc.) - ---------------- - -Module Contents ---------------- - - :class:`EnumType` - - The ``type`` for Enum and its subclasses. - - :class:`Enum` - - Base class for creating enumerated constants. - - :class:`IntEnum` - - Base class for creating enumerated constants that are also - subclasses of :class:`int`. (`Notes`_) - - :class:`StrEnum` - - Base class for creating enumerated constants that are also - subclasses of :class:`str`. (`Notes`_) - - :class:`Flag` - - Base class for creating enumerated constants that can be combined using - the bitwise operations without losing their :class:`Flag` membership. - - :class:`IntFlag` - - Base class for creating enumerated constants that can be combined using - the bitwise operators without losing their :class:`IntFlag` membership. - :class:`IntFlag` members are also subclasses of :class:`int`. (`Notes`_) - - :class:`ReprEnum` - - Used by :class:`IntEnum`, :class:`StrEnum`, and :class:`IntFlag` - to keep the :class:`str() ` of the mixed-in type. - - :class:`EnumCheck` - - An enumeration with the values ``CONTINUOUS``, ``NAMED_FLAGS``, and - ``UNIQUE``, for use with :func:`verify` to ensure various constraints - are met by a given enumeration. - - :class:`FlagBoundary` - - An enumeration with the values ``STRICT``, ``CONFORM``, ``EJECT``, and - ``KEEP`` which allows for more fine-grained control over how invalid values - are dealt with in an enumeration. - - :class:`auto` - - Instances are replaced with an appropriate value for Enum members. - :class:`StrEnum` defaults to the lower-cased version of the member name, - while other Enums default to 1 and increase from there. - - :func:`~enum.property` - - Allows :class:`Enum` members to have attributes without conflicting with - member names. - - :func:`unique` - - Enum class decorator that ensures only one name is bound to any one value. - - :func:`verify` - - Enum class decorator that checks user-selectable constraints on an - enumeration. - - :func:`member` - - Make ``obj`` a member. Can be used as a decorator. - - :func:`nonmember` - - Do not make ``obj`` a member. Can be used as a decorator. - - :func:`global_enum` - - Modify the :class:`str() ` and :func:`repr` of an enum - to show its members as belonging to the module instead of its class, - and export the enum members to the global namespace. - - :func:`show_flag_values` - - Return a list of all power-of-two integers contained in a flag. - - -.. versionadded:: 3.6 ``Flag``, ``IntFlag``, ``auto`` -.. versionadded:: 3.11 ``StrEnum``, ``EnumCheck``, ``ReprEnum``, ``FlagBoundary``, ``property``, ``member``, ``nonmember``, ``global_enum``, ``show_flag_values`` - ---------------- - -Data Types ----------- - - -.. class:: EnumType - - *EnumType* is the :term:`metaclass` for *enum* enumerations. It is possible - to subclass *EnumType* -- see :ref:`Subclassing EnumType ` - for details. - - *EnumType* is responsible for setting the correct :meth:`!__repr__`, - :meth:`!__str__`, :meth:`!__format__`, and :meth:`!__reduce__` methods on the - final *enum*, as well as creating the enum members, properly handling - duplicates, providing iteration over the enum class, etc. - - .. method:: EnumType.__call__(cls, value, names=None, *, module=None, qualname=None, type=None, start=1, boundary=None) - - This method is called in two different ways: - - * to look up an existing member: - - :cls: The enum class being called. - :value: The value to lookup. - - * to use the ``cls`` enum to create a new enum (only if the existing enum - does not have any members): - - :cls: The enum class being called. - :value: The name of the new Enum to create. - :names: The names/values of the members for the new Enum. - :module: The name of the module the new Enum is created in. - :qualname: The actual location in the module where this Enum can be found. - :type: A mix-in type for the new Enum. - :start: The first integer value for the Enum (used by :class:`auto`). - :boundary: How to handle out-of-range values from bit operations (:class:`Flag` only). - - .. method:: EnumType.__contains__(cls, member) - - Returns ``True`` if member belongs to the ``cls``:: - - >>> some_var = Color.RED - >>> some_var in Color - True - - .. note:: - - In Python 3.12 it will be possible to check for member values and not - just members; until then, a ``TypeError`` will be raised if a - non-Enum-member is used in a containment check. - - .. method:: EnumType.__dir__(cls) - - Returns ``['__class__', '__doc__', '__members__', '__module__']`` and the - names of the members in *cls*:: - - >>> dir(Color) - ['BLUE', 'GREEN', 'RED', '__class__', '__contains__', '__doc__', '__getitem__', '__init_subclass__', '__iter__', '__len__', '__members__', '__module__', '__name__', '__qualname__'] - - .. method:: EnumType.__getattr__(cls, name) - - Returns the Enum member in *cls* matching *name*, or raises an :exc:`AttributeError`:: - - >>> Color.GREEN - - - .. method:: EnumType.__getitem__(cls, name) - - Returns the Enum member in *cls* matching *name*, or raises a :exc:`KeyError`:: - - >>> Color['BLUE'] - - - .. method:: EnumType.__iter__(cls) - - Returns each member in *cls* in definition order:: - - >>> list(Color) - [, , ] - - .. method:: EnumType.__len__(cls) - - Returns the number of member in *cls*:: - - >>> len(Color) - 3 - - .. method:: EnumType.__reversed__(cls) - - Returns each member in *cls* in reverse definition order:: - - >>> list(reversed(Color)) - [, , ] - - .. versionadded:: 3.11 - - Before 3.11 ``enum`` used ``EnumMeta`` type, which is kept as an alias. - - -.. class:: Enum - - *Enum* is the base class for all *enum* enumerations. - - .. attribute:: Enum.name - - The name used to define the ``Enum`` member:: - - >>> Color.BLUE.name - 'BLUE' - - .. attribute:: Enum.value - - The value given to the ``Enum`` member:: - - >>> Color.RED.value - 1 - - .. note:: Enum member values - - Member values can be anything: :class:`int`, :class:`str`, etc. If - the exact value is unimportant you may use :class:`auto` instances and an - appropriate value will be chosen for you. See :class:`auto` for the - details. - - .. attribute:: Enum._ignore_ - - ``_ignore_`` is only used during creation and is removed from the - enumeration once creation is complete. - - ``_ignore_`` is a list of names that will not become members, and whose - names will also be removed from the completed enumeration. See - :ref:`TimePeriod ` for an example. - - .. method:: Enum.__dir__(self) - - Returns ``['__class__', '__doc__', '__module__', 'name', 'value']`` and - any public methods defined on *self.__class__*:: - - >>> from datetime import date - >>> class Weekday(Enum): - ... MONDAY = 1 - ... TUESDAY = 2 - ... WEDNESDAY = 3 - ... THURSDAY = 4 - ... FRIDAY = 5 - ... SATURDAY = 6 - ... SUNDAY = 7 - ... @classmethod - ... def today(cls): - ... print('today is %s' % cls(date.today().isoweekday()).name) - >>> dir(Weekday.SATURDAY) - ['__class__', '__doc__', '__eq__', '__hash__', '__module__', 'name', 'today', 'value'] - - .. method:: Enum._generate_next_value_(name, start, count, last_values) - - :name: The name of the member being defined (e.g. 'RED'). - :start: The start value for the Enum; the default is 1. - :count: The number of members currently defined, not including this one. - :last_values: A list of the previous values. - - A *staticmethod* that is used to determine the next value returned by - :class:`auto`:: - - >>> from enum import auto - >>> class PowersOfThree(Enum): - ... @staticmethod - ... def _generate_next_value_(name, start, count, last_values): - ... return 3 ** (count + 1) - ... FIRST = auto() - ... SECOND = auto() - >>> PowersOfThree.SECOND.value - 9 - - .. method:: Enum.__init_subclass__(cls, **kwds) - - A *classmethod* that is used to further configure subsequent subclasses. - By default, does nothing. - - .. method:: Enum._missing_(cls, value) - - A *classmethod* for looking up values not found in *cls*. By default it - does nothing, but can be overridden to implement custom search behavior:: - - >>> from enum import StrEnum - >>> class Build(StrEnum): - ... DEBUG = auto() - ... OPTIMIZED = auto() - ... @classmethod - ... def _missing_(cls, value): - ... value = value.lower() - ... for member in cls: - ... if member.value == value: - ... return member - ... return None - >>> Build.DEBUG.value - 'debug' - >>> Build('deBUG') - - - .. method:: Enum.__repr__(self) - - Returns the string used for *repr()* calls. By default, returns the - *Enum* name, member name, and value, but can be overridden:: - - >>> class OtherStyle(Enum): - ... ALTERNATE = auto() - ... OTHER = auto() - ... SOMETHING_ELSE = auto() - ... def __repr__(self): - ... cls_name = self.__class__.__name__ - ... return f'{cls_name}.{self.name}' - >>> OtherStyle.ALTERNATE, str(OtherStyle.ALTERNATE), f"{OtherStyle.ALTERNATE}" - (OtherStyle.ALTERNATE, 'OtherStyle.ALTERNATE', 'OtherStyle.ALTERNATE') - - .. method:: Enum.__str__(self) - - Returns the string used for *str()* calls. By default, returns the - *Enum* name and member name, but can be overridden:: - - >>> class OtherStyle(Enum): - ... ALTERNATE = auto() - ... OTHER = auto() - ... SOMETHING_ELSE = auto() - ... def __str__(self): - ... return f'{self.name}' - >>> OtherStyle.ALTERNATE, str(OtherStyle.ALTERNATE), f"{OtherStyle.ALTERNATE}" - (, 'ALTERNATE', 'ALTERNATE') - - .. method:: Enum.__format__(self) - - Returns the string used for *format()* and *f-string* calls. By default, - returns :meth:`__str__` return value, but can be overridden:: - - >>> class OtherStyle(Enum): - ... ALTERNATE = auto() - ... OTHER = auto() - ... SOMETHING_ELSE = auto() - ... def __format__(self, spec): - ... return f'{self.name}' - >>> OtherStyle.ALTERNATE, str(OtherStyle.ALTERNATE), f"{OtherStyle.ALTERNATE}" - (, 'OtherStyle.ALTERNATE', 'ALTERNATE') - - .. note:: - - Using :class:`auto` with :class:`Enum` results in integers of increasing value, - starting with ``1``. - - -.. class:: IntEnum - - *IntEnum* is the same as *Enum*, but its members are also integers and can be - used anywhere that an integer can be used. If any integer operation is performed - with an *IntEnum* member, the resulting value loses its enumeration status. - - >>> from enum import IntEnum - >>> class Number(IntEnum): - ... ONE = 1 - ... TWO = 2 - ... THREE = 3 - ... - >>> Number.THREE - - >>> Number.ONE + Number.TWO - 3 - >>> Number.THREE + 5 - 8 - >>> Number.THREE == 3 - True - - .. note:: - - Using :class:`auto` with :class:`IntEnum` results in integers of increasing - value, starting with ``1``. - - .. versionchanged:: 3.11 :meth:`~object.__str__` is now :meth:`!int.__str__` to - better support the *replacement of existing constants* use-case. - :meth:`~object.__format__` was already :meth:`!int.__format__` for that same reason. - - -.. class:: StrEnum - - *StrEnum* is the same as *Enum*, but its members are also strings and can be used - in most of the same places that a string can be used. The result of any string - operation performed on or with a *StrEnum* member is not part of the enumeration. - - .. note:: - - There are places in the stdlib that check for an exact :class:`str` - instead of a :class:`str` subclass (i.e. ``type(unknown) == str`` - instead of ``isinstance(unknown, str)``), and in those locations you - will need to use ``str(StrEnum.member)``. - - .. note:: - - Using :class:`auto` with :class:`StrEnum` results in the lower-cased member - name as the value. - - .. note:: - - :meth:`~object.__str__` is :meth:`!str.__str__` to better support the - *replacement of existing constants* use-case. :meth:`~object.__format__` is likewise - :meth:`!str.__format__` for that same reason. - - .. versionadded:: 3.11 - -.. class:: Flag - - *Flag* members support the bitwise operators ``&`` (*AND*), ``|`` (*OR*), - ``^`` (*XOR*), and ``~`` (*INVERT*); the results of those operators are members - of the enumeration. - - .. method:: __contains__(self, value) - - Returns *True* if value is in self:: - - >>> from enum import Flag, auto - >>> class Color(Flag): - ... RED = auto() - ... GREEN = auto() - ... BLUE = auto() - >>> purple = Color.RED | Color.BLUE - >>> white = Color.RED | Color.GREEN | Color.BLUE - >>> Color.GREEN in purple - False - >>> Color.GREEN in white - True - >>> purple in white - True - >>> white in purple - False - - .. method:: __iter__(self): - - Returns all contained non-alias members:: - - >>> list(Color.RED) - [] - >>> list(purple) - [, ] - - .. versionchanged:: 3.11 - - Aliases are no longer returned during iteration. - - .. method:: __len__(self): - - Returns number of members in flag:: - - >>> len(Color.GREEN) - 1 - >>> len(white) - 3 - - .. method:: __bool__(self): - - Returns *True* if any members in flag, *False* otherwise:: - - >>> bool(Color.GREEN) - True - >>> bool(white) - True - >>> black = Color(0) - >>> bool(black) - False - - .. method:: __or__(self, other) - - Returns current flag binary or'ed with other:: - - >>> Color.RED | Color.GREEN - - - .. method:: __and__(self, other) - - Returns current flag binary and'ed with other:: - - >>> purple & white - - >>> purple & Color.GREEN - - - .. method:: __xor__(self, other) - - Returns current flag binary xor'ed with other:: - - >>> purple ^ white - - >>> purple ^ Color.GREEN - - - .. method:: __invert__(self): - - Returns all the flags in *type(self)* that are not in self:: - - >>> ~white - - >>> ~purple - - >>> ~Color.RED - - - .. method:: _numeric_repr_ - - Function used to format any remaining unnamed numeric values. Default is - the value's repr; common choices are :func:`hex` and :func:`oct`. - - .. note:: - - Using :class:`auto` with :class:`Flag` results in integers that are powers - of two, starting with ``1``. - - .. versionchanged:: 3.11 The *repr()* of zero-valued flags has changed. It - is now:: - - >>> Color(0) # doctest: +SKIP - - -.. class:: IntFlag - - *IntFlag* is the same as *Flag*, but its members are also integers and can be - used anywhere that an integer can be used. - - >>> from enum import IntFlag, auto - >>> class Color(IntFlag): - ... RED = auto() - ... GREEN = auto() - ... BLUE = auto() - >>> Color.RED & 2 - - >>> Color.RED | 2 - - - If any integer operation is performed with an *IntFlag* member, the result is - not an *IntFlag*:: - - >>> Color.RED + 2 - 3 - - If a *Flag* operation is performed with an *IntFlag* member and: - - * the result is a valid *IntFlag*: an *IntFlag* is returned - * the result is not a valid *IntFlag*: the result depends on the *FlagBoundary* setting - - The *repr()* of unnamed zero-valued flags has changed. It is now: - - >>> Color(0) - - - .. note:: - - Using :class:`auto` with :class:`IntFlag` results in integers that are powers - of two, starting with ``1``. - - .. versionchanged:: 3.11 - - :meth:`~object.__str__` is now :meth:`!int.__str__` to better support the - *replacement of existing constants* use-case. :meth:`~object.__format__` was - already :meth:`!int.__format__` for that same reason. - - Inversion of an :class:`!IntFlag` now returns a positive value that is the - union of all flags not in the given flag, rather than a negative value. - This matches the existing :class:`Flag` behavior. - -.. class:: ReprEnum - - :class:`!ReprEnum` uses the :meth:`repr() ` of :class:`Enum`, - but the :class:`str() ` of the mixed-in data type: - - * :meth:`!int.__str__` for :class:`IntEnum` and :class:`IntFlag` - * :meth:`!str.__str__` for :class:`StrEnum` - - Inherit from :class:`!ReprEnum` to keep the :class:`str() ` / :func:`format` - of the mixed-in data type instead of using the - :class:`Enum`-default :meth:`str() `. - - - .. versionadded:: 3.11 - -.. class:: EnumCheck - - *EnumCheck* contains the options used by the :func:`verify` decorator to ensure - various constraints; failed constraints result in a :exc:`ValueError`. - - .. attribute:: UNIQUE - - Ensure that each value has only one name:: - - >>> from enum import Enum, verify, UNIQUE - >>> @verify(UNIQUE) - ... class Color(Enum): - ... RED = 1 - ... GREEN = 2 - ... BLUE = 3 - ... CRIMSON = 1 - Traceback (most recent call last): - ... - ValueError: aliases found in : CRIMSON -> RED - - - .. attribute:: CONTINUOUS - - Ensure that there are no missing values between the lowest-valued member - and the highest-valued member:: - - >>> from enum import Enum, verify, CONTINUOUS - >>> @verify(CONTINUOUS) - ... class Color(Enum): - ... RED = 1 - ... GREEN = 2 - ... BLUE = 5 - Traceback (most recent call last): - ... - ValueError: invalid enum 'Color': missing values 3, 4 - - .. attribute:: NAMED_FLAGS - - Ensure that any flag groups/masks contain only named flags -- useful when - values are specified instead of being generated by :func:`auto`:: - - >>> from enum import Flag, verify, NAMED_FLAGS - >>> @verify(NAMED_FLAGS) - ... class Color(Flag): - ... RED = 1 - ... GREEN = 2 - ... BLUE = 4 - ... WHITE = 15 - ... NEON = 31 - Traceback (most recent call last): - ... - ValueError: invalid Flag 'Color': aliases WHITE and NEON are missing combined values of 0x18 [use enum.show_flag_values(value) for details] - - .. note:: - - CONTINUOUS and NAMED_FLAGS are designed to work with integer-valued members. - - .. versionadded:: 3.11 - -.. class:: FlagBoundary - - *FlagBoundary* controls how out-of-range values are handled in *Flag* and its - subclasses. - - .. attribute:: STRICT - - Out-of-range values cause a :exc:`ValueError` to be raised. This is the - default for :class:`Flag`:: - - >>> from enum import Flag, STRICT, auto - >>> class StrictFlag(Flag, boundary=STRICT): - ... RED = auto() - ... GREEN = auto() - ... BLUE = auto() - >>> StrictFlag(2**2 + 2**4) - Traceback (most recent call last): - ... - ValueError: invalid value 20 - given 0b0 10100 - allowed 0b0 00111 - - .. attribute:: CONFORM - - Out-of-range values have invalid values removed, leaving a valid *Flag* - value:: - - >>> from enum import Flag, CONFORM, auto - >>> class ConformFlag(Flag, boundary=CONFORM): - ... RED = auto() - ... GREEN = auto() - ... BLUE = auto() - >>> ConformFlag(2**2 + 2**4) - - - .. attribute:: EJECT - - Out-of-range values lose their *Flag* membership and revert to :class:`int`. - - >>> from enum import Flag, EJECT, auto - >>> class EjectFlag(Flag, boundary=EJECT): - ... RED = auto() - ... GREEN = auto() - ... BLUE = auto() - >>> EjectFlag(2**2 + 2**4) - 20 - - .. attribute:: KEEP - - Out-of-range values are kept, and the *Flag* membership is kept. - This is the default for :class:`IntFlag`:: - - >>> from enum import Flag, KEEP, auto - >>> class KeepFlag(Flag, boundary=KEEP): - ... RED = auto() - ... GREEN = auto() - ... BLUE = auto() - >>> KeepFlag(2**2 + 2**4) - - -.. versionadded:: 3.11 - ---------------- - -Supported ``__dunder__`` names -"""""""""""""""""""""""""""""" - -:attr:`~EnumType.__members__` is a read-only ordered mapping of ``member_name``:``member`` -items. It is only available on the class. - -:meth:`~object.__new__`, if specified, must create and return the enum members; it is -also a very good idea to set the member's :attr:`!_value_` appropriately. Once -all the members are created it is no longer used. - - -Supported ``_sunder_`` names -"""""""""""""""""""""""""""" - -- ``_name_`` -- name of the member -- ``_value_`` -- value of the member; can be set / modified in ``__new__`` - -- ``_missing_`` -- a lookup function used when a value is not found; may be - overridden -- ``_ignore_`` -- a list of names, either as a :class:`list` or a :class:`str`, - that will not be transformed into members, and will be removed from the final - class -- ``_order_`` -- used in Python 2/3 code to ensure member order is consistent - (class attribute, removed during class creation) -- ``_generate_next_value_`` -- used to get an appropriate value for an enum - member; may be overridden - - .. note:: - - For standard :class:`Enum` classes the next value chosen is the last value seen - incremented by one. - - For :class:`Flag` classes the next value chosen will be the next highest - power-of-two, regardless of the last value seen. - -.. versionadded:: 3.6 ``_missing_``, ``_order_``, ``_generate_next_value_`` -.. versionadded:: 3.7 ``_ignore_`` - ---------------- - -Utilities and Decorators ------------------------- - -.. class:: auto - - *auto* can be used in place of a value. If used, the *Enum* machinery will - call an *Enum*'s :meth:`~Enum._generate_next_value_` to get an appropriate value. - For *Enum* and *IntEnum* that appropriate value will be the last value plus - one; for *Flag* and *IntFlag* it will be the first power-of-two greater - than the highest value; for *StrEnum* it will be the lower-cased version of - the member's name. Care must be taken if mixing *auto()* with manually - specified values. - - *auto* instances are only resolved when at the top level of an assignment: - - * ``FIRST = auto()`` will work (auto() is replaced with ``1``); - * ``SECOND = auto(), -2`` will work (auto is replaced with ``2``, so ``2, -2`` is - used to create the ``SECOND`` enum member; - * ``THREE = [auto(), -3]`` will *not* work (``, -3`` is used to - create the ``THREE`` enum member) - - .. versionchanged:: 3.11.1 - - In prior versions, ``auto()`` had to be the only thing - on the assignment line to work properly. - - ``_generate_next_value_`` can be overridden to customize the values used by - *auto*. - - .. note:: in 3.13 the default ``_generate_next_value_`` will always return - the highest member value incremented by 1, and will fail if any - member is an incompatible type. - -.. decorator:: property - - A decorator similar to the built-in *property*, but specifically for - enumerations. It allows member attributes to have the same names as members - themselves. - - .. note:: the *property* and the member must be defined in separate classes; - for example, the *value* and *name* attributes are defined in the - *Enum* class, and *Enum* subclasses can define members with the - names ``value`` and ``name``. - - .. versionadded:: 3.11 - -.. decorator:: unique - - A :keyword:`class` decorator specifically for enumerations. It searches an - enumeration's :attr:`~EnumType.__members__`, gathering any aliases it finds; if any are - found :exc:`ValueError` is raised with the details:: - - >>> from enum import Enum, unique - >>> @unique - ... class Mistake(Enum): - ... ONE = 1 - ... TWO = 2 - ... THREE = 3 - ... FOUR = 3 - ... - Traceback (most recent call last): - ... - ValueError: duplicate values found in : FOUR -> THREE - -.. decorator:: verify - - A :keyword:`class` decorator specifically for enumerations. Members from - :class:`EnumCheck` are used to specify which constraints should be checked - on the decorated enumeration. - - .. versionadded:: 3.11 - -.. decorator:: member - - A decorator for use in enums: its target will become a member. - - .. versionadded:: 3.11 - -.. decorator:: nonmember - - A decorator for use in enums: its target will not become a member. - - .. versionadded:: 3.11 - -.. decorator:: global_enum - - A decorator to change the :class:`str() ` and :func:`repr` of an enum - to show its members as belonging to the module instead of its class. - Should only be used when the enum members are exported - to the module global namespace (see :class:`re.RegexFlag` for an example). - - - .. versionadded:: 3.11 - -.. function:: show_flag_values(value) - - Return a list of all power-of-two integers contained in a flag *value*. - - .. versionadded:: 3.11 - ---------------- - -Notes ------ - -:class:`IntEnum`, :class:`StrEnum`, and :class:`IntFlag` - - These three enum types are designed to be drop-in replacements for existing - integer- and string-based values; as such, they have extra limitations: - - - ``__str__`` uses the value and not the name of the enum member - - - ``__format__``, because it uses ``__str__``, will also use the value of - the enum member instead of its name - - If you do not need/want those limitations, you can either create your own - base class by mixing in the ``int`` or ``str`` type yourself:: - - >>> from enum import Enum - >>> class MyIntEnum(int, Enum): - ... pass - - or you can reassign the appropriate :meth:`str`, etc., in your enum:: - - >>> from enum import Enum, IntEnum - >>> class MyIntEnum(IntEnum): - ... __str__ = Enum.__str__ diff --git a/Doc/library/errno.rst b/Doc/library/errno.rst deleted file mode 100644 index 283e8b013265d9..00000000000000 --- a/Doc/library/errno.rst +++ /dev/null @@ -1,696 +0,0 @@ -:mod:`errno` --- Standard errno system symbols -============================================== - -.. module:: errno - :synopsis: Standard errno system symbols. - ----------------- - -This module makes available standard ``errno`` system symbols. The value of each -symbol is the corresponding integer value. The names and descriptions are -borrowed from :file:`linux/include/errno.h`, which should be -all-inclusive. - - -.. data:: errorcode - - Dictionary providing a mapping from the errno value to the string name in the - underlying system. For instance, ``errno.errorcode[errno.EPERM]`` maps to - ``'EPERM'``. - -To translate a numeric error code to an error message, use :func:`os.strerror`. - -Of the following list, symbols that are not used on the current platform are not -defined by the module. The specific list of defined symbols is available as -``errno.errorcode.keys()``. Symbols available can include: - - -.. data:: EPERM - - Operation not permitted. This error is mapped to the exception - :exc:`PermissionError`. - - -.. data:: ENOENT - - No such file or directory. This error is mapped to the exception - :exc:`FileNotFoundError`. - - -.. data:: ESRCH - - No such process. This error is mapped to the exception - :exc:`ProcessLookupError`. - - -.. data:: EINTR - - Interrupted system call. This error is mapped to the exception - :exc:`InterruptedError`. - - -.. data:: EIO - - I/O error - - -.. data:: ENXIO - - No such device or address - - -.. data:: E2BIG - - Arg list too long - - -.. data:: ENOEXEC - - Exec format error - - -.. data:: EBADF - - Bad file number - - -.. data:: ECHILD - - No child processes. This error is mapped to the exception - :exc:`ChildProcessError`. - - -.. data:: EAGAIN - - Try again. This error is mapped to the exception :exc:`BlockingIOError`. - - -.. data:: ENOMEM - - Out of memory - - -.. data:: EACCES - - Permission denied. This error is mapped to the exception - :exc:`PermissionError`. - - -.. data:: EFAULT - - Bad address - - -.. data:: ENOTBLK - - Block device required - - -.. data:: EBUSY - - Device or resource busy - - -.. data:: EEXIST - - File exists. This error is mapped to the exception - :exc:`FileExistsError`. - - -.. data:: EXDEV - - Cross-device link - - -.. data:: ENODEV - - No such device - - -.. data:: ENOTDIR - - Not a directory. This error is mapped to the exception - :exc:`NotADirectoryError`. - - -.. data:: EISDIR - - Is a directory. This error is mapped to the exception - :exc:`IsADirectoryError`. - - -.. data:: EINVAL - - Invalid argument - - -.. data:: ENFILE - - File table overflow - - -.. data:: EMFILE - - Too many open files - - -.. data:: ENOTTY - - Not a typewriter - - -.. data:: ETXTBSY - - Text file busy - - -.. data:: EFBIG - - File too large - - -.. data:: ENOSPC - - No space left on device - - -.. data:: ESPIPE - - Illegal seek - - -.. data:: EROFS - - Read-only file system - - -.. data:: EMLINK - - Too many links - - -.. data:: EPIPE - - Broken pipe. This error is mapped to the exception - :exc:`BrokenPipeError`. - - -.. data:: EDOM - - Math argument out of domain of func - - -.. data:: ERANGE - - Math result not representable - - -.. data:: EDEADLK - - Resource deadlock would occur - - -.. data:: ENAMETOOLONG - - File name too long - - -.. data:: ENOLCK - - No record locks available - - -.. data:: ENOSYS - - Function not implemented - - -.. data:: ENOTEMPTY - - Directory not empty - - -.. data:: ELOOP - - Too many symbolic links encountered - - -.. data:: EWOULDBLOCK - - Operation would block. This error is mapped to the exception - :exc:`BlockingIOError`. - - -.. data:: ENOMSG - - No message of desired type - - -.. data:: EIDRM - - Identifier removed - - -.. data:: ECHRNG - - Channel number out of range - - -.. data:: EL2NSYNC - - Level 2 not synchronized - - -.. data:: EL3HLT - - Level 3 halted - - -.. data:: EL3RST - - Level 3 reset - - -.. data:: ELNRNG - - Link number out of range - - -.. data:: EUNATCH - - Protocol driver not attached - - -.. data:: ENOCSI - - No CSI structure available - - -.. data:: EL2HLT - - Level 2 halted - - -.. data:: EBADE - - Invalid exchange - - -.. data:: EBADR - - Invalid request descriptor - - -.. data:: EXFULL - - Exchange full - - -.. data:: ENOANO - - No anode - - -.. data:: EBADRQC - - Invalid request code - - -.. data:: EBADSLT - - Invalid slot - - -.. data:: EDEADLOCK - - File locking deadlock error - - -.. data:: EBFONT - - Bad font file format - - -.. data:: ENOSTR - - Device not a stream - - -.. data:: ENODATA - - No data available - - -.. data:: ETIME - - Timer expired - - -.. data:: ENOSR - - Out of streams resources - - -.. data:: ENONET - - Machine is not on the network - - -.. data:: ENOPKG - - Package not installed - - -.. data:: EREMOTE - - Object is remote - - -.. data:: ENOLINK - - Link has been severed - - -.. data:: EADV - - Advertise error - - -.. data:: ESRMNT - - Srmount error - - -.. data:: ECOMM - - Communication error on send - - -.. data:: EPROTO - - Protocol error - - -.. data:: EMULTIHOP - - Multihop attempted - - -.. data:: EDOTDOT - - RFS specific error - - -.. data:: EBADMSG - - Not a data message - - -.. data:: EOVERFLOW - - Value too large for defined data type - - -.. data:: ENOTUNIQ - - Name not unique on network - - -.. data:: EBADFD - - File descriptor in bad state - - -.. data:: EREMCHG - - Remote address changed - - -.. data:: ELIBACC - - Can not access a needed shared library - - -.. data:: ELIBBAD - - Accessing a corrupted shared library - - -.. data:: ELIBSCN - - .lib section in a.out corrupted - - -.. data:: ELIBMAX - - Attempting to link in too many shared libraries - - -.. data:: ELIBEXEC - - Cannot exec a shared library directly - - -.. data:: EILSEQ - - Illegal byte sequence - - -.. data:: ERESTART - - Interrupted system call should be restarted - - -.. data:: ESTRPIPE - - Streams pipe error - - -.. data:: EUSERS - - Too many users - - -.. data:: ENOTSOCK - - Socket operation on non-socket - - -.. data:: EDESTADDRREQ - - Destination address required - - -.. data:: EMSGSIZE - - Message too long - - -.. data:: EPROTOTYPE - - Protocol wrong type for socket - - -.. data:: ENOPROTOOPT - - Protocol not available - - -.. data:: EPROTONOSUPPORT - - Protocol not supported - - -.. data:: ESOCKTNOSUPPORT - - Socket type not supported - - -.. data:: EOPNOTSUPP - - Operation not supported on transport endpoint - - -.. data:: ENOTSUP - - Operation not supported - - .. versionadded:: 3.2 - - -.. data:: EPFNOSUPPORT - - Protocol family not supported - - -.. data:: EAFNOSUPPORT - - Address family not supported by protocol - - -.. data:: EADDRINUSE - - Address already in use - - -.. data:: EADDRNOTAVAIL - - Cannot assign requested address - - -.. data:: ENETDOWN - - Network is down - - -.. data:: ENETUNREACH - - Network is unreachable - - -.. data:: ENETRESET - - Network dropped connection because of reset - - -.. data:: ECONNABORTED - - Software caused connection abort. This error is mapped to the - exception :exc:`ConnectionAbortedError`. - - -.. data:: ECONNRESET - - Connection reset by peer. This error is mapped to the exception - :exc:`ConnectionResetError`. - - -.. data:: ENOBUFS - - No buffer space available - - -.. data:: EISCONN - - Transport endpoint is already connected - - -.. data:: ENOTCONN - - Transport endpoint is not connected - - -.. data:: ESHUTDOWN - - Cannot send after transport endpoint shutdown. This error is mapped - to the exception :exc:`BrokenPipeError`. - - -.. data:: ETOOMANYREFS - - Too many references: cannot splice - - -.. data:: ETIMEDOUT - - Connection timed out. This error is mapped to the exception - :exc:`TimeoutError`. - - -.. data:: ECONNREFUSED - - Connection refused. This error is mapped to the exception - :exc:`ConnectionRefusedError`. - - -.. data:: EHOSTDOWN - - Host is down - - -.. data:: EHOSTUNREACH - - No route to host - - -.. data:: EALREADY - - Operation already in progress. This error is mapped to the - exception :exc:`BlockingIOError`. - - -.. data:: EINPROGRESS - - Operation now in progress. This error is mapped to the exception - :exc:`BlockingIOError`. - - -.. data:: ESTALE - - Stale NFS file handle - - -.. data:: EUCLEAN - - Structure needs cleaning - - -.. data:: ENOTNAM - - Not a XENIX named type file - - -.. data:: ENAVAIL - - No XENIX semaphores available - - -.. data:: EISNAM - - Is a named type file - - -.. data:: EREMOTEIO - - Remote I/O error - - -.. data:: EDQUOT - - Quota exceeded - -.. data:: EQFULL - - Interface output queue is full - - .. versionadded:: 3.11 - -.. data:: ENOTCAPABLE - - Capabilities insufficient. This error is mapped to the exception - :exc:`PermissionError`. - - .. availability:: WASI, FreeBSD - - .. versionadded:: 3.11.1 - - -.. data:: ECANCELED - - Operation canceled - - .. versionadded:: 3.2 - - -.. data:: EOWNERDEAD - - Owner died - - .. versionadded:: 3.2 - - -.. data:: ENOTRECOVERABLE - - State not recoverable - - .. versionadded:: 3.2 diff --git a/Doc/library/exceptions.rst b/Doc/library/exceptions.rst deleted file mode 100644 index d210f82f3926f7..00000000000000 --- a/Doc/library/exceptions.rst +++ /dev/null @@ -1,1010 +0,0 @@ -.. _bltin-exceptions: - -Built-in Exceptions -=================== - -.. index:: - pair: statement; try - pair: statement; except - -In Python, all exceptions must be instances of a class that derives from -:class:`BaseException`. In a :keyword:`try` statement with an :keyword:`except` -clause that mentions a particular class, that clause also handles any exception -classes derived from that class (but not exception classes from which *it* is -derived). Two exception classes that are not related via subclassing are never -equivalent, even if they have the same name. - -.. index:: pair: statement; raise - -The built-in exceptions listed below can be generated by the interpreter or -built-in functions. Except where mentioned, they have an "associated value" -indicating the detailed cause of the error. This may be a string or a tuple of -several items of information (e.g., an error code and a string explaining the -code). The associated value is usually passed as arguments to the exception -class's constructor. - -User code can raise built-in exceptions. This can be used to test an exception -handler or to report an error condition "just like" the situation in which the -interpreter raises the same exception; but beware that there is nothing to -prevent user code from raising an inappropriate error. - -The built-in exception classes can be subclassed to define new exceptions; -programmers are encouraged to derive new exceptions from the :exc:`Exception` -class or one of its subclasses, and not from :exc:`BaseException`. More -information on defining exceptions is available in the Python Tutorial under -:ref:`tut-userexceptions`. - - -Exception context ------------------ - -When raising a new exception while another exception -is already being handled, the new exception's -:attr:`__context__` attribute is automatically set to the handled -exception. An exception may be handled when an :keyword:`except` or -:keyword:`finally` clause, or a :keyword:`with` statement, is used. - -This implicit exception context can be -supplemented with an explicit cause by using :keyword:`!from` with -:keyword:`raise`:: - - raise new_exc from original_exc - -The expression following :keyword:`from` must be an exception or ``None``. It -will be set as :attr:`__cause__` on the raised exception. Setting -:attr:`__cause__` also implicitly sets the :attr:`__suppress_context__` -attribute to ``True``, so that using ``raise new_exc from None`` -effectively replaces the old exception with the new one for display -purposes (e.g. converting :exc:`KeyError` to :exc:`AttributeError`), while -leaving the old exception available in :attr:`__context__` for introspection -when debugging. - -The default traceback display code shows these chained exceptions in -addition to the traceback for the exception itself. An explicitly chained -exception in :attr:`__cause__` is always shown when present. An implicitly -chained exception in :attr:`__context__` is shown only if :attr:`__cause__` -is :const:`None` and :attr:`__suppress_context__` is false. - -In either case, the exception itself is always shown after any chained -exceptions so that the final line of the traceback always shows the last -exception that was raised. - - -Inheriting from built-in exceptions ------------------------------------ - -User code can create subclasses that inherit from an exception type. -It's recommended to only subclass one exception type at a time to avoid -any possible conflicts between how the bases handle the ``args`` -attribute, as well as due to possible memory layout incompatibilities. - -.. impl-detail:: - - Most built-in exceptions are implemented in C for efficiency, see: - :source:`Objects/exceptions.c`. Some have custom memory layouts - which makes it impossible to create a subclass that inherits from - multiple exception types. The memory layout of a type is an implementation - detail and might change between Python versions, leading to new - conflicts in the future. Therefore, it's recommended to avoid - subclassing multiple exception types altogether. - - -Base classes ------------- - -The following exceptions are used mostly as base classes for other exceptions. - -.. exception:: BaseException - - The base class for all built-in exceptions. It is not meant to be directly - inherited by user-defined classes (for that, use :exc:`Exception`). If - :func:`str` is called on an instance of this class, the representation of - the argument(s) to the instance are returned, or the empty string when - there were no arguments. - - .. attribute:: args - - The tuple of arguments given to the exception constructor. Some built-in - exceptions (like :exc:`OSError`) expect a certain number of arguments and - assign a special meaning to the elements of this tuple, while others are - usually called only with a single string giving an error message. - - .. method:: with_traceback(tb) - - This method sets *tb* as the new traceback for the exception and returns - the exception object. It was more commonly used before the exception - chaining features of :pep:`3134` became available. The following example - shows how we can convert an instance of ``SomeException`` into an - instance of ``OtherException`` while preserving the traceback. Once - raised, the current frame is pushed onto the traceback of the - ``OtherException``, as would have happened to the traceback of the - original ``SomeException`` had we allowed it to propagate to the caller. :: - - try: - ... - except SomeException: - tb = sys.exception().__traceback__ - raise OtherException(...).with_traceback(tb) - - .. method:: add_note(note) - - Add the string ``note`` to the exception's notes which appear in the standard - traceback after the exception string. A :exc:`TypeError` is raised if ``note`` - is not a string. - - .. versionadded:: 3.11 - - .. attribute:: __notes__ - - A list of the notes of this exception, which were added with :meth:`add_note`. - This attribute is created when :meth:`add_note` is called. - - .. versionadded:: 3.11 - - -.. exception:: Exception - - All built-in, non-system-exiting exceptions are derived from this class. All - user-defined exceptions should also be derived from this class. - - -.. exception:: ArithmeticError - - The base class for those built-in exceptions that are raised for various - arithmetic errors: :exc:`OverflowError`, :exc:`ZeroDivisionError`, - :exc:`FloatingPointError`. - - -.. exception:: BufferError - - Raised when a :ref:`buffer ` related operation cannot be - performed. - - -.. exception:: LookupError - - The base class for the exceptions that are raised when a key or index used on - a mapping or sequence is invalid: :exc:`IndexError`, :exc:`KeyError`. This - can be raised directly by :func:`codecs.lookup`. - - -Concrete exceptions -------------------- - -The following exceptions are the exceptions that are usually raised. - -.. exception:: AssertionError - - .. index:: pair: statement; assert - - Raised when an :keyword:`assert` statement fails. - - -.. exception:: AttributeError - - Raised when an attribute reference (see :ref:`attribute-references`) or - assignment fails. (When an object does not support attribute references or - attribute assignments at all, :exc:`TypeError` is raised.) - - The :attr:`name` and :attr:`obj` attributes can be set using keyword-only - arguments to the constructor. When set they represent the name of the attribute - that was attempted to be accessed and the object that was accessed for said - attribute, respectively. - - .. versionchanged:: 3.10 - Added the :attr:`name` and :attr:`obj` attributes. - -.. exception:: EOFError - - Raised when the :func:`input` function hits an end-of-file condition (EOF) - without reading any data. (N.B.: the :meth:`io.IOBase.read` and - :meth:`io.IOBase.readline` methods return an empty string when they hit EOF.) - - -.. exception:: FloatingPointError - - Not currently used. - - -.. exception:: GeneratorExit - - Raised when a :term:`generator` or :term:`coroutine` is closed; - see :meth:`generator.close` and :meth:`coroutine.close`. It - directly inherits from :exc:`BaseException` instead of :exc:`Exception` since - it is technically not an error. - - -.. exception:: ImportError - - Raised when the :keyword:`import` statement has troubles trying to - load a module. Also raised when the "from list" in ``from ... import`` - has a name that cannot be found. - - The optional *name* and *path* keyword-only arguments - set the corresponding attributes: - - .. attribute:: name - - The name of the module that was attempted to be imported. - - .. attribute:: path - - The path to any file which triggered the exception. - - .. versionchanged:: 3.3 - Added the :attr:`name` and :attr:`path` attributes. - -.. exception:: ModuleNotFoundError - - A subclass of :exc:`ImportError` which is raised by :keyword:`import` - when a module could not be located. It is also raised when ``None`` - is found in :data:`sys.modules`. - - .. versionadded:: 3.6 - - -.. exception:: IndexError - - Raised when a sequence subscript is out of range. (Slice indices are - silently truncated to fall in the allowed range; if an index is not an - integer, :exc:`TypeError` is raised.) - - .. XXX xref to sequences - - -.. exception:: KeyError - - Raised when a mapping (dictionary) key is not found in the set of existing keys. - - .. XXX xref to mapping objects? - - -.. exception:: KeyboardInterrupt - - Raised when the user hits the interrupt key (normally :kbd:`Control-C` or - :kbd:`Delete`). During execution, a check for interrupts is made - regularly. The exception inherits from :exc:`BaseException` so as to not be - accidentally caught by code that catches :exc:`Exception` and thus prevent - the interpreter from exiting. - - .. note:: - - Catching a :exc:`KeyboardInterrupt` requires special consideration. - Because it can be raised at unpredictable points, it may, in some - circumstances, leave the running program in an inconsistent state. It is - generally best to allow :exc:`KeyboardInterrupt` to end the program as - quickly as possible or avoid raising it entirely. (See - :ref:`handlers-and-exceptions`.) - - -.. exception:: MemoryError - - Raised when an operation runs out of memory but the situation may still be - rescued (by deleting some objects). The associated value is a string indicating - what kind of (internal) operation ran out of memory. Note that because of the - underlying memory management architecture (C's :c:func:`malloc` function), the - interpreter may not always be able to completely recover from this situation; it - nevertheless raises an exception so that a stack traceback can be printed, in - case a run-away program was the cause. - - -.. exception:: NameError - - Raised when a local or global name is not found. This applies only to - unqualified names. The associated value is an error message that includes the - name that could not be found. - - The :attr:`name` attribute can be set using a keyword-only argument to the - constructor. When set it represent the name of the variable that was attempted - to be accessed. - - .. versionchanged:: 3.10 - Added the :attr:`name` attribute. - - -.. exception:: NotImplementedError - - This exception is derived from :exc:`RuntimeError`. In user defined base - classes, abstract methods should raise this exception when they require - derived classes to override the method, or while the class is being - developed to indicate that the real implementation still needs to be added. - - .. note:: - - It should not be used to indicate that an operator or method is not - meant to be supported at all -- in that case either leave the operator / - method undefined or, if a subclass, set it to :data:`None`. - - .. note:: - - ``NotImplementedError`` and ``NotImplemented`` are not interchangeable, - even though they have similar names and purposes. See - :data:`NotImplemented` for details on when to use it. - -.. exception:: OSError([arg]) - OSError(errno, strerror[, filename[, winerror[, filename2]]]) - - .. index:: pair: module; errno - - This exception is raised when a system function returns a system-related - error, including I/O failures such as "file not found" or "disk full" - (not for illegal argument types or other incidental errors). - - The second form of the constructor sets the corresponding attributes, - described below. The attributes default to :const:`None` if not - specified. For backwards compatibility, if three arguments are passed, - the :attr:`~BaseException.args` attribute contains only a 2-tuple - of the first two constructor arguments. - - The constructor often actually returns a subclass of :exc:`OSError`, as - described in `OS exceptions`_ below. The particular subclass depends on - the final :attr:`.errno` value. This behaviour only occurs when - constructing :exc:`OSError` directly or via an alias, and is not - inherited when subclassing. - - .. attribute:: errno - - A numeric error code from the C variable :c:data:`errno`. - - .. attribute:: winerror - - Under Windows, this gives you the native - Windows error code. The :attr:`.errno` attribute is then an approximate - translation, in POSIX terms, of that native error code. - - Under Windows, if the *winerror* constructor argument is an integer, - the :attr:`.errno` attribute is determined from the Windows error code, - and the *errno* argument is ignored. On other platforms, the - *winerror* argument is ignored, and the :attr:`winerror` attribute - does not exist. - - .. attribute:: strerror - - The corresponding error message, as provided by - the operating system. It is formatted by the C - functions :c:func:`perror` under POSIX, and :c:func:`FormatMessage` - under Windows. - - .. attribute:: filename - filename2 - - For exceptions that involve a file system path (such as :func:`open` or - :func:`os.unlink`), :attr:`filename` is the file name passed to the function. - For functions that involve two file system paths (such as - :func:`os.rename`), :attr:`filename2` corresponds to the second - file name passed to the function. - - - .. versionchanged:: 3.3 - :exc:`EnvironmentError`, :exc:`IOError`, :exc:`WindowsError`, - :exc:`socket.error`, :exc:`select.error` and - :exc:`mmap.error` have been merged into :exc:`OSError`, and the - constructor may return a subclass. - - .. versionchanged:: 3.4 - The :attr:`filename` attribute is now the original file name passed to - the function, instead of the name encoded to or decoded from the - :term:`filesystem encoding and error handler`. Also, the *filename2* - constructor argument and attribute was added. - - -.. exception:: OverflowError - - Raised when the result of an arithmetic operation is too large to be - represented. This cannot occur for integers (which would rather raise - :exc:`MemoryError` than give up). However, for historical reasons, - OverflowError is sometimes raised for integers that are outside a required - range. Because of the lack of standardization of floating point exception - handling in C, most floating point operations are not checked. - - -.. exception:: RecursionError - - This exception is derived from :exc:`RuntimeError`. It is raised when the - interpreter detects that the maximum recursion depth (see - :func:`sys.getrecursionlimit`) is exceeded. - - .. versionadded:: 3.5 - Previously, a plain :exc:`RuntimeError` was raised. - - -.. exception:: ReferenceError - - This exception is raised when a weak reference proxy, created by the - :func:`weakref.proxy` function, is used to access an attribute of the referent - after it has been garbage collected. For more information on weak references, - see the :mod:`weakref` module. - - -.. exception:: RuntimeError - - Raised when an error is detected that doesn't fall in any of the other - categories. The associated value is a string indicating what precisely went - wrong. - - -.. exception:: StopIteration - - Raised by built-in function :func:`next` and an :term:`iterator`\'s - :meth:`~iterator.__next__` method to signal that there are no further - items produced by the iterator. - - The exception object has a single attribute :attr:`value`, which is - given as an argument when constructing the exception, and defaults - to :const:`None`. - - When a :term:`generator` or :term:`coroutine` function - returns, a new :exc:`StopIteration` instance is - raised, and the value returned by the function is used as the - :attr:`value` parameter to the constructor of the exception. - - If a generator code directly or indirectly raises :exc:`StopIteration`, - it is converted into a :exc:`RuntimeError` (retaining the - :exc:`StopIteration` as the new exception's cause). - - .. versionchanged:: 3.3 - Added ``value`` attribute and the ability for generator functions to - use it to return a value. - - .. versionchanged:: 3.5 - Introduced the RuntimeError transformation via - ``from __future__ import generator_stop``, see :pep:`479`. - - .. versionchanged:: 3.7 - Enable :pep:`479` for all code by default: a :exc:`StopIteration` - error raised in a generator is transformed into a :exc:`RuntimeError`. - -.. exception:: StopAsyncIteration - - Must be raised by :meth:`~object.__anext__` method of an - :term:`asynchronous iterator` object to stop the iteration. - - .. versionadded:: 3.5 - -.. exception:: SyntaxError(message, details) - - Raised when the parser encounters a syntax error. This may occur in an - :keyword:`import` statement, in a call to the built-in functions - :func:`compile`, :func:`exec`, - or :func:`eval`, or when reading the initial script or standard input - (also interactively). - - The :func:`str` of the exception instance returns only the error message. - Details is a tuple whose members are also available as separate attributes. - - .. attribute:: filename - - The name of the file the syntax error occurred in. - - .. attribute:: lineno - - Which line number in the file the error occurred in. This is - 1-indexed: the first line in the file has a ``lineno`` of 1. - - .. attribute:: offset - - The column in the line where the error occurred. This is - 1-indexed: the first character in the line has an ``offset`` of 1. - - .. attribute:: text - - The source code text involved in the error. - - .. attribute:: end_lineno - - Which line number in the file the error occurred ends in. This is - 1-indexed: the first line in the file has a ``lineno`` of 1. - - .. attribute:: end_offset - - The column in the end line where the error occurred finishes. This is - 1-indexed: the first character in the line has an ``offset`` of 1. - - For errors in f-string fields, the message is prefixed by "f-string: " - and the offsets are offsets in a text constructed from the replacement - expression. For example, compiling f'Bad {a b} field' results in this - args attribute: ('f-string: ...', ('', 1, 2, '(a b)\n', 1, 5)). - - .. versionchanged:: 3.10 - Added the :attr:`end_lineno` and :attr:`end_offset` attributes. - -.. exception:: IndentationError - - Base class for syntax errors related to incorrect indentation. This is a - subclass of :exc:`SyntaxError`. - - -.. exception:: TabError - - Raised when indentation contains an inconsistent use of tabs and spaces. - This is a subclass of :exc:`IndentationError`. - - -.. exception:: SystemError - - Raised when the interpreter finds an internal error, but the situation does not - look so serious to cause it to abandon all hope. The associated value is a - string indicating what went wrong (in low-level terms). - - You should report this to the author or maintainer of your Python interpreter. - Be sure to report the version of the Python interpreter (``sys.version``; it is - also printed at the start of an interactive Python session), the exact error - message (the exception's associated value) and if possible the source of the - program that triggered the error. - - -.. exception:: SystemExit - - This exception is raised by the :func:`sys.exit` function. It inherits from - :exc:`BaseException` instead of :exc:`Exception` so that it is not accidentally - caught by code that catches :exc:`Exception`. This allows the exception to - properly propagate up and cause the interpreter to exit. When it is not - handled, the Python interpreter exits; no stack traceback is printed. The - constructor accepts the same optional argument passed to :func:`sys.exit`. - If the value is an integer, it specifies the system exit status (passed to - C's :c:func:`exit` function); if it is ``None``, the exit status is zero; if - it has another type (such as a string), the object's value is printed and - the exit status is one. - - A call to :func:`sys.exit` is translated into an exception so that clean-up - handlers (:keyword:`finally` clauses of :keyword:`try` statements) can be - executed, and so that a debugger can execute a script without running the risk - of losing control. The :func:`os._exit` function can be used if it is - absolutely positively necessary to exit immediately (for example, in the child - process after a call to :func:`os.fork`). - - .. attribute:: code - - The exit status or error message that is passed to the constructor. - (Defaults to ``None``.) - - -.. exception:: TypeError - - Raised when an operation or function is applied to an object of inappropriate - type. The associated value is a string giving details about the type mismatch. - - This exception may be raised by user code to indicate that an attempted - operation on an object is not supported, and is not meant to be. If an object - is meant to support a given operation but has not yet provided an - implementation, :exc:`NotImplementedError` is the proper exception to raise. - - Passing arguments of the wrong type (e.g. passing a :class:`list` when an - :class:`int` is expected) should result in a :exc:`TypeError`, but passing - arguments with the wrong value (e.g. a number outside expected boundaries) - should result in a :exc:`ValueError`. - -.. exception:: UnboundLocalError - - Raised when a reference is made to a local variable in a function or method, but - no value has been bound to that variable. This is a subclass of - :exc:`NameError`. - - -.. exception:: UnicodeError - - Raised when a Unicode-related encoding or decoding error occurs. It is a - subclass of :exc:`ValueError`. - - :exc:`UnicodeError` has attributes that describe the encoding or decoding - error. For example, ``err.object[err.start:err.end]`` gives the particular - invalid input that the codec failed on. - - .. attribute:: encoding - - The name of the encoding that raised the error. - - .. attribute:: reason - - A string describing the specific codec error. - - .. attribute:: object - - The object the codec was attempting to encode or decode. - - .. attribute:: start - - The first index of invalid data in :attr:`object`. - - .. attribute:: end - - The index after the last invalid data in :attr:`object`. - - -.. exception:: UnicodeEncodeError - - Raised when a Unicode-related error occurs during encoding. It is a subclass of - :exc:`UnicodeError`. - - -.. exception:: UnicodeDecodeError - - Raised when a Unicode-related error occurs during decoding. It is a subclass of - :exc:`UnicodeError`. - - -.. exception:: UnicodeTranslateError - - Raised when a Unicode-related error occurs during translating. It is a subclass - of :exc:`UnicodeError`. - - -.. exception:: ValueError - - Raised when an operation or function receives an argument that has the - right type but an inappropriate value, and the situation is not described by a - more precise exception such as :exc:`IndexError`. - - -.. exception:: ZeroDivisionError - - Raised when the second argument of a division or modulo operation is zero. The - associated value is a string indicating the type of the operands and the - operation. - - -The following exceptions are kept for compatibility with previous versions; -starting from Python 3.3, they are aliases of :exc:`OSError`. - -.. exception:: EnvironmentError - -.. exception:: IOError - -.. exception:: WindowsError - - Only available on Windows. - - -OS exceptions -^^^^^^^^^^^^^ - -The following exceptions are subclasses of :exc:`OSError`, they get raised -depending on the system error code. - -.. exception:: BlockingIOError - - Raised when an operation would block on an object (e.g. socket) set - for non-blocking operation. - Corresponds to :c:data:`errno` :py:const:`~errno.EAGAIN`, :py:const:`~errno.EALREADY`, - :py:const:`~errno.EWOULDBLOCK` and :py:const:`~errno.EINPROGRESS`. - - In addition to those of :exc:`OSError`, :exc:`BlockingIOError` can have - one more attribute: - - .. attribute:: characters_written - - An integer containing the number of characters written to the stream - before it blocked. This attribute is available when using the - buffered I/O classes from the :mod:`io` module. - -.. exception:: ChildProcessError - - Raised when an operation on a child process failed. - Corresponds to :c:data:`errno` :py:const:`~errno.ECHILD`. - -.. exception:: ConnectionError - - A base class for connection-related issues. - - Subclasses are :exc:`BrokenPipeError`, :exc:`ConnectionAbortedError`, - :exc:`ConnectionRefusedError` and :exc:`ConnectionResetError`. - -.. exception:: BrokenPipeError - - A subclass of :exc:`ConnectionError`, raised when trying to write on a - pipe while the other end has been closed, or trying to write on a socket - which has been shutdown for writing. - Corresponds to :c:data:`errno` :py:const:`~errno.EPIPE` and :py:const:`~errno.ESHUTDOWN`. - -.. exception:: ConnectionAbortedError - - A subclass of :exc:`ConnectionError`, raised when a connection attempt - is aborted by the peer. - Corresponds to :c:data:`errno` :py:const:`~errno.ECONNABORTED`. - -.. exception:: ConnectionRefusedError - - A subclass of :exc:`ConnectionError`, raised when a connection attempt - is refused by the peer. - Corresponds to :c:data:`errno` :py:const:`~errno.ECONNREFUSED`. - -.. exception:: ConnectionResetError - - A subclass of :exc:`ConnectionError`, raised when a connection is - reset by the peer. - Corresponds to :c:data:`errno` :py:const:`~errno.ECONNRESET`. - -.. exception:: FileExistsError - - Raised when trying to create a file or directory which already exists. - Corresponds to :c:data:`errno` :py:const:`~errno.EEXIST`. - -.. exception:: FileNotFoundError - - Raised when a file or directory is requested but doesn't exist. - Corresponds to :c:data:`errno` :py:const:`~errno.ENOENT`. - -.. exception:: InterruptedError - - Raised when a system call is interrupted by an incoming signal. - Corresponds to :c:data:`errno` :py:const:`~errno.EINTR`. - - .. versionchanged:: 3.5 - Python now retries system calls when a syscall is interrupted by a - signal, except if the signal handler raises an exception (see :pep:`475` - for the rationale), instead of raising :exc:`InterruptedError`. - -.. exception:: IsADirectoryError - - Raised when a file operation (such as :func:`os.remove`) is requested - on a directory. - Corresponds to :c:data:`errno` :py:const:`~errno.EISDIR`. - -.. exception:: NotADirectoryError - - Raised when a directory operation (such as :func:`os.listdir`) is requested on - something which is not a directory. On most POSIX platforms, it may also be - raised if an operation attempts to open or traverse a non-directory file as if - it were a directory. - Corresponds to :c:data:`errno` :py:const:`~errno.ENOTDIR`. - -.. exception:: PermissionError - - Raised when trying to run an operation without the adequate access - rights - for example filesystem permissions. - Corresponds to :c:data:`errno` :py:const:`~errno.EACCES`, - :py:const:`~errno.EPERM`, and :py:const:`~errno.ENOTCAPABLE`. - - .. versionchanged:: 3.11.1 - WASI's :py:const:`~errno.ENOTCAPABLE` is now mapped to - :exc:`PermissionError`. - -.. exception:: ProcessLookupError - - Raised when a given process doesn't exist. - Corresponds to :c:data:`errno` :py:const:`~errno.ESRCH`. - -.. exception:: TimeoutError - - Raised when a system function timed out at the system level. - Corresponds to :c:data:`errno` :py:const:`~errno.ETIMEDOUT`. - -.. versionadded:: 3.3 - All the above :exc:`OSError` subclasses were added. - - -.. seealso:: - - :pep:`3151` - Reworking the OS and IO exception hierarchy - - -.. _warning-categories-as-exceptions: - -Warnings --------- - -The following exceptions are used as warning categories; see the -:ref:`warning-categories` documentation for more details. - -.. exception:: Warning - - Base class for warning categories. - - -.. exception:: UserWarning - - Base class for warnings generated by user code. - - -.. exception:: DeprecationWarning - - Base class for warnings about deprecated features when those warnings are - intended for other Python developers. - - Ignored by the default warning filters, except in the ``__main__`` module - (:pep:`565`). Enabling the :ref:`Python Development Mode ` shows - this warning. - - The deprecation policy is described in :pep:`387`. - - -.. exception:: PendingDeprecationWarning - - Base class for warnings about features which are obsolete and - expected to be deprecated in the future, but are not deprecated - at the moment. - - This class is rarely used as emitting a warning about a possible - upcoming deprecation is unusual, and :exc:`DeprecationWarning` - is preferred for already active deprecations. - - Ignored by the default warning filters. Enabling the :ref:`Python - Development Mode ` shows this warning. - - The deprecation policy is described in :pep:`387`. - - -.. exception:: SyntaxWarning - - Base class for warnings about dubious syntax. - - -.. exception:: RuntimeWarning - - Base class for warnings about dubious runtime behavior. - - -.. exception:: FutureWarning - - Base class for warnings about deprecated features when those warnings are - intended for end users of applications that are written in Python. - - -.. exception:: ImportWarning - - Base class for warnings about probable mistakes in module imports. - - Ignored by the default warning filters. Enabling the :ref:`Python - Development Mode ` shows this warning. - - -.. exception:: UnicodeWarning - - Base class for warnings related to Unicode. - - -.. exception:: EncodingWarning - - Base class for warnings related to encodings. - - See :ref:`io-encoding-warning` for details. - - .. versionadded:: 3.10 - - -.. exception:: BytesWarning - - Base class for warnings related to :class:`bytes` and :class:`bytearray`. - - -.. exception:: ResourceWarning - - Base class for warnings related to resource usage. - - Ignored by the default warning filters. Enabling the :ref:`Python - Development Mode ` shows this warning. - - .. versionadded:: 3.2 - - -.. _lib-exception-groups: - -Exception groups ----------------- - -The following are used when it is necessary to raise multiple unrelated -exceptions. They are part of the exception hierarchy so they can be -handled with :keyword:`except` like all other exceptions. In addition, -they are recognised by :keyword:`except*`, which matches -their subgroups based on the types of the contained exceptions. - -.. exception:: ExceptionGroup(msg, excs) -.. exception:: BaseExceptionGroup(msg, excs) - - Both of these exception types wrap the exceptions in the sequence ``excs``. - The ``msg`` parameter must be a string. The difference between the two - classes is that :exc:`BaseExceptionGroup` extends :exc:`BaseException` and - it can wrap any exception, while :exc:`ExceptionGroup` extends :exc:`Exception` - and it can only wrap subclasses of :exc:`Exception`. This design is so that - ``except Exception`` catches an :exc:`ExceptionGroup` but not - :exc:`BaseExceptionGroup`. - - The :exc:`BaseExceptionGroup` constructor returns an :exc:`ExceptionGroup` - rather than a :exc:`BaseExceptionGroup` if all contained exceptions are - :exc:`Exception` instances, so it can be used to make the selection - automatic. The :exc:`ExceptionGroup` constructor, on the other hand, - raises a :exc:`TypeError` if any contained exception is not an - :exc:`Exception` subclass. - - .. attribute:: message - - The ``msg`` argument to the constructor. This is a read-only attribute. - - .. attribute:: exceptions - - A tuple of the exceptions in the ``excs`` sequence given to the - constructor. This is a read-only attribute. - - .. method:: subgroup(condition) - - Returns an exception group that contains only the exceptions from the - current group that match *condition*, or ``None`` if the result is empty. - - The condition can be either a function that accepts an exception and returns - true for those that should be in the subgroup, or it can be an exception type - or a tuple of exception types, which is used to check for a match using the - same check that is used in an ``except`` clause. - - The nesting structure of the current exception is preserved in the result, - as are the values of its :attr:`message`, :attr:`__traceback__`, - :attr:`__cause__`, :attr:`__context__` and :attr:`__notes__` fields. - Empty nested groups are omitted from the result. - - The condition is checked for all exceptions in the nested exception group, - including the top-level and any nested exception groups. If the condition is - true for such an exception group, it is included in the result in full. - - .. method:: split(condition) - - Like :meth:`subgroup`, but returns the pair ``(match, rest)`` where ``match`` - is ``subgroup(condition)`` and ``rest`` is the remaining non-matching - part. - - .. method:: derive(excs) - - Returns an exception group with the same :attr:`message`, but which - wraps the exceptions in ``excs``. - - This method is used by :meth:`subgroup` and :meth:`split`. A - subclass needs to override it in order to make :meth:`subgroup` - and :meth:`split` return instances of the subclass rather - than :exc:`ExceptionGroup`. - - :meth:`subgroup` and :meth:`split` copy the :attr:`__traceback__`, - :attr:`__cause__`, :attr:`__context__` and :attr:`__notes__` fields from - the original exception group to the one returned by :meth:`derive`, so - these fields do not need to be updated by :meth:`derive`. :: - - >>> class MyGroup(ExceptionGroup): - ... def derive(self, excs): - ... return MyGroup(self.message, excs) - ... - >>> e = MyGroup("eg", [ValueError(1), TypeError(2)]) - >>> e.add_note("a note") - >>> e.__context__ = Exception("context") - >>> e.__cause__ = Exception("cause") - >>> try: - ... raise e - ... except Exception as e: - ... exc = e - ... - >>> match, rest = exc.split(ValueError) - >>> exc, exc.__context__, exc.__cause__, exc.__notes__ - (MyGroup('eg', [ValueError(1), TypeError(2)]), Exception('context'), Exception('cause'), ['a note']) - >>> match, match.__context__, match.__cause__, match.__notes__ - (MyGroup('eg', [ValueError(1)]), Exception('context'), Exception('cause'), ['a note']) - >>> rest, rest.__context__, rest.__cause__, rest.__notes__ - (MyGroup('eg', [TypeError(2)]), Exception('context'), Exception('cause'), ['a note']) - >>> exc.__traceback__ is match.__traceback__ is rest.__traceback__ - True - - - Note that :exc:`BaseExceptionGroup` defines :meth:`__new__`, so - subclasses that need a different constructor signature need to - override that rather than :meth:`__init__`. For example, the following - defines an exception group subclass which accepts an exit_code and - and constructs the group's message from it. :: - - class Errors(ExceptionGroup): - def __new__(cls, errors, exit_code): - self = super().__new__(Errors, f"exit code: {exit_code}", errors) - self.exit_code = exit_code - return self - - def derive(self, excs): - return Errors(excs, self.exit_code) - - Like :exc:`ExceptionGroup`, any subclass of :exc:`BaseExceptionGroup` which - is also a subclass of :exc:`Exception` can only wrap instances of - :exc:`Exception`. - - .. versionadded:: 3.11 - - -Exception hierarchy -------------------- - -The class hierarchy for built-in exceptions is: - -.. literalinclude:: ../../Lib/test/exception_hierarchy.txt - :language: text diff --git a/Doc/library/faulthandler.rst b/Doc/library/faulthandler.rst deleted file mode 100644 index b80de69a79a8ab..00000000000000 --- a/Doc/library/faulthandler.rst +++ /dev/null @@ -1,189 +0,0 @@ -:mod:`faulthandler` --- Dump the Python traceback -================================================= - -.. module:: faulthandler - :synopsis: Dump the Python traceback. - -.. versionadded:: 3.3 - ----------------- - -This module contains functions to dump Python tracebacks explicitly, on a fault, -after a timeout, or on a user signal. Call :func:`faulthandler.enable` to -install fault handlers for the :const:`SIGSEGV`, :const:`SIGFPE`, -:const:`SIGABRT`, :const:`SIGBUS`, and :const:`SIGILL` signals. You can also -enable them at startup by setting the :envvar:`PYTHONFAULTHANDLER` environment -variable or by using the :option:`-X` ``faulthandler`` command line option. - -The fault handler is compatible with system fault handlers like Apport or the -Windows fault handler. The module uses an alternative stack for signal handlers -if the :c:func:`sigaltstack` function is available. This allows it to dump the -traceback even on a stack overflow. - -The fault handler is called on catastrophic cases and therefore can only use -signal-safe functions (e.g. it cannot allocate memory on the heap). Because of -this limitation traceback dumping is minimal compared to normal Python -tracebacks: - -* Only ASCII is supported. The ``backslashreplace`` error handler is used on - encoding. -* Each string is limited to 500 characters. -* Only the filename, the function name and the line number are - displayed. (no source code) -* It is limited to 100 frames and 100 threads. -* The order is reversed: the most recent call is shown first. - -By default, the Python traceback is written to :data:`sys.stderr`. To see -tracebacks, applications must be run in the terminal. A log file can -alternatively be passed to :func:`faulthandler.enable`. - -The module is implemented in C, so tracebacks can be dumped on a crash or when -Python is deadlocked. - -The :ref:`Python Development Mode ` calls :func:`faulthandler.enable` -at Python startup. - -.. seealso:: - - Module :mod:`pdb` - Interactive source code debugger for Python programs. - - Module :mod:`traceback` - Standard interface to extract, format and print stack traces of Python programs. - -Dumping the traceback ---------------------- - -.. function:: dump_traceback(file=sys.stderr, all_threads=True) - - Dump the tracebacks of all threads into *file*. If *all_threads* is - ``False``, dump only the current thread. - - .. seealso:: :func:`traceback.print_tb`, which can be used to print a traceback object. - - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - - -Fault handler state -------------------- - -.. function:: enable(file=sys.stderr, all_threads=True) - - Enable the fault handler: install handlers for the :const:`SIGSEGV`, - :const:`SIGFPE`, :const:`SIGABRT`, :const:`SIGBUS` and :const:`SIGILL` - signals to dump the Python traceback. If *all_threads* is ``True``, - produce tracebacks for every running thread. Otherwise, dump only the current - thread. - - The *file* must be kept open until the fault handler is disabled: see - :ref:`issue with file descriptors `. - - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - - .. versionchanged:: 3.6 - On Windows, a handler for Windows exception is also installed. - - .. versionchanged:: 3.10 - The dump now mentions if a garbage collector collection is running - if *all_threads* is true. - -.. function:: disable() - - Disable the fault handler: uninstall the signal handlers installed by - :func:`enable`. - -.. function:: is_enabled() - - Check if the fault handler is enabled. - - -Dumping the tracebacks after a timeout --------------------------------------- - -.. function:: dump_traceback_later(timeout, repeat=False, file=sys.stderr, exit=False) - - Dump the tracebacks of all threads, after a timeout of *timeout* seconds, or - every *timeout* seconds if *repeat* is ``True``. If *exit* is ``True``, call - :c:func:`_exit` with status=1 after dumping the tracebacks. (Note - :c:func:`_exit` exits the process immediately, which means it doesn't do any - cleanup like flushing file buffers.) If the function is called twice, the new - call replaces previous parameters and resets the timeout. The timer has a - sub-second resolution. - - The *file* must be kept open until the traceback is dumped or - :func:`cancel_dump_traceback_later` is called: see :ref:`issue with file - descriptors `. - - This function is implemented using a watchdog thread. - - .. versionchanged:: 3.7 - This function is now always available. - - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - -.. function:: cancel_dump_traceback_later() - - Cancel the last call to :func:`dump_traceback_later`. - - -Dumping the traceback on a user signal --------------------------------------- - -.. function:: register(signum, file=sys.stderr, all_threads=True, chain=False) - - Register a user signal: install a handler for the *signum* signal to dump - the traceback of all threads, or of the current thread if *all_threads* is - ``False``, into *file*. Call the previous handler if chain is ``True``. - - The *file* must be kept open until the signal is unregistered by - :func:`unregister`: see :ref:`issue with file descriptors `. - - Not available on Windows. - - .. versionchanged:: 3.5 - Added support for passing file descriptor to this function. - -.. function:: unregister(signum) - - Unregister a user signal: uninstall the handler of the *signum* signal - installed by :func:`register`. Return ``True`` if the signal was registered, - ``False`` otherwise. - - Not available on Windows. - - -.. _faulthandler-fd: - -Issue with file descriptors ---------------------------- - -:func:`enable`, :func:`dump_traceback_later` and :func:`register` keep the -file descriptor of their *file* argument. If the file is closed and its file -descriptor is reused by a new file, or if :func:`os.dup2` is used to replace -the file descriptor, the traceback will be written into a different file. Call -these functions again each time that the file is replaced. - - -Example -------- - -Example of a segmentation fault on Linux with and without enabling the fault -handler: - -.. code-block:: shell-session - - $ python3 -c "import ctypes; ctypes.string_at(0)" - Segmentation fault - - $ python3 -q -X faulthandler - >>> import ctypes - >>> ctypes.string_at(0) - Fatal Python error: Segmentation fault - - Current thread 0x00007fb899f39700 (most recent call first): - File "/home/python/cpython/Lib/ctypes/__init__.py", line 486 in string_at - File "", line 1 in - Segmentation fault diff --git a/Doc/library/fcntl.rst b/Doc/library/fcntl.rst deleted file mode 100644 index c417a84e048f8a..00000000000000 --- a/Doc/library/fcntl.rst +++ /dev/null @@ -1,201 +0,0 @@ -:mod:`fcntl` --- The ``fcntl`` and ``ioctl`` system calls -========================================================= - -.. module:: fcntl - :platform: Unix - :synopsis: The fcntl() and ioctl() system calls. - -.. sectionauthor:: Jaap Vermeulen - -.. index:: - pair: UNIX; file control - pair: UNIX; I/O control - ----------------- - -This module performs file control and I/O control on file descriptors. It is an -interface to the :c:func:`fcntl` and :c:func:`ioctl` Unix routines. For a -complete description of these calls, see :manpage:`fcntl(2)` and -:manpage:`ioctl(2)` Unix manual pages. - -.. availability:: Unix, not Emscripten, not WASI. - -All functions in this module take a file descriptor *fd* as their first -argument. This can be an integer file descriptor, such as returned by -``sys.stdin.fileno()``, or an :class:`io.IOBase` object, such as ``sys.stdin`` -itself, which provides a :meth:`~io.IOBase.fileno` that returns a genuine file -descriptor. - -.. versionchanged:: 3.3 - Operations in this module used to raise an :exc:`IOError` where they now - raise an :exc:`OSError`. - -.. versionchanged:: 3.8 - The fcntl module now contains ``F_ADD_SEALS``, ``F_GET_SEALS``, and - ``F_SEAL_*`` constants for sealing of :func:`os.memfd_create` file - descriptors. - -.. versionchanged:: 3.9 - On macOS, the fcntl module exposes the ``F_GETPATH`` constant, which obtains - the path of a file from a file descriptor. - On Linux(>=3.15), the fcntl module exposes the ``F_OFD_GETLK``, ``F_OFD_SETLK`` - and ``F_OFD_SETLKW`` constants, which are used when working with open file - description locks. - -.. versionchanged:: 3.10 - On Linux >= 2.6.11, the fcntl module exposes the ``F_GETPIPE_SZ`` and - ``F_SETPIPE_SZ`` constants, which allow to check and modify a pipe's size - respectively. - -.. versionchanged:: 3.11 - On FreeBSD, the fcntl module exposes the ``F_DUP2FD`` and ``F_DUP2FD_CLOEXEC`` - constants, which allow to duplicate a file descriptor, the latter setting - ``FD_CLOEXEC`` flag in addition. - -The module defines the following functions: - - -.. function:: fcntl(fd, cmd, arg=0) - - Perform the operation *cmd* on file descriptor *fd* (file objects providing - a :meth:`~io.IOBase.fileno` method are accepted as well). The values used - for *cmd* are operating system dependent, and are available as constants - in the :mod:`fcntl` module, using the same names as used in the relevant C - header files. The argument *arg* can either be an integer value, or a - :class:`bytes` object. With an integer value, the return value of this - function is the integer return value of the C :c:func:`fcntl` call. When - the argument is bytes it represents a binary structure, e.g. created by - :func:`struct.pack`. The binary data is copied to a buffer whose address is - passed to the C :c:func:`fcntl` call. The return value after a successful - call is the contents of the buffer, converted to a :class:`bytes` object. - The length of the returned object will be the same as the length of the - *arg* argument. This is limited to 1024 bytes. If the information returned - in the buffer by the operating system is larger than 1024 bytes, this is - most likely to result in a segmentation violation or a more subtle data - corruption. - - If the :c:func:`fcntl` fails, an :exc:`OSError` is raised. - - .. audit-event:: fcntl.fcntl fd,cmd,arg fcntl.fcntl - - -.. function:: ioctl(fd, request, arg=0, mutate_flag=True) - - This function is identical to the :func:`~fcntl.fcntl` function, except - that the argument handling is even more complicated. - - The *request* parameter is limited to values that can fit in 32-bits. - Additional constants of interest for use as the *request* argument can be - found in the :mod:`termios` module, under the same names as used in - the relevant C header files. - - The parameter *arg* can be one of an integer, an object supporting the - read-only buffer interface (like :class:`bytes`) or an object supporting - the read-write buffer interface (like :class:`bytearray`). - - In all but the last case, behaviour is as for the :func:`~fcntl.fcntl` - function. - - If a mutable buffer is passed, then the behaviour is determined by the value of - the *mutate_flag* parameter. - - If it is false, the buffer's mutability is ignored and behaviour is as for a - read-only buffer, except that the 1024 byte limit mentioned above is avoided -- - so long as the buffer you pass is at least as long as what the operating system - wants to put there, things should work. - - If *mutate_flag* is true (the default), then the buffer is (in effect) passed - to the underlying :func:`ioctl` system call, the latter's return code is - passed back to the calling Python, and the buffer's new contents reflect the - action of the :func:`ioctl`. This is a slight simplification, because if the - supplied buffer is less than 1024 bytes long it is first copied into a static - buffer 1024 bytes long which is then passed to :func:`ioctl` and copied back - into the supplied buffer. - - If the :c:func:`ioctl` fails, an :exc:`OSError` exception is raised. - - An example:: - - >>> import array, fcntl, struct, termios, os - >>> os.getpgrp() - 13341 - >>> struct.unpack('h', fcntl.ioctl(0, termios.TIOCGPGRP, " "))[0] - 13341 - >>> buf = array.array('h', [0]) - >>> fcntl.ioctl(0, termios.TIOCGPGRP, buf, 1) - 0 - >>> buf - array('h', [13341]) - - .. audit-event:: fcntl.ioctl fd,request,arg fcntl.ioctl - - -.. function:: flock(fd, operation) - - Perform the lock operation *operation* on file descriptor *fd* (file objects providing - a :meth:`~io.IOBase.fileno` method are accepted as well). See the Unix manual - :manpage:`flock(2)` for details. (On some systems, this function is emulated - using :c:func:`fcntl`.) - - If the :c:func:`flock` fails, an :exc:`OSError` exception is raised. - - .. audit-event:: fcntl.flock fd,operation fcntl.flock - - -.. function:: lockf(fd, cmd, len=0, start=0, whence=0) - - This is essentially a wrapper around the :func:`~fcntl.fcntl` locking calls. - *fd* is the file descriptor (file objects providing a :meth:`~io.IOBase.fileno` - method are accepted as well) of the file to lock or unlock, and *cmd* - is one of the following values: - - * :const:`LOCK_UN` -- unlock - * :const:`LOCK_SH` -- acquire a shared lock - * :const:`LOCK_EX` -- acquire an exclusive lock - - When *cmd* is :const:`LOCK_SH` or :const:`LOCK_EX`, it can also be - bitwise ORed with :const:`LOCK_NB` to avoid blocking on lock acquisition. - If :const:`LOCK_NB` is used and the lock cannot be acquired, an - :exc:`OSError` will be raised and the exception will have an *errno* - attribute set to :const:`EACCES` or :const:`EAGAIN` (depending on the - operating system; for portability, check for both values). On at least some - systems, :const:`LOCK_EX` can only be used if the file descriptor refers to a - file opened for writing. - - *len* is the number of bytes to lock, *start* is the byte offset at - which the lock starts, relative to *whence*, and *whence* is as with - :func:`io.IOBase.seek`, specifically: - - * ``0`` -- relative to the start of the file (:const:`os.SEEK_SET`) - * ``1`` -- relative to the current buffer position (:const:`os.SEEK_CUR`) - * ``2`` -- relative to the end of the file (:const:`os.SEEK_END`) - - The default for *start* is 0, which means to start at the beginning of the file. - The default for *len* is 0 which means to lock to the end of the file. The - default for *whence* is also 0. - - .. audit-event:: fcntl.lockf fd,cmd,len,start,whence fcntl.lockf - -Examples (all on a SVR4 compliant system):: - - import struct, fcntl, os - - f = open(...) - rv = fcntl.fcntl(f, fcntl.F_SETFL, os.O_NDELAY) - - lockdata = struct.pack('hhllhh', fcntl.F_WRLCK, 0, 0, 0, 0, 0) - rv = fcntl.fcntl(f, fcntl.F_SETLKW, lockdata) - -Note that in the first example the return value variable *rv* will hold an -integer value; in the second example it will hold a :class:`bytes` object. The -structure lay-out for the *lockdata* variable is system dependent --- therefore -using the :func:`flock` call may be better. - - -.. seealso:: - - Module :mod:`os` - If the locking flags :const:`~os.O_SHLOCK` and :const:`~os.O_EXLOCK` are - present in the :mod:`os` module (on BSD only), the :func:`os.open` - function provides an alternative to the :func:`lockf` and :func:`flock` - functions. diff --git a/Doc/library/filecmp.rst b/Doc/library/filecmp.rst deleted file mode 100644 index dfe4b7c59fd578..00000000000000 --- a/Doc/library/filecmp.rst +++ /dev/null @@ -1,207 +0,0 @@ -:mod:`filecmp` --- File and Directory Comparisons -================================================= - -.. module:: filecmp - :synopsis: Compare files efficiently. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/filecmp.py` - --------------- - -The :mod:`filecmp` module defines functions to compare files and directories, -with various optional time/correctness trade-offs. For comparing files, -see also the :mod:`difflib` module. - -The :mod:`filecmp` module defines the following functions: - - -.. function:: cmp(f1, f2, shallow=True) - - Compare the files named *f1* and *f2*, returning ``True`` if they seem equal, - ``False`` otherwise. - - If *shallow* is true and the :func:`os.stat` signatures (file type, size, and - modification time) of both files are identical, the files are taken to be - equal. - - Otherwise, the files are treated as different if their sizes or contents differ. - - Note that no external programs are called from this function, giving it - portability and efficiency. - - This function uses a cache for past comparisons and the results, - with cache entries invalidated if the :func:`os.stat` information for the - file changes. The entire cache may be cleared using :func:`clear_cache`. - - -.. function:: cmpfiles(dir1, dir2, common, shallow=True) - - Compare the files in the two directories *dir1* and *dir2* whose names are - given by *common*. - - Returns three lists of file names: *match*, *mismatch*, - *errors*. *match* contains the list of files that match, *mismatch* contains - the names of those that don't, and *errors* lists the names of files which - could not be compared. Files are listed in *errors* if they don't exist in - one of the directories, the user lacks permission to read them or if the - comparison could not be done for some other reason. - - The *shallow* parameter has the same meaning and default value as for - :func:`filecmp.cmp`. - - For example, ``cmpfiles('a', 'b', ['c', 'd/e'])`` will compare ``a/c`` with - ``b/c`` and ``a/d/e`` with ``b/d/e``. ``'c'`` and ``'d/e'`` will each be in - one of the three returned lists. - - -.. function:: clear_cache() - - Clear the filecmp cache. This may be useful if a file is compared so quickly - after it is modified that it is within the mtime resolution of - the underlying filesystem. - - .. versionadded:: 3.4 - - -.. _dircmp-objects: - -The :class:`dircmp` class -------------------------- - -.. class:: dircmp(a, b, ignore=None, hide=None) - - Construct a new directory comparison object, to compare the directories *a* - and *b*. *ignore* is a list of names to ignore, and defaults to - :const:`filecmp.DEFAULT_IGNORES`. *hide* is a list of names to hide, and - defaults to ``[os.curdir, os.pardir]``. - - The :class:`dircmp` class compares files by doing *shallow* comparisons - as described for :func:`filecmp.cmp`. - - The :class:`dircmp` class provides the following methods: - - .. method:: report() - - Print (to :data:`sys.stdout`) a comparison between *a* and *b*. - - .. method:: report_partial_closure() - - Print a comparison between *a* and *b* and common immediate - subdirectories. - - .. method:: report_full_closure() - - Print a comparison between *a* and *b* and common subdirectories - (recursively). - - The :class:`dircmp` class offers a number of interesting attributes that may be - used to get various bits of information about the directory trees being - compared. - - Note that via :meth:`~object.__getattr__` hooks, all attributes are computed lazily, - so there is no speed penalty if only those attributes which are lightweight - to compute are used. - - - .. attribute:: left - - The directory *a*. - - - .. attribute:: right - - The directory *b*. - - - .. attribute:: left_list - - Files and subdirectories in *a*, filtered by *hide* and *ignore*. - - - .. attribute:: right_list - - Files and subdirectories in *b*, filtered by *hide* and *ignore*. - - - .. attribute:: common - - Files and subdirectories in both *a* and *b*. - - - .. attribute:: left_only - - Files and subdirectories only in *a*. - - - .. attribute:: right_only - - Files and subdirectories only in *b*. - - - .. attribute:: common_dirs - - Subdirectories in both *a* and *b*. - - - .. attribute:: common_files - - Files in both *a* and *b*. - - - .. attribute:: common_funny - - Names in both *a* and *b*, such that the type differs between the - directories, or names for which :func:`os.stat` reports an error. - - - .. attribute:: same_files - - Files which are identical in both *a* and *b*, using the class's - file comparison operator. - - - .. attribute:: diff_files - - Files which are in both *a* and *b*, whose contents differ according - to the class's file comparison operator. - - - .. attribute:: funny_files - - Files which are in both *a* and *b*, but could not be compared. - - - .. attribute:: subdirs - - A dictionary mapping names in :attr:`common_dirs` to :class:`dircmp` - instances (or MyDirCmp instances if this instance is of type MyDirCmp, a - subclass of :class:`dircmp`). - - .. versionchanged:: 3.10 - Previously entries were always :class:`dircmp` instances. Now entries - are the same type as *self*, if *self* is a subclass of - :class:`dircmp`. - -.. attribute:: DEFAULT_IGNORES - - .. versionadded:: 3.4 - - List of directories ignored by :class:`dircmp` by default. - - -Here is a simplified example of using the ``subdirs`` attribute to search -recursively through two directories to show common different files:: - - >>> from filecmp import dircmp - >>> def print_diff_files(dcmp): - ... for name in dcmp.diff_files: - ... print("diff_file %s found in %s and %s" % (name, dcmp.left, - ... dcmp.right)) - ... for sub_dcmp in dcmp.subdirs.values(): - ... print_diff_files(sub_dcmp) - ... - >>> dcmp = dircmp('dir1', 'dir2') # doctest: +SKIP - >>> print_diff_files(dcmp) # doctest: +SKIP - diff --git a/Doc/library/fileformats.rst b/Doc/library/fileformats.rst deleted file mode 100644 index cab82b30ef42d7..00000000000000 --- a/Doc/library/fileformats.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. _fileformats: - -************ -File Formats -************ - -The modules described in this chapter parse various miscellaneous file formats -that aren't markup languages and are not related to e-mail. - - -.. toctree:: - - csv.rst - configparser.rst - tomllib.rst - netrc.rst - plistlib.rst diff --git a/Doc/library/fileinput.rst b/Doc/library/fileinput.rst deleted file mode 100644 index f93e9a58791eeb..00000000000000 --- a/Doc/library/fileinput.rst +++ /dev/null @@ -1,228 +0,0 @@ -:mod:`fileinput` --- Iterate over lines from multiple input streams -=================================================================== - -.. module:: fileinput - :synopsis: Loop over standard input or a list of files. - -.. moduleauthor:: Guido van Rossum -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/fileinput.py` - --------------- - -This module implements a helper class and functions to quickly write a -loop over standard input or a list of files. If you just want to read or -write one file see :func:`open`. - -The typical use is:: - - import fileinput - for line in fileinput.input(encoding="utf-8"): - process(line) - -This iterates over the lines of all files listed in ``sys.argv[1:]``, defaulting -to ``sys.stdin`` if the list is empty. If a filename is ``'-'``, it is also -replaced by ``sys.stdin`` and the optional arguments *mode* and *openhook* -are ignored. To specify an alternative list of filenames, pass it as the -first argument to :func:`.input`. A single file name is also allowed. - -All files are opened in text mode by default, but you can override this by -specifying the *mode* parameter in the call to :func:`.input` or -:class:`FileInput`. If an I/O error occurs during opening or reading a file, -:exc:`OSError` is raised. - -.. versionchanged:: 3.3 - :exc:`IOError` used to be raised; it is now an alias of :exc:`OSError`. - -If ``sys.stdin`` is used more than once, the second and further use will return -no lines, except perhaps for interactive use, or if it has been explicitly reset -(e.g. using ``sys.stdin.seek(0)``). - -Empty files are opened and immediately closed; the only time their presence in -the list of filenames is noticeable at all is when the last file opened is -empty. - -Lines are returned with any newlines intact, which means that the last line in -a file may not have one. - -You can control how files are opened by providing an opening hook via the -*openhook* parameter to :func:`fileinput.input` or :class:`FileInput()`. The -hook must be a function that takes two arguments, *filename* and *mode*, and -returns an accordingly opened file-like object. If *encoding* and/or *errors* -are specified, they will be passed to the hook as additional keyword arguments. -This module provides a :func:`hook_compressed` to support compressed files. - -The following function is the primary interface of this module: - - -.. function:: input(files=None, inplace=False, backup='', *, mode='r', openhook=None, encoding=None, errors=None) - - Create an instance of the :class:`FileInput` class. The instance will be used - as global state for the functions of this module, and is also returned to use - during iteration. The parameters to this function will be passed along to the - constructor of the :class:`FileInput` class. - - The :class:`FileInput` instance can be used as a context manager in the - :keyword:`with` statement. In this example, *input* is closed after the - :keyword:`!with` statement is exited, even if an exception occurs:: - - with fileinput.input(files=('spam.txt', 'eggs.txt'), encoding="utf-8") as f: - for line in f: - process(line) - - .. versionchanged:: 3.2 - Can be used as a context manager. - - .. versionchanged:: 3.8 - The keyword parameters *mode* and *openhook* are now keyword-only. - - .. versionchanged:: 3.10 - The keyword-only parameter *encoding* and *errors* are added. - - -The following functions use the global state created by :func:`fileinput.input`; -if there is no active state, :exc:`RuntimeError` is raised. - - -.. function:: filename() - - Return the name of the file currently being read. Before the first line has - been read, returns ``None``. - - -.. function:: fileno() - - Return the integer "file descriptor" for the current file. When no file is - opened (before the first line and between files), returns ``-1``. - - -.. function:: lineno() - - Return the cumulative line number of the line that has just been read. Before - the first line has been read, returns ``0``. After the last line of the last - file has been read, returns the line number of that line. - - -.. function:: filelineno() - - Return the line number in the current file. Before the first line has been - read, returns ``0``. After the last line of the last file has been read, - returns the line number of that line within the file. - - -.. function:: isfirstline() - - Return ``True`` if the line just read is the first line of its file, otherwise - return ``False``. - - -.. function:: isstdin() - - Return ``True`` if the last line was read from ``sys.stdin``, otherwise return - ``False``. - - -.. function:: nextfile() - - Close the current file so that the next iteration will read the first line from - the next file (if any); lines not read from the file will not count towards the - cumulative line count. The filename is not changed until after the first line - of the next file has been read. Before the first line has been read, this - function has no effect; it cannot be used to skip the first file. After the - last line of the last file has been read, this function has no effect. - - -.. function:: close() - - Close the sequence. - -The class which implements the sequence behavior provided by the module is -available for subclassing as well: - - -.. class:: FileInput(files=None, inplace=False, backup='', *, mode='r', openhook=None, encoding=None, errors=None) - - Class :class:`FileInput` is the implementation; its methods :meth:`filename`, - :meth:`fileno`, :meth:`lineno`, :meth:`filelineno`, :meth:`isfirstline`, - :meth:`isstdin`, :meth:`nextfile` and :meth:`close` correspond to the - functions of the same name in the module. In addition it is :term:`iterable` - and has a :meth:`~io.TextIOBase.readline` method which returns the next - input line. The sequence must be accessed in strictly sequential order; - random access and :meth:`~io.TextIOBase.readline` cannot be mixed. - - With *mode* you can specify which file mode will be passed to :func:`open`. It - must be one of ``'r'`` and ``'rb'``. - - The *openhook*, when given, must be a function that takes two arguments, - *filename* and *mode*, and returns an accordingly opened file-like object. You - cannot use *inplace* and *openhook* together. - - You can specify *encoding* and *errors* that is passed to :func:`open` or *openhook*. - - A :class:`FileInput` instance can be used as a context manager in the - :keyword:`with` statement. In this example, *input* is closed after the - :keyword:`!with` statement is exited, even if an exception occurs:: - - with FileInput(files=('spam.txt', 'eggs.txt')) as input: - process(input) - - .. versionchanged:: 3.2 - Can be used as a context manager. - - .. versionchanged:: 3.8 - The keyword parameter *mode* and *openhook* are now keyword-only. - - .. versionchanged:: 3.10 - The keyword-only parameter *encoding* and *errors* are added. - - .. versionchanged:: 3.11 - The ``'rU'`` and ``'U'`` modes and the :meth:`!__getitem__` method have - been removed. - - -**Optional in-place filtering:** if the keyword argument ``inplace=True`` is -passed to :func:`fileinput.input` or to the :class:`FileInput` constructor, the -file is moved to a backup file and standard output is directed to the input file -(if a file of the same name as the backup file already exists, it will be -replaced silently). This makes it possible to write a filter that rewrites its -input file in place. If the *backup* parameter is given (typically as -``backup='.'``), it specifies the extension for the backup file, -and the backup file remains around; by default, the extension is ``'.bak'`` and -it is deleted when the output file is closed. In-place filtering is disabled -when standard input is read. - - -The two following opening hooks are provided by this module: - -.. function:: hook_compressed(filename, mode, *, encoding=None, errors=None) - - Transparently opens files compressed with gzip and bzip2 (recognized by the - extensions ``'.gz'`` and ``'.bz2'``) using the :mod:`gzip` and :mod:`bz2` - modules. If the filename extension is not ``'.gz'`` or ``'.bz2'``, the file is - opened normally (ie, using :func:`open` without any decompression). - - The *encoding* and *errors* values are passed to :class:`io.TextIOWrapper` - for compressed files and open for normal files. - - Usage example: ``fi = fileinput.FileInput(openhook=fileinput.hook_compressed, encoding="utf-8")`` - - .. versionchanged:: 3.10 - The keyword-only parameter *encoding* and *errors* are added. - - -.. function:: hook_encoded(encoding, errors=None) - - Returns a hook which opens each file with :func:`open`, using the given - *encoding* and *errors* to read the file. - - Usage example: ``fi = - fileinput.FileInput(openhook=fileinput.hook_encoded("utf-8", - "surrogateescape"))`` - - .. versionchanged:: 3.6 - Added the optional *errors* parameter. - - .. deprecated:: 3.10 - This function is deprecated since :func:`fileinput.input` and :class:`FileInput` - now have *encoding* and *errors* parameters. diff --git a/Doc/library/filesys.rst b/Doc/library/filesys.rst deleted file mode 100644 index 0ccf2b7bf59a0f..00000000000000 --- a/Doc/library/filesys.rst +++ /dev/null @@ -1,38 +0,0 @@ -.. _filesys: - -************************* -File and Directory Access -************************* - -The modules described in this chapter deal with disk files and directories. For -example, there are modules for reading the properties of files, manipulating -paths in a portable way, and creating temporary files. The full list of modules -in this chapter is: - - -.. toctree:: - - pathlib.rst - os.path.rst - fileinput.rst - stat.rst - filecmp.rst - tempfile.rst - glob.rst - fnmatch.rst - linecache.rst - shutil.rst - - -.. seealso:: - - Module :mod:`os` - Operating system interfaces, including functions to work with files at a - lower level than Python :term:`file objects `. - - Module :mod:`io` - Python's built-in I/O library, including both abstract classes and - some concrete classes such as file I/O. - - Built-in function :func:`open` - The standard way to open files for reading and writing with Python. diff --git a/Doc/library/fnmatch.rst b/Doc/library/fnmatch.rst deleted file mode 100644 index aed8991d44772f..00000000000000 --- a/Doc/library/fnmatch.rst +++ /dev/null @@ -1,105 +0,0 @@ -:mod:`fnmatch` --- Unix filename pattern matching -================================================= - -.. module:: fnmatch - :synopsis: Unix shell style filename pattern matching. - -**Source code:** :source:`Lib/fnmatch.py` - -.. index:: single: filenames; wildcard expansion - -.. index:: pair: module; re - --------------- - -This module provides support for Unix shell-style wildcards, which are *not* the -same as regular expressions (which are documented in the :mod:`re` module). The -special characters used in shell-style wildcards are: - -.. index:: - single: * (asterisk); in glob-style wildcards - single: ? (question mark); in glob-style wildcards - single: [] (square brackets); in glob-style wildcards - single: ! (exclamation); in glob-style wildcards - single: - (minus); in glob-style wildcards - -+------------+------------------------------------+ -| Pattern | Meaning | -+============+====================================+ -| ``*`` | matches everything | -+------------+------------------------------------+ -| ``?`` | matches any single character | -+------------+------------------------------------+ -| ``[seq]`` | matches any character in *seq* | -+------------+------------------------------------+ -| ``[!seq]`` | matches any character not in *seq* | -+------------+------------------------------------+ - -For a literal match, wrap the meta-characters in brackets. -For example, ``'[?]'`` matches the character ``'?'``. - -.. index:: pair: module; glob - -Note that the filename separator (``'/'`` on Unix) is *not* special to this -module. See module :mod:`glob` for pathname expansion (:mod:`glob` uses -:func:`.filter` to match pathname segments). Similarly, filenames starting with -a period are not special for this module, and are matched by the ``*`` and ``?`` -patterns. - -Also note that :func:`functools.lru_cache` with the *maxsize* of 32768 is used to -cache the compiled regex patterns in the following functions: :func:`fnmatch`, -:func:`fnmatchcase`, :func:`.filter`. - -.. function:: fnmatch(filename, pattern) - - Test whether the *filename* string matches the *pattern* string, returning - :const:`True` or :const:`False`. Both parameters are case-normalized - using :func:`os.path.normcase`. :func:`fnmatchcase` can be used to perform a - case-sensitive comparison, regardless of whether that's standard for the - operating system. - - This example will print all file names in the current directory with the - extension ``.txt``:: - - import fnmatch - import os - - for file in os.listdir('.'): - if fnmatch.fnmatch(file, '*.txt'): - print(file) - - -.. function:: fnmatchcase(filename, pattern) - - Test whether *filename* matches *pattern*, returning :const:`True` or - :const:`False`; the comparison is case-sensitive and does not apply - :func:`os.path.normcase`. - - -.. function:: filter(names, pattern) - - Construct a list from those elements of the iterable *names* that match *pattern*. It is the same as - ``[n for n in names if fnmatch(n, pattern)]``, but implemented more efficiently. - - -.. function:: translate(pattern) - - Return the shell-style *pattern* converted to a regular expression for - using with :func:`re.match`. - - Example: - - >>> import fnmatch, re - >>> - >>> regex = fnmatch.translate('*.txt') - >>> regex - '(?s:.*\\.txt)\\Z' - >>> reobj = re.compile(regex) - >>> reobj.match('foobar.txt') - - - -.. seealso:: - - Module :mod:`glob` - Unix shell-style path expansion. diff --git a/Doc/library/fractions.rst b/Doc/library/fractions.rst deleted file mode 100644 index c751253a51c86b..00000000000000 --- a/Doc/library/fractions.rst +++ /dev/null @@ -1,191 +0,0 @@ -:mod:`fractions` --- Rational numbers -===================================== - -.. module:: fractions - :synopsis: Rational numbers. - -.. moduleauthor:: Jeffrey Yasskin -.. sectionauthor:: Jeffrey Yasskin - -**Source code:** :source:`Lib/fractions.py` - --------------- - -The :mod:`fractions` module provides support for rational number arithmetic. - - -A Fraction instance can be constructed from a pair of integers, from -another rational number, or from a string. - -.. class:: Fraction(numerator=0, denominator=1) - Fraction(other_fraction) - Fraction(float) - Fraction(decimal) - Fraction(string) - - The first version requires that *numerator* and *denominator* are instances - of :class:`numbers.Rational` and returns a new :class:`Fraction` instance - with value ``numerator/denominator``. If *denominator* is ``0``, it - raises a :exc:`ZeroDivisionError`. The second version requires that - *other_fraction* is an instance of :class:`numbers.Rational` and returns a - :class:`Fraction` instance with the same value. The next two versions accept - either a :class:`float` or a :class:`decimal.Decimal` instance, and return a - :class:`Fraction` instance with exactly the same value. Note that due to the - usual issues with binary floating-point (see :ref:`tut-fp-issues`), the - argument to ``Fraction(1.1)`` is not exactly equal to 11/10, and so - ``Fraction(1.1)`` does *not* return ``Fraction(11, 10)`` as one might expect. - (But see the documentation for the :meth:`limit_denominator` method below.) - The last version of the constructor expects a string or unicode instance. - The usual form for this instance is:: - - [sign] numerator ['/' denominator] - - where the optional ``sign`` may be either '+' or '-' and - ``numerator`` and ``denominator`` (if present) are strings of - decimal digits (underscores may be used to delimit digits as with - integral literals in code). In addition, any string that represents a finite - value and is accepted by the :class:`float` constructor is also - accepted by the :class:`Fraction` constructor. In either form the - input string may also have leading and/or trailing whitespace. - Here are some examples:: - - >>> from fractions import Fraction - >>> Fraction(16, -10) - Fraction(-8, 5) - >>> Fraction(123) - Fraction(123, 1) - >>> Fraction() - Fraction(0, 1) - >>> Fraction('3/7') - Fraction(3, 7) - >>> Fraction(' -3/7 ') - Fraction(-3, 7) - >>> Fraction('1.414213 \t\n') - Fraction(1414213, 1000000) - >>> Fraction('-.125') - Fraction(-1, 8) - >>> Fraction('7e-6') - Fraction(7, 1000000) - >>> Fraction(2.25) - Fraction(9, 4) - >>> Fraction(1.1) - Fraction(2476979795053773, 2251799813685248) - >>> from decimal import Decimal - >>> Fraction(Decimal('1.1')) - Fraction(11, 10) - - - The :class:`Fraction` class inherits from the abstract base class - :class:`numbers.Rational`, and implements all of the methods and - operations from that class. :class:`Fraction` instances are :term:`hashable`, - and should be treated as immutable. In addition, - :class:`Fraction` has the following properties and methods: - - .. versionchanged:: 3.2 - The :class:`Fraction` constructor now accepts :class:`float` and - :class:`decimal.Decimal` instances. - - .. versionchanged:: 3.9 - The :func:`math.gcd` function is now used to normalize the *numerator* - and *denominator*. :func:`math.gcd` always return a :class:`int` type. - Previously, the GCD type depended on *numerator* and *denominator*. - - .. versionchanged:: 3.11 - Underscores are now permitted when creating a :class:`Fraction` instance - from a string, following :PEP:`515` rules. - - .. versionchanged:: 3.11 - :class:`Fraction` implements ``__int__`` now to satisfy - ``typing.SupportsInt`` instance checks. - - .. attribute:: numerator - - Numerator of the Fraction in lowest term. - - .. attribute:: denominator - - Denominator of the Fraction in lowest term. - - - .. method:: as_integer_ratio() - - Return a tuple of two integers, whose ratio is equal - to the Fraction and with a positive denominator. - - .. versionadded:: 3.8 - - .. classmethod:: from_float(flt) - - Alternative constructor which only accepts instances of - :class:`float` or :class:`numbers.Integral`. Beware that - ``Fraction.from_float(0.3)`` is not the same value as ``Fraction(3, 10)``. - - .. note:: - - From Python 3.2 onwards, you can also construct a - :class:`Fraction` instance directly from a :class:`float`. - - - .. classmethod:: from_decimal(dec) - - Alternative constructor which only accepts instances of - :class:`decimal.Decimal` or :class:`numbers.Integral`. - - .. note:: - - From Python 3.2 onwards, you can also construct a - :class:`Fraction` instance directly from a :class:`decimal.Decimal` - instance. - - - .. method:: limit_denominator(max_denominator=1000000) - - Finds and returns the closest :class:`Fraction` to ``self`` that has - denominator at most max_denominator. This method is useful for finding - rational approximations to a given floating-point number: - - >>> from fractions import Fraction - >>> Fraction('3.1415926535897932').limit_denominator(1000) - Fraction(355, 113) - - or for recovering a rational number that's represented as a float: - - >>> from math import pi, cos - >>> Fraction(cos(pi/3)) - Fraction(4503599627370497, 9007199254740992) - >>> Fraction(cos(pi/3)).limit_denominator() - Fraction(1, 2) - >>> Fraction(1.1).limit_denominator() - Fraction(11, 10) - - - .. method:: __floor__() - - Returns the greatest :class:`int` ``<= self``. This method can - also be accessed through the :func:`math.floor` function: - - >>> from math import floor - >>> floor(Fraction(355, 113)) - 3 - - - .. method:: __ceil__() - - Returns the least :class:`int` ``>= self``. This method can - also be accessed through the :func:`math.ceil` function. - - - .. method:: __round__() - __round__(ndigits) - - The first version returns the nearest :class:`int` to ``self``, - rounding half to even. The second version rounds ``self`` to the - nearest multiple of ``Fraction(1, 10**ndigits)`` (logically, if - ``ndigits`` is negative), again rounding half toward even. This - method can also be accessed through the :func:`round` function. - - -.. seealso:: - - Module :mod:`numbers` - The abstract base classes making up the numeric tower. diff --git a/Doc/library/frameworks.rst b/Doc/library/frameworks.rst deleted file mode 100644 index 15ceeec9c255ed..00000000000000 --- a/Doc/library/frameworks.rst +++ /dev/null @@ -1,18 +0,0 @@ -.. _frameworks: - -****************** -Program Frameworks -****************** - -The modules described in this chapter are frameworks that will largely dictate -the structure of your program. Currently the modules described here are all -oriented toward writing command-line interfaces. - -The full list of modules described in this chapter is: - - -.. toctree:: - - turtle.rst - cmd.rst - shlex.rst diff --git a/Doc/library/ftplib.rst b/Doc/library/ftplib.rst deleted file mode 100644 index dcac83bbe777f6..00000000000000 --- a/Doc/library/ftplib.rst +++ /dev/null @@ -1,467 +0,0 @@ -:mod:`ftplib` --- FTP protocol client -===================================== - -.. module:: ftplib - :synopsis: FTP protocol client (requires sockets). - -**Source code:** :source:`Lib/ftplib.py` - -.. index:: - pair: FTP; protocol - single: FTP; ftplib (standard module) - --------------- - -This module defines the class :class:`FTP` and a few related items. The -:class:`FTP` class implements the client side of the FTP protocol. You can use -this to write Python programs that perform a variety of automated FTP jobs, such -as mirroring other FTP servers. It is also used by the module -:mod:`urllib.request` to handle URLs that use FTP. For more information on FTP -(File Transfer Protocol), see internet :rfc:`959`. - -The default encoding is UTF-8, following :rfc:`2640`. - -.. include:: ../includes/wasm-notavail.rst - -Here's a sample session using the :mod:`ftplib` module:: - - >>> from ftplib import FTP - >>> ftp = FTP('ftp.us.debian.org') # connect to host, default port - >>> ftp.login() # user anonymous, passwd anonymous@ - '230 Login successful.' - >>> ftp.cwd('debian') # change into "debian" directory - '250 Directory successfully changed.' - >>> ftp.retrlines('LIST') # list directory contents - -rw-rw-r-- 1 1176 1176 1063 Jun 15 10:18 README - ... - drwxr-sr-x 5 1176 1176 4096 Dec 19 2000 pool - drwxr-sr-x 4 1176 1176 4096 Nov 17 2008 project - drwxr-xr-x 3 1176 1176 4096 Oct 10 2012 tools - '226 Directory send OK.' - >>> with open('README', 'wb') as fp: - >>> ftp.retrbinary('RETR README', fp.write) - '226 Transfer complete.' - >>> ftp.quit() - '221 Goodbye.' - - -The module defines the following items: - -.. class:: FTP(host='', user='', passwd='', acct='', timeout=None, source_address=None, *, encoding='utf-8') - - Return a new instance of the :class:`FTP` class. When *host* is given, the - method call ``connect(host)`` is made. When *user* is given, additionally - the method call ``login(user, passwd, acct)`` is made (where *passwd* and - *acct* default to the empty string when not given). The optional *timeout* - parameter specifies a timeout in seconds for blocking operations like the - connection attempt (if is not specified, the global default timeout setting - will be used). *source_address* is a 2-tuple ``(host, port)`` for the socket - to bind to as its source address before connecting. The *encoding* parameter - specifies the encoding for directories and filenames. - - The :class:`FTP` class supports the :keyword:`with` statement, e.g.: - - >>> from ftplib import FTP - >>> with FTP("ftp1.at.proftpd.org") as ftp: - ... ftp.login() - ... ftp.dir() - ... # doctest: +SKIP - '230 Anonymous login ok, restrictions apply.' - dr-xr-xr-x 9 ftp ftp 154 May 6 10:43 . - dr-xr-xr-x 9 ftp ftp 154 May 6 10:43 .. - dr-xr-xr-x 5 ftp ftp 4096 May 6 10:43 CentOS - dr-xr-xr-x 3 ftp ftp 18 Jul 10 2008 Fedora - >>> - - .. versionchanged:: 3.2 - Support for the :keyword:`with` statement was added. - - .. versionchanged:: 3.3 - *source_address* parameter was added. - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - The *encoding* parameter was added, and the default was changed from - Latin-1 to UTF-8 to follow :rfc:`2640`. - -.. class:: FTP_TLS(host='', user='', passwd='', acct='', keyfile=None, certfile=None, context=None, timeout=None, source_address=None, *, encoding='utf-8') - - A :class:`FTP` subclass which adds TLS support to FTP as described in - :rfc:`4217`. - Connect as usual to port 21 implicitly securing the FTP control connection - before authenticating. Securing the data connection requires the user to - explicitly ask for it by calling the :meth:`prot_p` method. *context* - is a :class:`ssl.SSLContext` object which allows bundling SSL configuration - options, certificates and private keys into a single (potentially - long-lived) structure. Please read :ref:`ssl-security` for best practices. - - *keyfile* and *certfile* are a legacy alternative to *context* -- they - can point to PEM-formatted private key and certificate chain files - (respectively) for the SSL connection. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - *source_address* parameter was added. - - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :const:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - The *encoding* parameter was added, and the default was changed from - Latin-1 to UTF-8 to follow :rfc:`2640`. - - Here's a sample session using the :class:`FTP_TLS` class:: - - >>> ftps = FTP_TLS('ftp.pureftpd.org') - >>> ftps.login() - '230 Anonymous user logged in' - >>> ftps.prot_p() - '200 Data protection level set to "private"' - >>> ftps.nlst() - ['6jack', 'OpenBSD', 'antilink', 'blogbench', 'bsdcam', 'clockspeed', 'djbdns-jedi', 'docs', 'eaccelerator-jedi', 'favicon.ico', 'francotone', 'fugu', 'ignore', 'libpuzzle', 'metalog', 'minidentd', 'misc', 'mysql-udf-global-user-variables', 'php-jenkins-hash', 'php-skein-hash', 'php-webdav', 'phpaudit', 'phpbench', 'pincaster', 'ping', 'posto', 'pub', 'public', 'public_keys', 'pure-ftpd', 'qscan', 'qtc', 'sharedance', 'skycache', 'sound', 'tmp', 'ucarp'] - - -.. exception:: error_reply - - Exception raised when an unexpected reply is received from the server. - - -.. exception:: error_temp - - Exception raised when an error code signifying a temporary error (response - codes in the range 400--499) is received. - - -.. exception:: error_perm - - Exception raised when an error code signifying a permanent error (response - codes in the range 500--599) is received. - - -.. exception:: error_proto - - Exception raised when a reply is received from the server that does not fit - the response specifications of the File Transfer Protocol, i.e. begin with a - digit in the range 1--5. - - -.. data:: all_errors - - The set of all exceptions (as a tuple) that methods of :class:`FTP` - instances may raise as a result of problems with the FTP connection (as - opposed to programming errors made by the caller). This set includes the - four exceptions listed above as well as :exc:`OSError` and :exc:`EOFError`. - - -.. seealso:: - - Module :mod:`netrc` - Parser for the :file:`.netrc` file format. The file :file:`.netrc` is - typically used by FTP clients to load user authentication information - before prompting the user. - - -.. _ftp-objects: - -FTP Objects ------------ - -Several methods are available in two flavors: one for handling text files and -another for binary files. These are named for the command which is used -followed by ``lines`` for the text version or ``binary`` for the binary version. - -:class:`FTP` instances have the following methods: - - -.. method:: FTP.set_debuglevel(level) - - Set the instance's debugging level. This controls the amount of debugging - output printed. The default, ``0``, produces no debugging output. A value of - ``1`` produces a moderate amount of debugging output, generally a single line - per request. A value of ``2`` or higher produces the maximum amount of - debugging output, logging each line sent and received on the control connection. - - -.. method:: FTP.connect(host='', port=0, timeout=None, source_address=None) - - Connect to the given host and port. The default port number is ``21``, as - specified by the FTP protocol specification. It is rarely needed to specify a - different port number. This function should be called only once for each - instance; it should not be called at all if a host was given when the instance - was created. All other methods can only be used after a connection has been - made. - The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt. If no *timeout* is passed, the global default timeout - setting will be used. - *source_address* is a 2-tuple ``(host, port)`` for the socket to bind to as - its source address before connecting. - - .. audit-event:: ftplib.connect self,host,port ftplib.FTP.connect - - .. versionchanged:: 3.3 - *source_address* parameter was added. - - -.. method:: FTP.getwelcome() - - Return the welcome message sent by the server in reply to the initial - connection. (This message sometimes contains disclaimers or help information - that may be relevant to the user.) - - -.. method:: FTP.login(user='anonymous', passwd='', acct='') - - Log in as the given *user*. The *passwd* and *acct* parameters are optional and - default to the empty string. If no *user* is specified, it defaults to - ``'anonymous'``. If *user* is ``'anonymous'``, the default *passwd* is - ``'anonymous@'``. This function should be called only once for each instance, - after a connection has been established; it should not be called at all if a - host and user were given when the instance was created. Most FTP commands are - only allowed after the client has logged in. The *acct* parameter supplies - "accounting information"; few systems implement this. - - -.. method:: FTP.abort() - - Abort a file transfer that is in progress. Using this does not always work, but - it's worth a try. - - -.. method:: FTP.sendcmd(cmd) - - Send a simple command string to the server and return the response string. - - .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.sendcmd - - -.. method:: FTP.voidcmd(cmd) - - Send a simple command string to the server and handle the response. Return - nothing if a response code corresponding to success (codes in the range - 200--299) is received. Raise :exc:`error_reply` otherwise. - - .. audit-event:: ftplib.sendcmd self,cmd ftplib.FTP.voidcmd - - -.. method:: FTP.retrbinary(cmd, callback, blocksize=8192, rest=None) - - Retrieve a file in binary transfer mode. *cmd* should be an appropriate - ``RETR`` command: ``'RETR filename'``. The *callback* function is called for - each block of data received, with a single bytes argument giving the data - block. The optional *blocksize* argument specifies the maximum chunk size to - read on the low-level socket object created to do the actual transfer (which - will also be the largest size of the data blocks passed to *callback*). A - reasonable default is chosen. *rest* means the same thing as in the - :meth:`transfercmd` method. - - -.. method:: FTP.retrlines(cmd, callback=None) - - Retrieve a file or directory listing in the encoding specified by the - *encoding* parameter at initialization. - *cmd* should be an appropriate ``RETR`` command (see :meth:`retrbinary`) or - a command such as ``LIST`` or ``NLST`` (usually just the string ``'LIST'``). - ``LIST`` retrieves a list of files and information about those files. - ``NLST`` retrieves a list of file names. - The *callback* function is called for each line with a string argument - containing the line with the trailing CRLF stripped. The default *callback* - prints the line to ``sys.stdout``. - - -.. method:: FTP.set_pasv(val) - - Enable "passive" mode if *val* is true, otherwise disable passive mode. - Passive mode is on by default. - - -.. method:: FTP.storbinary(cmd, fp, blocksize=8192, callback=None, rest=None) - - Store a file in binary transfer mode. *cmd* should be an appropriate - ``STOR`` command: ``"STOR filename"``. *fp* is a :term:`file object` - (opened in binary mode) which is read until EOF using its :meth:`~io.IOBase.read` - method in blocks of size *blocksize* to provide the data to be stored. - The *blocksize* argument defaults to 8192. *callback* is an optional single - parameter callable that is called on each block of data after it is sent. - *rest* means the same thing as in the :meth:`transfercmd` method. - - .. versionchanged:: 3.2 - *rest* parameter added. - - -.. method:: FTP.storlines(cmd, fp, callback=None) - - Store a file in line mode. *cmd* should be an appropriate - ``STOR`` command (see :meth:`storbinary`). Lines are read until EOF from the - :term:`file object` *fp* (opened in binary mode) using its :meth:`~io.IOBase.readline` - method to provide the data to be stored. *callback* is an optional single - parameter callable that is called on each line after it is sent. - - -.. method:: FTP.transfercmd(cmd, rest=None) - - Initiate a transfer over the data connection. If the transfer is active, send an - ``EPRT`` or ``PORT`` command and the transfer command specified by *cmd*, and - accept the connection. If the server is passive, send an ``EPSV`` or ``PASV`` - command, connect to it, and start the transfer command. Either way, return the - socket for the connection. - - If optional *rest* is given, a ``REST`` command is sent to the server, passing - *rest* as an argument. *rest* is usually a byte offset into the requested file, - telling the server to restart sending the file's bytes at the requested offset, - skipping over the initial bytes. Note however that the :meth:`transfercmd` - method converts *rest* to a string with the *encoding* parameter specified - at initialization, but no check is performed on the string's contents. If the - server does not recognize the ``REST`` command, an :exc:`error_reply` exception - will be raised. If this happens, simply call :meth:`transfercmd` without a - *rest* argument. - - -.. method:: FTP.ntransfercmd(cmd, rest=None) - - Like :meth:`transfercmd`, but returns a tuple of the data connection and the - expected size of the data. If the expected size could not be computed, ``None`` - will be returned as the expected size. *cmd* and *rest* means the same thing as - in :meth:`transfercmd`. - - -.. method:: FTP.mlsd(path="", facts=[]) - - List a directory in a standardized format by using ``MLSD`` command - (:rfc:`3659`). If *path* is omitted the current directory is assumed. - *facts* is a list of strings representing the type of information desired - (e.g. ``["type", "size", "perm"]``). Return a generator object yielding a - tuple of two elements for every file found in path. First element is the - file name, the second one is a dictionary containing facts about the file - name. Content of this dictionary might be limited by the *facts* argument - but server is not guaranteed to return all requested facts. - - .. versionadded:: 3.3 - - -.. method:: FTP.nlst(argument[, ...]) - - Return a list of file names as returned by the ``NLST`` command. The - optional *argument* is a directory to list (default is the current server - directory). Multiple arguments can be used to pass non-standard options to - the ``NLST`` command. - - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. - - -.. method:: FTP.dir(argument[, ...]) - - Produce a directory listing as returned by the ``LIST`` command, printing it to - standard output. The optional *argument* is a directory to list (default is the - current server directory). Multiple arguments can be used to pass non-standard - options to the ``LIST`` command. If the last argument is a function, it is used - as a *callback* function as for :meth:`retrlines`; the default prints to - ``sys.stdout``. This method returns ``None``. - - .. note:: If your server supports the command, :meth:`mlsd` offers a better API. - - -.. method:: FTP.rename(fromname, toname) - - Rename file *fromname* on the server to *toname*. - - -.. method:: FTP.delete(filename) - - Remove the file named *filename* from the server. If successful, returns the - text of the response, otherwise raises :exc:`error_perm` on permission errors or - :exc:`error_reply` on other errors. - - -.. method:: FTP.cwd(pathname) - - Set the current directory on the server. - - -.. method:: FTP.mkd(pathname) - - Create a new directory on the server. - - -.. method:: FTP.pwd() - - Return the pathname of the current directory on the server. - - -.. method:: FTP.rmd(dirname) - - Remove the directory named *dirname* on the server. - - -.. method:: FTP.size(filename) - - Request the size of the file named *filename* on the server. On success, the - size of the file is returned as an integer, otherwise ``None`` is returned. - Note that the ``SIZE`` command is not standardized, but is supported by many - common server implementations. - - -.. method:: FTP.quit() - - Send a ``QUIT`` command to the server and close the connection. This is the - "polite" way to close a connection, but it may raise an exception if the server - responds with an error to the ``QUIT`` command. This implies a call to the - :meth:`close` method which renders the :class:`FTP` instance useless for - subsequent calls (see below). - - -.. method:: FTP.close() - - Close the connection unilaterally. This should not be applied to an already - closed connection such as after a successful call to :meth:`~FTP.quit`. - After this call the :class:`FTP` instance should not be used any more (after - a call to :meth:`close` or :meth:`~FTP.quit` you cannot reopen the - connection by issuing another :meth:`login` method). - - -FTP_TLS Objects ---------------- - -:class:`FTP_TLS` class inherits from :class:`FTP`, defining these additional objects: - -.. attribute:: FTP_TLS.ssl_version - - The SSL version to use (defaults to :data:`ssl.PROTOCOL_SSLv23`). - -.. method:: FTP_TLS.auth() - - Set up a secure control connection by using TLS or SSL, depending on what - is specified in the :attr:`ssl_version` attribute. - - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :const:`ssl.HAS_SNI`). - -.. method:: FTP_TLS.ccc() - - Revert control channel back to plaintext. This can be useful to take - advantage of firewalls that know how to handle NAT with non-secure FTP - without opening fixed ports. - - .. versionadded:: 3.3 - -.. method:: FTP_TLS.prot_p() - - Set up secure data connection. - -.. method:: FTP_TLS.prot_c() - - Set up clear text data connection. diff --git a/Doc/library/functional.rst b/Doc/library/functional.rst deleted file mode 100644 index 5b6185a65cd12c..00000000000000 --- a/Doc/library/functional.rst +++ /dev/null @@ -1,15 +0,0 @@ -****************************** -Functional Programming Modules -****************************** - -The modules described in this chapter provide functions and classes that support -a functional programming style, and general operations on callables. - -The following modules are documented in this chapter: - - -.. toctree:: - - itertools.rst - functools.rst - operator.rst diff --git a/Doc/library/functions.rst b/Doc/library/functions.rst deleted file mode 100644 index c287f52156ba63..00000000000000 --- a/Doc/library/functions.rst +++ /dev/null @@ -1,2061 +0,0 @@ -.. XXX document all delegations to __special__ methods -.. _built-in-funcs: - -Built-in Functions -================== - -The Python interpreter has a number of functions and types built into it that -are always available. They are listed here in alphabetical order. - -+---------------------------------------------------------------------------------------------------+ -| Built-in Functions | -+=========================+=======================+=======================+=========================+ -| | **A** | | **E** | | **L** | | **R** | -| | :func:`abs` | | :func:`enumerate` | | :func:`len` | | |func-range|_ | -| | :func:`aiter` | | :func:`eval` | | |func-list|_ | | :func:`repr` | -| | :func:`all` | | :func:`exec` | | :func:`locals` | | :func:`reversed` | -| | :func:`anext` | | | | | | :func:`round` | -| | :func:`any` | | **F** | | **M** | | | -| | :func:`ascii` | | :func:`filter` | | :func:`map` | | **S** | -| | | | :func:`float` | | :func:`max` | | |func-set|_ | -| | **B** | | :func:`format` | | |func-memoryview|_ | | :func:`setattr` | -| | :func:`bin` | | |func-frozenset|_ | | :func:`min` | | :func:`slice` | -| | :func:`bool` | | | | | | :func:`sorted` | -| | :func:`breakpoint` | | **G** | | **N** | | :func:`staticmethod` | -| | |func-bytearray|_ | | :func:`getattr` | | :func:`next` | | |func-str|_ | -| | |func-bytes|_ | | :func:`globals` | | | | :func:`sum` | -| | | | | | **O** | | :func:`super` | -| | **C** | | **H** | | :func:`object` | | | -| | :func:`callable` | | :func:`hasattr` | | :func:`oct` | | **T** | -| | :func:`chr` | | :func:`hash` | | :func:`open` | | |func-tuple|_ | -| | :func:`classmethod` | | :func:`help` | | :func:`ord` | | :func:`type` | -| | :func:`compile` | | :func:`hex` | | | | | -| | :func:`complex` | | | | **P** | | **V** | -| | | | **I** | | :func:`pow` | | :func:`vars` | -| | **D** | | :func:`id` | | :func:`print` | | | -| | :func:`delattr` | | :func:`input` | | :func:`property` | | **Z** | -| | |func-dict|_ | | :func:`int` | | | | :func:`zip` | -| | :func:`dir` | | :func:`isinstance` | | | | | -| | :func:`divmod` | | :func:`issubclass` | | | | **_** | -| | | | :func:`iter` | | | | :func:`__import__` | -+-------------------------+-----------------------+-----------------------+-------------------------+ - -.. using :func:`dict` would create a link to another page, so local targets are - used, with replacement texts to make the output in the table consistent - -.. |func-dict| replace:: ``dict()`` -.. |func-frozenset| replace:: ``frozenset()`` -.. |func-memoryview| replace:: ``memoryview()`` -.. |func-set| replace:: ``set()`` -.. |func-list| replace:: ``list()`` -.. |func-str| replace:: ``str()`` -.. |func-tuple| replace:: ``tuple()`` -.. |func-range| replace:: ``range()`` -.. |func-bytearray| replace:: ``bytearray()`` -.. |func-bytes| replace:: ``bytes()`` - -.. function:: abs(x) - - Return the absolute value of a number. The argument may be an - integer, a floating point number, or an object implementing :meth:`__abs__`. - If the argument is a complex number, its magnitude is returned. - - -.. function:: aiter(async_iterable) - - Return an :term:`asynchronous iterator` for an :term:`asynchronous iterable`. - Equivalent to calling ``x.__aiter__()``. - - Note: Unlike :func:`iter`, :func:`aiter` has no 2-argument variant. - - .. versionadded:: 3.10 - -.. function:: all(iterable) - - Return ``True`` if all elements of the *iterable* are true (or if the iterable - is empty). Equivalent to:: - - def all(iterable): - for element in iterable: - if not element: - return False - return True - - -.. awaitablefunction:: anext(async_iterator) - anext(async_iterator, default) - - When awaited, return the next item from the given :term:`asynchronous - iterator`, or *default* if given and the iterator is exhausted. - - This is the async variant of the :func:`next` builtin, and behaves - similarly. - - This calls the :meth:`~object.__anext__` method of *async_iterator*, - returning an :term:`awaitable`. Awaiting this returns the next value of the - iterator. If *default* is given, it is returned if the iterator is exhausted, - otherwise :exc:`StopAsyncIteration` is raised. - - .. versionadded:: 3.10 - -.. function:: any(iterable) - - Return ``True`` if any element of the *iterable* is true. If the iterable - is empty, return ``False``. Equivalent to:: - - def any(iterable): - for element in iterable: - if element: - return True - return False - - -.. function:: ascii(object) - - As :func:`repr`, return a string containing a printable representation of an - object, but escape the non-ASCII characters in the string returned by - :func:`repr` using ``\x``, ``\u``, or ``\U`` escapes. This generates a string - similar to that returned by :func:`repr` in Python 2. - - -.. function:: bin(x) - - Convert an integer number to a binary string prefixed with "0b". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it - has to define an :meth:`~object.__index__` method that returns an integer. Some - examples: - - >>> bin(3) - '0b11' - >>> bin(-10) - '-0b1010' - - If the prefix "0b" is desired or not, you can use either of the following ways. - - >>> format(14, '#b'), format(14, 'b') - ('0b1110', '1110') - >>> f'{14:#b}', f'{14:b}' - ('0b1110', '1110') - - See also :func:`format` for more information. - - -.. class:: bool(x=False) - - Return a Boolean value, i.e. one of ``True`` or ``False``. *x* is converted - using the standard :ref:`truth testing procedure `. If *x* is false - or omitted, this returns ``False``; otherwise, it returns ``True``. The - :class:`bool` class is a subclass of :class:`int` (see :ref:`typesnumeric`). - It cannot be subclassed further. Its only instances are ``False`` and - ``True`` (see :ref:`bltin-boolean-values`). - - .. index:: pair: Boolean; type - - .. versionchanged:: 3.7 - *x* is now a positional-only parameter. - -.. function:: breakpoint(*args, **kws) - - This function drops you into the debugger at the call site. Specifically, - it calls :func:`sys.breakpointhook`, passing ``args`` and ``kws`` straight - through. By default, ``sys.breakpointhook()`` calls - :func:`pdb.set_trace()` expecting no arguments. In this case, it is - purely a convenience function so you don't have to explicitly import - :mod:`pdb` or type as much code to enter the debugger. However, - :func:`sys.breakpointhook` can be set to some other function and - :func:`breakpoint` will automatically call that, allowing you to drop into - the debugger of choice. - If :func:`sys.breakpointhook` is not accessible, this function will - raise :exc:`RuntimeError`. - - By default, the behavior of :func:`breakpoint` can be changed with - the :envvar:`PYTHONBREAKPOINT` environment variable. - See :func:`sys.breakpointhook` for usage details. - - Note that this is not guaranteed if :func:`sys.breakpointhook` - has been replaced. - - .. audit-event:: builtins.breakpoint breakpointhook breakpoint - - .. versionadded:: 3.7 - -.. _func-bytearray: -.. class:: bytearray(source=b'') - bytearray(source, encoding) - bytearray(source, encoding, errors) - :noindex: - - Return a new array of bytes. The :class:`bytearray` class is a mutable - sequence of integers in the range 0 <= x < 256. It has most of the usual - methods of mutable sequences, described in :ref:`typesseq-mutable`, as well - as most methods that the :class:`bytes` type has, see :ref:`bytes-methods`. - - The optional *source* parameter can be used to initialize the array in a few - different ways: - - * If it is a *string*, you must also give the *encoding* (and optionally, - *errors*) parameters; :func:`bytearray` then converts the string to - bytes using :meth:`str.encode`. - - * If it is an *integer*, the array will have that size and will be - initialized with null bytes. - - * If it is an object conforming to the :ref:`buffer interface `, - a read-only buffer of the object will be used to initialize the bytes array. - - * If it is an *iterable*, it must be an iterable of integers in the range - ``0 <= x < 256``, which are used as the initial contents of the array. - - Without an argument, an array of size 0 is created. - - See also :ref:`binaryseq` and :ref:`typebytearray`. - - -.. _func-bytes: -.. class:: bytes(source=b'') - bytes(source, encoding) - bytes(source, encoding, errors) - :noindex: - - Return a new "bytes" object which is an immutable sequence of integers in - the range ``0 <= x < 256``. :class:`bytes` is an immutable version of - :class:`bytearray` -- it has the same non-mutating methods and the same - indexing and slicing behavior. - - Accordingly, constructor arguments are interpreted as for :func:`bytearray`. - - Bytes objects can also be created with literals, see :ref:`strings`. - - See also :ref:`binaryseq`, :ref:`typebytes`, and :ref:`bytes-methods`. - - -.. function:: callable(object) - - Return :const:`True` if the *object* argument appears callable, - :const:`False` if not. If this returns ``True``, it is still possible that a - call fails, but if it is ``False``, calling *object* will never succeed. - Note that classes are callable (calling a class returns a new instance); - instances are callable if their class has a :meth:`__call__` method. - - .. versionadded:: 3.2 - This function was first removed in Python 3.0 and then brought back - in Python 3.2. - - -.. function:: chr(i) - - Return the string representing a character whose Unicode code point is the - integer *i*. For example, ``chr(97)`` returns the string ``'a'``, while - ``chr(8364)`` returns the string ``'€'``. This is the inverse of :func:`ord`. - - The valid range for the argument is from 0 through 1,114,111 (0x10FFFF in - base 16). :exc:`ValueError` will be raised if *i* is outside that range. - - -.. decorator:: classmethod - - Transform a method into a class method. - - A class method receives the class as an implicit first argument, just like an - instance method receives the instance. To declare a class method, use this - idiom:: - - class C: - @classmethod - def f(cls, arg1, arg2): ... - - The ``@classmethod`` form is a function :term:`decorator` -- see - :ref:`function` for details. - - A class method can be called either on the class (such as ``C.f()``) or on an instance (such - as ``C().f()``). The instance is ignored except for its class. If a class - method is called for a derived class, the derived class object is passed as the - implied first argument. - - Class methods are different than C++ or Java static methods. If you want those, - see :func:`staticmethod` in this section. - For more information on class methods, see :ref:`types`. - - .. versionchanged:: 3.9 - Class methods can now wrap other :term:`descriptors ` such as - :func:`property`. - - .. versionchanged:: 3.10 - Class methods now inherit the method attributes (``__module__``, - ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``) and - have a new ``__wrapped__`` attribute. - - .. versionchanged:: 3.11 - Class methods can no longer wrap other :term:`descriptors ` such as - :func:`property`. - - -.. function:: compile(source, filename, mode, flags=0, dont_inherit=False, optimize=-1) - - Compile the *source* into a code or AST object. Code objects can be executed - by :func:`exec` or :func:`eval`. *source* can either be a normal string, a - byte string, or an AST object. Refer to the :mod:`ast` module documentation - for information on how to work with AST objects. - - The *filename* argument should give the file from which the code was read; - pass some recognizable value if it wasn't read from a file (``''`` is - commonly used). - - The *mode* argument specifies what kind of code must be compiled; it can be - ``'exec'`` if *source* consists of a sequence of statements, ``'eval'`` if it - consists of a single expression, or ``'single'`` if it consists of a single - interactive statement (in the latter case, expression statements that - evaluate to something other than ``None`` will be printed). - - The optional arguments *flags* and *dont_inherit* control which - :ref:`compiler options ` should be activated - and which :ref:`future features ` should be allowed. If neither - is present (or both are zero) the code is compiled with the same flags that - affect the code that is calling :func:`compile`. If the *flags* - argument is given and *dont_inherit* is not (or is zero) then the compiler - options and the future statements specified by the *flags* argument are used - in addition to those that would be used anyway. If *dont_inherit* is a - non-zero integer then the *flags* argument is it -- the flags (future - features and compiler options) in the surrounding code are ignored. - - Compiler options and future statements are specified by bits which can be - bitwise ORed together to specify multiple options. The bitfield required to - specify a given future feature can be found as the - :attr:`~__future__._Feature.compiler_flag` attribute on the - :class:`~__future__._Feature` instance in the :mod:`__future__` module. - :ref:`Compiler flags ` can be found in :mod:`ast` - module, with ``PyCF_`` prefix. - - The argument *optimize* specifies the optimization level of the compiler; the - default value of ``-1`` selects the optimization level of the interpreter as - given by :option:`-O` options. Explicit levels are ``0`` (no optimization; - ``__debug__`` is true), ``1`` (asserts are removed, ``__debug__`` is false) - or ``2`` (docstrings are removed too). - - This function raises :exc:`SyntaxError` if the compiled source is invalid, - and :exc:`ValueError` if the source contains null bytes. - - If you want to parse Python code into its AST representation, see - :func:`ast.parse`. - - .. audit-event:: compile source,filename compile - - Raises an :ref:`auditing event ` ``compile`` with arguments - ``source`` and ``filename``. This event may also be raised by implicit - compilation. - - .. note:: - - When compiling a string with multi-line code in ``'single'`` or - ``'eval'`` mode, input must be terminated by at least one newline - character. This is to facilitate detection of incomplete and complete - statements in the :mod:`code` module. - - .. warning:: - - It is possible to crash the Python interpreter with a - sufficiently large/complex string when compiling to an AST - object due to stack depth limitations in Python's AST compiler. - - .. versionchanged:: 3.2 - Allowed use of Windows and Mac newlines. Also, input in ``'exec'`` mode - does not have to end in a newline anymore. Added the *optimize* parameter. - - .. versionchanged:: 3.5 - Previously, :exc:`TypeError` was raised when null bytes were encountered - in *source*. - - .. versionadded:: 3.8 - ``ast.PyCF_ALLOW_TOP_LEVEL_AWAIT`` can now be passed in flags to enable - support for top-level ``await``, ``async for``, and ``async with``. - - -.. class:: complex(real=0, imag=0) - complex(string) - - Return a complex number with the value *real* + *imag*\*1j or convert a string - or number to a complex number. If the first parameter is a string, it will - be interpreted as a complex number and the function must be called without a - second parameter. The second parameter can never be a string. Each argument - may be any numeric type (including complex). If *imag* is omitted, it - defaults to zero and the constructor serves as a numeric conversion like - :class:`int` and :class:`float`. If both arguments are omitted, returns - ``0j``. - - For a general Python object ``x``, ``complex(x)`` delegates to - ``x.__complex__()``. If :meth:`~object.__complex__` is not defined then it falls back - to :meth:`~object.__float__`. If :meth:`!__float__` is not defined then it falls back - to :meth:`~object.__index__`. - - .. note:: - - When converting from a string, the string must not contain whitespace - around the central ``+`` or ``-`` operator. For example, - ``complex('1+2j')`` is fine, but ``complex('1 + 2j')`` raises - :exc:`ValueError`. - - The complex type is described in :ref:`typesnumeric`. - - .. versionchanged:: 3.6 - Grouping digits with underscores as in code literals is allowed. - - .. versionchanged:: 3.8 - Falls back to :meth:`~object.__index__` if :meth:`~object.__complex__` and - :meth:`~object.__float__` are not defined. - - -.. function:: delattr(object, name) - - This is a relative of :func:`setattr`. The arguments are an object and a - string. The string must be the name of one of the object's attributes. The - function deletes the named attribute, provided the object allows it. For - example, ``delattr(x, 'foobar')`` is equivalent to ``del x.foobar``. - *name* need not be a Python identifier (see :func:`setattr`). - - -.. _func-dict: -.. class:: dict(**kwarg) - dict(mapping, **kwarg) - dict(iterable, **kwarg) - :noindex: - - Create a new dictionary. The :class:`dict` object is the dictionary class. - See :class:`dict` and :ref:`typesmapping` for documentation about this class. - - For other containers see the built-in :class:`list`, :class:`set`, and - :class:`tuple` classes, as well as the :mod:`collections` module. - - -.. function:: dir() - dir(object) - - Without arguments, return the list of names in the current local scope. With an - argument, attempt to return a list of valid attributes for that object. - - If the object has a method named :meth:`__dir__`, this method will be called and - must return the list of attributes. This allows objects that implement a custom - :func:`__getattr__` or :func:`__getattribute__` function to customize the way - :func:`dir` reports their attributes. - - If the object does not provide :meth:`__dir__`, the function tries its best to - gather information from the object's :attr:`~object.__dict__` attribute, if defined, and - from its type object. The resulting list is not necessarily complete and may - be inaccurate when the object has a custom :func:`__getattr__`. - - The default :func:`dir` mechanism behaves differently with different types of - objects, as it attempts to produce the most relevant, rather than complete, - information: - - * If the object is a module object, the list contains the names of the module's - attributes. - - * If the object is a type or class object, the list contains the names of its - attributes, and recursively of the attributes of its bases. - - * Otherwise, the list contains the object's attributes' names, the names of its - class's attributes, and recursively of the attributes of its class's base - classes. - - The resulting list is sorted alphabetically. For example: - - >>> import struct - >>> dir() # show the names in the module namespace # doctest: +SKIP - ['__builtins__', '__name__', 'struct'] - >>> dir(struct) # show the names in the struct module # doctest: +SKIP - ['Struct', '__all__', '__builtins__', '__cached__', '__doc__', '__file__', - '__initializing__', '__loader__', '__name__', '__package__', - '_clearcache', 'calcsize', 'error', 'pack', 'pack_into', - 'unpack', 'unpack_from'] - >>> class Shape: - ... def __dir__(self): - ... return ['area', 'perimeter', 'location'] - >>> s = Shape() - >>> dir(s) - ['area', 'location', 'perimeter'] - - .. note:: - - Because :func:`dir` is supplied primarily as a convenience for use at an - interactive prompt, it tries to supply an interesting set of names more - than it tries to supply a rigorously or consistently defined set of names, - and its detailed behavior may change across releases. For example, - metaclass attributes are not in the result list when the argument is a - class. - - -.. function:: divmod(a, b) - - Take two (non-complex) numbers as arguments and return a pair of numbers - consisting of their quotient and remainder when using integer division. With - mixed operand types, the rules for binary arithmetic operators apply. For - integers, the result is the same as ``(a // b, a % b)``. For floating point - numbers the result is ``(q, a % b)``, where *q* is usually ``math.floor(a / - b)`` but may be 1 less than that. In any case ``q * b + a % b`` is very - close to *a*, if ``a % b`` is non-zero it has the same sign as *b*, and ``0 - <= abs(a % b) < abs(b)``. - - -.. function:: enumerate(iterable, start=0) - - Return an enumerate object. *iterable* must be a sequence, an - :term:`iterator`, or some other object which supports iteration. - The :meth:`~iterator.__next__` method of the iterator returned by - :func:`enumerate` returns a tuple containing a count (from *start* which - defaults to 0) and the values obtained from iterating over *iterable*. - - >>> seasons = ['Spring', 'Summer', 'Fall', 'Winter'] - >>> list(enumerate(seasons)) - [(0, 'Spring'), (1, 'Summer'), (2, 'Fall'), (3, 'Winter')] - >>> list(enumerate(seasons, start=1)) - [(1, 'Spring'), (2, 'Summer'), (3, 'Fall'), (4, 'Winter')] - - Equivalent to:: - - def enumerate(iterable, start=0): - n = start - for elem in iterable: - yield n, elem - n += 1 - -.. _func-eval: - -.. function:: eval(expression, globals=None, locals=None) - - The arguments are a string and optional globals and locals. If provided, - *globals* must be a dictionary. If provided, *locals* can be any mapping - object. - - The *expression* argument is parsed and evaluated as a Python expression - (technically speaking, a condition list) using the *globals* and *locals* - dictionaries as global and local namespace. If the *globals* dictionary is - present and does not contain a value for the key ``__builtins__``, a - reference to the dictionary of the built-in module :mod:`builtins` is - inserted under that key before *expression* is parsed. That way you can - control what builtins are available to the executed code by inserting your - own ``__builtins__`` dictionary into *globals* before passing it to - :func:`eval`. If the *locals* dictionary is omitted it defaults to the - *globals* dictionary. If both dictionaries are omitted, the expression is - executed with the *globals* and *locals* in the environment where - :func:`eval` is called. Note, *eval()* does not have access to the - :term:`nested scopes ` (non-locals) in the enclosing - environment. - - The return value is the result of - the evaluated expression. Syntax errors are reported as exceptions. Example: - - >>> x = 1 - >>> eval('x+1') - 2 - - This function can also be used to execute arbitrary code objects (such as - those created by :func:`compile`). In this case, pass a code object instead - of a string. If the code object has been compiled with ``'exec'`` as the - *mode* argument, :func:`eval`\'s return value will be ``None``. - - Hints: dynamic execution of statements is supported by the :func:`exec` - function. The :func:`globals` and :func:`locals` functions - return the current global and local dictionary, respectively, which may be - useful to pass around for use by :func:`eval` or :func:`exec`. - - If the given source is a string, then leading and trailing spaces and tabs - are stripped. - - See :func:`ast.literal_eval` for a function that can safely evaluate strings - with expressions containing only literals. - - .. audit-event:: exec code_object eval - - Raises an :ref:`auditing event ` ``exec`` with the code object - as the argument. Code compilation events may also be raised. - -.. index:: pair: built-in function; exec - -.. function:: exec(object, globals=None, locals=None, /, *, closure=None) - - This function supports dynamic execution of Python code. *object* must be - either a string or a code object. If it is a string, the string is parsed as - a suite of Python statements which is then executed (unless a syntax error - occurs). [#]_ If it is a code object, it is simply executed. In all cases, - the code that's executed is expected to be valid as file input (see the - section :ref:`file-input` in the Reference Manual). Be aware that the - :keyword:`nonlocal`, :keyword:`yield`, and :keyword:`return` - statements may not be used outside of - function definitions even within the context of code passed to the - :func:`exec` function. The return value is ``None``. - - In all cases, if the optional parts are omitted, the code is executed in the - current scope. If only *globals* is provided, it must be a dictionary - (and not a subclass of dictionary), which - will be used for both the global and the local variables. If *globals* and - *locals* are given, they are used for the global and local variables, - respectively. If provided, *locals* can be any mapping object. Remember - that at the module level, globals and locals are the same dictionary. If exec - gets two separate objects as *globals* and *locals*, the code will be - executed as if it were embedded in a class definition. - - If the *globals* dictionary does not contain a value for the key - ``__builtins__``, a reference to the dictionary of the built-in module - :mod:`builtins` is inserted under that key. That way you can control what - builtins are available to the executed code by inserting your own - ``__builtins__`` dictionary into *globals* before passing it to :func:`exec`. - - The *closure* argument specifies a closure--a tuple of cellvars. - It's only valid when the *object* is a code object containing free variables. - The length of the tuple must exactly match the number of free variables - referenced by the code object. - - .. audit-event:: exec code_object exec - - Raises an :ref:`auditing event ` ``exec`` with the code object - as the argument. Code compilation events may also be raised. - - .. note:: - - The built-in functions :func:`globals` and :func:`locals` return the current - global and local dictionary, respectively, which may be useful to pass around - for use as the second and third argument to :func:`exec`. - - .. note:: - - The default *locals* act as described for function :func:`locals` below: - modifications to the default *locals* dictionary should not be attempted. - Pass an explicit *locals* dictionary if you need to see effects of the - code on *locals* after function :func:`exec` returns. - - .. versionchanged:: 3.11 - Added the *closure* parameter. - - -.. function:: filter(function, iterable) - - Construct an iterator from those elements of *iterable* for which *function* - is true. *iterable* may be either a sequence, a container which - supports iteration, or an iterator. If *function* is ``None``, the identity - function is assumed, that is, all elements of *iterable* that are false are - removed. - - Note that ``filter(function, iterable)`` is equivalent to the generator - expression ``(item for item in iterable if function(item))`` if function is - not ``None`` and ``(item for item in iterable if item)`` if function is - ``None``. - - See :func:`itertools.filterfalse` for the complementary function that returns - elements of *iterable* for which *function* is false. - - -.. class:: float(x=0.0) - - .. index:: - single: NaN - single: Infinity - - Return a floating point number constructed from a number or string *x*. - - If the argument is a string, it should contain a decimal number, optionally - preceded by a sign, and optionally embedded in whitespace. The optional - sign may be ``'+'`` or ``'-'``; a ``'+'`` sign has no effect on the value - produced. The argument may also be a string representing a NaN - (not-a-number), or positive or negative infinity. More precisely, the - input must conform to the ``floatvalue`` production rule in the following - grammar, after leading and trailing whitespace characters are removed: - - .. productionlist:: float - sign: "+" | "-" - infinity: "Infinity" | "inf" - nan: "nan" - digitpart: `digit` (["_"] `digit`)* - number: [`digitpart`] "." `digitpart` | `digitpart` ["."] - exponent: ("e" | "E") ["+" | "-"] `digitpart` - floatnumber: number [`exponent`] - floatvalue: [`sign`] (`floatnumber` | `infinity` | `nan`) - - Here ``digit`` is a Unicode decimal digit (character in the Unicode general - category ``Nd``). Case is not significant, so, for example, "inf", "Inf", - "INFINITY", and "iNfINity" are all acceptable spellings for positive - infinity. - - Otherwise, if the argument is an integer or a floating point number, a - floating point number with the same value (within Python's floating point - precision) is returned. If the argument is outside the range of a Python - float, an :exc:`OverflowError` will be raised. - - For a general Python object ``x``, ``float(x)`` delegates to - ``x.__float__()``. If :meth:`~object.__float__` is not defined then it falls back - to :meth:`~object.__index__`. - - If no argument is given, ``0.0`` is returned. - - Examples:: - - >>> float('+1.23') - 1.23 - >>> float(' -12345\n') - -12345.0 - >>> float('1e-003') - 0.001 - >>> float('+1E6') - 1000000.0 - >>> float('-Infinity') - -inf - - The float type is described in :ref:`typesnumeric`. - - .. versionchanged:: 3.6 - Grouping digits with underscores as in code literals is allowed. - - .. versionchanged:: 3.7 - *x* is now a positional-only parameter. - - .. versionchanged:: 3.8 - Falls back to :meth:`~object.__index__` if :meth:`~object.__float__` is not defined. - - -.. index:: - single: __format__ - single: string; format() (built-in function) - -.. function:: format(value, format_spec="") - - Convert a *value* to a "formatted" representation, as controlled by - *format_spec*. The interpretation of *format_spec* will depend on the type - of the *value* argument; however, there is a standard formatting syntax that - is used by most built-in types: :ref:`formatspec`. - - The default *format_spec* is an empty string which usually gives the same - effect as calling :func:`str(value) `. - - A call to ``format(value, format_spec)`` is translated to - ``type(value).__format__(value, format_spec)`` which bypasses the instance - dictionary when searching for the value's :meth:`__format__` method. A - :exc:`TypeError` exception is raised if the method search reaches - :mod:`object` and the *format_spec* is non-empty, or if either the - *format_spec* or the return value are not strings. - - .. versionchanged:: 3.4 - ``object().__format__(format_spec)`` raises :exc:`TypeError` - if *format_spec* is not an empty string. - - -.. _func-frozenset: -.. class:: frozenset(iterable=set()) - :noindex: - - Return a new :class:`frozenset` object, optionally with elements taken from - *iterable*. ``frozenset`` is a built-in class. See :class:`frozenset` and - :ref:`types-set` for documentation about this class. - - For other containers see the built-in :class:`set`, :class:`list`, - :class:`tuple`, and :class:`dict` classes, as well as the :mod:`collections` - module. - - -.. function:: getattr(object, name) - getattr(object, name, default) - - Return the value of the named attribute of *object*. *name* must be a string. - If the string is the name of one of the object's attributes, the result is the - value of that attribute. For example, ``getattr(x, 'foobar')`` is equivalent to - ``x.foobar``. If the named attribute does not exist, *default* is returned if - provided, otherwise :exc:`AttributeError` is raised. - *name* need not be a Python identifier (see :func:`setattr`). - - .. note:: - - Since :ref:`private name mangling ` happens at - compilation time, one must manually mangle a private attribute's - (attributes with two leading underscores) name in order to retrieve it with - :func:`getattr`. - - -.. function:: globals() - - Return the dictionary implementing the current module namespace. For code within - functions, this is set when the function is defined and remains the same - regardless of where the function is called. - - -.. function:: hasattr(object, name) - - The arguments are an object and a string. The result is ``True`` if the - string is the name of one of the object's attributes, ``False`` if not. (This - is implemented by calling ``getattr(object, name)`` and seeing whether it - raises an :exc:`AttributeError` or not.) - - -.. function:: hash(object) - - Return the hash value of the object (if it has one). Hash values are - integers. They are used to quickly compare dictionary keys during a - dictionary lookup. Numeric values that compare equal have the same hash - value (even if they are of different types, as is the case for 1 and 1.0). - - .. note:: - - For objects with custom :meth:`__hash__` methods, note that :func:`hash` - truncates the return value based on the bit width of the host machine. - See :meth:`__hash__ ` for details. - -.. function:: help() - help(request) - - Invoke the built-in help system. (This function is intended for interactive - use.) If no argument is given, the interactive help system starts on the - interpreter console. If the argument is a string, then the string is looked up - as the name of a module, function, class, method, keyword, or documentation - topic, and a help page is printed on the console. If the argument is any other - kind of object, a help page on the object is generated. - - Note that if a slash(/) appears in the parameter list of a function when - invoking :func:`help`, it means that the parameters prior to the slash are - positional-only. For more info, see - :ref:`the FAQ entry on positional-only parameters `. - - This function is added to the built-in namespace by the :mod:`site` module. - - .. versionchanged:: 3.4 - Changes to :mod:`pydoc` and :mod:`inspect` mean that the reported - signatures for callables are now more comprehensive and consistent. - - -.. function:: hex(x) - - Convert an integer number to a lowercase hexadecimal string prefixed with - "0x". If *x* is not a Python :class:`int` object, it has to define an - :meth:`~object.__index__` method that returns an integer. Some examples: - - >>> hex(255) - '0xff' - >>> hex(-42) - '-0x2a' - - If you want to convert an integer number to an uppercase or lower hexadecimal - string with prefix or not, you can use either of the following ways: - - >>> '%#x' % 255, '%x' % 255, '%X' % 255 - ('0xff', 'ff', 'FF') - >>> format(255, '#x'), format(255, 'x'), format(255, 'X') - ('0xff', 'ff', 'FF') - >>> f'{255:#x}', f'{255:x}', f'{255:X}' - ('0xff', 'ff', 'FF') - - See also :func:`format` for more information. - - See also :func:`int` for converting a hexadecimal string to an - integer using a base of 16. - - .. note:: - - To obtain a hexadecimal string representation for a float, use the - :meth:`float.hex` method. - - -.. function:: id(object) - - Return the "identity" of an object. This is an integer which - is guaranteed to be unique and constant for this object during its lifetime. - Two objects with non-overlapping lifetimes may have the same :func:`id` - value. - - .. impl-detail:: This is the address of the object in memory. - - .. audit-event:: builtins.id id id - - -.. function:: input() - input(prompt) - - If the *prompt* argument is present, it is written to standard output without - a trailing newline. The function then reads a line from input, converts it - to a string (stripping a trailing newline), and returns that. When EOF is - read, :exc:`EOFError` is raised. Example:: - - >>> s = input('--> ') # doctest: +SKIP - --> Monty Python's Flying Circus - >>> s # doctest: +SKIP - "Monty Python's Flying Circus" - - If the :mod:`readline` module was loaded, then :func:`input` will use it - to provide elaborate line editing and history features. - - .. audit-event:: builtins.input prompt input - - Raises an :ref:`auditing event ` ``builtins.input`` with - argument ``prompt`` before reading input - - .. audit-event:: builtins.input/result result input - - Raises an :ref:`auditing event ` ``builtins.input/result`` - with the result after successfully reading input. - - -.. class:: int(x=0) - int(x, base=10) - - Return an integer object constructed from a number or string *x*, or return - ``0`` if no arguments are given. If *x* defines :meth:`~object.__int__`, - ``int(x)`` returns ``x.__int__()``. If *x* defines :meth:`~object.__index__`, - it returns ``x.__index__()``. If *x* defines :meth:`~object.__trunc__`, - it returns ``x.__trunc__()``. - For floating point numbers, this truncates towards zero. - - If *x* is not a number or if *base* is given, then *x* must be a string, - :class:`bytes`, or :class:`bytearray` instance representing an integer - in radix *base*. Optionally, the string can be preceded by ``+`` or ``-`` - (with no space in between), have leading zeros, be surrounded by whitespace, - and have single underscores interspersed between digits. - - A base-n integer string contains digits, each representing a value from 0 to - n-1. The values 0--9 can be represented by any Unicode decimal digit. The - values 10--35 can be represented by ``a`` to ``z`` (or ``A`` to ``Z``). The - default *base* is 10. The allowed bases are 0 and 2--36. Base-2, -8, and -16 - strings can be optionally prefixed with ``0b``/``0B``, ``0o``/``0O``, or - ``0x``/``0X``, as with integer literals in code. For base 0, the string is - interpreted in a similar way to an :ref:`integer literal in code `, - in that the actual base is 2, 8, 10, or 16 as determined by the prefix. Base - 0 also disallows leading zeros: ``int('010', 0)`` is not legal, while - ``int('010')`` and ``int('010', 8)`` are. - - The integer type is described in :ref:`typesnumeric`. - - .. versionchanged:: 3.4 - If *base* is not an instance of :class:`int` and the *base* object has a - :meth:`base.__index__ ` method, that method is called - to obtain an integer for the base. Previous versions used - :meth:`base.__int__ ` instead of :meth:`base.__index__ - `. - - .. versionchanged:: 3.6 - Grouping digits with underscores as in code literals is allowed. - - .. versionchanged:: 3.7 - *x* is now a positional-only parameter. - - .. versionchanged:: 3.8 - Falls back to :meth:`~object.__index__` if :meth:`~object.__int__` is not defined. - - .. versionchanged:: 3.11 - The delegation to :meth:`~object.__trunc__` is deprecated. - - .. versionchanged:: 3.11 - :class:`int` string inputs and string representations can be limited to - help avoid denial of service attacks. A :exc:`ValueError` is raised when - the limit is exceeded while converting a string *x* to an :class:`int` or - when converting an :class:`int` into a string would exceed the limit. - See the :ref:`integer string conversion length limitation - ` documentation. - -.. function:: isinstance(object, classinfo) - - Return ``True`` if the *object* argument is an instance of the *classinfo* - argument, or of a (direct, indirect, or :term:`virtual `) subclass thereof. If *object* is not - an object of the given type, the function always returns ``False``. - If *classinfo* is a tuple of type objects (or recursively, other such - tuples) or a :ref:`types-union` of multiple types, return ``True`` if - *object* is an instance of any of the types. - If *classinfo* is not a type or tuple of types and such tuples, - a :exc:`TypeError` exception is raised. :exc:`TypeError` may not be - raised for an invalid type if an earlier check succeeds. - - .. versionchanged:: 3.10 - *classinfo* can be a :ref:`types-union`. - - -.. function:: issubclass(class, classinfo) - - Return ``True`` if *class* is a subclass (direct, indirect, or :term:`virtual - `) of *classinfo*. A - class is considered a subclass of itself. *classinfo* may be a tuple of class - objects (or recursively, other such tuples) - or a :ref:`types-union`, in which case return ``True`` if *class* is a - subclass of any entry in *classinfo*. In any other case, a :exc:`TypeError` - exception is raised. - - .. versionchanged:: 3.10 - *classinfo* can be a :ref:`types-union`. - - -.. function:: iter(object) - iter(object, sentinel) - - Return an :term:`iterator` object. The first argument is interpreted very - differently depending on the presence of the second argument. Without a - second argument, *object* must be a collection object which supports the - :term:`iterable` protocol (the :meth:`__iter__` method), or it must support - the sequence protocol (the :meth:`~object.__getitem__` method with integer arguments - starting at ``0``). If it does not support either of those protocols, - :exc:`TypeError` is raised. If the second argument, *sentinel*, is given, - then *object* must be a callable object. The iterator created in this case - will call *object* with no arguments for each call to its - :meth:`~iterator.__next__` method; if the value returned is equal to - *sentinel*, :exc:`StopIteration` will be raised, otherwise the value will - be returned. - - See also :ref:`typeiter`. - - One useful application of the second form of :func:`iter` is to build a - block-reader. For example, reading fixed-width blocks from a binary - database file until the end of file is reached:: - - from functools import partial - with open('mydata.db', 'rb') as f: - for block in iter(partial(f.read, 64), b''): - process_block(block) - - -.. function:: len(s) - - Return the length (the number of items) of an object. The argument may be a - sequence (such as a string, bytes, tuple, list, or range) or a collection - (such as a dictionary, set, or frozen set). - - .. impl-detail:: - - ``len`` raises :exc:`OverflowError` on lengths larger than - :data:`sys.maxsize`, such as :class:`range(2 ** 100) `. - - -.. _func-list: -.. class:: list() - list(iterable) - :noindex: - - Rather than being a function, :class:`list` is actually a mutable - sequence type, as documented in :ref:`typesseq-list` and :ref:`typesseq`. - - -.. function:: locals() - - Update and return a dictionary representing the current local symbol table. - Free variables are returned by :func:`locals` when it is called in function - blocks, but not in class blocks. Note that at the module level, :func:`locals` - and :func:`globals` are the same dictionary. - - .. note:: - The contents of this dictionary should not be modified; changes may not - affect the values of local and free variables used by the interpreter. - -.. function:: map(function, iterable, *iterables) - - Return an iterator that applies *function* to every item of *iterable*, - yielding the results. If additional *iterables* arguments are passed, - *function* must take that many arguments and is applied to the items from all - iterables in parallel. With multiple iterables, the iterator stops when the - shortest iterable is exhausted. For cases where the function inputs are - already arranged into argument tuples, see :func:`itertools.starmap`\. - - -.. function:: max(iterable, *, key=None) - max(iterable, *, default, key=None) - max(arg1, arg2, *args, key=None) - - Return the largest item in an iterable or the largest of two or more - arguments. - - If one positional argument is provided, it should be an :term:`iterable`. - The largest item in the iterable is returned. If two or more positional - arguments are provided, the largest of the positional arguments is - returned. - - There are two optional keyword-only arguments. The *key* argument specifies - a one-argument ordering function like that used for :meth:`list.sort`. The - *default* argument specifies an object to return if the provided iterable is - empty. If the iterable is empty and *default* is not provided, a - :exc:`ValueError` is raised. - - If multiple items are maximal, the function returns the first one - encountered. This is consistent with other sort-stability preserving tools - such as ``sorted(iterable, key=keyfunc, reverse=True)[0]`` and - ``heapq.nlargest(1, iterable, key=keyfunc)``. - - .. versionadded:: 3.4 - The *default* keyword-only argument. - - .. versionchanged:: 3.8 - The *key* can be ``None``. - - -.. _func-memoryview: -.. class:: memoryview(object) - :noindex: - - Return a "memory view" object created from the given argument. See - :ref:`typememoryview` for more information. - - -.. function:: min(iterable, *, key=None) - min(iterable, *, default, key=None) - min(arg1, arg2, *args, key=None) - - Return the smallest item in an iterable or the smallest of two or more - arguments. - - If one positional argument is provided, it should be an :term:`iterable`. - The smallest item in the iterable is returned. If two or more positional - arguments are provided, the smallest of the positional arguments is - returned. - - There are two optional keyword-only arguments. The *key* argument specifies - a one-argument ordering function like that used for :meth:`list.sort`. The - *default* argument specifies an object to return if the provided iterable is - empty. If the iterable is empty and *default* is not provided, a - :exc:`ValueError` is raised. - - If multiple items are minimal, the function returns the first one - encountered. This is consistent with other sort-stability preserving tools - such as ``sorted(iterable, key=keyfunc)[0]`` and ``heapq.nsmallest(1, - iterable, key=keyfunc)``. - - .. versionadded:: 3.4 - The *default* keyword-only argument. - - .. versionchanged:: 3.8 - The *key* can be ``None``. - - -.. function:: next(iterator) - next(iterator, default) - - Retrieve the next item from the :term:`iterator` by calling its - :meth:`~iterator.__next__` method. If *default* is given, it is returned - if the iterator is exhausted, otherwise :exc:`StopIteration` is raised. - - -.. class:: object() - - Return a new featureless object. :class:`object` is a base for all classes. - It has methods that are common to all instances of Python classes. This - function does not accept any arguments. - - .. note:: - - :class:`object` does *not* have a :attr:`~object.__dict__`, so you can't - assign arbitrary attributes to an instance of the :class:`object` class. - - -.. function:: oct(x) - - Convert an integer number to an octal string prefixed with "0o". The result - is a valid Python expression. If *x* is not a Python :class:`int` object, it - has to define an :meth:`~object.__index__` method that returns an integer. For - example: - - >>> oct(8) - '0o10' - >>> oct(-56) - '-0o70' - - If you want to convert an integer number to an octal string either with the prefix - "0o" or not, you can use either of the following ways. - - >>> '%#o' % 10, '%o' % 10 - ('0o12', '12') - >>> format(10, '#o'), format(10, 'o') - ('0o12', '12') - >>> f'{10:#o}', f'{10:o}' - ('0o12', '12') - - See also :func:`format` for more information. - -.. index:: - single: file object; open() built-in function - -.. function:: open(file, mode='r', buffering=-1, encoding=None, errors=None, newline=None, closefd=True, opener=None) - - Open *file* and return a corresponding :term:`file object`. If the file - cannot be opened, an :exc:`OSError` is raised. See - :ref:`tut-files` for more examples of how to use this function. - - *file* is a :term:`path-like object` giving the pathname (absolute or - relative to the current working directory) of the file to be opened or an - integer file descriptor of the file to be wrapped. (If a file descriptor is - given, it is closed when the returned I/O object is closed unless *closefd* - is set to ``False``.) - - *mode* is an optional string that specifies the mode in which the file is - opened. It defaults to ``'r'`` which means open for reading in text mode. - Other common values are ``'w'`` for writing (truncating the file if it - already exists), ``'x'`` for exclusive creation, and ``'a'`` for appending - (which on *some* Unix systems, means that *all* writes append to the end of - the file regardless of the current seek position). In text mode, if - *encoding* is not specified the encoding used is platform-dependent: - :func:`locale.getencoding()` is called to get the current locale encoding. - (For reading and writing raw bytes use binary mode and leave - *encoding* unspecified.) The available modes are: - - .. _filemodes: - - .. index:: - pair: file; modes - - ========= =============================================================== - Character Meaning - ========= =============================================================== - ``'r'`` open for reading (default) - ``'w'`` open for writing, truncating the file first - ``'x'`` open for exclusive creation, failing if the file already exists - ``'a'`` open for writing, appending to the end of file if it exists - ``'b'`` binary mode - ``'t'`` text mode (default) - ``'+'`` open for updating (reading and writing) - ========= =============================================================== - - The default mode is ``'r'`` (open for reading text, a synonym of ``'rt'``). - Modes ``'w+'`` and ``'w+b'`` open and truncate the file. Modes ``'r+'`` - and ``'r+b'`` open the file with no truncation. - - As mentioned in the :ref:`io-overview`, Python distinguishes between binary - and text I/O. Files opened in binary mode (including ``'b'`` in the *mode* - argument) return contents as :class:`bytes` objects without any decoding. In - text mode (the default, or when ``'t'`` is included in the *mode* argument), - the contents of the file are returned as :class:`str`, the bytes having been - first decoded using a platform-dependent encoding or using the specified - *encoding* if given. - - .. note:: - - Python doesn't depend on the underlying operating system's notion of text - files; all the processing is done by Python itself, and is therefore - platform-independent. - - *buffering* is an optional integer used to set the buffering policy. Pass 0 - to switch buffering off (only allowed in binary mode), 1 to select line - buffering (only usable in text mode), and an integer > 1 to indicate the size - in bytes of a fixed-size chunk buffer. Note that specifying a buffer size this - way applies for binary buffered I/O, but ``TextIOWrapper`` (i.e., files opened - with ``mode='r+'``) would have another buffering. To disable buffering in - ``TextIOWrapper``, consider using the ``write_through`` flag for - :func:`io.TextIOWrapper.reconfigure`. When no *buffering* argument is - given, the default buffering policy works as follows: - - * Binary files are buffered in fixed-size chunks; the size of the buffer is - chosen using a heuristic trying to determine the underlying device's "block - size" and falling back on :const:`io.DEFAULT_BUFFER_SIZE`. On many systems, - the buffer will typically be 4096 or 8192 bytes long. - - * "Interactive" text files (files for which :meth:`~io.IOBase.isatty` - returns ``True``) use line buffering. Other text files use the policy - described above for binary files. - - *encoding* is the name of the encoding used to decode or encode the file. - This should only be used in text mode. The default encoding is platform - dependent (whatever :func:`locale.getencoding` returns), but any - :term:`text encoding` supported by Python can be used. - See the :mod:`codecs` module for the list of supported encodings. - - *errors* is an optional string that specifies how encoding and decoding - errors are to be handled—this cannot be used in binary mode. - A variety of standard error handlers are available - (listed under :ref:`error-handlers`), though any - error handling name that has been registered with - :func:`codecs.register_error` is also valid. The standard names - include: - - * ``'strict'`` to raise a :exc:`ValueError` exception if there is - an encoding error. The default value of ``None`` has the same - effect. - - * ``'ignore'`` ignores errors. Note that ignoring encoding errors - can lead to data loss. - - * ``'replace'`` causes a replacement marker (such as ``'?'``) to be inserted - where there is malformed data. - - * ``'surrogateescape'`` will represent any incorrect bytes as low - surrogate code units ranging from U+DC80 to U+DCFF. - These surrogate code units will then be turned back into - the same bytes when the ``surrogateescape`` error handler is used - when writing data. This is useful for processing files in an - unknown encoding. - - * ``'xmlcharrefreplace'`` is only supported when writing to a file. - Characters not supported by the encoding are replaced with the - appropriate XML character reference :samp:`&#{nnn};`. - - * ``'backslashreplace'`` replaces malformed data by Python's backslashed - escape sequences. - - * ``'namereplace'`` (also only supported when writing) - replaces unsupported characters with ``\N{...}`` escape sequences. - - .. index:: - single: universal newlines; open() built-in function - - .. _open-newline-parameter: - - *newline* determines how to parse newline characters from the stream. - It can be ``None``, ``''``, ``'\n'``, ``'\r'``, and - ``'\r\n'``. It works as follows: - - * When reading input from the stream, if *newline* is ``None``, universal - newlines mode is enabled. Lines in the input can end in ``'\n'``, - ``'\r'``, or ``'\r\n'``, and these are translated into ``'\n'`` before - being returned to the caller. If it is ``''``, universal newlines mode is - enabled, but line endings are returned to the caller untranslated. If it - has any of the other legal values, input lines are only terminated by the - given string, and the line ending is returned to the caller untranslated. - - * When writing output to the stream, if *newline* is ``None``, any ``'\n'`` - characters written are translated to the system default line separator, - :data:`os.linesep`. If *newline* is ``''`` or ``'\n'``, no translation - takes place. If *newline* is any of the other legal values, any ``'\n'`` - characters written are translated to the given string. - - If *closefd* is ``False`` and a file descriptor rather than a filename was - given, the underlying file descriptor will be kept open when the file is - closed. If a filename is given *closefd* must be ``True`` (the default); - otherwise, an error will be raised. - - A custom opener can be used by passing a callable as *opener*. The underlying - file descriptor for the file object is then obtained by calling *opener* with - (*file*, *flags*). *opener* must return an open file descriptor (passing - :mod:`os.open` as *opener* results in functionality similar to passing - ``None``). - - The newly created file is :ref:`non-inheritable `. - - The following example uses the :ref:`dir_fd ` parameter of the - :func:`os.open` function to open a file relative to a given directory:: - - >>> import os - >>> dir_fd = os.open('somedir', os.O_RDONLY) - >>> def opener(path, flags): - ... return os.open(path, flags, dir_fd=dir_fd) - ... - >>> with open('spamspam.txt', 'w', opener=opener) as f: - ... print('This will be written to somedir/spamspam.txt', file=f) - ... - >>> os.close(dir_fd) # don't leak a file descriptor - - The type of :term:`file object` returned by the :func:`open` function - depends on the mode. When :func:`open` is used to open a file in a text - mode (``'w'``, ``'r'``, ``'wt'``, ``'rt'``, etc.), it returns a subclass of - :class:`io.TextIOBase` (specifically :class:`io.TextIOWrapper`). When used - to open a file in a binary mode with buffering, the returned class is a - subclass of :class:`io.BufferedIOBase`. The exact class varies: in read - binary mode, it returns an :class:`io.BufferedReader`; in write binary and - append binary modes, it returns an :class:`io.BufferedWriter`, and in - read/write mode, it returns an :class:`io.BufferedRandom`. When buffering is - disabled, the raw stream, a subclass of :class:`io.RawIOBase`, - :class:`io.FileIO`, is returned. - - .. index:: - single: line-buffered I/O - single: unbuffered I/O - single: buffer size, I/O - single: I/O control; buffering - single: binary mode - single: text mode - pair: module; sys - - See also the file handling modules, such as :mod:`fileinput`, :mod:`io` - (where :func:`open` is declared), :mod:`os`, :mod:`os.path`, :mod:`tempfile`, - and :mod:`shutil`. - - .. audit-event:: open file,mode,flags open - - The ``mode`` and ``flags`` arguments may have been modified or inferred from - the original call. - - .. versionchanged:: 3.3 - - * The *opener* parameter was added. - * The ``'x'`` mode was added. - * :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. - * :exc:`FileExistsError` is now raised if the file opened in exclusive - creation mode (``'x'``) already exists. - - .. versionchanged:: 3.4 - - * The file is now non-inheritable. - - .. versionchanged:: 3.5 - - * If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - * The ``'namereplace'`` error handler was added. - - .. versionchanged:: 3.6 - - * Support added to accept objects implementing :class:`os.PathLike`. - * On Windows, opening a console buffer may return a subclass of - :class:`io.RawIOBase` other than :class:`io.FileIO`. - - .. versionchanged:: 3.11 - The ``'U'`` mode has been removed. - -.. function:: ord(c) - - Given a string representing one Unicode character, return an integer - representing the Unicode code point of that character. For example, - ``ord('a')`` returns the integer ``97`` and ``ord('€')`` (Euro sign) - returns ``8364``. This is the inverse of :func:`chr`. - - -.. function:: pow(base, exp, mod=None) - - Return *base* to the power *exp*; if *mod* is present, return *base* to the - power *exp*, modulo *mod* (computed more efficiently than - ``pow(base, exp) % mod``). The two-argument form ``pow(base, exp)`` is - equivalent to using the power operator: ``base**exp``. - - The arguments must have numeric types. With mixed operand types, the - coercion rules for binary arithmetic operators apply. For :class:`int` - operands, the result has the same type as the operands (after coercion) - unless the second argument is negative; in that case, all arguments are - converted to float and a float result is delivered. For example, ``pow(10, 2)`` - returns ``100``, but ``pow(10, -2)`` returns ``0.01``. For a negative base of - type :class:`int` or :class:`float` and a non-integral exponent, a complex - result is delivered. For example, ``pow(-9, 0.5)`` returns a value close - to ``3j``. - - For :class:`int` operands *base* and *exp*, if *mod* is present, *mod* must - also be of integer type and *mod* must be nonzero. If *mod* is present and - *exp* is negative, *base* must be relatively prime to *mod*. In that case, - ``pow(inv_base, -exp, mod)`` is returned, where *inv_base* is an inverse to - *base* modulo *mod*. - - Here's an example of computing an inverse for ``38`` modulo ``97``:: - - >>> pow(38, -1, mod=97) - 23 - >>> 23 * 38 % 97 == 1 - True - - .. versionchanged:: 3.8 - For :class:`int` operands, the three-argument form of ``pow`` now allows - the second argument to be negative, permitting computation of modular - inverses. - - .. versionchanged:: 3.8 - Allow keyword arguments. Formerly, only positional arguments were - supported. - - -.. function:: print(*objects, sep=' ', end='\n', file=None, flush=False) - - Print *objects* to the text stream *file*, separated by *sep* and followed - by *end*. *sep*, *end*, *file*, and *flush*, if present, must be given as keyword - arguments. - - All non-keyword arguments are converted to strings like :func:`str` does and - written to the stream, separated by *sep* and followed by *end*. Both *sep* - and *end* must be strings; they can also be ``None``, which means to use the - default values. If no *objects* are given, :func:`print` will just write - *end*. - - The *file* argument must be an object with a ``write(string)`` method; if it - is not present or ``None``, :data:`sys.stdout` will be used. Since printed - arguments are converted to text strings, :func:`print` cannot be used with - binary mode file objects. For these, use ``file.write(...)`` instead. - - Output buffering is usually determined by *file*. - However, if *flush* is true, the stream is forcibly flushed. - - - .. versionchanged:: 3.3 - Added the *flush* keyword argument. - - -.. class:: property(fget=None, fset=None, fdel=None, doc=None) - - Return a property attribute. - - *fget* is a function for getting an attribute value. *fset* is a function - for setting an attribute value. *fdel* is a function for deleting an attribute - value. And *doc* creates a docstring for the attribute. - - A typical use is to define a managed attribute ``x``:: - - class C: - def __init__(self): - self._x = None - - def getx(self): - return self._x - - def setx(self, value): - self._x = value - - def delx(self): - del self._x - - x = property(getx, setx, delx, "I'm the 'x' property.") - - If *c* is an instance of *C*, ``c.x`` will invoke the getter, - ``c.x = value`` will invoke the setter, and ``del c.x`` the deleter. - - If given, *doc* will be the docstring of the property attribute. Otherwise, the - property will copy *fget*'s docstring (if it exists). This makes it possible to - create read-only properties easily using :func:`property` as a :term:`decorator`:: - - class Parrot: - def __init__(self): - self._voltage = 100000 - - @property - def voltage(self): - """Get the current voltage.""" - return self._voltage - - The ``@property`` decorator turns the :meth:`voltage` method into a "getter" - for a read-only attribute with the same name, and it sets the docstring for - *voltage* to "Get the current voltage." - - A property object has :attr:`~property.getter`, :attr:`~property.setter`, - and :attr:`~property.deleter` methods usable as decorators that create a - copy of the property with the corresponding accessor function set to the - decorated function. This is best explained with an example:: - - class C: - def __init__(self): - self._x = None - - @property - def x(self): - """I'm the 'x' property.""" - return self._x - - @x.setter - def x(self, value): - self._x = value - - @x.deleter - def x(self): - del self._x - - This code is exactly equivalent to the first example. Be sure to give the - additional functions the same name as the original property (``x`` in this - case.) - - The returned property object also has the attributes ``fget``, ``fset``, and - ``fdel`` corresponding to the constructor arguments. - - .. versionchanged:: 3.5 - The docstrings of property objects are now writeable. - - -.. _func-range: -.. class:: range(stop) - range(start, stop, step=1) - :noindex: - - Rather than being a function, :class:`range` is actually an immutable - sequence type, as documented in :ref:`typesseq-range` and :ref:`typesseq`. - - -.. function:: repr(object) - - Return a string containing a printable representation of an object. For many - types, this function makes an attempt to return a string that would yield an - object with the same value when passed to :func:`eval`; otherwise, the - representation is a string enclosed in angle brackets that contains the name - of the type of the object together with additional information often - including the name and address of the object. A class can control what this - function returns for its instances by defining a :meth:`__repr__` method. - If :func:`sys.displayhook` is not accessible, this function will raise - :exc:`RuntimeError`. - - -.. function:: reversed(seq) - - Return a reverse :term:`iterator`. *seq* must be an object which has - a :meth:`__reversed__` method or supports the sequence protocol (the - :meth:`__len__` method and the :meth:`~object.__getitem__` method with integer - arguments starting at ``0``). - - -.. function:: round(number, ndigits=None) - - Return *number* rounded to *ndigits* precision after the decimal - point. If *ndigits* is omitted or is ``None``, it returns the - nearest integer to its input. - - For the built-in types supporting :func:`round`, values are rounded to the - closest multiple of 10 to the power minus *ndigits*; if two multiples are - equally close, rounding is done toward the even choice (so, for example, - both ``round(0.5)`` and ``round(-0.5)`` are ``0``, and ``round(1.5)`` is - ``2``). Any integer value is valid for *ndigits* (positive, zero, or - negative). The return value is an integer if *ndigits* is omitted or - ``None``. - Otherwise, the return value has the same type as *number*. - - For a general Python object ``number``, ``round`` delegates to - ``number.__round__``. - - .. note:: - - The behavior of :func:`round` for floats can be surprising: for example, - ``round(2.675, 2)`` gives ``2.67`` instead of the expected ``2.68``. - This is not a bug: it's a result of the fact that most decimal fractions - can't be represented exactly as a float. See :ref:`tut-fp-issues` for - more information. - - -.. _func-set: -.. class:: set() - set(iterable) - :noindex: - - Return a new :class:`set` object, optionally with elements taken from - *iterable*. ``set`` is a built-in class. See :class:`set` and - :ref:`types-set` for documentation about this class. - - For other containers see the built-in :class:`frozenset`, :class:`list`, - :class:`tuple`, and :class:`dict` classes, as well as the :mod:`collections` - module. - - -.. function:: setattr(object, name, value) - - This is the counterpart of :func:`getattr`. The arguments are an object, a - string, and an arbitrary value. The string may name an existing attribute or a - new attribute. The function assigns the value to the attribute, provided the - object allows it. For example, ``setattr(x, 'foobar', 123)`` is equivalent to - ``x.foobar = 123``. - - *name* need not be a Python identifier as defined in :ref:`identifiers` - unless the object chooses to enforce that, for example in a custom - :meth:`~object.__getattribute__` or via :attr:`~object.__slots__`. - An attribute whose name is not an identifier will not be accessible using - the dot notation, but is accessible through :func:`getattr` etc.. - - .. note:: - - Since :ref:`private name mangling ` happens at - compilation time, one must manually mangle a private attribute's - (attributes with two leading underscores) name in order to set it with - :func:`setattr`. - - -.. class:: slice(stop) - slice(start, stop, step=None) - - Return a :term:`slice` object representing the set of indices specified by - ``range(start, stop, step)``. The *start* and *step* arguments default to - ``None``. Slice objects have read-only data attributes :attr:`~slice.start`, - :attr:`~slice.stop`, and :attr:`~slice.step` which merely return the argument - values (or their default). They have no other explicit functionality; - however, they are used by NumPy and other third-party packages. - Slice objects are also generated when extended indexing syntax is used. For - example: ``a[start:stop:step]`` or ``a[start:stop, i]``. See - :func:`itertools.islice` for an alternate version that returns an iterator. - - -.. function:: sorted(iterable, /, *, key=None, reverse=False) - - Return a new sorted list from the items in *iterable*. - - Has two optional arguments which must be specified as keyword arguments. - - *key* specifies a function of one argument that is used to extract a comparison - key from each element in *iterable* (for example, ``key=str.lower``). The - default value is ``None`` (compare the elements directly). - - *reverse* is a boolean value. If set to ``True``, then the list elements are - sorted as if each comparison were reversed. - - Use :func:`functools.cmp_to_key` to convert an old-style *cmp* function to a - *key* function. - - The built-in :func:`sorted` function is guaranteed to be stable. A sort is - stable if it guarantees not to change the relative order of elements that - compare equal --- this is helpful for sorting in multiple passes (for - example, sort by department, then by salary grade). - - The sort algorithm uses only ``<`` comparisons between items. While - defining an :meth:`~object.__lt__` method will suffice for sorting, - :PEP:`8` recommends that all six :ref:`rich comparisons - ` be implemented. This will help avoid bugs when using - the same data with other ordering tools such as :func:`max` that rely - on a different underlying method. Implementing all six comparisons - also helps avoid confusion for mixed type comparisons which can call - reflected the :meth:`~object.__gt__` method. - - For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. - -.. decorator:: staticmethod - - Transform a method into a static method. - - A static method does not receive an implicit first argument. To declare a static - method, use this idiom:: - - class C: - @staticmethod - def f(arg1, arg2, argN): ... - - The ``@staticmethod`` form is a function :term:`decorator` -- see - :ref:`function` for details. - - A static method can be called either on the class (such as ``C.f()``) or on - an instance (such as ``C().f()``). Moreover, they can be called as regular - functions (such as ``f()``). - - Static methods in Python are similar to those found in Java or C++. Also, see - :func:`classmethod` for a variant that is useful for creating alternate class - constructors. - - Like all decorators, it is also possible to call ``staticmethod`` as - a regular function and do something with its result. This is needed - in some cases where you need a reference to a function from a class - body and you want to avoid the automatic transformation to instance - method. For these cases, use this idiom:: - - def regular_function(): - ... - - class C: - method = staticmethod(regular_function) - - For more information on static methods, see :ref:`types`. - - .. versionchanged:: 3.10 - Static methods now inherit the method attributes (``__module__``, - ``__name__``, ``__qualname__``, ``__doc__`` and ``__annotations__``), - have a new ``__wrapped__`` attribute, and are now callable as regular - functions. - - -.. index:: - single: string; str() (built-in function) - -.. _func-str: -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') - :noindex: - - Return a :class:`str` version of *object*. See :func:`str` for details. - - ``str`` is the built-in string :term:`class`. For general information - about strings, see :ref:`textseq`. - - -.. function:: sum(iterable, /, start=0) - - Sums *start* and the items of an *iterable* from left to right and returns the - total. The *iterable*'s items are normally numbers, and the start value is not - allowed to be a string. - - For some use cases, there are good alternatives to :func:`sum`. - The preferred, fast way to concatenate a sequence of strings is by calling - ``''.join(sequence)``. To add floating point values with extended precision, - see :func:`math.fsum`\. To concatenate a series of iterables, consider using - :func:`itertools.chain`. - - .. versionchanged:: 3.8 - The *start* parameter can be specified as a keyword argument. - -.. class:: super() - super(type, object_or_type=None) - - Return a proxy object that delegates method calls to a parent or sibling - class of *type*. This is useful for accessing inherited methods that have - been overridden in a class. - - The *object_or_type* determines the :term:`method resolution order` - to be searched. The search starts from the class right after the - *type*. - - For example, if :attr:`~class.__mro__` of *object_or_type* is - ``D -> B -> C -> A -> object`` and the value of *type* is ``B``, - then :func:`super` searches ``C -> A -> object``. - - The :attr:`~class.__mro__` attribute of the *object_or_type* lists the method - resolution search order used by both :func:`getattr` and :func:`super`. The - attribute is dynamic and can change whenever the inheritance hierarchy is - updated. - - If the second argument is omitted, the super object returned is unbound. If - the second argument is an object, ``isinstance(obj, type)`` must be true. If - the second argument is a type, ``issubclass(type2, type)`` must be true (this - is useful for classmethods). - - There are two typical use cases for *super*. In a class hierarchy with - single inheritance, *super* can be used to refer to parent classes without - naming them explicitly, thus making the code more maintainable. This use - closely parallels the use of *super* in other programming languages. - - The second use case is to support cooperative multiple inheritance in a - dynamic execution environment. This use case is unique to Python and is - not found in statically compiled languages or languages that only support - single inheritance. This makes it possible to implement "diamond diagrams" - where multiple base classes implement the same method. Good design dictates - that such implementations have the same calling signature in every case (because the - order of calls is determined at runtime, because that order adapts - to changes in the class hierarchy, and because that order can include - sibling classes that are unknown prior to runtime). - - For both use cases, a typical superclass call looks like this:: - - class C(B): - def method(self, arg): - super().method(arg) # This does the same thing as: - # super(C, self).method(arg) - - In addition to method lookups, :func:`super` also works for attribute - lookups. One possible use case for this is calling :term:`descriptors ` - in a parent or sibling class. - - Note that :func:`super` is implemented as part of the binding process for - explicit dotted attribute lookups such as ``super().__getitem__(name)``. - It does so by implementing its own :meth:`__getattribute__` method for searching - classes in a predictable order that supports cooperative multiple inheritance. - Accordingly, :func:`super` is undefined for implicit lookups using statements or - operators such as ``super()[name]``. - - Also note that, aside from the zero argument form, :func:`super` is not - limited to use inside methods. The two argument form specifies the - arguments exactly and makes the appropriate references. The zero - argument form only works inside a class definition, as the compiler fills - in the necessary details to correctly retrieve the class being defined, - as well as accessing the current instance for ordinary methods. - - For practical suggestions on how to design cooperative classes using - :func:`super`, see `guide to using super() - `_. - - -.. _func-tuple: -.. class:: tuple() - tuple(iterable) - :noindex: - - Rather than being a function, :class:`tuple` is actually an immutable - sequence type, as documented in :ref:`typesseq-tuple` and :ref:`typesseq`. - - -.. class:: type(object) - type(name, bases, dict, **kwds) - - .. index:: pair: object; type - - With one argument, return the type of an *object*. The return value is a - type object and generally the same object as returned by - :attr:`object.__class__ `. - - The :func:`isinstance` built-in function is recommended for testing the type - of an object, because it takes subclasses into account. - - - With three arguments, return a new type object. This is essentially a - dynamic form of the :keyword:`class` statement. The *name* string is - the class name and becomes the :attr:`~definition.__name__` attribute. - The *bases* tuple contains the base classes and becomes the - :attr:`~class.__bases__` attribute; if empty, :class:`object`, the - ultimate base of all classes, is added. The *dict* dictionary contains - attribute and method definitions for the class body; it may be copied - or wrapped before becoming the :attr:`~object.__dict__` attribute. - The following two statements create identical :class:`type` objects: - - >>> class X: - ... a = 1 - ... - >>> X = type('X', (), dict(a=1)) - - See also :ref:`bltin-type-objects`. - - Keyword arguments provided to the three argument form are passed to the - appropriate metaclass machinery (usually :meth:`~object.__init_subclass__`) - in the same way that keywords in a class - definition (besides *metaclass*) would. - - See also :ref:`class-customization`. - - .. versionchanged:: 3.6 - Subclasses of :class:`type` which don't override ``type.__new__`` may no - longer use the one-argument form to get the type of an object. - -.. function:: vars() - vars(object) - - Return the :attr:`~object.__dict__` attribute for a module, class, instance, - or any other object with a :attr:`~object.__dict__` attribute. - - Objects such as modules and instances have an updateable :attr:`~object.__dict__` - attribute; however, other objects may have write restrictions on their - :attr:`~object.__dict__` attributes (for example, classes use a - :class:`types.MappingProxyType` to prevent direct dictionary updates). - - Without an argument, :func:`vars` acts like :func:`locals`. Note, the - locals dictionary is only useful for reads since updates to the locals - dictionary are ignored. - - A :exc:`TypeError` exception is raised if an object is specified but - it doesn't have a :attr:`~object.__dict__` attribute (for example, if - its class defines the :attr:`~object.__slots__` attribute). - -.. function:: zip(*iterables, strict=False) - - Iterate over several iterables in parallel, producing tuples with an item - from each one. - - Example:: - - >>> for item in zip([1, 2, 3], ['sugar', 'spice', 'everything nice']): - ... print(item) - ... - (1, 'sugar') - (2, 'spice') - (3, 'everything nice') - - More formally: :func:`zip` returns an iterator of tuples, where the *i*-th - tuple contains the *i*-th element from each of the argument iterables. - - Another way to think of :func:`zip` is that it turns rows into columns, and - columns into rows. This is similar to `transposing a matrix - `_. - - :func:`zip` is lazy: The elements won't be processed until the iterable is - iterated on, e.g. by a :keyword:`!for` loop or by wrapping in a - :class:`list`. - - One thing to consider is that the iterables passed to :func:`zip` could have - different lengths; sometimes by design, and sometimes because of a bug in - the code that prepared these iterables. Python offers three different - approaches to dealing with this issue: - - * By default, :func:`zip` stops when the shortest iterable is exhausted. - It will ignore the remaining items in the longer iterables, cutting off - the result to the length of the shortest iterable:: - - >>> list(zip(range(3), ['fee', 'fi', 'fo', 'fum'])) - [(0, 'fee'), (1, 'fi'), (2, 'fo')] - - * :func:`zip` is often used in cases where the iterables are assumed to be - of equal length. In such cases, it's recommended to use the ``strict=True`` - option. Its output is the same as regular :func:`zip`:: - - >>> list(zip(('a', 'b', 'c'), (1, 2, 3), strict=True)) - [('a', 1), ('b', 2), ('c', 3)] - - Unlike the default behavior, it raises a :exc:`ValueError` if one iterable - is exhausted before the others: - - >>> for item in zip(range(3), ['fee', 'fi', 'fo', 'fum'], strict=True): # doctest: +SKIP - ... print(item) - ... - (0, 'fee') - (1, 'fi') - (2, 'fo') - Traceback (most recent call last): - ... - ValueError: zip() argument 2 is longer than argument 1 - - .. - This doctest is disabled because doctest does not support capturing - output and exceptions in the same code unit. - https://github.com/python/cpython/issues/65382 - - Without the ``strict=True`` argument, any bug that results in iterables of - different lengths will be silenced, possibly manifesting as a hard-to-find - bug in another part of the program. - - * Shorter iterables can be padded with a constant value to make all the - iterables have the same length. This is done by - :func:`itertools.zip_longest`. - - Edge cases: With a single iterable argument, :func:`zip` returns an - iterator of 1-tuples. With no arguments, it returns an empty iterator. - - Tips and tricks: - - * The left-to-right evaluation order of the iterables is guaranteed. This - makes possible an idiom for clustering a data series into n-length groups - using ``zip(*[iter(s)]*n, strict=True)``. This repeats the *same* iterator - ``n`` times so that each output tuple has the result of ``n`` calls to the - iterator. This has the effect of dividing the input into n-length chunks. - - * :func:`zip` in conjunction with the ``*`` operator can be used to unzip a - list:: - - >>> x = [1, 2, 3] - >>> y = [4, 5, 6] - >>> list(zip(x, y)) - [(1, 4), (2, 5), (3, 6)] - >>> x2, y2 = zip(*zip(x, y)) - >>> x == list(x2) and y == list(y2) - True - - .. versionchanged:: 3.10 - Added the ``strict`` argument. - - -.. function:: __import__(name, globals=None, locals=None, fromlist=(), level=0) - - .. index:: - pair: statement; import - pair: module; builtins - - .. note:: - - This is an advanced function that is not needed in everyday Python - programming, unlike :func:`importlib.import_module`. - - This function is invoked by the :keyword:`import` statement. It can be - replaced (by importing the :mod:`builtins` module and assigning to - ``builtins.__import__``) in order to change semantics of the - :keyword:`!import` statement, but doing so is **strongly** discouraged as it - is usually simpler to use import hooks (see :pep:`302`) to attain the same - goals and does not cause issues with code which assumes the default import - implementation is in use. Direct use of :func:`__import__` is also - discouraged in favor of :func:`importlib.import_module`. - - The function imports the module *name*, potentially using the given *globals* - and *locals* to determine how to interpret the name in a package context. - The *fromlist* gives the names of objects or submodules that should be - imported from the module given by *name*. The standard implementation does - not use its *locals* argument at all and uses its *globals* only to - determine the package context of the :keyword:`import` statement. - - *level* specifies whether to use absolute or relative imports. ``0`` (the - default) means only perform absolute imports. Positive values for - *level* indicate the number of parent directories to search relative to the - directory of the module calling :func:`__import__` (see :pep:`328` for the - details). - - When the *name* variable is of the form ``package.module``, normally, the - top-level package (the name up till the first dot) is returned, *not* the - module named by *name*. However, when a non-empty *fromlist* argument is - given, the module named by *name* is returned. - - For example, the statement ``import spam`` results in bytecode resembling the - following code:: - - spam = __import__('spam', globals(), locals(), [], 0) - - The statement ``import spam.ham`` results in this call:: - - spam = __import__('spam.ham', globals(), locals(), [], 0) - - Note how :func:`__import__` returns the toplevel module here because this is - the object that is bound to a name by the :keyword:`import` statement. - - On the other hand, the statement ``from spam.ham import eggs, sausage as - saus`` results in :: - - _temp = __import__('spam.ham', globals(), locals(), ['eggs', 'sausage'], 0) - eggs = _temp.eggs - saus = _temp.sausage - - Here, the ``spam.ham`` module is returned from :func:`__import__`. From this - object, the names to import are retrieved and assigned to their respective - names. - - If you simply want to import a module (potentially within a package) by name, - use :func:`importlib.import_module`. - - .. versionchanged:: 3.3 - Negative values for *level* are no longer supported (which also changes - the default value to 0). - - .. versionchanged:: 3.9 - When the command line options :option:`-E` or :option:`-I` are being used, - the environment variable :envvar:`PYTHONCASEOK` is now ignored. - -.. rubric:: Footnotes - -.. [#] Note that the parser only accepts the Unix-style end of line convention. - If you are reading the code from a file, make sure to use newline conversion - mode to convert Windows or Mac-style newlines. diff --git a/Doc/library/functools.rst b/Doc/library/functools.rst deleted file mode 100644 index 9dadbb69fc2f24..00000000000000 --- a/Doc/library/functools.rst +++ /dev/null @@ -1,732 +0,0 @@ -:mod:`functools` --- Higher-order functions and operations on callable objects -============================================================================== - -.. module:: functools - :synopsis: Higher-order functions and operations on callable objects. - -.. moduleauthor:: Peter Harris -.. moduleauthor:: Raymond Hettinger -.. moduleauthor:: Nick Coghlan -.. moduleauthor:: Łukasz Langa -.. moduleauthor:: Pablo Galindo -.. sectionauthor:: Peter Harris - -**Source code:** :source:`Lib/functools.py` - -.. testsetup:: default - - import functools - from functools import * - --------------- - -The :mod:`functools` module is for higher-order functions: functions that act on -or return other functions. In general, any callable object can be treated as a -function for the purposes of this module. - -The :mod:`functools` module defines the following functions: - -.. decorator:: cache(user_function) - - Simple lightweight unbounded function cache. Sometimes called - `"memoize" `_. - - Returns the same as ``lru_cache(maxsize=None)``, creating a thin - wrapper around a dictionary lookup for the function arguments. Because it - never needs to evict old values, this is smaller and faster than - :func:`lru_cache()` with a size limit. - - For example:: - - @cache - def factorial(n): - return n * factorial(n-1) if n else 1 - - >>> factorial(10) # no previously cached result, makes 11 recursive calls - 3628800 - >>> factorial(5) # just looks up cached value result - 120 - >>> factorial(12) # makes two new recursive calls, the other 10 are cached - 479001600 - - The cache is threadsafe so that the wrapped function can be used in - multiple threads. This means that the underlying data structure will - remain coherent during concurrent updates. - - It is possible for the wrapped function to be called more than once if - another thread makes an additional call before the initial call has been - completed and cached. - - .. versionadded:: 3.9 - - -.. decorator:: cached_property(func) - - Transform a method of a class into a property whose value is computed once - and then cached as a normal attribute for the life of the instance. Similar - to :func:`property`, with the addition of caching. Useful for expensive - computed properties of instances that are otherwise effectively immutable. - - Example:: - - class DataSet: - - def __init__(self, sequence_of_numbers): - self._data = tuple(sequence_of_numbers) - - @cached_property - def stdev(self): - return statistics.stdev(self._data) - - The mechanics of :func:`cached_property` are somewhat different from - :func:`property`. A regular property blocks attribute writes unless a - setter is defined. In contrast, a *cached_property* allows writes. - - The *cached_property* decorator only runs on lookups and only when an - attribute of the same name doesn't exist. When it does run, the - *cached_property* writes to the attribute with the same name. Subsequent - attribute reads and writes take precedence over the *cached_property* - method and it works like a normal attribute. - - The cached value can be cleared by deleting the attribute. This - allows the *cached_property* method to run again. - - Note, this decorator interferes with the operation of :pep:`412` - key-sharing dictionaries. This means that instance dictionaries - can take more space than usual. - - Also, this decorator requires that the ``__dict__`` attribute on each instance - be a mutable mapping. This means it will not work with some types, such as - metaclasses (since the ``__dict__`` attributes on type instances are - read-only proxies for the class namespace), and those that specify - ``__slots__`` without including ``__dict__`` as one of the defined slots - (as such classes don't provide a ``__dict__`` attribute at all). - - If a mutable mapping is not available or if space-efficient key sharing is - desired, an effect similar to :func:`cached_property` can also be achieved by - stacking :func:`property` on top of :func:`lru_cache`. See - :ref:`faq-cache-method-calls` for more details on how this differs from :func:`cached_property`. - - .. versionadded:: 3.8 - - -.. function:: cmp_to_key(func) - - Transform an old-style comparison function to a :term:`key function`. Used - with tools that accept key functions (such as :func:`sorted`, :func:`min`, - :func:`max`, :func:`heapq.nlargest`, :func:`heapq.nsmallest`, - :func:`itertools.groupby`). This function is primarily used as a transition - tool for programs being converted from Python 2 which supported the use of - comparison functions. - - A comparison function is any callable that accepts two arguments, compares them, - and returns a negative number for less-than, zero for equality, or a positive - number for greater-than. A key function is a callable that accepts one - argument and returns another value to be used as the sort key. - - Example:: - - sorted(iterable, key=cmp_to_key(locale.strcoll)) # locale-aware sort order - - For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. - - .. versionadded:: 3.2 - - -.. decorator:: lru_cache(user_function) - lru_cache(maxsize=128, typed=False) - - Decorator to wrap a function with a memoizing callable that saves up to the - *maxsize* most recent calls. It can save time when an expensive or I/O bound - function is periodically called with the same arguments. - - The cache is threadsafe so that the wrapped function can be used in - multiple threads. This means that the underlying data structure will - remain coherent during concurrent updates. - - It is possible for the wrapped function to be called more than once if - another thread makes an additional call before the initial call has been - completed and cached. - - Since a dictionary is used to cache results, the positional and keyword - arguments to the function must be :term:`hashable`. - - Distinct argument patterns may be considered to be distinct calls with - separate cache entries. For example, ``f(a=1, b=2)`` and ``f(b=2, a=1)`` - differ in their keyword argument order and may have two separate cache - entries. - - If *user_function* is specified, it must be a callable. This allows the - *lru_cache* decorator to be applied directly to a user function, leaving - the *maxsize* at its default value of 128:: - - @lru_cache - def count_vowels(sentence): - return sum(sentence.count(vowel) for vowel in 'AEIOUaeiou') - - If *maxsize* is set to ``None``, the LRU feature is disabled and the cache can - grow without bound. - - If *typed* is set to true, function arguments of different types will be - cached separately. If *typed* is false, the implementation will usually - regard them as equivalent calls and only cache a single result. (Some - types such as *str* and *int* may be cached separately even when *typed* - is false.) - - Note, type specificity applies only to the function's immediate arguments - rather than their contents. The scalar arguments, ``Decimal(42)`` and - ``Fraction(42)`` are be treated as distinct calls with distinct results. - In contrast, the tuple arguments ``('answer', Decimal(42))`` and - ``('answer', Fraction(42))`` are treated as equivalent. - - The wrapped function is instrumented with a :func:`cache_parameters` - function that returns a new :class:`dict` showing the values for *maxsize* - and *typed*. This is for information purposes only. Mutating the values - has no effect. - - To help measure the effectiveness of the cache and tune the *maxsize* - parameter, the wrapped function is instrumented with a :func:`cache_info` - function that returns a :term:`named tuple` showing *hits*, *misses*, - *maxsize* and *currsize*. - - The decorator also provides a :func:`cache_clear` function for clearing or - invalidating the cache. - - The original underlying function is accessible through the - :attr:`__wrapped__` attribute. This is useful for introspection, for - bypassing the cache, or for rewrapping the function with a different cache. - - The cache keeps references to the arguments and return values until they age - out of the cache or until the cache is cleared. - - If a method is cached, the ``self`` instance argument is included in the - cache. See :ref:`faq-cache-method-calls` - - An `LRU (least recently used) cache - `_ - works best when the most recent calls are the best predictors of upcoming - calls (for example, the most popular articles on a news server tend to - change each day). The cache's size limit assures that the cache does not - grow without bound on long-running processes such as web servers. - - In general, the LRU cache should only be used when you want to reuse - previously computed values. Accordingly, it doesn't make sense to cache - functions with side-effects, functions that need to create - distinct mutable objects on each call (such as generators and async functions), - or impure functions such as time() or random(). - - Example of an LRU cache for static web content:: - - @lru_cache(maxsize=32) - def get_pep(num): - 'Retrieve text of a Python Enhancement Proposal' - resource = 'https://peps.python.org/pep-%04d/' % num - try: - with urllib.request.urlopen(resource) as s: - return s.read() - except urllib.error.HTTPError: - return 'Not Found' - - >>> for n in 8, 290, 308, 320, 8, 218, 320, 279, 289, 320, 9991: - ... pep = get_pep(n) - ... print(n, len(pep)) - - >>> get_pep.cache_info() - CacheInfo(hits=3, misses=8, maxsize=32, currsize=8) - - Example of efficiently computing - `Fibonacci numbers `_ - using a cache to implement a - `dynamic programming `_ - technique:: - - @lru_cache(maxsize=None) - def fib(n): - if n < 2: - return n - return fib(n-1) + fib(n-2) - - >>> [fib(n) for n in range(16)] - [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89, 144, 233, 377, 610] - - >>> fib.cache_info() - CacheInfo(hits=28, misses=16, maxsize=None, currsize=16) - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - Added the *typed* option. - - .. versionchanged:: 3.8 - Added the *user_function* option. - - .. versionadded:: 3.9 - Added the function :func:`cache_parameters` - -.. decorator:: total_ordering - - Given a class defining one or more rich comparison ordering methods, this - class decorator supplies the rest. This simplifies the effort involved - in specifying all of the possible rich comparison operations: - - The class must define one of :meth:`__lt__`, :meth:`__le__`, - :meth:`__gt__`, or :meth:`__ge__`. - In addition, the class should supply an :meth:`__eq__` method. - - For example:: - - @total_ordering - class Student: - def _is_valid_operand(self, other): - return (hasattr(other, "lastname") and - hasattr(other, "firstname")) - def __eq__(self, other): - if not self._is_valid_operand(other): - return NotImplemented - return ((self.lastname.lower(), self.firstname.lower()) == - (other.lastname.lower(), other.firstname.lower())) - def __lt__(self, other): - if not self._is_valid_operand(other): - return NotImplemented - return ((self.lastname.lower(), self.firstname.lower()) < - (other.lastname.lower(), other.firstname.lower())) - - .. note:: - - While this decorator makes it easy to create well behaved totally - ordered types, it *does* come at the cost of slower execution and - more complex stack traces for the derived comparison methods. If - performance benchmarking indicates this is a bottleneck for a given - application, implementing all six rich comparison methods instead is - likely to provide an easy speed boost. - - .. note:: - - This decorator makes no attempt to override methods that have been - declared in the class *or its superclasses*. Meaning that if a - superclass defines a comparison operator, *total_ordering* will not - implement it again, even if the original method is abstract. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Returning NotImplemented from the underlying comparison function for - unrecognised types is now supported. - -.. function:: partial(func, /, *args, **keywords) - - Return a new :ref:`partial object` which when called - will behave like *func* called with the positional arguments *args* - and keyword arguments *keywords*. If more arguments are supplied to the - call, they are appended to *args*. If additional keyword arguments are - supplied, they extend and override *keywords*. - Roughly equivalent to:: - - def partial(func, /, *args, **keywords): - def newfunc(*fargs, **fkeywords): - newkeywords = {**keywords, **fkeywords} - return func(*args, *fargs, **newkeywords) - newfunc.func = func - newfunc.args = args - newfunc.keywords = keywords - return newfunc - - The :func:`partial` is used for partial function application which "freezes" - some portion of a function's arguments and/or keywords resulting in a new object - with a simplified signature. For example, :func:`partial` can be used to create - a callable that behaves like the :func:`int` function where the *base* argument - defaults to two: - - >>> from functools import partial - >>> basetwo = partial(int, base=2) - >>> basetwo.__doc__ = 'Convert base 2 string to an int.' - >>> basetwo('10010') - 18 - - -.. class:: partialmethod(func, /, *args, **keywords) - - Return a new :class:`partialmethod` descriptor which behaves - like :class:`partial` except that it is designed to be used as a method - definition rather than being directly callable. - - *func* must be a :term:`descriptor` or a callable (objects which are both, - like normal functions, are handled as descriptors). - - When *func* is a descriptor (such as a normal Python function, - :func:`classmethod`, :func:`staticmethod`, :func:`abstractmethod` or - another instance of :class:`partialmethod`), calls to ``__get__`` are - delegated to the underlying descriptor, and an appropriate - :ref:`partial object` returned as the result. - - When *func* is a non-descriptor callable, an appropriate bound method is - created dynamically. This behaves like a normal Python function when - used as a method: the *self* argument will be inserted as the first - positional argument, even before the *args* and *keywords* supplied to - the :class:`partialmethod` constructor. - - Example:: - - >>> class Cell: - ... def __init__(self): - ... self._alive = False - ... @property - ... def alive(self): - ... return self._alive - ... def set_state(self, state): - ... self._alive = bool(state) - ... set_alive = partialmethod(set_state, True) - ... set_dead = partialmethod(set_state, False) - ... - >>> c = Cell() - >>> c.alive - False - >>> c.set_alive() - >>> c.alive - True - - .. versionadded:: 3.4 - - -.. function:: reduce(function, iterable[, initializer]) - - Apply *function* of two arguments cumulatively to the items of *iterable*, from - left to right, so as to reduce the iterable to a single value. For example, - ``reduce(lambda x, y: x+y, [1, 2, 3, 4, 5])`` calculates ``((((1+2)+3)+4)+5)``. - The left argument, *x*, is the accumulated value and the right argument, *y*, is - the update value from the *iterable*. If the optional *initializer* is present, - it is placed before the items of the iterable in the calculation, and serves as - a default when the iterable is empty. If *initializer* is not given and - *iterable* contains only one item, the first item is returned. - - Roughly equivalent to:: - - def reduce(function, iterable, initializer=None): - it = iter(iterable) - if initializer is None: - value = next(it) - else: - value = initializer - for element in it: - value = function(value, element) - return value - - See :func:`itertools.accumulate` for an iterator that yields all intermediate - values. - -.. decorator:: singledispatch - - Transform a function into a :term:`single-dispatch ` :term:`generic function`. - - To define a generic function, decorate it with the ``@singledispatch`` - decorator. When defining a function using ``@singledispatch``, note that the - dispatch happens on the type of the first argument:: - - >>> from functools import singledispatch - >>> @singledispatch - ... def fun(arg, verbose=False): - ... if verbose: - ... print("Let me just say,", end=" ") - ... print(arg) - - To add overloaded implementations to the function, use the :func:`register` - attribute of the generic function, which can be used as a decorator. For - functions annotated with types, the decorator will infer the type of the - first argument automatically:: - - >>> @fun.register - ... def _(arg: int, verbose=False): - ... if verbose: - ... print("Strength in numbers, eh?", end=" ") - ... print(arg) - ... - >>> @fun.register - ... def _(arg: list, verbose=False): - ... if verbose: - ... print("Enumerate this:") - ... for i, elem in enumerate(arg): - ... print(i, elem) - - :data:`types.UnionType` and :data:`typing.Union` can also be used:: - - >>> @fun.register - ... def _(arg: int | float, verbose=False): - ... if verbose: - ... print("Strength in numbers, eh?", end=" ") - ... print(arg) - ... - >>> from typing import Union - >>> @fun.register - ... def _(arg: Union[list, set], verbose=False): - ... if verbose: - ... print("Enumerate this:") - ... for i, elem in enumerate(arg): - ... print(i, elem) - ... - - For code which doesn't use type annotations, the appropriate type - argument can be passed explicitly to the decorator itself:: - - >>> @fun.register(complex) - ... def _(arg, verbose=False): - ... if verbose: - ... print("Better than complicated.", end=" ") - ... print(arg.real, arg.imag) - ... - - - To enable registering :term:`lambdas` and pre-existing functions, - the :func:`register` attribute can also be used in a functional form:: - - >>> def nothing(arg, verbose=False): - ... print("Nothing.") - ... - >>> fun.register(type(None), nothing) - - The :func:`register` attribute returns the undecorated function. This - enables decorator stacking, :mod:`pickling`, and the creation - of unit tests for each variant independently:: - - >>> @fun.register(float) - ... @fun.register(Decimal) - ... def fun_num(arg, verbose=False): - ... if verbose: - ... print("Half of your number:", end=" ") - ... print(arg / 2) - ... - >>> fun_num is fun - False - - When called, the generic function dispatches on the type of the first - argument:: - - >>> fun("Hello, world.") - Hello, world. - >>> fun("test.", verbose=True) - Let me just say, test. - >>> fun(42, verbose=True) - Strength in numbers, eh? 42 - >>> fun(['spam', 'spam', 'eggs', 'spam'], verbose=True) - Enumerate this: - 0 spam - 1 spam - 2 eggs - 3 spam - >>> fun(None) - Nothing. - >>> fun(1.23) - 0.615 - - Where there is no registered implementation for a specific type, its - method resolution order is used to find a more generic implementation. - The original function decorated with ``@singledispatch`` is registered - for the base :class:`object` type, which means it is used if no better - implementation is found. - - If an implementation is registered to an :term:`abstract base class`, - virtual subclasses of the base class will be dispatched to that - implementation:: - - >>> from collections.abc import Mapping - >>> @fun.register - ... def _(arg: Mapping, verbose=False): - ... if verbose: - ... print("Keys & Values") - ... for key, value in arg.items(): - ... print(key, "=>", value) - ... - >>> fun({"a": "b"}) - a => b - - To check which implementation the generic function will choose for - a given type, use the ``dispatch()`` attribute:: - - >>> fun.dispatch(float) - - >>> fun.dispatch(dict) # note: default implementation - - - To access all registered implementations, use the read-only ``registry`` - attribute:: - - >>> fun.registry.keys() - dict_keys([, , , - , , - ]) - >>> fun.registry[float] - - >>> fun.registry[object] - - - .. versionadded:: 3.4 - - .. versionchanged:: 3.7 - The :func:`register` attribute now supports using type annotations. - - .. versionchanged:: 3.11 - The :func:`register` attribute now supports :data:`types.UnionType` - and :data:`typing.Union` as type annotations. - - -.. class:: singledispatchmethod(func) - - Transform a method into a :term:`single-dispatch ` :term:`generic function`. - - To define a generic method, decorate it with the ``@singledispatchmethod`` - decorator. When defining a function using ``@singledispatchmethod``, note - that the dispatch happens on the type of the first non-*self* or non-*cls* - argument:: - - class Negator: - @singledispatchmethod - def neg(self, arg): - raise NotImplementedError("Cannot negate a") - - @neg.register - def _(self, arg: int): - return -arg - - @neg.register - def _(self, arg: bool): - return not arg - - ``@singledispatchmethod`` supports nesting with other decorators such as - :func:`@classmethod`. Note that to allow for - ``dispatcher.register``, ``singledispatchmethod`` must be the *outer most* - decorator. Here is the ``Negator`` class with the ``neg`` methods bound to - the class, rather than an instance of the class:: - - class Negator: - @singledispatchmethod - @classmethod - def neg(cls, arg): - raise NotImplementedError("Cannot negate a") - - @neg.register - @classmethod - def _(cls, arg: int): - return -arg - - @neg.register - @classmethod - def _(cls, arg: bool): - return not arg - - The same pattern can be used for other similar decorators: - :func:`@staticmethod`, - :func:`@abstractmethod`, and others. - - .. versionadded:: 3.8 - - -.. function:: update_wrapper(wrapper, wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) - - Update a *wrapper* function to look like the *wrapped* function. The optional - arguments are tuples to specify which attributes of the original function are - assigned directly to the matching attributes on the wrapper function and which - attributes of the wrapper function are updated with the corresponding attributes - from the original function. The default values for these arguments are the - module level constants ``WRAPPER_ASSIGNMENTS`` (which assigns to the wrapper - function's ``__module__``, ``__name__``, ``__qualname__``, ``__annotations__`` - and ``__doc__``, the documentation string) and ``WRAPPER_UPDATES`` (which - updates the wrapper function's ``__dict__``, i.e. the instance dictionary). - - To allow access to the original function for introspection and other purposes - (e.g. bypassing a caching decorator such as :func:`lru_cache`), this function - automatically adds a ``__wrapped__`` attribute to the wrapper that refers to - the function being wrapped. - - The main intended use for this function is in :term:`decorator` functions which - wrap the decorated function and return the wrapper. If the wrapper function is - not updated, the metadata of the returned function will reflect the wrapper - definition rather than the original function definition, which is typically less - than helpful. - - :func:`update_wrapper` may be used with callables other than functions. Any - attributes named in *assigned* or *updated* that are missing from the object - being wrapped are ignored (i.e. this function will not attempt to set them - on the wrapper function). :exc:`AttributeError` is still raised if the - wrapper function itself is missing any attributes named in *updated*. - - .. versionadded:: 3.2 - Automatic addition of the ``__wrapped__`` attribute. - - .. versionadded:: 3.2 - Copying of the ``__annotations__`` attribute by default. - - .. versionchanged:: 3.2 - Missing attributes no longer trigger an :exc:`AttributeError`. - - .. versionchanged:: 3.4 - The ``__wrapped__`` attribute now always refers to the wrapped - function, even if that function defined a ``__wrapped__`` attribute. - (see :issue:`17482`) - - -.. decorator:: wraps(wrapped, assigned=WRAPPER_ASSIGNMENTS, updated=WRAPPER_UPDATES) - - This is a convenience function for invoking :func:`update_wrapper` as a - function decorator when defining a wrapper function. It is equivalent to - ``partial(update_wrapper, wrapped=wrapped, assigned=assigned, updated=updated)``. - For example:: - - >>> from functools import wraps - >>> def my_decorator(f): - ... @wraps(f) - ... def wrapper(*args, **kwds): - ... print('Calling decorated function') - ... return f(*args, **kwds) - ... return wrapper - ... - >>> @my_decorator - ... def example(): - ... """Docstring""" - ... print('Called example function') - ... - >>> example() - Calling decorated function - Called example function - >>> example.__name__ - 'example' - >>> example.__doc__ - 'Docstring' - - Without the use of this decorator factory, the name of the example function - would have been ``'wrapper'``, and the docstring of the original :func:`example` - would have been lost. - - -.. _partial-objects: - -:class:`partial` Objects ------------------------- - -:class:`partial` objects are callable objects created by :func:`partial`. They -have three read-only attributes: - - -.. attribute:: partial.func - - A callable object or function. Calls to the :class:`partial` object will be - forwarded to :attr:`func` with new arguments and keywords. - - -.. attribute:: partial.args - - The leftmost positional arguments that will be prepended to the positional - arguments provided to a :class:`partial` object call. - - -.. attribute:: partial.keywords - - The keyword arguments that will be supplied when the :class:`partial` object is - called. - -:class:`partial` objects are like :class:`function` objects in that they are -callable, weak referencable, and can have attributes. There are some important -differences. For instance, the :attr:`~definition.__name__` and :attr:`__doc__` attributes -are not created automatically. Also, :class:`partial` objects defined in -classes behave like static methods and do not transform into bound methods -during instance attribute look-up. diff --git a/Doc/library/gc.rst b/Doc/library/gc.rst deleted file mode 100644 index 0ecd93813c9193..00000000000000 --- a/Doc/library/gc.rst +++ /dev/null @@ -1,335 +0,0 @@ -:mod:`gc` --- Garbage Collector interface -========================================= - -.. module:: gc - :synopsis: Interface to the cycle-detecting garbage collector. - -.. moduleauthor:: Neil Schemenauer -.. sectionauthor:: Neil Schemenauer - --------------- - -This module provides an interface to the optional garbage collector. It -provides the ability to disable the collector, tune the collection frequency, -and set debugging options. It also provides access to unreachable objects that -the collector found but cannot free. Since the collector supplements the -reference counting already used in Python, you can disable the collector if you -are sure your program does not create reference cycles. Automatic collection -can be disabled by calling ``gc.disable()``. To debug a leaking program call -``gc.set_debug(gc.DEBUG_LEAK)``. Notice that this includes -``gc.DEBUG_SAVEALL``, causing garbage-collected objects to be saved in -gc.garbage for inspection. - -The :mod:`gc` module provides the following functions: - - -.. function:: enable() - - Enable automatic garbage collection. - - -.. function:: disable() - - Disable automatic garbage collection. - - -.. function:: isenabled() - - Return ``True`` if automatic collection is enabled. - - -.. function:: collect(generation=2) - - With no arguments, run a full collection. The optional argument *generation* - may be an integer specifying which generation to collect (from 0 to 2). A - :exc:`ValueError` is raised if the generation number is invalid. The number of - unreachable objects found is returned. - - The free lists maintained for a number of built-in types are cleared - whenever a full collection or collection of the highest generation (2) - is run. Not all items in some free lists may be freed due to the - particular implementation, in particular :class:`float`. - - The effect of calling ``gc.collect()`` while the interpreter is already - performing a collection is undefined. - - -.. function:: set_debug(flags) - - Set the garbage collection debugging flags. Debugging information will be - written to ``sys.stderr``. See below for a list of debugging flags which can be - combined using bit operations to control debugging. - - -.. function:: get_debug() - - Return the debugging flags currently set. - - -.. function:: get_objects(generation=None) - - Returns a list of all objects tracked by the collector, excluding the list - returned. If *generation* is not None, return only the objects tracked by - the collector that are in that generation. - - .. versionchanged:: 3.8 - New *generation* parameter. - - .. audit-event:: gc.get_objects generation gc.get_objects - -.. function:: get_stats() - - Return a list of three per-generation dictionaries containing collection - statistics since interpreter start. The number of keys may change - in the future, but currently each dictionary will contain the following - items: - - * ``collections`` is the number of times this generation was collected; - - * ``collected`` is the total number of objects collected inside this - generation; - - * ``uncollectable`` is the total number of objects which were found - to be uncollectable (and were therefore moved to the :data:`garbage` - list) inside this generation. - - .. versionadded:: 3.4 - - -.. function:: set_threshold(threshold0[, threshold1[, threshold2]]) - - Set the garbage collection thresholds (the collection frequency). Setting - *threshold0* to zero disables collection. - - The GC classifies objects into three generations depending on how many - collection sweeps they have survived. New objects are placed in the youngest - generation (generation ``0``). If an object survives a collection it is moved - into the next older generation. Since generation ``2`` is the oldest - generation, objects in that generation remain there after a collection. In - order to decide when to run, the collector keeps track of the number object - allocations and deallocations since the last collection. When the number of - allocations minus the number of deallocations exceeds *threshold0*, collection - starts. Initially only generation ``0`` is examined. If generation ``0`` has - been examined more than *threshold1* times since generation ``1`` has been - examined, then generation ``1`` is examined as well. - With the third generation, things are a bit more complicated, - see `Collecting the oldest generation `_ for more information. - - -.. function:: get_count() - - Return the current collection counts as a tuple of ``(count0, count1, - count2)``. - - -.. function:: get_threshold() - - Return the current collection thresholds as a tuple of ``(threshold0, - threshold1, threshold2)``. - - -.. function:: get_referrers(*objs) - - Return the list of objects that directly refer to any of objs. This function - will only locate those containers which support garbage collection; extension - types which do refer to other objects but do not support garbage collection will - not be found. - - Note that objects which have already been dereferenced, but which live in cycles - and have not yet been collected by the garbage collector can be listed among the - resulting referrers. To get only currently live objects, call :func:`collect` - before calling :func:`get_referrers`. - - .. warning:: - Care must be taken when using objects returned by :func:`get_referrers` because - some of them could still be under construction and hence in a temporarily - invalid state. Avoid using :func:`get_referrers` for any purpose other than - debugging. - - .. audit-event:: gc.get_referrers objs gc.get_referrers - - -.. function:: get_referents(*objs) - - Return a list of objects directly referred to by any of the arguments. The - referents returned are those objects visited by the arguments' C-level - :c:member:`~PyTypeObject.tp_traverse` methods (if any), and may not be all objects actually - directly reachable. :c:member:`~PyTypeObject.tp_traverse` methods are supported only by objects - that support garbage collection, and are only required to visit objects that may - be involved in a cycle. So, for example, if an integer is directly reachable - from an argument, that integer object may or may not appear in the result list. - - .. audit-event:: gc.get_referents objs gc.get_referents - -.. function:: is_tracked(obj) - - Returns ``True`` if the object is currently tracked by the garbage collector, - ``False`` otherwise. As a general rule, instances of atomic types aren't - tracked and instances of non-atomic types (containers, user-defined - objects...) are. However, some type-specific optimizations can be present - in order to suppress the garbage collector footprint of simple instances - (e.g. dicts containing only atomic keys and values):: - - >>> gc.is_tracked(0) - False - >>> gc.is_tracked("a") - False - >>> gc.is_tracked([]) - True - >>> gc.is_tracked({}) - False - >>> gc.is_tracked({"a": 1}) - False - >>> gc.is_tracked({"a": []}) - True - - .. versionadded:: 3.1 - - -.. function:: is_finalized(obj) - - Returns ``True`` if the given object has been finalized by the - garbage collector, ``False`` otherwise. :: - - >>> x = None - >>> class Lazarus: - ... def __del__(self): - ... global x - ... x = self - ... - >>> lazarus = Lazarus() - >>> gc.is_finalized(lazarus) - False - >>> del lazarus - >>> gc.is_finalized(x) - True - - .. versionadded:: 3.9 - - -.. function:: freeze() - - Freeze all the objects tracked by the garbage collector; move them to a - permanent generation and ignore them in all the future collections. - - If a process will ``fork()`` without ``exec()``, avoiding unnecessary - copy-on-write in child processes will maximize memory sharing and reduce - overall memory usage. This requires both avoiding creation of freed "holes" - in memory pages in the parent process and ensuring that GC collections in - child processes won't touch the ``gc_refs`` counter of long-lived objects - originating in the parent process. To accomplish both, call ``gc.disable()`` - early in the parent process, ``gc.freeze()`` right before ``fork()``, and - ``gc.enable()`` early in child processes. - - .. versionadded:: 3.7 - - -.. function:: unfreeze() - - Unfreeze the objects in the permanent generation, put them back into the - oldest generation. - - .. versionadded:: 3.7 - - -.. function:: get_freeze_count() - - Return the number of objects in the permanent generation. - - .. versionadded:: 3.7 - - -The following variables are provided for read-only access (you can mutate the -values but should not rebind them): - -.. data:: garbage - - A list of objects which the collector found to be unreachable but could - not be freed (uncollectable objects). Starting with Python 3.4, this - list should be empty most of the time, except when using instances of - C extension types with a non-``NULL`` ``tp_del`` slot. - - If :const:`DEBUG_SAVEALL` is set, then all unreachable objects will be - added to this list rather than freed. - - .. versionchanged:: 3.2 - If this list is non-empty at :term:`interpreter shutdown`, a - :exc:`ResourceWarning` is emitted, which is silent by default. If - :const:`DEBUG_UNCOLLECTABLE` is set, in addition all uncollectable objects - are printed. - - .. versionchanged:: 3.4 - Following :pep:`442`, objects with a :meth:`__del__` method don't end - up in :data:`gc.garbage` anymore. - -.. data:: callbacks - - A list of callbacks that will be invoked by the garbage collector before and - after collection. The callbacks will be called with two arguments, - *phase* and *info*. - - *phase* can be one of two values: - - "start": The garbage collection is about to start. - - "stop": The garbage collection has finished. - - *info* is a dict providing more information for the callback. The following - keys are currently defined: - - "generation": The oldest generation being collected. - - "collected": When *phase* is "stop", the number of objects - successfully collected. - - "uncollectable": When *phase* is "stop", the number of objects - that could not be collected and were put in :data:`garbage`. - - Applications can add their own callbacks to this list. The primary - use cases are: - - Gathering statistics about garbage collection, such as how often - various generations are collected, and how long the collection - takes. - - Allowing applications to identify and clear their own uncollectable - types when they appear in :data:`garbage`. - - .. versionadded:: 3.3 - - -The following constants are provided for use with :func:`set_debug`: - - -.. data:: DEBUG_STATS - - Print statistics during collection. This information can be useful when tuning - the collection frequency. - - -.. data:: DEBUG_COLLECTABLE - - Print information on collectable objects found. - - -.. data:: DEBUG_UNCOLLECTABLE - - Print information of uncollectable objects found (objects which are not - reachable but cannot be freed by the collector). These objects will be added - to the ``garbage`` list. - - .. versionchanged:: 3.2 - Also print the contents of the :data:`garbage` list at - :term:`interpreter shutdown`, if it isn't empty. - -.. data:: DEBUG_SAVEALL - - When set, all unreachable objects found will be appended to *garbage* rather - than being freed. This can be useful for debugging a leaking program. - - -.. data:: DEBUG_LEAK - - The debugging flags necessary for the collector to print information about a - leaking program (equal to ``DEBUG_COLLECTABLE | DEBUG_UNCOLLECTABLE | - DEBUG_SAVEALL``). diff --git a/Doc/library/getopt.rst b/Doc/library/getopt.rst deleted file mode 100644 index 336deab28cb8a4..00000000000000 --- a/Doc/library/getopt.rst +++ /dev/null @@ -1,164 +0,0 @@ -:mod:`getopt` --- C-style parser for command line options -========================================================= - -.. module:: getopt - :synopsis: Portable parser for command line options; support both short and - long option names. - -**Source code:** :source:`Lib/getopt.py` - -.. note:: - - The :mod:`getopt` module is a parser for command line options whose API is - designed to be familiar to users of the C :c:func:`getopt` function. Users who - are unfamiliar with the C :c:func:`getopt` function or who would like to write - less code and get better help and error messages should consider using the - :mod:`argparse` module instead. - --------------- - -This module helps scripts to parse the command line arguments in ``sys.argv``. -It supports the same conventions as the Unix :c:func:`getopt` function (including -the special meanings of arguments of the form '``-``' and '``--``'). Long -options similar to those supported by GNU software may be used as well via an -optional third argument. - -This module provides two functions and an -exception: - - -.. function:: getopt(args, shortopts, longopts=[]) - - Parses command line options and parameter list. *args* is the argument list to - be parsed, without the leading reference to the running program. Typically, this - means ``sys.argv[1:]``. *shortopts* is the string of option letters that the - script wants to recognize, with options that require an argument followed by a - colon (``':'``; i.e., the same format that Unix :c:func:`getopt` uses). - - .. note:: - - Unlike GNU :c:func:`getopt`, after a non-option argument, all further - arguments are considered also non-options. This is similar to the way - non-GNU Unix systems work. - - *longopts*, if specified, must be a list of strings with the names of the - long options which should be supported. The leading ``'--'`` characters - should not be included in the option name. Long options which require an - argument should be followed by an equal sign (``'='``). Optional arguments - are not supported. To accept only long options, *shortopts* should be an - empty string. Long options on the command line can be recognized so long as - they provide a prefix of the option name that matches exactly one of the - accepted options. For example, if *longopts* is ``['foo', 'frob']``, the - option ``--fo`` will match as ``--foo``, but ``--f`` will - not match uniquely, so :exc:`GetoptError` will be raised. - - The return value consists of two elements: the first is a list of ``(option, - value)`` pairs; the second is the list of program arguments left after the - option list was stripped (this is a trailing slice of *args*). Each - option-and-value pair returned has the option as its first element, prefixed - with a hyphen for short options (e.g., ``'-x'``) or two hyphens for long - options (e.g., ``'--long-option'``), and the option argument as its - second element, or an empty string if the option has no argument. The - options occur in the list in the same order in which they were found, thus - allowing multiple occurrences. Long and short options may be mixed. - - -.. function:: gnu_getopt(args, shortopts, longopts=[]) - - This function works like :func:`getopt`, except that GNU style scanning mode is - used by default. This means that option and non-option arguments may be - intermixed. The :func:`getopt` function stops processing options as soon as a - non-option argument is encountered. - - If the first character of the option string is ``'+'``, or if the environment - variable :envvar:`POSIXLY_CORRECT` is set, then option processing stops as - soon as a non-option argument is encountered. - - -.. exception:: GetoptError - - This is raised when an unrecognized option is found in the argument list or when - an option requiring an argument is given none. The argument to the exception is - a string indicating the cause of the error. For long options, an argument given - to an option which does not require one will also cause this exception to be - raised. The attributes :attr:`msg` and :attr:`opt` give the error message and - related option; if there is no specific option to which the exception relates, - :attr:`opt` is an empty string. - -.. XXX deprecated? -.. exception:: error - - Alias for :exc:`GetoptError`; for backward compatibility. - -An example using only Unix style options: - - >>> import getopt - >>> args = '-a -b -cfoo -d bar a1 a2'.split() - >>> args - ['-a', '-b', '-cfoo', '-d', 'bar', 'a1', 'a2'] - >>> optlist, args = getopt.getopt(args, 'abc:d:') - >>> optlist - [('-a', ''), ('-b', ''), ('-c', 'foo'), ('-d', 'bar')] - >>> args - ['a1', 'a2'] - -Using long option names is equally easy: - - >>> s = '--condition=foo --testing --output-file abc.def -x a1 a2' - >>> args = s.split() - >>> args - ['--condition=foo', '--testing', '--output-file', 'abc.def', '-x', 'a1', 'a2'] - >>> optlist, args = getopt.getopt(args, 'x', [ - ... 'condition=', 'output-file=', 'testing']) - >>> optlist - [('--condition', 'foo'), ('--testing', ''), ('--output-file', 'abc.def'), ('-x', '')] - >>> args - ['a1', 'a2'] - -In a script, typical usage is something like this:: - - import getopt, sys - - def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], "ho:v", ["help", "output="]) - except getopt.GetoptError as err: - # print help information and exit: - print(err) # will print something like "option -a not recognized" - usage() - sys.exit(2) - output = None - verbose = False - for o, a in opts: - if o == "-v": - verbose = True - elif o in ("-h", "--help"): - usage() - sys.exit() - elif o in ("-o", "--output"): - output = a - else: - assert False, "unhandled option" - # ... - - if __name__ == "__main__": - main() - -Note that an equivalent command line interface could be produced with less code -and more informative help and error messages by using the :mod:`argparse` module:: - - import argparse - - if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('-o', '--output') - parser.add_argument('-v', dest='verbose', action='store_true') - args = parser.parse_args() - # ... do something with args.output ... - # ... do something with args.verbose .. - -.. seealso:: - - Module :mod:`argparse` - Alternative command line option and argument parsing library. - diff --git a/Doc/library/getpass.rst b/Doc/library/getpass.rst deleted file mode 100644 index 5c79daf0f47d8e..00000000000000 --- a/Doc/library/getpass.rst +++ /dev/null @@ -1,52 +0,0 @@ -:mod:`getpass` --- Portable password input -========================================== - -.. module:: getpass - :synopsis: Portable reading of passwords and retrieval of the userid. - -.. moduleauthor:: Piers Lauder -.. sectionauthor:: Fred L. Drake, Jr. -.. Windows (& Mac?) support by Guido van Rossum. - -**Source code:** :source:`Lib/getpass.py` - --------------- - -.. include:: ../includes/wasm-notavail.rst - -The :mod:`getpass` module provides two functions: - -.. function:: getpass(prompt='Password: ', stream=None) - - Prompt the user for a password without echoing. The user is prompted using - the string *prompt*, which defaults to ``'Password: '``. On Unix, the - prompt is written to the file-like object *stream* using the replace error - handler if needed. *stream* defaults to the controlling terminal - (:file:`/dev/tty`) or if that is unavailable to ``sys.stderr`` (this - argument is ignored on Windows). - - If echo free input is unavailable getpass() falls back to printing - a warning message to *stream* and reading from ``sys.stdin`` and - issuing a :exc:`GetPassWarning`. - - .. note:: - If you call getpass from within IDLE, the input may be done in the - terminal you launched IDLE from rather than the idle window itself. - -.. exception:: GetPassWarning - - A :exc:`UserWarning` subclass issued when password input may be echoed. - - -.. function:: getuser() - - Return the "login name" of the user. - - This function checks the environment variables :envvar:`LOGNAME`, - :envvar:`USER`, :envvar:`!LNAME` and :envvar:`USERNAME`, in order, and - returns the value of the first one which is set to a non-empty string. If - none are set, the login name from the password database is returned on - systems which support the :mod:`pwd` module, otherwise, an exception is - raised. - - In general, this function should be preferred over :func:`os.getlogin()`. diff --git a/Doc/library/gettext.rst b/Doc/library/gettext.rst deleted file mode 100644 index dc6cf5533fccbe..00000000000000 --- a/Doc/library/gettext.rst +++ /dev/null @@ -1,647 +0,0 @@ -:mod:`gettext` --- Multilingual internationalization services -============================================================= - -.. module:: gettext - :synopsis: Multilingual internationalization services. - -.. moduleauthor:: Barry A. Warsaw -.. sectionauthor:: Barry A. Warsaw - -**Source code:** :source:`Lib/gettext.py` - --------------- - -The :mod:`gettext` module provides internationalization (I18N) and localization -(L10N) services for your Python modules and applications. It supports both the -GNU :program:`gettext` message catalog API and a higher level, class-based API that may -be more appropriate for Python files. The interface described below allows you -to write your module and application messages in one natural language, and -provide a catalog of translated messages for running under different natural -languages. - -Some hints on localizing your Python modules and applications are also given. - - -GNU :program:`gettext` API --------------------------- - -The :mod:`gettext` module defines the following API, which is very similar to -the GNU :program:`gettext` API. If you use this API you will affect the -translation of your entire application globally. Often this is what you want if -your application is monolingual, with the choice of language dependent on the -locale of your user. If you are localizing a Python module, or if your -application needs to switch languages on the fly, you probably want to use the -class-based API instead. - - -.. function:: bindtextdomain(domain, localedir=None) - - Bind the *domain* to the locale directory *localedir*. More concretely, - :mod:`gettext` will look for binary :file:`.mo` files for the given domain using - the path (on Unix): :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo`, where - *language* is searched for in the environment variables :envvar:`LANGUAGE`, - :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and :envvar:`LANG` respectively. - - If *localedir* is omitted or ``None``, then the current binding for *domain* is - returned. [#]_ - - -.. function:: textdomain(domain=None) - - Change or query the current global domain. If *domain* is ``None``, then the - current global domain is returned, otherwise the global domain is set to - *domain*, which is returned. - - -.. index:: single: _ (underscore); gettext -.. function:: gettext(message) - - Return the localized translation of *message*, based on the current global - domain, language, and locale directory. This function is usually aliased as - :func:`!_` in the local namespace (see examples below). - - -.. function:: dgettext(domain, message) - - Like :func:`.gettext`, but look the message up in the specified *domain*. - - -.. function:: ngettext(singular, plural, n) - - Like :func:`.gettext`, but consider plural forms. If a translation is found, - apply the plural formula to *n*, and return the resulting message (some - languages have more than two plural forms). If no translation is found, return - *singular* if *n* is 1; return *plural* otherwise. - - The Plural formula is taken from the catalog header. It is a C or Python - expression that has a free variable *n*; the expression evaluates to the index - of the plural in the catalog. See - `the GNU gettext documentation `__ - for the precise syntax to be used in :file:`.po` files and the - formulas for a variety of languages. - - -.. function:: dngettext(domain, singular, plural, n) - - Like :func:`ngettext`, but look the message up in the specified *domain*. - - -.. function:: pgettext(context, message) -.. function:: dpgettext(domain, context, message) -.. function:: npgettext(context, singular, plural, n) -.. function:: dnpgettext(domain, context, singular, plural, n) - - Similar to the corresponding functions without the ``p`` in the prefix (that - is, :func:`gettext`, :func:`dgettext`, :func:`ngettext`, :func:`dngettext`), - but the translation is restricted to the given message *context*. - - .. versionadded:: 3.8 - - -Note that GNU :program:`gettext` also defines a :func:`!dcgettext` method, but -this was deemed not useful and so it is currently unimplemented. - -Here's an example of typical usage for this API:: - - import gettext - gettext.bindtextdomain('myapplication', '/path/to/my/language/directory') - gettext.textdomain('myapplication') - _ = gettext.gettext - # ... - print(_('This is a translatable string.')) - - -Class-based API ---------------- - -The class-based API of the :mod:`gettext` module gives you more flexibility and -greater convenience than the GNU :program:`gettext` API. It is the recommended -way of localizing your Python applications and modules. :mod:`!gettext` defines -a :class:`GNUTranslations` class which implements the parsing of GNU :file:`.mo` format -files, and has methods for returning strings. Instances of this class can also -install themselves in the built-in namespace as the function :func:`!_`. - - -.. function:: find(domain, localedir=None, languages=None, all=False) - - This function implements the standard :file:`.mo` file search algorithm. It - takes a *domain*, identical to what :func:`textdomain` takes. Optional - *localedir* is as in :func:`bindtextdomain`. Optional *languages* is a list of - strings, where each string is a language code. - - If *localedir* is not given, then the default system locale directory is used. - [#]_ If *languages* is not given, then the following environment variables are - searched: :envvar:`LANGUAGE`, :envvar:`LC_ALL`, :envvar:`LC_MESSAGES`, and - :envvar:`LANG`. The first one returning a non-empty value is used for the - *languages* variable. The environment variables should contain a colon separated - list of languages, which will be split on the colon to produce the expected list - of language code strings. - - :func:`find` then expands and normalizes the languages, and then iterates - through them, searching for an existing file built of these components: - - :file:`{localedir}/{language}/LC_MESSAGES/{domain}.mo` - - The first such file name that exists is returned by :func:`find`. If no such - file is found, then ``None`` is returned. If *all* is given, it returns a list - of all file names, in the order in which they appear in the languages list or - the environment variables. - - -.. function:: translation(domain, localedir=None, languages=None, class_=None, fallback=False) - - Return a ``*Translations`` instance based on the *domain*, *localedir*, - and *languages*, which are first passed to :func:`find` to get a list of the - associated :file:`.mo` file paths. Instances with identical :file:`.mo` file - names are cached. The actual class instantiated is *class_* if - provided, otherwise :class:`GNUTranslations`. The class's constructor must - take a single :term:`file object` argument. - - If multiple files are found, later files are used as fallbacks for earlier ones. - To allow setting the fallback, :func:`copy.copy` is used to clone each - translation object from the cache; the actual instance data is still shared with - the cache. - - If no :file:`.mo` file is found, this function raises :exc:`OSError` if - *fallback* is false (which is the default), and returns a - :class:`NullTranslations` instance if *fallback* is true. - - .. versionchanged:: 3.3 - :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. - - .. versionchanged:: 3.11 - *codeset* parameter is removed. - -.. function:: install(domain, localedir=None, *, names=None) - - This installs the function :func:`!_` in Python's builtins namespace, based on - *domain* and *localedir* which are passed to the function :func:`translation`. - - For the *names* parameter, please see the description of the translation - object's :meth:`~NullTranslations.install` method. - - As seen below, you usually mark the strings in your application that are - candidates for translation, by wrapping them in a call to the :func:`!_` - function, like this:: - - print(_('This string will be translated.')) - - For convenience, you want the :func:`!_` function to be installed in Python's - builtins namespace, so it is easily accessible in all modules of your - application. - - .. versionchanged:: 3.11 - *names* is now a keyword-only parameter. - -The :class:`NullTranslations` class -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Translation classes are what actually implement the translation of original -source file message strings to translated message strings. The base class used -by all translation classes is :class:`NullTranslations`; this provides the basic -interface you can use to write your own specialized translation classes. Here -are the methods of :class:`!NullTranslations`: - - -.. class:: NullTranslations(fp=None) - - Takes an optional :term:`file object` *fp*, which is ignored by the base class. - Initializes "protected" instance variables *_info* and *_charset* which are set - by derived classes, as well as *_fallback*, which is set through - :meth:`add_fallback`. It then calls ``self._parse(fp)`` if *fp* is not - ``None``. - - .. method:: _parse(fp) - - No-op in the base class, this method takes file object *fp*, and reads - the data from the file, initializing its message catalog. If you have an - unsupported message catalog file format, you should override this method - to parse your format. - - - .. method:: add_fallback(fallback) - - Add *fallback* as the fallback object for the current translation object. - A translation object should consult the fallback if it cannot provide a - translation for a given message. - - - .. method:: gettext(message) - - If a fallback has been set, forward :meth:`!gettext` to the fallback. - Otherwise, return *message*. Overridden in derived classes. - - - .. method:: ngettext(singular, plural, n) - - If a fallback has been set, forward :meth:`!ngettext` to the fallback. - Otherwise, return *singular* if *n* is 1; return *plural* otherwise. - Overridden in derived classes. - - - .. method:: pgettext(context, message) - - If a fallback has been set, forward :meth:`pgettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. - - .. versionadded:: 3.8 - - - .. method:: npgettext(context, singular, plural, n) - - If a fallback has been set, forward :meth:`npgettext` to the fallback. - Otherwise, return the translated message. Overridden in derived classes. - - .. versionadded:: 3.8 - - - .. method:: info() - - Return the "protected" :attr:`_info` variable, a dictionary containing - the metadata found in the message catalog file. - - - .. method:: charset() - - Return the encoding of the message catalog file. - - - .. method:: install(names=None) - - This method installs :meth:`.gettext` into the built-in namespace, - binding it to ``_``. - - If the *names* parameter is given, it must be a sequence containing the - names of functions you want to install in the builtins namespace in - addition to :func:`!_`. Supported names are ``'gettext'``, ``'ngettext'``, - ``'pgettext'``, and ``'npgettext'``. - - Note that this is only one way, albeit the most convenient way, to make - the :func:`!_` function available to your application. Because it affects - the entire application globally, and specifically the built-in namespace, - localized modules should never install :func:`!_`. Instead, they should use - this code to make :func:`!_` available to their module:: - - import gettext - t = gettext.translation('mymodule', ...) - _ = t.gettext - - This puts :func:`!_` only in the module's global namespace and so only - affects calls within this module. - - .. versionchanged:: 3.8 - Added ``'pgettext'`` and ``'npgettext'``. - - -The :class:`GNUTranslations` class -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :mod:`gettext` module provides one additional class derived from -:class:`NullTranslations`: :class:`GNUTranslations`. This class overrides -:meth:`_parse` to enable reading GNU :program:`gettext` format :file:`.mo` files -in both big-endian and little-endian format. - -:class:`GNUTranslations` parses optional metadata out of the translation -catalog. It is convention with GNU :program:`gettext` to include metadata as -the translation for the empty string. This metadata is in :rfc:`822`\ -style -``key: value`` pairs, and should contain the ``Project-Id-Version`` key. If the -key ``Content-Type`` is found, then the ``charset`` property is used to -initialize the "protected" :attr:`_charset` instance variable, defaulting to -``None`` if not found. If the charset encoding is specified, then all message -ids and message strings read from the catalog are converted to Unicode using -this encoding, else ASCII is assumed. - -Since message ids are read as Unicode strings too, all ``*gettext()`` methods -will assume message ids as Unicode strings, not byte strings. - -The entire set of key/value pairs are placed into a dictionary and set as the -"protected" :attr:`_info` instance variable. - -If the :file:`.mo` file's magic number is invalid, the major version number is -unexpected, or if other problems occur while reading the file, instantiating a -:class:`GNUTranslations` class can raise :exc:`OSError`. - -.. class:: GNUTranslations - - The following methods are overridden from the base class implementation: - - .. method:: gettext(message) - - Look up the *message* id in the catalog and return the corresponding message - string, as a Unicode string. If there is no entry in the catalog for the - *message* id, and a fallback has been set, the look up is forwarded to the - fallback's :meth:`~NullTranslations.gettext` method. Otherwise, the - *message* id is returned. - - - .. method:: ngettext(singular, plural, n) - - Do a plural-forms lookup of a message id. *singular* is used as the message id - for purposes of lookup in the catalog, while *n* is used to determine which - plural form to use. The returned message string is a Unicode string. - - If the message id is not found in the catalog, and a fallback is specified, - the request is forwarded to the fallback's :meth:`~NullTranslations.ngettext` - method. Otherwise, when *n* is 1 *singular* is returned, and *plural* is - returned in all other cases. - - Here is an example:: - - n = len(os.listdir('.')) - cat = GNUTranslations(somefile) - message = cat.ngettext( - 'There is %(num)d file in this directory', - 'There are %(num)d files in this directory', - n) % {'num': n} - - - .. method:: pgettext(context, message) - - Look up the *context* and *message* id in the catalog and return the - corresponding message string, as a Unicode string. If there is no - entry in the catalog for the *message* id and *context*, and a fallback - has been set, the look up is forwarded to the fallback's - :meth:`pgettext` method. Otherwise, the *message* id is returned. - - .. versionadded:: 3.8 - - - .. method:: npgettext(context, singular, plural, n) - - Do a plural-forms lookup of a message id. *singular* is used as the - message id for purposes of lookup in the catalog, while *n* is used to - determine which plural form to use. - - If the message id for *context* is not found in the catalog, and a - fallback is specified, the request is forwarded to the fallback's - :meth:`npgettext` method. Otherwise, when *n* is 1 *singular* is - returned, and *plural* is returned in all other cases. - - .. versionadded:: 3.8 - - -Solaris message catalog support -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The Solaris operating system defines its own binary :file:`.mo` file format, but -since no documentation can be found on this format, it is not supported at this -time. - - -The Catalog constructor -^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: single: GNOME - -GNOME uses a version of the :mod:`gettext` module by James Henstridge, but this -version has a slightly different API. Its documented usage was:: - - import gettext - cat = gettext.Catalog(domain, localedir) - _ = cat.gettext - print(_('hello world')) - -For compatibility with this older module, the function :func:`!Catalog` is an -alias for the :func:`translation` function described above. - -One difference between this module and Henstridge's: his catalog objects -supported access through a mapping API, but this appears to be unused and so is -not currently supported. - -.. _i18n-howto: - -Internationalizing your programs and modules --------------------------------------------- - -Internationalization (I18N) refers to the operation by which a program is made -aware of multiple languages. Localization (L10N) refers to the adaptation of -your program, once internationalized, to the local language and cultural habits. -In order to provide multilingual messages for your Python programs, you need to -take the following steps: - -#. prepare your program or module by specially marking translatable strings - -#. run a suite of tools over your marked files to generate raw messages catalogs - -#. create language-specific translations of the message catalogs - -#. use the :mod:`gettext` module so that message strings are properly translated - -In order to prepare your code for I18N, you need to look at all the strings in -your files. Any string that needs to be translated should be marked by wrapping -it in ``_('...')`` --- that is, a call to the function :func:`_ `. For example:: - - filename = 'mylog.txt' - message = _('writing a log message') - with open(filename, 'w') as fp: - fp.write(message) - -In this example, the string ``'writing a log message'`` is marked as a candidate -for translation, while the strings ``'mylog.txt'`` and ``'w'`` are not. - -There are a few tools to extract the strings meant for translation. -The original GNU :program:`gettext` only supported C or C++ source -code but its extended version :program:`xgettext` scans code written -in a number of languages, including Python, to find strings marked as -translatable. `Babel `__ is a Python -internationalization library that includes a :file:`pybabel` script to -extract and compile message catalogs. François Pinard's program -called :program:`xpot` does a similar job and is available as part of -his `po-utils package `__. - -(Python also includes pure-Python versions of these programs, called -:program:`pygettext.py` and :program:`msgfmt.py`; some Python distributions -will install them for you. :program:`pygettext.py` is similar to -:program:`xgettext`, but only understands Python source code and -cannot handle other programming languages such as C or C++. -:program:`pygettext.py` supports a command-line interface similar to -:program:`xgettext`; for details on its use, run ``pygettext.py ---help``. :program:`msgfmt.py` is binary compatible with GNU -:program:`msgfmt`. With these two programs, you may not need the GNU -:program:`gettext` package to internationalize your Python -applications.) - -:program:`xgettext`, :program:`pygettext`, and similar tools generate -:file:`.po` files that are message catalogs. They are structured -human-readable files that contain every marked string in the source -code, along with a placeholder for the translated versions of these -strings. - -Copies of these :file:`.po` files are then handed over to the -individual human translators who write translations for every -supported natural language. They send back the completed -language-specific versions as a :file:`.po` file that's -compiled into a machine-readable :file:`.mo` binary catalog file using -the :program:`msgfmt` program. The :file:`.mo` files are used by the -:mod:`gettext` module for the actual translation processing at -run-time. - -How you use the :mod:`gettext` module in your code depends on whether you are -internationalizing a single module or your entire application. The next two -sections will discuss each case. - - -Localizing your module -^^^^^^^^^^^^^^^^^^^^^^ - -If you are localizing your module, you must take care not to make global -changes, e.g. to the built-in namespace. You should not use the GNU :program:`gettext` -API but instead the class-based API. - -Let's say your module is called "spam" and the module's various natural language -translation :file:`.mo` files reside in :file:`/usr/share/locale` in GNU -:program:`gettext` format. Here's what you would put at the top of your -module:: - - import gettext - t = gettext.translation('spam', '/usr/share/locale') - _ = t.gettext - - -Localizing your application -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you are localizing your application, you can install the :func:`!_` function -globally into the built-in namespace, usually in the main driver file of your -application. This will let all your application-specific files just use -``_('...')`` without having to explicitly install it in each file. - -In the simple case then, you need only add the following bit of code to the main -driver file of your application:: - - import gettext - gettext.install('myapplication') - -If you need to set the locale directory, you can pass it into the -:func:`install` function:: - - import gettext - gettext.install('myapplication', '/usr/share/locale') - - -Changing languages on the fly -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If your program needs to support many languages at the same time, you may want -to create multiple translation instances and then switch between them -explicitly, like so:: - - import gettext - - lang1 = gettext.translation('myapplication', languages=['en']) - lang2 = gettext.translation('myapplication', languages=['fr']) - lang3 = gettext.translation('myapplication', languages=['de']) - - # start by using language1 - lang1.install() - - # ... time goes by, user selects language 2 - lang2.install() - - # ... more time goes by, user selects language 3 - lang3.install() - - -Deferred translations -^^^^^^^^^^^^^^^^^^^^^ - -In most coding situations, strings are translated where they are coded. -Occasionally however, you need to mark strings for translation, but defer actual -translation until later. A classic example is:: - - animals = ['mollusk', - 'albatross', - 'rat', - 'penguin', - 'python', ] - # ... - for a in animals: - print(a) - -Here, you want to mark the strings in the ``animals`` list as being -translatable, but you don't actually want to translate them until they are -printed. - -Here is one way you can handle this situation:: - - def _(message): return message - - animals = [_('mollusk'), - _('albatross'), - _('rat'), - _('penguin'), - _('python'), ] - - del _ - - # ... - for a in animals: - print(_(a)) - -This works because the dummy definition of :func:`!_` simply returns the string -unchanged. And this dummy definition will temporarily override any definition -of :func:`!_` in the built-in namespace (until the :keyword:`del` command). Take -care, though if you have a previous definition of :func:`!_` in the local -namespace. - -Note that the second use of :func:`!_` will not identify "a" as being -translatable to the :program:`gettext` program, because the parameter -is not a string literal. - -Another way to handle this is with the following example:: - - def N_(message): return message - - animals = [N_('mollusk'), - N_('albatross'), - N_('rat'), - N_('penguin'), - N_('python'), ] - - # ... - for a in animals: - print(_(a)) - -In this case, you are marking translatable strings with the function -:func:`!N_`, which won't conflict with any definition of :func:`!_`. -However, you will need to teach your message extraction program to -look for translatable strings marked with :func:`!N_`. :program:`xgettext`, -:program:`pygettext`, ``pybabel extract``, and :program:`xpot` all -support this through the use of the :option:`!-k` command-line switch. -The choice of :func:`!N_` here is totally arbitrary; it could have just -as easily been :func:`!MarkThisStringForTranslation`. - - -Acknowledgements ----------------- - -The following people contributed code, feedback, design suggestions, previous -implementations, and valuable experience to the creation of this module: - -* Peter Funk - -* James Henstridge - -* Juan David Ibáñez Palomar - -* Marc-André Lemburg - -* Martin von Löwis - -* François Pinard - -* Barry Warsaw - -* Gustavo Niemeyer - -.. rubric:: Footnotes - -.. [#] The default locale directory is system dependent; for example, on RedHat Linux - it is :file:`/usr/share/locale`, but on Solaris it is :file:`/usr/lib/locale`. - The :mod:`gettext` module does not try to support these system dependent - defaults; instead its default is :file:`{sys.base_prefix}/share/locale` (see - :data:`sys.base_prefix`). For this reason, it is always best to call - :func:`bindtextdomain` with an explicit absolute path at the start of your - application. - -.. [#] See the footnote for :func:`bindtextdomain` above. diff --git a/Doc/library/glob.rst b/Doc/library/glob.rst deleted file mode 100644 index 5af15c41a25c9c..00000000000000 --- a/Doc/library/glob.rst +++ /dev/null @@ -1,151 +0,0 @@ -:mod:`glob` --- Unix style pathname pattern expansion -===================================================== - -.. module:: glob - :synopsis: Unix shell style pathname pattern expansion. - -**Source code:** :source:`Lib/glob.py` - -.. index:: single: filenames; pathname expansion - --------------- - -.. index:: - single: * (asterisk); in glob-style wildcards - single: ? (question mark); in glob-style wildcards - single: [] (square brackets); in glob-style wildcards - single: ! (exclamation); in glob-style wildcards - single: - (minus); in glob-style wildcards - single: . (dot); in glob-style wildcards - -The :mod:`glob` module finds all the pathnames matching a specified pattern -according to the rules used by the Unix shell, although results are returned in -arbitrary order. No tilde expansion is done, but ``*``, ``?``, and character -ranges expressed with ``[]`` will be correctly matched. This is done by using -the :func:`os.scandir` and :func:`fnmatch.fnmatch` functions in concert, and -not by actually invoking a subshell. - -Note that files beginning with a dot (``.``) can only be matched by -patterns that also start with a dot, -unlike :func:`fnmatch.fnmatch` or :func:`pathlib.Path.glob`. -(For tilde and shell variable expansion, use :func:`os.path.expanduser` and -:func:`os.path.expandvars`.) - -For a literal match, wrap the meta-characters in brackets. -For example, ``'[?]'`` matches the character ``'?'``. - - -.. seealso:: - The :mod:`pathlib` module offers high-level path objects. - - -.. function:: glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, \ - include_hidden=False) - - Return a possibly empty list of path names that match *pathname*, which must be - a string containing a path specification. *pathname* can be either absolute - (like :file:`/usr/src/Python-1.5/Makefile`) or relative (like - :file:`../../Tools/\*/\*.gif`), and can contain shell-style wildcards. Broken - symlinks are included in the results (as in the shell). Whether or not the - results are sorted depends on the file system. If a file that satisfies - conditions is removed or added during the call of this function, whether - a path name for that file be included is unspecified. - - If *root_dir* is not ``None``, it should be a :term:`path-like object` - specifying the root directory for searching. It has the same effect on - :func:`glob` as changing the current directory before calling it. If - *pathname* is relative, the result will contain paths relative to - *root_dir*. - - This function can support :ref:`paths relative to directory descriptors - ` with the *dir_fd* parameter. - - .. index:: - single: **; in glob-style wildcards - - If *recursive* is true, the pattern "``**``" will match any files and zero or - more directories, subdirectories and symbolic links to directories. If the - pattern is followed by an :data:`os.sep` or :data:`os.altsep` then files will not - match. - - If *include_hidden* is true, "``**``" pattern will match hidden directories. - - .. audit-event:: glob.glob pathname,recursive glob.glob - .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.glob - - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. - - .. versionchanged:: 3.5 - Support for recursive globs using "``**``". - - .. versionchanged:: 3.10 - Added the *root_dir* and *dir_fd* parameters. - - .. versionchanged:: 3.11 - Added the *include_hidden* parameter. - - -.. function:: iglob(pathname, *, root_dir=None, dir_fd=None, recursive=False, \ - include_hidden=False) - - Return an :term:`iterator` which yields the same values as :func:`glob` - without actually storing them all simultaneously. - - .. audit-event:: glob.glob pathname,recursive glob.iglob - .. audit-event:: glob.glob/2 pathname,recursive,root_dir,dir_fd glob.iglob - - .. versionchanged:: 3.5 - Support for recursive globs using "``**``". - - .. versionchanged:: 3.10 - Added the *root_dir* and *dir_fd* parameters. - - .. versionchanged:: 3.11 - Added the *include_hidden* parameter. - - -.. function:: escape(pathname) - - Escape all special characters (``'?'``, ``'*'`` and ``'['``). - This is useful if you want to match an arbitrary literal string that may - have special characters in it. Special characters in drive/UNC - sharepoints are not escaped, e.g. on Windows - ``escape('//?/c:/Quo vadis?.txt')`` returns ``'//?/c:/Quo vadis[?].txt'``. - - .. versionadded:: 3.4 - - -For example, consider a directory containing the following files: -:file:`1.gif`, :file:`2.txt`, :file:`card.gif` and a subdirectory :file:`sub` -which contains only the file :file:`3.txt`. :func:`glob` will produce -the following results. Notice how any leading components of the path are -preserved. :: - - >>> import glob - >>> glob.glob('./[0-9].*') - ['./1.gif', './2.txt'] - >>> glob.glob('*.gif') - ['1.gif', 'card.gif'] - >>> glob.glob('?.gif') - ['1.gif'] - >>> glob.glob('**/*.txt', recursive=True) - ['2.txt', 'sub/3.txt'] - >>> glob.glob('./**/', recursive=True) - ['./', './sub/'] - -If the directory contains files starting with ``.`` they won't be matched by -default. For example, consider a directory containing :file:`card.gif` and -:file:`.card.gif`:: - - >>> import glob - >>> glob.glob('*.gif') - ['card.gif'] - >>> glob.glob('.c*') - ['.card.gif'] - -.. seealso:: - - Module :mod:`fnmatch` - Shell-style filename (not path) expansion diff --git a/Doc/library/graphlib.rst b/Doc/library/graphlib.rst deleted file mode 100644 index 5414d6370b78ce..00000000000000 --- a/Doc/library/graphlib.rst +++ /dev/null @@ -1,210 +0,0 @@ -:mod:`graphlib` --- Functionality to operate with graph-like structures -========================================================================= - -.. module:: graphlib - :synopsis: Functionality to operate with graph-like structures - - -**Source code:** :source:`Lib/graphlib.py` - -.. testsetup:: default - - import graphlib - from graphlib import * - --------------- - - -.. class:: TopologicalSorter(graph=None) - - Provides functionality to topologically sort a graph of :term:`hashable` nodes. - - A topological order is a linear ordering of the vertices in a graph such that - for every directed edge u -> v from vertex u to vertex v, vertex u comes - before vertex v in the ordering. For instance, the vertices of the graph may - represent tasks to be performed, and the edges may represent constraints that - one task must be performed before another; in this example, a topological - ordering is just a valid sequence for the tasks. A complete topological - ordering is possible if and only if the graph has no directed cycles, that - is, if it is a directed acyclic graph. - - If the optional *graph* argument is provided it must be a dictionary - representing a directed acyclic graph where the keys are nodes and the values - are iterables of all predecessors of that node in the graph (the nodes that - have edges that point to the value in the key). Additional nodes can be added - to the graph using the :meth:`~TopologicalSorter.add` method. - - In the general case, the steps required to perform the sorting of a given - graph are as follows: - - * Create an instance of the :class:`TopologicalSorter` with an optional - initial graph. - * Add additional nodes to the graph. - * Call :meth:`~TopologicalSorter.prepare` on the graph. - * While :meth:`~TopologicalSorter.is_active` is ``True``, iterate over - the nodes returned by :meth:`~TopologicalSorter.get_ready` and - process them. Call :meth:`~TopologicalSorter.done` on each node as it - finishes processing. - - In case just an immediate sorting of the nodes in the graph is required and - no parallelism is involved, the convenience method - :meth:`TopologicalSorter.static_order` can be used directly: - - .. doctest:: - - >>> graph = {"D": {"B", "C"}, "C": {"A"}, "B": {"A"}} - >>> ts = TopologicalSorter(graph) - >>> tuple(ts.static_order()) - ('A', 'C', 'B', 'D') - - The class is designed to easily support parallel processing of the nodes as - they become ready. For instance:: - - topological_sorter = TopologicalSorter() - - # Add nodes to 'topological_sorter'... - - topological_sorter.prepare() - while topological_sorter.is_active(): - for node in topological_sorter.get_ready(): - # Worker threads or processes take nodes to work on off the - # 'task_queue' queue. - task_queue.put(node) - - # When the work for a node is done, workers put the node in - # 'finalized_tasks_queue' so we can get more nodes to work on. - # The definition of 'is_active()' guarantees that, at this point, at - # least one node has been placed on 'task_queue' that hasn't yet - # been passed to 'done()', so this blocking 'get()' must (eventually) - # succeed. After calling 'done()', we loop back to call 'get_ready()' - # again, so put newly freed nodes on 'task_queue' as soon as - # logically possible. - node = finalized_tasks_queue.get() - topological_sorter.done(node) - - .. method:: add(node, *predecessors) - - Add a new node and its predecessors to the graph. Both the *node* and all - elements in *predecessors* must be :term:`hashable`. - - If called multiple times with the same node argument, the set of - dependencies will be the union of all dependencies passed in. - - It is possible to add a node with no dependencies (*predecessors* is not - provided) or to provide a dependency twice. If a node that has not been - provided before is included among *predecessors* it will be automatically - added to the graph with no predecessors of its own. - - Raises :exc:`ValueError` if called after :meth:`~TopologicalSorter.prepare`. - - .. method:: prepare() - - Mark the graph as finished and check for cycles in the graph. If any cycle - is detected, :exc:`CycleError` will be raised, but - :meth:`~TopologicalSorter.get_ready` can still be used to obtain as many - nodes as possible until cycles block more progress. After a call to this - function, the graph cannot be modified, and therefore no more nodes can be - added using :meth:`~TopologicalSorter.add`. - - .. method:: is_active() - - Returns ``True`` if more progress can be made and ``False`` otherwise. - Progress can be made if cycles do not block the resolution and either - there are still nodes ready that haven't yet been returned by - :meth:`TopologicalSorter.get_ready` or the number of nodes marked - :meth:`TopologicalSorter.done` is less than the number that have been - returned by :meth:`TopologicalSorter.get_ready`. - - The :meth:`~object.__bool__` method of this class defers to - this function, so instead of:: - - if ts.is_active(): - ... - - it is possible to simply do:: - - if ts: - ... - - Raises :exc:`ValueError` if called without calling - :meth:`~TopologicalSorter.prepare` previously. - - .. method:: done(*nodes) - - Marks a set of nodes returned by :meth:`TopologicalSorter.get_ready` as - processed, unblocking any successor of each node in *nodes* for being - returned in the future by a call to :meth:`TopologicalSorter.get_ready`. - - Raises :exc:`ValueError` if any node in *nodes* has already been marked as - processed by a previous call to this method or if a node was not added to - the graph by using :meth:`TopologicalSorter.add`, if called without - calling :meth:`~TopologicalSorter.prepare` or if node has not yet been - returned by :meth:`~TopologicalSorter.get_ready`. - - .. method:: get_ready() - - Returns a ``tuple`` with all the nodes that are ready. Initially it - returns all nodes with no predecessors, and once those are marked as - processed by calling :meth:`TopologicalSorter.done`, further calls will - return all new nodes that have all their predecessors already processed. - Once no more progress can be made, empty tuples are returned. - - Raises :exc:`ValueError` if called without calling - :meth:`~TopologicalSorter.prepare` previously. - - .. method:: static_order() - - Returns an iterator object which will iterate over nodes in a topological - order. When using this method, :meth:`~TopologicalSorter.prepare` and - :meth:`~TopologicalSorter.done` should not be called. This method is - equivalent to:: - - def static_order(self): - self.prepare() - while self.is_active(): - node_group = self.get_ready() - yield from node_group - self.done(*node_group) - - The particular order that is returned may depend on the specific order in - which the items were inserted in the graph. For example: - - .. doctest:: - - >>> ts = TopologicalSorter() - >>> ts.add(3, 2, 1) - >>> ts.add(1, 0) - >>> print([*ts.static_order()]) - [2, 0, 1, 3] - - >>> ts2 = TopologicalSorter() - >>> ts2.add(1, 0) - >>> ts2.add(3, 2, 1) - >>> print([*ts2.static_order()]) - [0, 2, 1, 3] - - This is due to the fact that "0" and "2" are in the same level in the - graph (they would have been returned in the same call to - :meth:`~TopologicalSorter.get_ready`) and the order between them is - determined by the order of insertion. - - - If any cycle is detected, :exc:`CycleError` will be raised. - - .. versionadded:: 3.9 - - -Exceptions ----------- -The :mod:`graphlib` module defines the following exception classes: - -.. exception:: CycleError - - Subclass of :exc:`ValueError` raised by :meth:`TopologicalSorter.prepare` if cycles exist - in the working graph. If multiple cycles exist, only one undefined choice among them will - be reported and included in the exception. - - The detected cycle can be accessed via the second element in the :attr:`~BaseException.args` - attribute of the exception instance and consists in a list of nodes, such that each node is, - in the graph, an immediate predecessor of the next node in the list. In the reported list, - the first and the last node will be the same, to make it clear that it is cyclic. diff --git a/Doc/library/grp.rst b/Doc/library/grp.rst deleted file mode 100644 index ee55b12ea8643a..00000000000000 --- a/Doc/library/grp.rst +++ /dev/null @@ -1,69 +0,0 @@ -:mod:`grp` --- The group database -================================= - -.. module:: grp - :platform: Unix - :synopsis: The group database (getgrnam() and friends). - --------------- - -This module provides access to the Unix group database. It is available on all -Unix versions. - -.. availability:: Unix, not Emscripten, not WASI. - -Group database entries are reported as a tuple-like object, whose attributes -correspond to the members of the ``group`` structure (Attribute field below, see -````): - -+-------+-----------+---------------------------------+ -| Index | Attribute | Meaning | -+=======+===========+=================================+ -| 0 | gr_name | the name of the group | -+-------+-----------+---------------------------------+ -| 1 | gr_passwd | the (encrypted) group password; | -| | | often empty | -+-------+-----------+---------------------------------+ -| 2 | gr_gid | the numerical group ID | -+-------+-----------+---------------------------------+ -| 3 | gr_mem | all the group member's user | -| | | names | -+-------+-----------+---------------------------------+ - -The gid is an integer, name and password are strings, and the member list is a -list of strings. (Note that most users are not explicitly listed as members of -the group they are in according to the password database. Check both databases -to get complete membership information. Also note that a ``gr_name`` that -starts with a ``+`` or ``-`` is likely to be a YP/NIS reference and may not be -accessible via :func:`getgrnam` or :func:`getgrgid`.) - -It defines the following items: - - -.. function:: getgrgid(id) - - Return the group database entry for the given numeric group ID. :exc:`KeyError` - is raised if the entry asked for cannot be found. - - .. versionchanged:: 3.10 - :exc:`TypeError` is raised for non-integer arguments like floats or strings. - -.. function:: getgrnam(name) - - Return the group database entry for the given group name. :exc:`KeyError` is - raised if the entry asked for cannot be found. - - -.. function:: getgrall() - - Return a list of all available group entries, in arbitrary order. - - -.. seealso:: - - Module :mod:`pwd` - An interface to the user database, similar to this. - - Module :mod:`spwd` - An interface to the shadow password database, similar to this. - diff --git a/Doc/library/gzip.rst b/Doc/library/gzip.rst deleted file mode 100644 index fdb546236aeadc..00000000000000 --- a/Doc/library/gzip.rst +++ /dev/null @@ -1,286 +0,0 @@ -:mod:`gzip` --- Support for :program:`gzip` files -================================================= - -.. module:: gzip - :synopsis: Interfaces for gzip compression and decompression using file objects. - -**Source code:** :source:`Lib/gzip.py` - --------------- - -This module provides a simple interface to compress and decompress files just -like the GNU programs :program:`gzip` and :program:`gunzip` would. - -The data compression is provided by the :mod:`zlib` module. - -The :mod:`gzip` module provides the :class:`GzipFile` class, as well as the -:func:`.open`, :func:`compress` and :func:`decompress` convenience functions. -The :class:`GzipFile` class reads and writes :program:`gzip`\ -format files, -automatically compressing or decompressing the data so that it looks like an -ordinary :term:`file object`. - -Note that additional file formats which can be decompressed by the -:program:`gzip` and :program:`gunzip` programs, such as those produced by -:program:`compress` and :program:`pack`, are not supported by this module. - -The module defines the following items: - - -.. function:: open(filename, mode='rb', compresslevel=9, encoding=None, errors=None, newline=None) - - Open a gzip-compressed file in binary or text mode, returning a :term:`file - object`. - - The *filename* argument can be an actual filename (a :class:`str` or - :class:`bytes` object), or an existing file object to read from or write to. - - The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, - ``'w'``, ``'wb'``, ``'x'`` or ``'xb'`` for binary mode, or ``'rt'``, - ``'at'``, ``'wt'``, or ``'xt'`` for text mode. The default is ``'rb'``. - - The *compresslevel* argument is an integer from 0 to 9, as for the - :class:`GzipFile` constructor. - - For binary mode, this function is equivalent to the :class:`GzipFile` - constructor: ``GzipFile(filename, mode, compresslevel)``. In this case, the - *encoding*, *errors* and *newline* arguments must not be provided. - - For text mode, a :class:`GzipFile` object is created, and wrapped in an - :class:`io.TextIOWrapper` instance with the specified encoding, error - handling behavior, and line ending(s). - - .. versionchanged:: 3.3 - Added support for *filename* being a file object, support for text mode, - and the *encoding*, *errors* and *newline* arguments. - - .. versionchanged:: 3.4 - Added support for the ``'x'``, ``'xb'`` and ``'xt'`` modes. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. exception:: BadGzipFile - - An exception raised for invalid gzip files. It inherits :exc:`OSError`. - :exc:`EOFError` and :exc:`zlib.error` can also be raised for invalid gzip - files. - - .. versionadded:: 3.8 - -.. class:: GzipFile(filename=None, mode=None, compresslevel=9, fileobj=None, mtime=None) - - Constructor for the :class:`GzipFile` class, which simulates most of the - methods of a :term:`file object`, with the exception of the :meth:`~io.IOBase.truncate` - method. At least one of *fileobj* and *filename* must be given a non-trivial - value. - - The new class instance is based on *fileobj*, which can be a regular file, an - :class:`io.BytesIO` object, or any other object which simulates a file. It - defaults to ``None``, in which case *filename* is opened to provide a file - object. - - When *fileobj* is not ``None``, the *filename* argument is only used to be - included in the :program:`gzip` file header, which may include the original - filename of the uncompressed file. It defaults to the filename of *fileobj*, if - discernible; otherwise, it defaults to the empty string, and in this case the - original filename is not included in the header. - - The *mode* argument can be any of ``'r'``, ``'rb'``, ``'a'``, ``'ab'``, ``'w'``, - ``'wb'``, ``'x'``, or ``'xb'``, depending on whether the file will be read or - written. The default is the mode of *fileobj* if discernible; otherwise, the - default is ``'rb'``. In future Python releases the mode of *fileobj* will - not be used. It is better to always specify *mode* for writing. - - Note that the file is always opened in binary mode. To open a compressed file - in text mode, use :func:`.open` (or wrap your :class:`GzipFile` with an - :class:`io.TextIOWrapper`). - - The *compresslevel* argument is an integer from ``0`` to ``9`` controlling - the level of compression; ``1`` is fastest and produces the least - compression, and ``9`` is slowest and produces the most compression. ``0`` - is no compression. The default is ``9``. - - The *mtime* argument is an optional numeric timestamp to be written to - the last modification time field in the stream when compressing. It - should only be provided in compression mode. If omitted or ``None``, the - current time is used. See the :attr:`mtime` attribute for more details. - - Calling a :class:`GzipFile` object's :meth:`close` method does not close - *fileobj*, since you might wish to append more material after the compressed - data. This also allows you to pass an :class:`io.BytesIO` object opened for - writing as *fileobj*, and retrieve the resulting memory buffer using the - :class:`io.BytesIO` object's :meth:`~io.BytesIO.getvalue` method. - - :class:`GzipFile` supports the :class:`io.BufferedIOBase` interface, - including iteration and the :keyword:`with` statement. Only the - :meth:`~io.IOBase.truncate` method isn't implemented. - - :class:`GzipFile` also provides the following method and attribute: - - .. method:: peek(n) - - Read *n* uncompressed bytes without advancing the file position. - At most one single read on the compressed stream is done to satisfy - the call. The number of bytes returned may be more or less than - requested. - - .. note:: While calling :meth:`peek` does not change the file position of - the :class:`GzipFile`, it may change the position of the underlying - file object (e.g. if the :class:`GzipFile` was constructed with the - *fileobj* parameter). - - .. versionadded:: 3.2 - - .. attribute:: mtime - - When decompressing, the value of the last modification time field in - the most recently read header may be read from this attribute, as an - integer. The initial value before reading any headers is ``None``. - - All :program:`gzip` compressed streams are required to contain this - timestamp field. Some programs, such as :program:`gunzip`\ , make use - of the timestamp. The format is the same as the return value of - :func:`time.time` and the :attr:`~os.stat_result.st_mtime` attribute of - the object returned by :func:`os.stat`. - - .. attribute:: name - - The path to the gzip file on disk, as a :class:`str` or :class:`bytes`. - Equivalent to the output of :func:`os.fspath` on the original input path, - with no other normalization, resolution or expansion. - - .. versionchanged:: 3.1 - Support for the :keyword:`with` statement was added, along with the - *mtime* constructor argument and :attr:`mtime` attribute. - - .. versionchanged:: 3.2 - Support for zero-padded and unseekable files was added. - - .. versionchanged:: 3.3 - The :meth:`io.BufferedIOBase.read1` method is now implemented. - - .. versionchanged:: 3.4 - Added support for the ``'x'`` and ``'xb'`` modes. - - .. versionchanged:: 3.5 - Added support for writing arbitrary - :term:`bytes-like objects `. - The :meth:`~io.BufferedIOBase.read` method now accepts an argument of - ``None``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. deprecated:: 3.9 - Opening :class:`GzipFile` for writing without specifying the *mode* - argument is deprecated. - - -.. function:: compress(data, compresslevel=9, *, mtime=None) - - Compress the *data*, returning a :class:`bytes` object containing - the compressed data. *compresslevel* and *mtime* have the same meaning as in - the :class:`GzipFile` constructor above. When *mtime* is set to ``0``, this - function is equivalent to :func:`zlib.compress` with *wbits* set to ``31``. - The zlib function is faster. - - .. versionadded:: 3.2 - .. versionchanged:: 3.8 - Added the *mtime* parameter for reproducible output. - .. versionchanged:: 3.11 - Speed is improved by compressing all data at once instead of in a - streamed fashion. Calls with *mtime* set to ``0`` are delegated to - :func:`zlib.compress` for better speed. - -.. function:: decompress(data) - - Decompress the *data*, returning a :class:`bytes` object containing the - uncompressed data. This function is capable of decompressing multi-member - gzip data (multiple gzip blocks concatenated together). When the data is - certain to contain only one member the :func:`zlib.decompress` function with - *wbits* set to 31 is faster. - - .. versionadded:: 3.2 - .. versionchanged:: 3.11 - Speed is improved by decompressing members at once in memory instead of in - a streamed fashion. - -.. _gzip-usage-examples: - -Examples of usage ------------------ - -Example of how to read a compressed file:: - - import gzip - with gzip.open('/home/joe/file.txt.gz', 'rb') as f: - file_content = f.read() - -Example of how to create a compressed GZIP file:: - - import gzip - content = b"Lots of content here" - with gzip.open('/home/joe/file.txt.gz', 'wb') as f: - f.write(content) - -Example of how to GZIP compress an existing file:: - - import gzip - import shutil - with open('/home/joe/file.txt', 'rb') as f_in: - with gzip.open('/home/joe/file.txt.gz', 'wb') as f_out: - shutil.copyfileobj(f_in, f_out) - -Example of how to GZIP compress a binary string:: - - import gzip - s_in = b"Lots of content here" - s_out = gzip.compress(s_in) - -.. seealso:: - - Module :mod:`zlib` - The basic data compression module needed to support the :program:`gzip` file - format. - - -.. program:: gzip - -.. _gzip-cli: - -Command Line Interface ----------------------- - -The :mod:`gzip` module provides a simple command line interface to compress or -decompress files. - -Once executed the :mod:`gzip` module keeps the input file(s). - -.. versionchanged:: 3.8 - - Add a new command line interface with a usage. - By default, when you will execute the CLI, the default compression level is 6. - -Command line options -^^^^^^^^^^^^^^^^^^^^ - -.. option:: file - - If *file* is not specified, read from :data:`sys.stdin`. - -.. option:: --fast - - Indicates the fastest compression method (less compression). - -.. option:: --best - - Indicates the slowest compression method (best compression). - -.. option:: -d, --decompress - - Decompress the given file. - -.. option:: -h, --help - - Show the help message. - diff --git a/Doc/library/hashlib-blake2-tree.png b/Doc/library/hashlib-blake2-tree.png deleted file mode 100644 index faef21b55f96b1..00000000000000 Binary files a/Doc/library/hashlib-blake2-tree.png and /dev/null differ diff --git a/Doc/library/hashlib.rst b/Doc/library/hashlib.rst deleted file mode 100644 index 957d51978d6d59..00000000000000 --- a/Doc/library/hashlib.rst +++ /dev/null @@ -1,848 +0,0 @@ -:mod:`hashlib` --- Secure hashes and message digests -==================================================== - -.. module:: hashlib - :synopsis: Secure hash and message digest algorithms. - -.. moduleauthor:: Gregory P. Smith -.. sectionauthor:: Gregory P. Smith - -**Source code:** :source:`Lib/hashlib.py` - -.. index:: - single: message digest, MD5 - single: secure hash algorithm, SHA1, SHA2, SHA224, SHA256, SHA384, SHA512, SHA3, Shake, Blake2 - -.. testsetup:: - - import hashlib - - --------------- - -This module implements a common interface to many different secure hash and -message digest algorithms. Included are the FIPS secure hash algorithms SHA1, -SHA224, SHA256, SHA384, SHA512, (defined in `the FIPS 180-4 standard`_), -the SHA-3 series (defined in `the FIPS 202 standard`_) as well as RSA's MD5 -algorithm (defined in internet :rfc:`1321`). The terms "secure hash" and -"message digest" are interchangeable. Older algorithms were called message -digests. The modern term is secure hash. - -.. note:: - - If you want the adler32 or crc32 hash functions, they are available in - the :mod:`zlib` module. - - -.. _hash-algorithms: - -Hash algorithms ---------------- - -There is one constructor method named for each type of :dfn:`hash`. All return -a hash object with the same simple interface. For example: use :func:`sha256` -to create a SHA-256 hash object. You can now feed this object with -:term:`bytes-like objects ` (normally :class:`bytes`) using -the :meth:`update` method. At any point you can ask it for the -:dfn:`digest` of the concatenation of the data fed to it so far using the -:meth:`digest()` or :meth:`hexdigest()` methods. - -To allow multithreading, the Python :term:`GIL` is released while computing a -hash supplied more than 2047 bytes of data at once in its constructor or -:meth:`.update` method. - - -.. index:: single: OpenSSL; (use in module hashlib) - -Constructors for hash algorithms that are always present in this module are -:func:`sha1`, :func:`sha224`, :func:`sha256`, :func:`sha384`, :func:`sha512`, -:func:`sha3_224`, :func:`sha3_256`, :func:`sha3_384`, :func:`sha3_512`, -:func:`shake_128`, :func:`shake_256`, :func:`blake2b`, and :func:`blake2s`. -:func:`md5` is normally available as well, though it may be missing or blocked -if you are using a rare "FIPS compliant" build of Python. -These correspond to :data:`algorithms_guaranteed`. - -Additional algorithms may also be available if your Python distribution's -:mod:`hashlib` was linked against a build of OpenSSL that provides others. -Others *are not guaranteed available* on all installations and will only be -accessible by name via :func:`new`. See :data:`algorithms_available`. - -.. warning:: - - Some algorithms have known hash collision weaknesses (including MD5 and - SHA1). Refer to `Attacks on cryptographic hash algorithms`_ and the - `hashlib-seealso`_ section at the end of this document. - -.. versionadded:: 3.6 - SHA3 (Keccak) and SHAKE constructors :func:`sha3_224`, :func:`sha3_256`, - :func:`sha3_384`, :func:`sha3_512`, :func:`shake_128`, :func:`shake_256` - were added. - -.. versionadded:: 3.6 - :func:`blake2b` and :func:`blake2s` were added. - -.. _hashlib-usedforsecurity: - -.. versionchanged:: 3.9 - All hashlib constructors take a keyword-only argument *usedforsecurity* - with default value ``True``. A false value allows the use of insecure and - blocked hashing algorithms in restricted environments. ``False`` indicates - that the hashing algorithm is not used in a security context, e.g. as a - non-cryptographic one-way compression function. - -.. versionchanged:: 3.9 - Hashlib now uses SHA3 and SHAKE from OpenSSL if it provides it. - -Usage ------ - -To obtain the digest of the byte string ``b"Nobody inspects the spammish -repetition"``:: - - >>> import hashlib - >>> m = hashlib.sha256() - >>> m.update(b"Nobody inspects") - >>> m.update(b" the spammish repetition") - >>> m.digest() - b'\x03\x1e\xdd}Ae\x15\x93\xc5\xfe\\\x00o\xa5u+7\xfd\xdf\xf7\xbcN\x84:\xa6\xaf\x0c\x95\x0fK\x94\x06' - >>> m.hexdigest() - '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' - -More condensed: - - >>> hashlib.sha256(b"Nobody inspects the spammish repetition").hexdigest() - '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' - -Constructors ------------- - -.. function:: new(name[, data], \*, usedforsecurity=True) - - Is a generic constructor that takes the string *name* of the desired - algorithm as its first parameter. It also exists to allow access to the - above listed hashes as well as any other algorithms that your OpenSSL - library may offer. - -Using :func:`new` with an algorithm name: - - >>> h = hashlib.new('sha256') - >>> h.update(b"Nobody inspects the spammish repetition") - >>> h.hexdigest() - '031edd7d41651593c5fe5c006fa5752b37fddff7bc4e843aa6af0c950f4b9406' - - -.. function:: md5([, data], *, usedforsecurity=True) -.. function:: sha1([, data], *, usedforsecurity=True) -.. function:: sha224([, data], *, usedforsecurity=True) -.. function:: sha256([, data], *, usedforsecurity=True) -.. function:: sha384([, data], *, usedforsecurity=True) -.. function:: sha512([, data], *, usedforsecurity=True) -.. function:: sha3_224([, data], *, usedforsecurity=True) -.. function:: sha3_256([, data], *, usedforsecurity=True) -.. function:: sha3_384([, data], *, usedforsecurity=True) -.. function:: sha3_512([, data], *, usedforsecurity=True) - -Named constructors such as these are faster than passing an algorithm name to -:func:`new`. - -Attributes ----------- - -Hashlib provides the following constant module attributes: - -.. data:: algorithms_guaranteed - - A set containing the names of the hash algorithms guaranteed to be supported - by this module on all platforms. Note that 'md5' is in this list despite - some upstream vendors offering an odd "FIPS compliant" Python build that - excludes it. - - .. versionadded:: 3.2 - -.. data:: algorithms_available - - A set containing the names of the hash algorithms that are available in the - running Python interpreter. These names will be recognized when passed to - :func:`new`. :attr:`algorithms_guaranteed` will always be a subset. The - same algorithm may appear multiple times in this set under different names - (thanks to OpenSSL). - - .. versionadded:: 3.2 - -Hash Objects ------------- - -The following values are provided as constant attributes of the hash objects -returned by the constructors: - -.. data:: hash.digest_size - - The size of the resulting hash in bytes. - -.. data:: hash.block_size - - The internal block size of the hash algorithm in bytes. - -A hash object has the following attributes: - -.. attribute:: hash.name - - The canonical name of this hash, always lowercase and always suitable as a - parameter to :func:`new` to create another hash of this type. - - .. versionchanged:: 3.4 - The name attribute has been present in CPython since its inception, but - until Python 3.4 was not formally specified, so may not exist on some - platforms. - -A hash object has the following methods: - - -.. method:: hash.update(data) - - Update the hash object with the :term:`bytes-like object`. - Repeated calls are equivalent to a single call with the - concatenation of all the arguments: ``m.update(a); m.update(b)`` is - equivalent to ``m.update(a+b)``. - - .. versionchanged:: 3.1 - The Python GIL is released to allow other threads to run while hash - updates on data larger than 2047 bytes is taking place when using hash - algorithms supplied by OpenSSL. - - -.. method:: hash.digest() - - Return the digest of the data passed to the :meth:`update` method so far. - This is a bytes object of size :attr:`digest_size` which may contain bytes in - the whole range from 0 to 255. - - -.. method:: hash.hexdigest() - - Like :meth:`digest` except the digest is returned as a string object of - double length, containing only hexadecimal digits. This may be used to - exchange the value safely in email or other non-binary environments. - - -.. method:: hash.copy() - - Return a copy ("clone") of the hash object. This can be used to efficiently - compute the digests of data sharing a common initial substring. - - -SHAKE variable length digests ------------------------------ - -.. function:: shake_128([, data], *, usedforsecurity=True) -.. function:: shake_256([, data], *, usedforsecurity=True) - -The :func:`shake_128` and :func:`shake_256` algorithms provide variable -length digests with length_in_bits//2 up to 128 or 256 bits of security. -As such, their digest methods require a length. Maximum length is not limited -by the SHAKE algorithm. - -.. method:: shake.digest(length) - - Return the digest of the data passed to the :meth:`~hash.update` method so far. - This is a bytes object of size *length* which may contain bytes in - the whole range from 0 to 255. - - -.. method:: shake.hexdigest(length) - - Like :meth:`digest` except the digest is returned as a string object of - double length, containing only hexadecimal digits. This may be used to - exchange the value in email or other non-binary environments. - -Example use: - - >>> h = hashlib.shake_256(b'Nobody inspects the spammish repetition') - >>> h.hexdigest(20) - '44709d6fcb83d92a76dcb0b668c98e1b1d3dafe7' - -File hashing ------------- - -The hashlib module provides a helper function for efficient hashing of -a file or file-like object. - -.. function:: file_digest(fileobj, digest, /) - - Return a digest object that has been updated with contents of file object. - - *fileobj* must be a file-like object opened for reading in binary mode. - It accepts file objects from builtin :func:`open`, :class:`~io.BytesIO` - instances, SocketIO objects from :meth:`socket.socket.makefile`, and - similar. The function may bypass Python's I/O and use the file descriptor - from :meth:`~io.IOBase.fileno` directly. *fileobj* must be assumed to be - in an unknown state after this function returns or raises. It is up to - the caller to close *fileobj*. - - *digest* must either be a hash algorithm name as a *str*, a hash - constructor, or a callable that returns a hash object. - - Example: - - >>> import io, hashlib, hmac - >>> with open(hashlib.__file__, "rb") as f: - ... digest = hashlib.file_digest(f, "sha256") - ... - >>> digest.hexdigest() # doctest: +ELLIPSIS - '...' - - >>> buf = io.BytesIO(b"somedata") - >>> mac1 = hmac.HMAC(b"key", digestmod=hashlib.sha512) - >>> digest = hashlib.file_digest(buf, lambda: mac1) - - >>> digest is mac1 - True - >>> mac2 = hmac.HMAC(b"key", b"somedata", digestmod=hashlib.sha512) - >>> mac1.digest() == mac2.digest() - True - - .. versionadded:: 3.11 - - -Key derivation --------------- - -Key derivation and key stretching algorithms are designed for secure password -hashing. Naive algorithms such as ``sha1(password)`` are not resistant against -brute-force attacks. A good password hashing function must be tunable, slow, and -include a `salt `_. - - -.. function:: pbkdf2_hmac(hash_name, password, salt, iterations, dklen=None) - - The function provides PKCS#5 password-based key derivation function 2. It - uses HMAC as pseudorandom function. - - The string *hash_name* is the desired name of the hash digest algorithm for - HMAC, e.g. 'sha1' or 'sha256'. *password* and *salt* are interpreted as - buffers of bytes. Applications and libraries should limit *password* to - a sensible length (e.g. 1024). *salt* should be about 16 or more bytes from - a proper source, e.g. :func:`os.urandom`. - - The number of *iterations* should be chosen based on the hash algorithm and - computing power. As of 2022, hundreds of thousands of iterations of SHA-256 - are suggested. For rationale as to why and how to choose what is best for - your application, read *Appendix A.2.2* of NIST-SP-800-132_. The answers - on the `stackexchange pbkdf2 iterations question`_ explain in detail. - - *dklen* is the length of the derived key. If *dklen* is ``None`` then the - digest size of the hash algorithm *hash_name* is used, e.g. 64 for SHA-512. - - >>> from hashlib import pbkdf2_hmac - >>> our_app_iters = 500_000 # Application specific, read above. - >>> dk = pbkdf2_hmac('sha256', b'password', b'bad salt'*2, our_app_iters) - >>> dk.hex() - '15530bba69924174860db778f2c6f8104d3aaf9d26241840c8c4a641c8d000a9' - - .. versionadded:: 3.4 - - .. note:: - - A fast implementation of *pbkdf2_hmac* is available with OpenSSL. The - Python implementation uses an inline version of :mod:`hmac`. It is about - three times slower and doesn't release the GIL. - - .. deprecated:: 3.10 - - Slow Python implementation of *pbkdf2_hmac* is deprecated. In the - future the function will only be available when Python is compiled - with OpenSSL. - -.. function:: scrypt(password, *, salt, n, r, p, maxmem=0, dklen=64) - - The function provides scrypt password-based key derivation function as - defined in :rfc:`7914`. - - *password* and *salt* must be :term:`bytes-like objects - `. Applications and libraries should limit *password* - to a sensible length (e.g. 1024). *salt* should be about 16 or more - bytes from a proper source, e.g. :func:`os.urandom`. - - *n* is the CPU/Memory cost factor, *r* the block size, *p* parallelization - factor and *maxmem* limits memory (OpenSSL 1.1.0 defaults to 32 MiB). - *dklen* is the length of the derived key. - - .. versionadded:: 3.6 - - -BLAKE2 ------- - -.. sectionauthor:: Dmitry Chestnykh - -.. index:: - single: blake2b, blake2s - -BLAKE2_ is a cryptographic hash function defined in :rfc:`7693` that comes in two -flavors: - -* **BLAKE2b**, optimized for 64-bit platforms and produces digests of any size - between 1 and 64 bytes, - -* **BLAKE2s**, optimized for 8- to 32-bit platforms and produces digests of any - size between 1 and 32 bytes. - -BLAKE2 supports **keyed mode** (a faster and simpler replacement for HMAC_), -**salted hashing**, **personalization**, and **tree hashing**. - -Hash objects from this module follow the API of standard library's -:mod:`hashlib` objects. - - -Creating hash objects -^^^^^^^^^^^^^^^^^^^^^ - -New hash objects are created by calling constructor functions: - - -.. function:: blake2b(data=b'', *, digest_size=64, key=b'', salt=b'', \ - person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ - node_depth=0, inner_size=0, last_node=False, \ - usedforsecurity=True) - -.. function:: blake2s(data=b'', *, digest_size=32, key=b'', salt=b'', \ - person=b'', fanout=1, depth=1, leaf_size=0, node_offset=0, \ - node_depth=0, inner_size=0, last_node=False, \ - usedforsecurity=True) - - -These functions return the corresponding hash objects for calculating -BLAKE2b or BLAKE2s. They optionally take these general parameters: - -* *data*: initial chunk of data to hash, which must be - :term:`bytes-like object`. It can be passed only as positional argument. - -* *digest_size*: size of output digest in bytes. - -* *key*: key for keyed hashing (up to 64 bytes for BLAKE2b, up to 32 bytes for - BLAKE2s). - -* *salt*: salt for randomized hashing (up to 16 bytes for BLAKE2b, up to 8 - bytes for BLAKE2s). - -* *person*: personalization string (up to 16 bytes for BLAKE2b, up to 8 bytes - for BLAKE2s). - -The following table shows limits for general parameters (in bytes): - -======= =========== ======== ========= =========== -Hash digest_size len(key) len(salt) len(person) -======= =========== ======== ========= =========== -BLAKE2b 64 64 16 16 -BLAKE2s 32 32 8 8 -======= =========== ======== ========= =========== - -.. note:: - - BLAKE2 specification defines constant lengths for salt and personalization - parameters, however, for convenience, this implementation accepts byte - strings of any size up to the specified length. If the length of the - parameter is less than specified, it is padded with zeros, thus, for - example, ``b'salt'`` and ``b'salt\x00'`` is the same value. (This is not - the case for *key*.) - -These sizes are available as module `constants`_ described below. - -Constructor functions also accept the following tree hashing parameters: - -* *fanout*: fanout (0 to 255, 0 if unlimited, 1 in sequential mode). - -* *depth*: maximal depth of tree (1 to 255, 255 if unlimited, 1 in - sequential mode). - -* *leaf_size*: maximal byte length of leaf (0 to ``2**32-1``, 0 if unlimited or in - sequential mode). - -* *node_offset*: node offset (0 to ``2**64-1`` for BLAKE2b, 0 to ``2**48-1`` for - BLAKE2s, 0 for the first, leftmost, leaf, or in sequential mode). - -* *node_depth*: node depth (0 to 255, 0 for leaves, or in sequential mode). - -* *inner_size*: inner digest size (0 to 64 for BLAKE2b, 0 to 32 for - BLAKE2s, 0 in sequential mode). - -* *last_node*: boolean indicating whether the processed node is the last - one (``False`` for sequential mode). - -.. figure:: hashlib-blake2-tree.png - :alt: Explanation of tree mode parameters. - :class: invert-in-dark-mode - -See section 2.10 in `BLAKE2 specification -`_ for comprehensive review of tree -hashing. - - -Constants -^^^^^^^^^ - -.. data:: blake2b.SALT_SIZE -.. data:: blake2s.SALT_SIZE - -Salt length (maximum length accepted by constructors). - - -.. data:: blake2b.PERSON_SIZE -.. data:: blake2s.PERSON_SIZE - -Personalization string length (maximum length accepted by constructors). - - -.. data:: blake2b.MAX_KEY_SIZE -.. data:: blake2s.MAX_KEY_SIZE - -Maximum key size. - - -.. data:: blake2b.MAX_DIGEST_SIZE -.. data:: blake2s.MAX_DIGEST_SIZE - -Maximum digest size that the hash function can output. - - -Examples -^^^^^^^^ - -Simple hashing -"""""""""""""" - -To calculate hash of some data, you should first construct a hash object by -calling the appropriate constructor function (:func:`blake2b` or -:func:`blake2s`), then update it with the data by calling :meth:`~hash.update` on the -object, and, finally, get the digest out of the object by calling -:meth:`~hash.digest` (or :meth:`~hash.hexdigest` for hex-encoded string). - - >>> from hashlib import blake2b - >>> h = blake2b() - >>> h.update(b'Hello world') - >>> h.hexdigest() - '6ff843ba685842aa82031d3f53c48b66326df7639a63d128974c5c14f31a0f33343a8c65551134ed1ae0f2b0dd2bb495dc81039e3eeb0aa1bb0388bbeac29183' - - -As a shortcut, you can pass the first chunk of data to update directly to the -constructor as the positional argument: - - >>> from hashlib import blake2b - >>> blake2b(b'Hello world').hexdigest() - '6ff843ba685842aa82031d3f53c48b66326df7639a63d128974c5c14f31a0f33343a8c65551134ed1ae0f2b0dd2bb495dc81039e3eeb0aa1bb0388bbeac29183' - -You can call :meth:`hash.update` as many times as you need to iteratively -update the hash: - - >>> from hashlib import blake2b - >>> items = [b'Hello', b' ', b'world'] - >>> h = blake2b() - >>> for item in items: - ... h.update(item) - >>> h.hexdigest() - '6ff843ba685842aa82031d3f53c48b66326df7639a63d128974c5c14f31a0f33343a8c65551134ed1ae0f2b0dd2bb495dc81039e3eeb0aa1bb0388bbeac29183' - - -Using different digest sizes -"""""""""""""""""""""""""""" - -BLAKE2 has configurable size of digests up to 64 bytes for BLAKE2b and up to 32 -bytes for BLAKE2s. For example, to replace SHA-1 with BLAKE2b without changing -the size of output, we can tell BLAKE2b to produce 20-byte digests: - - >>> from hashlib import blake2b - >>> h = blake2b(digest_size=20) - >>> h.update(b'Replacing SHA1 with the more secure function') - >>> h.hexdigest() - 'd24f26cf8de66472d58d4e1b1774b4c9158b1f4c' - >>> h.digest_size - 20 - >>> len(h.digest()) - 20 - -Hash objects with different digest sizes have completely different outputs -(shorter hashes are *not* prefixes of longer hashes); BLAKE2b and BLAKE2s -produce different outputs even if the output length is the same: - - >>> from hashlib import blake2b, blake2s - >>> blake2b(digest_size=10).hexdigest() - '6fa1d8fcfd719046d762' - >>> blake2b(digest_size=11).hexdigest() - 'eb6ec15daf9546254f0809' - >>> blake2s(digest_size=10).hexdigest() - '1bf21a98c78a1c376ae9' - >>> blake2s(digest_size=11).hexdigest() - '567004bf96e4a25773ebf4' - - -Keyed hashing -""""""""""""" - -Keyed hashing can be used for authentication as a faster and simpler -replacement for `Hash-based message authentication code -`_ (HMAC). -BLAKE2 can be securely used in prefix-MAC mode thanks to the -indifferentiability property inherited from BLAKE. - -This example shows how to get a (hex-encoded) 128-bit authentication code for -message ``b'message data'`` with key ``b'pseudorandom key'``:: - - >>> from hashlib import blake2b - >>> h = blake2b(key=b'pseudorandom key', digest_size=16) - >>> h.update(b'message data') - >>> h.hexdigest() - '3d363ff7401e02026f4a4687d4863ced' - - -As a practical example, a web application can symmetrically sign cookies sent -to users and later verify them to make sure they weren't tampered with:: - - >>> from hashlib import blake2b - >>> from hmac import compare_digest - >>> - >>> SECRET_KEY = b'pseudorandomly generated server secret key' - >>> AUTH_SIZE = 16 - >>> - >>> def sign(cookie): - ... h = blake2b(digest_size=AUTH_SIZE, key=SECRET_KEY) - ... h.update(cookie) - ... return h.hexdigest().encode('utf-8') - >>> - >>> def verify(cookie, sig): - ... good_sig = sign(cookie) - ... return compare_digest(good_sig, sig) - >>> - >>> cookie = b'user-alice' - >>> sig = sign(cookie) - >>> print("{0},{1}".format(cookie.decode('utf-8'), sig)) - user-alice,b'43b3c982cf697e0c5ab22172d1ca7421' - >>> verify(cookie, sig) - True - >>> verify(b'user-bob', sig) - False - >>> verify(cookie, b'0102030405060708090a0b0c0d0e0f00') - False - -Even though there's a native keyed hashing mode, BLAKE2 can, of course, be used -in HMAC construction with :mod:`hmac` module:: - - >>> import hmac, hashlib - >>> m = hmac.new(b'secret key', digestmod=hashlib.blake2s) - >>> m.update(b'message') - >>> m.hexdigest() - 'e3c8102868d28b5ff85fc35dda07329970d1a01e273c37481326fe0c861c8142' - - -Randomized hashing -"""""""""""""""""" - -By setting *salt* parameter users can introduce randomization to the hash -function. Randomized hashing is useful for protecting against collision attacks -on the hash function used in digital signatures. - - Randomized hashing is designed for situations where one party, the message - preparer, generates all or part of a message to be signed by a second - party, the message signer. If the message preparer is able to find - cryptographic hash function collisions (i.e., two messages producing the - same hash value), then they might prepare meaningful versions of the message - that would produce the same hash value and digital signature, but with - different results (e.g., transferring $1,000,000 to an account, rather than - $10). Cryptographic hash functions have been designed with collision - resistance as a major goal, but the current concentration on attacking - cryptographic hash functions may result in a given cryptographic hash - function providing less collision resistance than expected. Randomized - hashing offers the signer additional protection by reducing the likelihood - that a preparer can generate two or more messages that ultimately yield the - same hash value during the digital signature generation process --- even if - it is practical to find collisions for the hash function. However, the use - of randomized hashing may reduce the amount of security provided by a - digital signature when all portions of the message are prepared - by the signer. - - (`NIST SP-800-106 "Randomized Hashing for Digital Signatures" - `_) - -In BLAKE2 the salt is processed as a one-time input to the hash function during -initialization, rather than as an input to each compression function. - -.. warning:: - - *Salted hashing* (or just hashing) with BLAKE2 or any other general-purpose - cryptographic hash function, such as SHA-256, is not suitable for hashing - passwords. See `BLAKE2 FAQ `_ for more - information. -.. - - >>> import os - >>> from hashlib import blake2b - >>> msg = b'some message' - >>> # Calculate the first hash with a random salt. - >>> salt1 = os.urandom(blake2b.SALT_SIZE) - >>> h1 = blake2b(salt=salt1) - >>> h1.update(msg) - >>> # Calculate the second hash with a different random salt. - >>> salt2 = os.urandom(blake2b.SALT_SIZE) - >>> h2 = blake2b(salt=salt2) - >>> h2.update(msg) - >>> # The digests are different. - >>> h1.digest() != h2.digest() - True - - -Personalization -""""""""""""""" - -Sometimes it is useful to force hash function to produce different digests for -the same input for different purposes. Quoting the authors of the Skein hash -function: - - We recommend that all application designers seriously consider doing this; - we have seen many protocols where a hash that is computed in one part of - the protocol can be used in an entirely different part because two hash - computations were done on similar or related data, and the attacker can - force the application to make the hash inputs the same. Personalizing each - hash function used in the protocol summarily stops this type of attack. - - (`The Skein Hash Function Family - `_, - p. 21) - -BLAKE2 can be personalized by passing bytes to the *person* argument:: - - >>> from hashlib import blake2b - >>> FILES_HASH_PERSON = b'MyApp Files Hash' - >>> BLOCK_HASH_PERSON = b'MyApp Block Hash' - >>> h = blake2b(digest_size=32, person=FILES_HASH_PERSON) - >>> h.update(b'the same content') - >>> h.hexdigest() - '20d9cd024d4fb086aae819a1432dd2466de12947831b75c5a30cf2676095d3b4' - >>> h = blake2b(digest_size=32, person=BLOCK_HASH_PERSON) - >>> h.update(b'the same content') - >>> h.hexdigest() - 'cf68fb5761b9c44e7878bfb2c4c9aea52264a80b75005e65619778de59f383a3' - -Personalization together with the keyed mode can also be used to derive different -keys from a single one. - - >>> from hashlib import blake2s - >>> from base64 import b64decode, b64encode - >>> orig_key = b64decode(b'Rm5EPJai72qcK3RGBpW3vPNfZy5OZothY+kHY6h21KM=') - >>> enc_key = blake2s(key=orig_key, person=b'kEncrypt').digest() - >>> mac_key = blake2s(key=orig_key, person=b'kMAC').digest() - >>> print(b64encode(enc_key).decode('utf-8')) - rbPb15S/Z9t+agffno5wuhB77VbRi6F9Iv2qIxU7WHw= - >>> print(b64encode(mac_key).decode('utf-8')) - G9GtHFE1YluXY1zWPlYk1e/nWfu0WSEb0KRcjhDeP/o= - -Tree mode -""""""""" - -Here's an example of hashing a minimal tree with two leaf nodes:: - - 10 - / \ - 00 01 - -This example uses 64-byte internal digests, and returns the 32-byte final -digest:: - - >>> from hashlib import blake2b - >>> - >>> FANOUT = 2 - >>> DEPTH = 2 - >>> LEAF_SIZE = 4096 - >>> INNER_SIZE = 64 - >>> - >>> buf = bytearray(6000) - >>> - >>> # Left leaf - ... h00 = blake2b(buf[0:LEAF_SIZE], fanout=FANOUT, depth=DEPTH, - ... leaf_size=LEAF_SIZE, inner_size=INNER_SIZE, - ... node_offset=0, node_depth=0, last_node=False) - >>> # Right leaf - ... h01 = blake2b(buf[LEAF_SIZE:], fanout=FANOUT, depth=DEPTH, - ... leaf_size=LEAF_SIZE, inner_size=INNER_SIZE, - ... node_offset=1, node_depth=0, last_node=True) - >>> # Root node - ... h10 = blake2b(digest_size=32, fanout=FANOUT, depth=DEPTH, - ... leaf_size=LEAF_SIZE, inner_size=INNER_SIZE, - ... node_offset=0, node_depth=1, last_node=True) - >>> h10.update(h00.digest()) - >>> h10.update(h01.digest()) - >>> h10.hexdigest() - '3ad2a9b37c6070e374c7a8c508fe20ca86b6ed54e286e93a0318e95e881db5aa' - -Credits -^^^^^^^ - -BLAKE2_ was designed by *Jean-Philippe Aumasson*, *Samuel Neves*, *Zooko -Wilcox-O'Hearn*, and *Christian Winnerlein* based on SHA-3_ finalist BLAKE_ -created by *Jean-Philippe Aumasson*, *Luca Henzen*, *Willi Meier*, and -*Raphael C.-W. Phan*. - -It uses core algorithm from ChaCha_ cipher designed by *Daniel J. Bernstein*. - -The stdlib implementation is based on pyblake2_ module. It was written by -*Dmitry Chestnykh* based on C implementation written by *Samuel Neves*. The -documentation was copied from pyblake2_ and written by *Dmitry Chestnykh*. - -The C code was partly rewritten for Python by *Christian Heimes*. - -The following public domain dedication applies for both C hash function -implementation, extension code, and this documentation: - - To the extent possible under law, the author(s) have dedicated all copyright - and related and neighboring rights to this software to the public domain - worldwide. This software is distributed without any warranty. - - You should have received a copy of the CC0 Public Domain Dedication along - with this software. If not, see - https://creativecommons.org/publicdomain/zero/1.0/. - -The following people have helped with development or contributed their changes -to the project and the public domain according to the Creative Commons Public -Domain Dedication 1.0 Universal: - -* *Alexandr Sokolovskiy* - -.. _BLAKE2: https://www.blake2.net -.. _HMAC: https://en.wikipedia.org/wiki/Hash-based_message_authentication_code -.. _BLAKE: https://web.archive.org/web/20200918190133/https://131002.net/blake/ -.. _SHA-3: https://en.wikipedia.org/wiki/Secure_Hash_Algorithms -.. _ChaCha: https://cr.yp.to/chacha.html -.. _pyblake2: https://pythonhosted.org/pyblake2/ -.. _NIST-SP-800-132: https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-132.pdf -.. _stackexchange pbkdf2 iterations question: https://security.stackexchange.com/questions/3959/recommended-of-iterations-when-using-pbkdf2-sha256/ -.. _Attacks on cryptographic hash algorithms: https://en.wikipedia.org/wiki/Cryptographic_hash_function#Attacks_on_cryptographic_hash_algorithms -.. _the FIPS 180-4 standard: https://csrc.nist.gov/publications/detail/fips/180/4/final -.. _the FIPS 202 standard: https://csrc.nist.gov/publications/detail/fips/202/final - - -.. _hashlib-seealso: - -.. seealso:: - - Module :mod:`hmac` - A module to generate message authentication codes using hashes. - - Module :mod:`base64` - Another way to encode binary hashes for non-binary environments. - - https://nvlpubs.nist.gov/nistpubs/fips/nist.fips.180-4.pdf - The FIPS 180-4 publication on Secure Hash Algorithms. - - https://csrc.nist.gov/publications/detail/fips/202/final - The FIPS 202 publication on the SHA-3 Standard. - - https://www.blake2.net/ - Official BLAKE2 website. - - https://en.wikipedia.org/wiki/Cryptographic_hash_function - Wikipedia article with information on which algorithms have known issues - and what that means regarding their use. - - https://www.ietf.org/rfc/rfc8018.txt - PKCS #5: Password-Based Cryptography Specification Version 2.1 - - https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-132.pdf - NIST Recommendation for Password-Based Key Derivation. diff --git a/Doc/library/heapq.rst b/Doc/library/heapq.rst deleted file mode 100644 index 8b00f7b27959b6..00000000000000 --- a/Doc/library/heapq.rst +++ /dev/null @@ -1,322 +0,0 @@ -:mod:`heapq` --- Heap queue algorithm -===================================== - -.. module:: heapq - :synopsis: Heap queue algorithm (a.k.a. priority queue). - -.. moduleauthor:: Kevin O'Connor -.. sectionauthor:: Guido van Rossum -.. sectionauthor:: François Pinard -.. sectionauthor:: Raymond Hettinger - -**Source code:** :source:`Lib/heapq.py` - --------------- - -This module provides an implementation of the heap queue algorithm, also known -as the priority queue algorithm. - -Heaps are binary trees for which every parent node has a value less than or -equal to any of its children. This implementation uses arrays for which -``heap[k] <= heap[2*k+1]`` and ``heap[k] <= heap[2*k+2]`` for all *k*, counting -elements from zero. For the sake of comparison, non-existing elements are -considered to be infinite. The interesting property of a heap is that its -smallest element is always the root, ``heap[0]``. - -The API below differs from textbook heap algorithms in two aspects: (a) We use -zero-based indexing. This makes the relationship between the index for a node -and the indexes for its children slightly less obvious, but is more suitable -since Python uses zero-based indexing. (b) Our pop method returns the smallest -item, not the largest (called a "min heap" in textbooks; a "max heap" is more -common in texts because of its suitability for in-place sorting). - -These two make it possible to view the heap as a regular Python list without -surprises: ``heap[0]`` is the smallest item, and ``heap.sort()`` maintains the -heap invariant! - -To create a heap, use a list initialized to ``[]``, or you can transform a -populated list into a heap via function :func:`heapify`. - -The following functions are provided: - - -.. function:: heappush(heap, item) - - Push the value *item* onto the *heap*, maintaining the heap invariant. - - -.. function:: heappop(heap) - - Pop and return the smallest item from the *heap*, maintaining the heap - invariant. If the heap is empty, :exc:`IndexError` is raised. To access the - smallest item without popping it, use ``heap[0]``. - - -.. function:: heappushpop(heap, item) - - Push *item* on the heap, then pop and return the smallest item from the - *heap*. The combined action runs more efficiently than :func:`heappush` - followed by a separate call to :func:`heappop`. - - -.. function:: heapify(x) - - Transform list *x* into a heap, in-place, in linear time. - - -.. function:: heapreplace(heap, item) - - Pop and return the smallest item from the *heap*, and also push the new *item*. - The heap size doesn't change. If the heap is empty, :exc:`IndexError` is raised. - - This one step operation is more efficient than a :func:`heappop` followed by - :func:`heappush` and can be more appropriate when using a fixed-size heap. - The pop/push combination always returns an element from the heap and replaces - it with *item*. - - The value returned may be larger than the *item* added. If that isn't - desired, consider using :func:`heappushpop` instead. Its push/pop - combination returns the smaller of the two values, leaving the larger value - on the heap. - - -The module also offers three general purpose functions based on heaps. - - -.. function:: merge(*iterables, key=None, reverse=False) - - Merge multiple sorted inputs into a single sorted output (for example, merge - timestamped entries from multiple log files). Returns an :term:`iterator` - over the sorted values. - - Similar to ``sorted(itertools.chain(*iterables))`` but returns an iterable, does - not pull the data into memory all at once, and assumes that each of the input - streams is already sorted (smallest to largest). - - Has two optional arguments which must be specified as keyword arguments. - - *key* specifies a :term:`key function` of one argument that is used to - extract a comparison key from each input element. The default value is - ``None`` (compare the elements directly). - - *reverse* is a boolean value. If set to ``True``, then the input elements - are merged as if each comparison were reversed. To achieve behavior similar - to ``sorted(itertools.chain(*iterables), reverse=True)``, all iterables must - be sorted from largest to smallest. - - .. versionchanged:: 3.5 - Added the optional *key* and *reverse* parameters. - - -.. function:: nlargest(n, iterable, key=None) - - Return a list with the *n* largest elements from the dataset defined by - *iterable*. *key*, if provided, specifies a function of one argument that is - used to extract a comparison key from each element in *iterable* (for example, - ``key=str.lower``). Equivalent to: ``sorted(iterable, key=key, - reverse=True)[:n]``. - - -.. function:: nsmallest(n, iterable, key=None) - - Return a list with the *n* smallest elements from the dataset defined by - *iterable*. *key*, if provided, specifies a function of one argument that is - used to extract a comparison key from each element in *iterable* (for example, - ``key=str.lower``). Equivalent to: ``sorted(iterable, key=key)[:n]``. - - -The latter two functions perform best for smaller values of *n*. For larger -values, it is more efficient to use the :func:`sorted` function. Also, when -``n==1``, it is more efficient to use the built-in :func:`min` and :func:`max` -functions. If repeated usage of these functions is required, consider turning -the iterable into an actual heap. - - -Basic Examples --------------- - -A `heapsort `_ can be implemented by -pushing all values onto a heap and then popping off the smallest values one at a -time:: - - >>> def heapsort(iterable): - ... h = [] - ... for value in iterable: - ... heappush(h, value) - ... return [heappop(h) for i in range(len(h))] - ... - >>> heapsort([1, 3, 5, 7, 9, 2, 4, 6, 8, 0]) - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -This is similar to ``sorted(iterable)``, but unlike :func:`sorted`, this -implementation is not stable. - -Heap elements can be tuples. This is useful for assigning comparison values -(such as task priorities) alongside the main record being tracked:: - - >>> h = [] - >>> heappush(h, (5, 'write code')) - >>> heappush(h, (7, 'release product')) - >>> heappush(h, (1, 'write spec')) - >>> heappush(h, (3, 'create tests')) - >>> heappop(h) - (1, 'write spec') - - -Priority Queue Implementation Notes ------------------------------------ - -A `priority queue `_ is common use -for a heap, and it presents several implementation challenges: - -* Sort stability: how do you get two tasks with equal priorities to be returned - in the order they were originally added? - -* Tuple comparison breaks for (priority, task) pairs if the priorities are equal - and the tasks do not have a default comparison order. - -* If the priority of a task changes, how do you move it to a new position in - the heap? - -* Or if a pending task needs to be deleted, how do you find it and remove it - from the queue? - -A solution to the first two challenges is to store entries as 3-element list -including the priority, an entry count, and the task. The entry count serves as -a tie-breaker so that two tasks with the same priority are returned in the order -they were added. And since no two entry counts are the same, the tuple -comparison will never attempt to directly compare two tasks. - -Another solution to the problem of non-comparable tasks is to create a wrapper -class that ignores the task item and only compares the priority field:: - - from dataclasses import dataclass, field - from typing import Any - - @dataclass(order=True) - class PrioritizedItem: - priority: int - item: Any=field(compare=False) - -The remaining challenges revolve around finding a pending task and making -changes to its priority or removing it entirely. Finding a task can be done -with a dictionary pointing to an entry in the queue. - -Removing the entry or changing its priority is more difficult because it would -break the heap structure invariants. So, a possible solution is to mark the -entry as removed and add a new entry with the revised priority:: - - pq = [] # list of entries arranged in a heap - entry_finder = {} # mapping of tasks to entries - REMOVED = '' # placeholder for a removed task - counter = itertools.count() # unique sequence count - - def add_task(task, priority=0): - 'Add a new task or update the priority of an existing task' - if task in entry_finder: - remove_task(task) - count = next(counter) - entry = [priority, count, task] - entry_finder[task] = entry - heappush(pq, entry) - - def remove_task(task): - 'Mark an existing task as REMOVED. Raise KeyError if not found.' - entry = entry_finder.pop(task) - entry[-1] = REMOVED - - def pop_task(): - 'Remove and return the lowest priority task. Raise KeyError if empty.' - while pq: - priority, count, task = heappop(pq) - if task is not REMOVED: - del entry_finder[task] - return task - raise KeyError('pop from an empty priority queue') - - -Theory ------- - -Heaps are arrays for which ``a[k] <= a[2*k+1]`` and ``a[k] <= a[2*k+2]`` for all -*k*, counting elements from 0. For the sake of comparison, non-existing -elements are considered to be infinite. The interesting property of a heap is -that ``a[0]`` is always its smallest element. - -The strange invariant above is meant to be an efficient memory representation -for a tournament. The numbers below are *k*, not ``a[k]``:: - - 0 - - 1 2 - - 3 4 5 6 - - 7 8 9 10 11 12 13 14 - - 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 - -In the tree above, each cell *k* is topping ``2*k+1`` and ``2*k+2``. In a usual -binary tournament we see in sports, each cell is the winner over the two cells -it tops, and we can trace the winner down the tree to see all opponents s/he -had. However, in many computer applications of such tournaments, we do not need -to trace the history of a winner. To be more memory efficient, when a winner is -promoted, we try to replace it by something else at a lower level, and the rule -becomes that a cell and the two cells it tops contain three different items, but -the top cell "wins" over the two topped cells. - -If this heap invariant is protected at all time, index 0 is clearly the overall -winner. The simplest algorithmic way to remove it and find the "next" winner is -to move some loser (let's say cell 30 in the diagram above) into the 0 position, -and then percolate this new 0 down the tree, exchanging values, until the -invariant is re-established. This is clearly logarithmic on the total number of -items in the tree. By iterating over all items, you get an O(n log n) sort. - -A nice feature of this sort is that you can efficiently insert new items while -the sort is going on, provided that the inserted items are not "better" than the -last 0'th element you extracted. This is especially useful in simulation -contexts, where the tree holds all incoming events, and the "win" condition -means the smallest scheduled time. When an event schedules other events for -execution, they are scheduled into the future, so they can easily go into the -heap. So, a heap is a good structure for implementing schedulers (this is what -I used for my MIDI sequencer :-). - -Various structures for implementing schedulers have been extensively studied, -and heaps are good for this, as they are reasonably speedy, the speed is almost -constant, and the worst case is not much different than the average case. -However, there are other representations which are more efficient overall, yet -the worst cases might be terrible. - -Heaps are also very useful in big disk sorts. You most probably all know that a -big sort implies producing "runs" (which are pre-sorted sequences, whose size is -usually related to the amount of CPU memory), followed by a merging passes for -these runs, which merging is often very cleverly organised [#]_. It is very -important that the initial sort produces the longest runs possible. Tournaments -are a good way to achieve that. If, using all the memory available to hold a -tournament, you replace and percolate items that happen to fit the current run, -you'll produce runs which are twice the size of the memory for random input, and -much better for input fuzzily ordered. - -Moreover, if you output the 0'th item on disk and get an input which may not fit -in the current tournament (because the value "wins" over the last output value), -it cannot fit in the heap, so the size of the heap decreases. The freed memory -could be cleverly reused immediately for progressively building a second heap, -which grows at exactly the same rate the first heap is melting. When the first -heap completely vanishes, you switch heaps and start a new run. Clever and -quite effective! - -In a word, heaps are useful memory structures to know. I use them in a few -applications, and I think it is good to keep a 'heap' module around. :-) - -.. rubric:: Footnotes - -.. [#] The disk balancing algorithms which are current, nowadays, are more annoying - than clever, and this is a consequence of the seeking capabilities of the disks. - On devices which cannot seek, like big tape drives, the story was quite - different, and one had to be very clever to ensure (far in advance) that each - tape movement will be the most effective possible (that is, will best - participate at "progressing" the merge). Some tapes were even able to read - backwards, and this was also used to avoid the rewinding time. Believe me, real - good tape sorts were quite spectacular to watch! From all times, sorting has - always been a Great Art! :-) - diff --git a/Doc/library/hmac.rst b/Doc/library/hmac.rst deleted file mode 100644 index b2ca0455d3745c..00000000000000 --- a/Doc/library/hmac.rst +++ /dev/null @@ -1,150 +0,0 @@ -:mod:`hmac` --- Keyed-Hashing for Message Authentication -======================================================== - -.. module:: hmac - :synopsis: Keyed-Hashing for Message Authentication (HMAC) implementation - -.. moduleauthor:: Gerhard Häring -.. sectionauthor:: Gerhard Häring - -**Source code:** :source:`Lib/hmac.py` - --------------- - -This module implements the HMAC algorithm as described by :rfc:`2104`. - - -.. function:: new(key, msg=None, digestmod='') - - Return a new hmac object. *key* is a bytes or bytearray object giving the - secret key. If *msg* is present, the method call ``update(msg)`` is made. - *digestmod* is the digest name, digest constructor or module for the HMAC - object to use. It may be any name suitable to :func:`hashlib.new`. - Despite its argument position, it is required. - - .. versionchanged:: 3.4 - Parameter *key* can be a bytes or bytearray object. - Parameter *msg* can be of any type supported by :mod:`hashlib`. - Parameter *digestmod* can be the name of a hash algorithm. - - .. deprecated-removed:: 3.4 3.8 - MD5 as implicit default digest for *digestmod* is deprecated. - The digestmod parameter is now required. Pass it as a keyword - argument to avoid awkwardness when you do not have an initial msg. - - -.. function:: digest(key, msg, digest) - - Return digest of *msg* for given secret *key* and *digest*. The - function is equivalent to ``HMAC(key, msg, digest).digest()``, but - uses an optimized C or inline implementation, which is faster for messages - that fit into memory. The parameters *key*, *msg*, and *digest* have - the same meaning as in :func:`~hmac.new`. - - CPython implementation detail, the optimized C implementation is only used - when *digest* is a string and name of a digest algorithm, which is - supported by OpenSSL. - - .. versionadded:: 3.7 - - -An HMAC object has the following methods: - -.. method:: HMAC.update(msg) - - Update the hmac object with *msg*. Repeated calls are equivalent to a - single call with the concatenation of all the arguments: - ``m.update(a); m.update(b)`` is equivalent to ``m.update(a + b)``. - - .. versionchanged:: 3.4 - Parameter *msg* can be of any type supported by :mod:`hashlib`. - - -.. method:: HMAC.digest() - - Return the digest of the bytes passed to the :meth:`update` method so far. - This bytes object will be the same length as the *digest_size* of the digest - given to the constructor. It may contain non-ASCII bytes, including NUL - bytes. - - .. warning:: - - When comparing the output of :meth:`digest` to an externally supplied - digest during a verification routine, it is recommended to use the - :func:`compare_digest` function instead of the ``==`` operator - to reduce the vulnerability to timing attacks. - - -.. method:: HMAC.hexdigest() - - Like :meth:`digest` except the digest is returned as a string twice the - length containing only hexadecimal digits. This may be used to exchange the - value safely in email or other non-binary environments. - - .. warning:: - - When comparing the output of :meth:`hexdigest` to an externally supplied - digest during a verification routine, it is recommended to use the - :func:`compare_digest` function instead of the ``==`` operator - to reduce the vulnerability to timing attacks. - - -.. method:: HMAC.copy() - - Return a copy ("clone") of the hmac object. This can be used to efficiently - compute the digests of strings that share a common initial substring. - - -A hash object has the following attributes: - -.. attribute:: HMAC.digest_size - - The size of the resulting HMAC digest in bytes. - -.. attribute:: HMAC.block_size - - The internal block size of the hash algorithm in bytes. - - .. versionadded:: 3.4 - -.. attribute:: HMAC.name - - The canonical name of this HMAC, always lowercase, e.g. ``hmac-md5``. - - .. versionadded:: 3.4 - - -.. deprecated:: 3.9 - - The undocumented attributes ``HMAC.digest_cons``, ``HMAC.inner``, and - ``HMAC.outer`` are internal implementation details and will be removed in - Python 3.10. - -This module also provides the following helper function: - -.. function:: compare_digest(a, b) - - Return ``a == b``. This function uses an approach designed to prevent - timing analysis by avoiding content-based short circuiting behaviour, - making it appropriate for cryptography. *a* and *b* must both be of the - same type: either :class:`str` (ASCII only, as e.g. returned by - :meth:`HMAC.hexdigest`), or a :term:`bytes-like object`. - - .. note:: - - If *a* and *b* are of different lengths, or if an error occurs, - a timing attack could theoretically reveal information about the - types and lengths of *a* and *b*—but not their values. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.10 - - The function uses OpenSSL's ``CRYPTO_memcmp()`` internally when - available. - - -.. seealso:: - - Module :mod:`hashlib` - The Python module providing secure hash functions. diff --git a/Doc/library/html.entities.rst b/Doc/library/html.entities.rst deleted file mode 100644 index cb8b623781c5e1..00000000000000 --- a/Doc/library/html.entities.rst +++ /dev/null @@ -1,47 +0,0 @@ -:mod:`html.entities` --- Definitions of HTML general entities -============================================================= - -.. module:: html.entities - :synopsis: Definitions of HTML general entities. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/html/entities.py` - --------------- - -This module defines four dictionaries, :data:`html5`, -:data:`name2codepoint`, :data:`codepoint2name`, and :data:`entitydefs`. - - -.. data:: html5 - - A dictionary that maps HTML5 named character references [#]_ to the - equivalent Unicode character(s), e.g. ``html5['gt;'] == '>'``. - Note that the trailing semicolon is included in the name (e.g. ``'gt;'``), - however some of the names are accepted by the standard even without the - semicolon: in this case the name is present with and without the ``';'``. - See also :func:`html.unescape`. - - .. versionadded:: 3.3 - - -.. data:: entitydefs - - A dictionary mapping XHTML 1.0 entity definitions to their replacement text in - ISO Latin-1. - - -.. data:: name2codepoint - - A dictionary that maps HTML entity names to the Unicode code points. - - -.. data:: codepoint2name - - A dictionary that maps Unicode code points to HTML entity names. - - -.. rubric:: Footnotes - -.. [#] See https://html.spec.whatwg.org/multipage/named-characters.html#named-character-references diff --git a/Doc/library/html.parser.rst b/Doc/library/html.parser.rst deleted file mode 100644 index d35090111e0822..00000000000000 --- a/Doc/library/html.parser.rst +++ /dev/null @@ -1,340 +0,0 @@ -:mod:`html.parser` --- Simple HTML and XHTML parser -=================================================== - -.. module:: html.parser - :synopsis: A simple parser that can handle HTML and XHTML. - -**Source code:** :source:`Lib/html/parser.py` - -.. index:: - single: HTML - single: XHTML - --------------- - -This module defines a class :class:`HTMLParser` which serves as the basis for -parsing text files formatted in HTML (HyperText Mark-up Language) and XHTML. - -.. class:: HTMLParser(*, convert_charrefs=True) - - Create a parser instance able to parse invalid markup. - - If *convert_charrefs* is ``True`` (the default), all character - references (except the ones in ``script``/``style`` elements) are - automatically converted to the corresponding Unicode characters. - - An :class:`.HTMLParser` instance is fed HTML data and calls handler methods - when start tags, end tags, text, comments, and other markup elements are - encountered. The user should subclass :class:`.HTMLParser` and override its - methods to implement the desired behavior. - - This parser does not check that end tags match start tags or call the end-tag - handler for elements which are closed implicitly by closing an outer element. - - .. versionchanged:: 3.4 - *convert_charrefs* keyword argument added. - - .. versionchanged:: 3.5 - The default value for argument *convert_charrefs* is now ``True``. - - -Example HTML Parser Application -------------------------------- - -As a basic example, below is a simple HTML parser that uses the -:class:`HTMLParser` class to print out start tags, end tags, and data -as they are encountered:: - - from html.parser import HTMLParser - - class MyHTMLParser(HTMLParser): - def handle_starttag(self, tag, attrs): - print("Encountered a start tag:", tag) - - def handle_endtag(self, tag): - print("Encountered an end tag :", tag) - - def handle_data(self, data): - print("Encountered some data :", data) - - parser = MyHTMLParser() - parser.feed('Test' - '

    Parse me!

    ') - -The output will then be: - -.. code-block:: none - - Encountered a start tag: html - Encountered a start tag: head - Encountered a start tag: title - Encountered some data : Test - Encountered an end tag : title - Encountered an end tag : head - Encountered a start tag: body - Encountered a start tag: h1 - Encountered some data : Parse me! - Encountered an end tag : h1 - Encountered an end tag : body - Encountered an end tag : html - - -:class:`.HTMLParser` Methods ----------------------------- - -:class:`HTMLParser` instances have the following methods: - - -.. method:: HTMLParser.feed(data) - - Feed some text to the parser. It is processed insofar as it consists of - complete elements; incomplete data is buffered until more data is fed or - :meth:`close` is called. *data* must be :class:`str`. - - -.. method:: HTMLParser.close() - - Force processing of all buffered data as if it were followed by an end-of-file - mark. This method may be redefined by a derived class to define additional - processing at the end of the input, but the redefined version should always call - the :class:`HTMLParser` base class method :meth:`close`. - - -.. method:: HTMLParser.reset() - - Reset the instance. Loses all unprocessed data. This is called implicitly at - instantiation time. - - -.. method:: HTMLParser.getpos() - - Return current line number and offset. - - -.. method:: HTMLParser.get_starttag_text() - - Return the text of the most recently opened start tag. This should not normally - be needed for structured processing, but may be useful in dealing with HTML "as - deployed" or for re-generating input with minimal changes (whitespace between - attributes can be preserved, etc.). - - -The following methods are called when data or markup elements are encountered -and they are meant to be overridden in a subclass. The base class -implementations do nothing (except for :meth:`~HTMLParser.handle_startendtag`): - - -.. method:: HTMLParser.handle_starttag(tag, attrs) - - This method is called to handle the start tag of an element (e.g. ``
    ``). - - The *tag* argument is the name of the tag converted to lower case. The *attrs* - argument is a list of ``(name, value)`` pairs containing the attributes found - inside the tag's ``<>`` brackets. The *name* will be translated to lower case, - and quotes in the *value* have been removed, and character and entity references - have been replaced. - - For instance, for the tag ````, this method - would be called as ``handle_starttag('a', [('href', 'https://www.cwi.nl/')])``. - - All entity references from :mod:`html.entities` are replaced in the attribute - values. - - -.. method:: HTMLParser.handle_endtag(tag) - - This method is called to handle the end tag of an element (e.g. ``
    ``). - - The *tag* argument is the name of the tag converted to lower case. - - -.. method:: HTMLParser.handle_startendtag(tag, attrs) - - Similar to :meth:`handle_starttag`, but called when the parser encounters an - XHTML-style empty tag (````). This method may be overridden by - subclasses which require this particular lexical information; the default - implementation simply calls :meth:`handle_starttag` and :meth:`handle_endtag`. - - -.. method:: HTMLParser.handle_data(data) - - This method is called to process arbitrary data (e.g. text nodes and the - content of ```` and ````). - - -.. method:: HTMLParser.handle_entityref(name) - - This method is called to process a named character reference of the form - ``&name;`` (e.g. ``>``), where *name* is a general entity reference - (e.g. ``'gt'``). This method is never called if *convert_charrefs* is - ``True``. - - -.. method:: HTMLParser.handle_charref(name) - - This method is called to process decimal and hexadecimal numeric character - references of the form :samp:`&#{NNN};` and :samp:`&#x{NNN};`. For example, the decimal - equivalent for ``>`` is ``>``, whereas the hexadecimal is ``>``; - in this case the method will receive ``'62'`` or ``'x3E'``. This method - is never called if *convert_charrefs* is ``True``. - - -.. method:: HTMLParser.handle_comment(data) - - This method is called when a comment is encountered (e.g. ````). - - For example, the comment ```` will cause this method to be - called with the argument ``' comment '``. - - The content of Internet Explorer conditional comments (condcoms) will also be - sent to this method, so, for ````, - this method will receive ``'[if IE 9]>IE9-specific content``). - - The *decl* parameter will be the entire contents of the declaration inside - the ```` markup (e.g. ``'DOCTYPE html'``). - - -.. method:: HTMLParser.handle_pi(data) - - Method called when a processing instruction is encountered. The *data* - parameter will contain the entire processing instruction. For example, for the - processing instruction ````, this method would be called as - ``handle_pi("proc color='red'")``. It is intended to be overridden by a derived - class; the base class implementation does nothing. - - .. note:: - - The :class:`HTMLParser` class uses the SGML syntactic rules for processing - instructions. An XHTML processing instruction using the trailing ``'?'`` will - cause the ``'?'`` to be included in *data*. - - -.. method:: HTMLParser.unknown_decl(data) - - This method is called when an unrecognized declaration is read by the parser. - - The *data* parameter will be the entire contents of the declaration inside - the ```` markup. It is sometimes useful to be overridden by a - derived class. The base class implementation does nothing. - - -.. _htmlparser-examples: - -Examples --------- - -The following class implements a parser that will be used to illustrate more -examples:: - - from html.parser import HTMLParser - from html.entities import name2codepoint - - class MyHTMLParser(HTMLParser): - def handle_starttag(self, tag, attrs): - print("Start tag:", tag) - for attr in attrs: - print(" attr:", attr) - - def handle_endtag(self, tag): - print("End tag :", tag) - - def handle_data(self, data): - print("Data :", data) - - def handle_comment(self, data): - print("Comment :", data) - - def handle_entityref(self, name): - c = chr(name2codepoint[name]) - print("Named ent:", c) - - def handle_charref(self, name): - if name.startswith('x'): - c = chr(int(name[1:], 16)) - else: - c = chr(int(name)) - print("Num ent :", c) - - def handle_decl(self, data): - print("Decl :", data) - - parser = MyHTMLParser() - -Parsing a doctype:: - - >>> parser.feed('') - Decl : DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" "http://www.w3.org/TR/html4/strict.dtd" - -Parsing an element with a few attributes and a title:: - - >>> parser.feed('The Python logo') - Start tag: img - attr: ('src', 'python-logo.png') - attr: ('alt', 'The Python logo') - >>> - >>> parser.feed('

    Python

    ') - Start tag: h1 - Data : Python - End tag : h1 - -The content of ``script`` and ``style`` elements is returned as is, without -further parsing:: - - >>> parser.feed('') - Start tag: style - attr: ('type', 'text/css') - Data : #python { color: green } - End tag : style - - >>> parser.feed('') - Start tag: script - attr: ('type', 'text/javascript') - Data : alert("hello!"); - End tag : script - -Parsing comments:: - - >>> parser.feed('' - ... '') - Comment : a comment - Comment : [if IE 9]>IE-specific content'``):: - - >>> parser.feed('>>>') - Named ent: > - Num ent : > - Num ent : > - -Feeding incomplete chunks to :meth:`~HTMLParser.feed` works, but -:meth:`~HTMLParser.handle_data` might be called more than once -(unless *convert_charrefs* is set to ``True``):: - - >>> for chunk in ['buff', 'ered ', 'text']: - ... parser.feed(chunk) - ... - Start tag: span - Data : buff - Data : ered - Data : text - End tag : span - -Parsing invalid HTML (e.g. unquoted attributes) also works:: - - >>> parser.feed('

    tag soup

    ') - Start tag: p - Start tag: a - attr: ('class', 'link') - attr: ('href', '#main') - Data : tag soup - End tag : p - End tag : a diff --git a/Doc/library/html.rst b/Doc/library/html.rst deleted file mode 100644 index c2b01e14ea7555..00000000000000 --- a/Doc/library/html.rst +++ /dev/null @@ -1,39 +0,0 @@ -:mod:`html` --- HyperText Markup Language support -================================================= - -.. module:: html - :synopsis: Helpers for manipulating HTML. - -**Source code:** :source:`Lib/html/__init__.py` - --------------- - -This module defines utilities to manipulate HTML. - -.. function:: escape(s, quote=True) - - Convert the characters ``&``, ``<`` and ``>`` in string *s* to HTML-safe - sequences. Use this if you need to display text that might contain such - characters in HTML. If the optional flag *quote* is true, the characters - (``"``) and (``'``) are also translated; this helps for inclusion in an HTML - attribute value delimited by quotes, as in ````. - - .. versionadded:: 3.2 - - -.. function:: unescape(s) - - Convert all named and numeric character references (e.g. ``>``, - ``>``, ``>``) in the string *s* to the corresponding Unicode - characters. This function uses the rules defined by the HTML 5 standard - for both valid and invalid character references, and the :data:`list of - HTML 5 named character references `. - - .. versionadded:: 3.4 - --------------- - -Submodules in the ``html`` package are: - -* :mod:`html.parser` -- HTML/XHTML parser with lenient parsing mode -* :mod:`html.entities` -- HTML entity definitions diff --git a/Doc/library/http.client.rst b/Doc/library/http.client.rst deleted file mode 100644 index 87433ddbd74971..00000000000000 --- a/Doc/library/http.client.rst +++ /dev/null @@ -1,639 +0,0 @@ -:mod:`http.client` --- HTTP protocol client -=========================================== - -.. module:: http.client - :synopsis: HTTP and HTTPS protocol client (requires sockets). - -**Source code:** :source:`Lib/http/client.py` - -.. index:: - pair: HTTP; protocol - single: HTTP; http.client (standard module) - -.. index:: pair: module; urllib.request - --------------- - -This module defines classes that implement the client side of the HTTP and -HTTPS protocols. It is normally not used directly --- the module -:mod:`urllib.request` uses it to handle URLs that use HTTP and HTTPS. - -.. seealso:: - - The `Requests package `_ - is recommended for a higher-level HTTP client interface. - -.. note:: - - HTTPS support is only available if Python was compiled with SSL support - (through the :mod:`ssl` module). - -.. include:: ../includes/wasm-notavail.rst - -The module provides the following classes: - - -.. class:: HTTPConnection(host, port=None[, timeout], source_address=None, \ - blocksize=8192) - - An :class:`HTTPConnection` instance represents one transaction with an HTTP - server. It should be instantiated by passing it a host and optional port - number. If no port number is passed, the port is extracted from the host - string if it has the form ``host:port``, else the default HTTP port (80) is - used. If the optional *timeout* parameter is given, blocking - operations (like connection attempts) will timeout after that many seconds - (if it is not given, the global default timeout setting is used). - The optional *source_address* parameter may be a tuple of a (host, port) - to use as the source address the HTTP connection is made from. - The optional *blocksize* parameter sets the buffer size in bytes for - sending a file-like message body. - - For example, the following calls all create instances that connect to the server - at the same host and port:: - - >>> h1 = http.client.HTTPConnection('www.python.org') - >>> h2 = http.client.HTTPConnection('www.python.org:80') - >>> h3 = http.client.HTTPConnection('www.python.org', 80) - >>> h4 = http.client.HTTPConnection('www.python.org', 80, timeout=10) - - .. versionchanged:: 3.2 - *source_address* was added. - - .. versionchanged:: 3.4 - The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are - no longer supported. - - .. versionchanged:: 3.7 - *blocksize* parameter was added. - - -.. class:: HTTPSConnection(host, port=None, key_file=None, \ - cert_file=None[, timeout], \ - source_address=None, *, context=None, \ - check_hostname=None, blocksize=8192) - - A subclass of :class:`HTTPConnection` that uses SSL for communication with - secure servers. Default port is ``443``. If *context* is specified, it - must be a :class:`ssl.SSLContext` instance describing the various SSL - options. - - Please read :ref:`ssl-security` for more information on best practices. - - .. versionchanged:: 3.2 - *source_address*, *context* and *check_hostname* were added. - - .. versionchanged:: 3.2 - This class now supports HTTPS virtual hosts if possible (that is, - if :const:`ssl.HAS_SNI` is true). - - .. versionchanged:: 3.4 - The *strict* parameter was removed. HTTP 0.9-style "Simple Responses" are - no longer supported. - - .. versionchanged:: 3.4.3 - This class now performs all the necessary certificate and hostname checks - by default. To revert to the previous, unverified, behavior - :func:`ssl._create_unverified_context` can be passed to the *context* - parameter. - - .. versionchanged:: 3.8 - This class now enables TLS 1.3 - :attr:`ssl.SSLContext.post_handshake_auth` for the default *context* or - when *cert_file* is passed with a custom *context*. - - .. versionchanged:: 3.10 - This class now sends an ALPN extension with protocol indicator - ``http/1.1`` when no *context* is given. Custom *context* should set - ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. - - .. deprecated:: 3.6 - - *key_file* and *cert_file* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - The *check_hostname* parameter is also deprecated; the - :attr:`ssl.SSLContext.check_hostname` attribute of *context* should - be used instead. - - -.. class:: HTTPResponse(sock, debuglevel=0, method=None, url=None) - - Class whose instances are returned upon successful connection. Not - instantiated directly by user. - - .. versionchanged:: 3.4 - The *strict* parameter was removed. HTTP 0.9 style "Simple Responses" are - no longer supported. - -This module provides the following function: - -.. function:: parse_headers(fp) - - Parse the headers from a file pointer *fp* representing a HTTP - request/response. The file has to be a :class:`BufferedIOBase` reader - (i.e. not text) and must provide a valid :rfc:`2822` style header. - - This function returns an instance of :class:`http.client.HTTPMessage` - that holds the header fields, but no payload - (the same as :attr:`HTTPResponse.msg` - and :attr:`http.server.BaseHTTPRequestHandler.headers`). - After returning, the file pointer *fp* is ready to read the HTTP body. - - .. note:: - :meth:`parse_headers` does not parse the start-line of a HTTP message; - it only parses the ``Name: value`` lines. The file has to be ready to - read these field lines, so the first line should already be consumed - before calling the function. - -The following exceptions are raised as appropriate: - - -.. exception:: HTTPException - - The base class of the other exceptions in this module. It is a subclass of - :exc:`Exception`. - - -.. exception:: NotConnected - - A subclass of :exc:`HTTPException`. - - -.. exception:: InvalidURL - - A subclass of :exc:`HTTPException`, raised if a port is given and is either - non-numeric or empty. - - -.. exception:: UnknownProtocol - - A subclass of :exc:`HTTPException`. - - -.. exception:: UnknownTransferEncoding - - A subclass of :exc:`HTTPException`. - - -.. exception:: UnimplementedFileMode - - A subclass of :exc:`HTTPException`. - - -.. exception:: IncompleteRead - - A subclass of :exc:`HTTPException`. - - -.. exception:: ImproperConnectionState - - A subclass of :exc:`HTTPException`. - - -.. exception:: CannotSendRequest - - A subclass of :exc:`ImproperConnectionState`. - - -.. exception:: CannotSendHeader - - A subclass of :exc:`ImproperConnectionState`. - - -.. exception:: ResponseNotReady - - A subclass of :exc:`ImproperConnectionState`. - - -.. exception:: BadStatusLine - - A subclass of :exc:`HTTPException`. Raised if a server responds with a HTTP - status code that we don't understand. - - -.. exception:: LineTooLong - - A subclass of :exc:`HTTPException`. Raised if an excessively long line - is received in the HTTP protocol from the server. - - -.. exception:: RemoteDisconnected - - A subclass of :exc:`ConnectionResetError` and :exc:`BadStatusLine`. Raised - by :meth:`HTTPConnection.getresponse` when the attempt to read the response - results in no data read from the connection, indicating that the remote end - has closed the connection. - - .. versionadded:: 3.5 - Previously, :exc:`BadStatusLine`\ ``('')`` was raised. - - -The constants defined in this module are: - -.. data:: HTTP_PORT - - The default port for the HTTP protocol (always ``80``). - -.. data:: HTTPS_PORT - - The default port for the HTTPS protocol (always ``443``). - -.. data:: responses - - This dictionary maps the HTTP 1.1 status codes to the W3C names. - - Example: ``http.client.responses[http.client.NOT_FOUND]`` is ``'Not Found'``. - -See :ref:`http-status-codes` for a list of HTTP status codes that are -available in this module as constants. - - -.. _httpconnection-objects: - -HTTPConnection Objects ----------------------- - -:class:`HTTPConnection` instances have the following methods: - - -.. method:: HTTPConnection.request(method, url, body=None, headers={}, *, \ - encode_chunked=False) - - This will send a request to the server using the HTTP request - method *method* and the request URI *url*. The provided *url* must be - an absolute path to conform with :rfc:`RFC 2616 §5.1.2 <2616#section-5.1.2>` - (unless connecting to an HTTP proxy server or using the ``OPTIONS`` or - ``CONNECT`` methods). - - If *body* is specified, the specified data is sent after the headers are - finished. It may be a :class:`str`, a :term:`bytes-like object`, an - open :term:`file object`, or an iterable of :class:`bytes`. If *body* - is a string, it is encoded as ISO-8859-1, the default for HTTP. If it - is a bytes-like object, the bytes are sent as is. If it is a :term:`file - object`, the contents of the file is sent; this file object should - support at least the ``read()`` method. If the file object is an - instance of :class:`io.TextIOBase`, the data returned by the ``read()`` - method will be encoded as ISO-8859-1, otherwise the data returned by - ``read()`` is sent as is. If *body* is an iterable, the elements of the - iterable are sent as is until the iterable is exhausted. - - The *headers* argument should be a mapping of extra HTTP headers to send - with the request. A :rfc:`Host header <2616#section-14.23>` - must be provided to conform with :rfc:`RFC 2616 §5.1.2 <2616#section-5.1.2>` - (unless connecting to an HTTP proxy server or using the ``OPTIONS`` or - ``CONNECT`` methods). - - If *headers* contains neither Content-Length nor Transfer-Encoding, - but there is a request body, one of those - header fields will be added automatically. If - *body* is ``None``, the Content-Length header is set to ``0`` for - methods that expect a body (``PUT``, ``POST``, and ``PATCH``). If - *body* is a string or a bytes-like object that is not also a - :term:`file `, the Content-Length header is - set to its length. Any other type of *body* (files - and iterables in general) will be chunk-encoded, and the - Transfer-Encoding header will automatically be set instead of - Content-Length. - - The *encode_chunked* argument is only relevant if Transfer-Encoding is - specified in *headers*. If *encode_chunked* is ``False``, the - HTTPConnection object assumes that all encoding is handled by the - calling code. If it is ``True``, the body will be chunk-encoded. - - For example, to perform a ``GET`` request to ``https://docs.python.org/3/``:: - - >>> import http.client - >>> host = "docs.python.org" - >>> conn = http.client.HTTPSConnection(host) - >>> conn.request("GET", "/3/", headers={"Host": host}) - >>> response = conn.getresponse() - >>> print(response.status, response.reason) - 200 OK - - .. note:: - Chunked transfer encoding has been added to the HTTP protocol - version 1.1. Unless the HTTP server is known to handle HTTP 1.1, - the caller must either specify the Content-Length, or must pass a - :class:`str` or bytes-like object that is not also a file as the - body representation. - - .. versionadded:: 3.2 - *body* can now be an iterable. - - .. versionchanged:: 3.6 - If neither Content-Length nor Transfer-Encoding are set in - *headers*, file and iterable *body* objects are now chunk-encoded. - The *encode_chunked* argument was added. - No attempt is made to determine the Content-Length for file - objects. - -.. method:: HTTPConnection.getresponse() - - Should be called after a request is sent to get the response from the server. - Returns an :class:`HTTPResponse` instance. - - .. note:: - - Note that you must have read the whole response before you can send a new - request to the server. - - .. versionchanged:: 3.5 - If a :exc:`ConnectionError` or subclass is raised, the - :class:`HTTPConnection` object will be ready to reconnect when - a new request is sent. - - -.. method:: HTTPConnection.set_debuglevel(level) - - Set the debugging level. The default debug level is ``0``, meaning no - debugging output is printed. Any value greater than ``0`` will cause all - currently defined debug output to be printed to stdout. The ``debuglevel`` - is passed to any new :class:`HTTPResponse` objects that are created. - - .. versionadded:: 3.1 - - -.. method:: HTTPConnection.set_tunnel(host, port=None, headers=None) - - Set the host and the port for HTTP Connect Tunnelling. This allows running - the connection through a proxy server. - - The host and port arguments specify the endpoint of the tunneled connection - (i.e. the address included in the CONNECT request, *not* the address of the - proxy server). - - The headers argument should be a mapping of extra HTTP headers to send with - the CONNECT request. - - For example, to tunnel through a HTTPS proxy server running locally on port - 8080, we would pass the address of the proxy to the :class:`HTTPSConnection` - constructor, and the address of the host that we eventually want to reach to - the :meth:`~HTTPConnection.set_tunnel` method:: - - >>> import http.client - >>> conn = http.client.HTTPSConnection("localhost", 8080) - >>> conn.set_tunnel("www.python.org") - >>> conn.request("HEAD","/index.html") - - .. versionadded:: 3.2 - - -.. method:: HTTPConnection.connect() - - Connect to the server specified when the object was created. By default, - this is called automatically when making a request if the client does not - already have a connection. - - .. audit-event:: http.client.connect self,host,port http.client.HTTPConnection.connect - - -.. method:: HTTPConnection.close() - - Close the connection to the server. - - -.. attribute:: HTTPConnection.blocksize - - Buffer size in bytes for sending a file-like message body. - - .. versionadded:: 3.7 - - -As an alternative to using the :meth:`request` method described above, you can -also send your request step by step, by using the four functions below. - - -.. method:: HTTPConnection.putrequest(method, url, skip_host=False, \ - skip_accept_encoding=False) - - This should be the first call after the connection to the server has been - made. It sends a line to the server consisting of the *method* string, - the *url* string, and the HTTP version (``HTTP/1.1``). To disable automatic - sending of ``Host:`` or ``Accept-Encoding:`` headers (for example to accept - additional content encodings), specify *skip_host* or *skip_accept_encoding* - with non-False values. - - -.. method:: HTTPConnection.putheader(header, argument[, ...]) - - Send an :rfc:`822`\ -style header to the server. It sends a line to the server - consisting of the header, a colon and a space, and the first argument. If more - arguments are given, continuation lines are sent, each consisting of a tab and - an argument. - - -.. method:: HTTPConnection.endheaders(message_body=None, *, encode_chunked=False) - - Send a blank line to the server, signalling the end of the headers. The - optional *message_body* argument can be used to pass a message body - associated with the request. - - If *encode_chunked* is ``True``, the result of each iteration of - *message_body* will be chunk-encoded as specified in :rfc:`7230`, - Section 3.3.1. How the data is encoded is dependent on the type of - *message_body*. If *message_body* implements the :ref:`buffer interface - ` the encoding will result in a single chunk. - If *message_body* is a :class:`collections.abc.Iterable`, each iteration - of *message_body* will result in a chunk. If *message_body* is a - :term:`file object`, each call to ``.read()`` will result in a chunk. - The method automatically signals the end of the chunk-encoded data - immediately after *message_body*. - - .. note:: Due to the chunked encoding specification, empty chunks - yielded by an iterator body will be ignored by the chunk-encoder. - This is to avoid premature termination of the read of the request by - the target server due to malformed encoding. - - .. versionadded:: 3.6 - Chunked encoding support. The *encode_chunked* parameter was - added. - - -.. method:: HTTPConnection.send(data) - - Send data to the server. This should be used directly only after the - :meth:`endheaders` method has been called and before :meth:`getresponse` is - called. - - .. audit-event:: http.client.send self,data http.client.HTTPConnection.send - - -.. _httpresponse-objects: - -HTTPResponse Objects --------------------- - -An :class:`HTTPResponse` instance wraps the HTTP response from the -server. It provides access to the request headers and the entity -body. The response is an iterable object and can be used in a with -statement. - -.. versionchanged:: 3.5 - The :class:`io.BufferedIOBase` interface is now implemented and - all of its reader operations are supported. - - -.. method:: HTTPResponse.read([amt]) - - Reads and returns the response body, or up to the next *amt* bytes. - -.. method:: HTTPResponse.readinto(b) - - Reads up to the next len(b) bytes of the response body into the buffer *b*. - Returns the number of bytes read. - - .. versionadded:: 3.3 - -.. method:: HTTPResponse.getheader(name, default=None) - - Return the value of the header *name*, or *default* if there is no header - matching *name*. If there is more than one header with the name *name*, - return all of the values joined by ', '. If *default* is any iterable other - than a single string, its elements are similarly returned joined by commas. - -.. method:: HTTPResponse.getheaders() - - Return a list of (header, value) tuples. - -.. method:: HTTPResponse.fileno() - - Return the ``fileno`` of the underlying socket. - -.. attribute:: HTTPResponse.msg - - A :class:`http.client.HTTPMessage` instance containing the response - headers. :class:`http.client.HTTPMessage` is a subclass of - :class:`email.message.Message`. - -.. attribute:: HTTPResponse.version - - HTTP protocol version used by server. 10 for HTTP/1.0, 11 for HTTP/1.1. - -.. attribute:: HTTPResponse.url - - URL of the resource retrieved, commonly used to determine if a redirect was followed. - -.. attribute:: HTTPResponse.headers - - Headers of the response in the form of an :class:`email.message.EmailMessage` instance. - -.. attribute:: HTTPResponse.status - - Status code returned by server. - -.. attribute:: HTTPResponse.reason - - Reason phrase returned by server. - -.. attribute:: HTTPResponse.debuglevel - - A debugging hook. If :attr:`debuglevel` is greater than zero, messages - will be printed to stdout as the response is read and parsed. - -.. attribute:: HTTPResponse.closed - - Is ``True`` if the stream is closed. - -.. method:: HTTPResponse.geturl() - - .. deprecated:: 3.9 - Deprecated in favor of :attr:`~HTTPResponse.url`. - -.. method:: HTTPResponse.info() - - .. deprecated:: 3.9 - Deprecated in favor of :attr:`~HTTPResponse.headers`. - -.. method:: HTTPResponse.getcode() - - .. deprecated:: 3.9 - Deprecated in favor of :attr:`~HTTPResponse.status`. - -Examples --------- - -Here is an example session that uses the ``GET`` method:: - - >>> import http.client - >>> conn = http.client.HTTPSConnection("www.python.org") - >>> conn.request("GET", "/") - >>> r1 = conn.getresponse() - >>> print(r1.status, r1.reason) - 200 OK - >>> data1 = r1.read() # This will return entire content. - >>> # The following example demonstrates reading data in chunks. - >>> conn.request("GET", "/") - >>> r1 = conn.getresponse() - >>> while chunk := r1.read(200): - ... print(repr(chunk)) - b'\n 10 11 12 13 14 ...`` -:func:`cycle` p p0, p1, ... plast, p0, p1, ... ``cycle('ABCD') --> A B C D A B C D ...`` -:func:`repeat` elem [,n] elem, elem, elem, ... endlessly or up to n times ``repeat(10, 3) --> 10 10 10`` -================== ================= ================================================= ========================================= - -**Iterators terminating on the shortest input sequence:** - -============================ ============================ ================================================= ============================================================= -Iterator Arguments Results Example -============================ ============================ ================================================= ============================================================= -:func:`accumulate` p [,func] p0, p0+p1, p0+p1+p2, ... ``accumulate([1,2,3,4,5]) --> 1 3 6 10 15`` -:func:`chain` p, q, ... p0, p1, ... plast, q0, q1, ... ``chain('ABC', 'DEF') --> A B C D E F`` -:func:`chain.from_iterable` iterable p0, p1, ... plast, q0, q1, ... ``chain.from_iterable(['ABC', 'DEF']) --> A B C D E F`` -:func:`compress` data, selectors (d[0] if s[0]), (d[1] if s[1]), ... ``compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F`` -:func:`dropwhile` pred, seq seq[n], seq[n+1], starting when pred fails ``dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1`` -:func:`filterfalse` pred, seq elements of seq where pred(elem) is false ``filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8`` -:func:`groupby` iterable[, key] sub-iterators grouped by value of key(v) -:func:`islice` seq, [start,] stop [, step] elements from seq[start:stop:step] ``islice('ABCDEFG', 2, None) --> C D E F G`` -:func:`pairwise` iterable (p[0], p[1]), (p[1], p[2]) ``pairwise('ABCDEFG') --> AB BC CD DE EF FG`` -:func:`starmap` func, seq func(\*seq[0]), func(\*seq[1]), ... ``starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000`` -:func:`takewhile` pred, seq seq[0], seq[1], until pred fails ``takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4`` -:func:`tee` it, n it1, it2, ... itn splits one iterator into n -:func:`zip_longest` p, q, ... (p[0], q[0]), (p[1], q[1]), ... ``zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D-`` -============================ ============================ ================================================= ============================================================= - -**Combinatoric iterators:** - -============================================== ==================== ============================================================= -Iterator Arguments Results -============================================== ==================== ============================================================= -:func:`product` p, q, ... [repeat=1] cartesian product, equivalent to a nested for-loop -:func:`permutations` p[, r] r-length tuples, all possible orderings, no repeated elements -:func:`combinations` p, r r-length tuples, in sorted order, no repeated elements -:func:`combinations_with_replacement` p, r r-length tuples, in sorted order, with repeated elements -============================================== ==================== ============================================================= - -============================================== ============================================================= -Examples Results -============================================== ============================================================= -``product('ABCD', repeat=2)`` ``AA AB AC AD BA BB BC BD CA CB CC CD DA DB DC DD`` -``permutations('ABCD', 2)`` ``AB AC AD BA BC BD CA CB CD DA DB DC`` -``combinations('ABCD', 2)`` ``AB AC AD BC BD CD`` -``combinations_with_replacement('ABCD', 2)`` ``AA AB AC AD BB BC BD CC CD DD`` -============================================== ============================================================= - - -.. _itertools-functions: - -Itertool functions ------------------- - -The following module functions all construct and return iterators. Some provide -streams of infinite length, so they should only be accessed by functions or -loops that truncate the stream. - -.. function:: accumulate(iterable[, func, *, initial=None]) - - Make an iterator that returns accumulated sums, or accumulated - results of other binary functions (specified via the optional - *func* argument). - - If *func* is supplied, it should be a function - of two arguments. Elements of the input *iterable* may be any type - that can be accepted as arguments to *func*. (For example, with - the default operation of addition, elements may be any addable - type including :class:`~decimal.Decimal` or - :class:`~fractions.Fraction`.) - - Usually, the number of elements output matches the input iterable. - However, if the keyword argument *initial* is provided, the - accumulation leads off with the *initial* value so that the output - has one more element than the input iterable. - - Roughly equivalent to:: - - def accumulate(iterable, func=operator.add, *, initial=None): - 'Return running totals' - # accumulate([1,2,3,4,5]) --> 1 3 6 10 15 - # accumulate([1,2,3,4,5], initial=100) --> 100 101 103 106 110 115 - # accumulate([1,2,3,4,5], operator.mul) --> 1 2 6 24 120 - it = iter(iterable) - total = initial - if initial is None: - try: - total = next(it) - except StopIteration: - return - yield total - for element in it: - total = func(total, element) - yield total - - There are a number of uses for the *func* argument. It can be set to - :func:`min` for a running minimum, :func:`max` for a running maximum, or - :func:`operator.mul` for a running product. Amortization tables can be - built by accumulating interest and applying payments: - - .. doctest:: - - >>> data = [3, 4, 6, 2, 1, 9, 0, 7, 5, 8] - >>> list(accumulate(data, operator.mul)) # running product - [3, 12, 72, 144, 144, 1296, 0, 0, 0, 0] - >>> list(accumulate(data, max)) # running maximum - [3, 4, 6, 6, 6, 9, 9, 9, 9, 9] - - # Amortize a 5% loan of 1000 with 4 annual payments of 90 - >>> cashflows = [1000, -90, -90, -90, -90] - >>> list(accumulate(cashflows, lambda bal, pmt: bal*1.05 + pmt)) - [1000, 960.0, 918.0, 873.9000000000001, 827.5950000000001] - - See :func:`functools.reduce` for a similar function that returns only the - final accumulated value. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - Added the optional *func* parameter. - - .. versionchanged:: 3.8 - Added the optional *initial* parameter. - -.. function:: chain(*iterables) - - Make an iterator that returns elements from the first iterable until it is - exhausted, then proceeds to the next iterable, until all of the iterables are - exhausted. Used for treating consecutive sequences as a single sequence. - Roughly equivalent to:: - - def chain(*iterables): - # chain('ABC', 'DEF') --> A B C D E F - for it in iterables: - for element in it: - yield element - - -.. classmethod:: chain.from_iterable(iterable) - - Alternate constructor for :func:`chain`. Gets chained inputs from a - single iterable argument that is evaluated lazily. Roughly equivalent to:: - - def from_iterable(iterables): - # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F - for it in iterables: - for element in it: - yield element - - -.. function:: combinations(iterable, r) - - Return *r* length subsequences of elements from the input *iterable*. - - The combination tuples are emitted in lexicographic ordering according to - the order of the input *iterable*. So, if the input *iterable* is sorted, - the output tuples will be produced in sorted order. - - Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated - values in each combination. - - Roughly equivalent to:: - - def combinations(iterable, r): - # combinations('ABCD', 2) --> AB AC AD BC BD CD - # combinations(range(4), 3) --> 012 013 023 123 - pool = tuple(iterable) - n = len(pool) - if r > n: - return - indices = list(range(r)) - yield tuple(pool[i] for i in indices) - while True: - for i in reversed(range(r)): - if indices[i] != i + n - r: - break - else: - return - indices[i] += 1 - for j in range(i+1, r): - indices[j] = indices[j-1] + 1 - yield tuple(pool[i] for i in indices) - - The code for :func:`combinations` can be also expressed as a subsequence - of :func:`permutations` after filtering entries where the elements are not - in sorted order (according to their position in the input pool):: - - def combinations(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in permutations(range(n), r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / r! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. - -.. function:: combinations_with_replacement(iterable, r) - - Return *r* length subsequences of elements from the input *iterable* - allowing individual elements to be repeated more than once. - - The combination tuples are emitted in lexicographic ordering according to - the order of the input *iterable*. So, if the input *iterable* is sorted, - the output tuples will be produced in sorted order. - - Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, the generated combinations - will also be unique. - - Roughly equivalent to:: - - def combinations_with_replacement(iterable, r): - # combinations_with_replacement('ABC', 2) --> AA AB AC BB BC CC - pool = tuple(iterable) - n = len(pool) - if not n and r: - return - indices = [0] * r - yield tuple(pool[i] for i in indices) - while True: - for i in reversed(range(r)): - if indices[i] != n - 1: - break - else: - return - indices[i:] = [indices[i] + 1] * (r - i) - yield tuple(pool[i] for i in indices) - - The code for :func:`combinations_with_replacement` can be also expressed as - a subsequence of :func:`product` after filtering entries where the elements - are not in sorted order (according to their position in the input pool):: - - def combinations_with_replacement(iterable, r): - pool = tuple(iterable) - n = len(pool) - for indices in product(range(n), repeat=r): - if sorted(indices) == list(indices): - yield tuple(pool[i] for i in indices) - - The number of items returned is ``(n+r-1)! / r! / (n-1)!`` when ``n > 0``. - - .. versionadded:: 3.1 - - -.. function:: compress(data, selectors) - - Make an iterator that filters elements from *data* returning only those that - have a corresponding element in *selectors* that evaluates to ``True``. - Stops when either the *data* or *selectors* iterables has been exhausted. - Roughly equivalent to:: - - def compress(data, selectors): - # compress('ABCDEF', [1,0,1,0,1,1]) --> A C E F - return (d for d, s in zip(data, selectors) if s) - - .. versionadded:: 3.1 - - -.. function:: count(start=0, step=1) - - Make an iterator that returns evenly spaced values starting with number *start*. Often - used as an argument to :func:`map` to generate consecutive data points. - Also, used with :func:`zip` to add sequence numbers. Roughly equivalent to:: - - def count(start=0, step=1): - # count(10) --> 10 11 12 13 14 ... - # count(2.5, 0.5) --> 2.5 3.0 3.5 ... - n = start - while True: - yield n - n += step - - When counting with floating point numbers, better accuracy can sometimes be - achieved by substituting multiplicative code such as: ``(start + step * i - for i in count())``. - - .. versionchanged:: 3.1 - Added *step* argument and allowed non-integer arguments. - -.. function:: cycle(iterable) - - Make an iterator returning elements from the iterable and saving a copy of each. - When the iterable is exhausted, return elements from the saved copy. Repeats - indefinitely. Roughly equivalent to:: - - def cycle(iterable): - # cycle('ABCD') --> A B C D A B C D A B C D ... - saved = [] - for element in iterable: - yield element - saved.append(element) - while saved: - for element in saved: - yield element - - Note, this member of the toolkit may require significant auxiliary storage - (depending on the length of the iterable). - - -.. function:: dropwhile(predicate, iterable) - - Make an iterator that drops elements from the iterable as long as the predicate - is true; afterwards, returns every element. Note, the iterator does not produce - *any* output until the predicate first becomes false, so it may have a lengthy - start-up time. Roughly equivalent to:: - - def dropwhile(predicate, iterable): - # dropwhile(lambda x: x<5, [1,4,6,4,1]) --> 6 4 1 - iterable = iter(iterable) - for x in iterable: - if not predicate(x): - yield x - break - for x in iterable: - yield x - -.. function:: filterfalse(predicate, iterable) - - Make an iterator that filters elements from iterable returning only those for - which the predicate is false. If *predicate* is ``None``, return the items - that are false. Roughly equivalent to:: - - def filterfalse(predicate, iterable): - # filterfalse(lambda x: x%2, range(10)) --> 0 2 4 6 8 - if predicate is None: - predicate = bool - for x in iterable: - if not predicate(x): - yield x - - -.. function:: groupby(iterable, key=None) - - Make an iterator that returns consecutive keys and groups from the *iterable*. - The *key* is a function computing a key value for each element. If not - specified or is ``None``, *key* defaults to an identity function and returns - the element unchanged. Generally, the iterable needs to already be sorted on - the same key function. - - The operation of :func:`groupby` is similar to the ``uniq`` filter in Unix. It - generates a break or new group every time the value of the key function changes - (which is why it is usually necessary to have sorted the data using the same key - function). That behavior differs from SQL's GROUP BY which aggregates common - elements regardless of their input order. - - The returned group is itself an iterator that shares the underlying iterable - with :func:`groupby`. Because the source is shared, when the :func:`groupby` - object is advanced, the previous group is no longer visible. So, if that data - is needed later, it should be stored as a list:: - - groups = [] - uniquekeys = [] - data = sorted(data, key=keyfunc) - for k, g in groupby(data, keyfunc): - groups.append(list(g)) # Store group iterator as a list - uniquekeys.append(k) - - :func:`groupby` is roughly equivalent to:: - - class groupby: - # [k for k, g in groupby('AAAABBBCCDAABBB')] --> A B C D A B - # [list(g) for k, g in groupby('AAAABBBCCD')] --> AAAA BBB CC D - - def __init__(self, iterable, key=None): - if key is None: - key = lambda x: x - self.keyfunc = key - self.it = iter(iterable) - self.tgtkey = self.currkey = self.currvalue = object() - - def __iter__(self): - return self - - def __next__(self): - self.id = object() - while self.currkey == self.tgtkey: - self.currvalue = next(self.it) # Exit on StopIteration - self.currkey = self.keyfunc(self.currvalue) - self.tgtkey = self.currkey - return (self.currkey, self._grouper(self.tgtkey, self.id)) - - def _grouper(self, tgtkey, id): - while self.id is id and self.currkey == tgtkey: - yield self.currvalue - try: - self.currvalue = next(self.it) - except StopIteration: - return - self.currkey = self.keyfunc(self.currvalue) - - -.. function:: islice(iterable, stop) - islice(iterable, start, stop[, step]) - - Make an iterator that returns selected elements from the iterable. If *start* is - non-zero, then elements from the iterable are skipped until start is reached. - Afterward, elements are returned consecutively unless *step* is set higher than - one which results in items being skipped. If *stop* is ``None``, then iteration - continues until the iterator is exhausted, if at all; otherwise, it stops at the - specified position. - - If *start* is ``None``, then iteration starts at zero. If *step* is ``None``, - then the step defaults to one. - - Unlike regular slicing, :func:`islice` does not support negative values for - *start*, *stop*, or *step*. Can be used to extract related fields from - data where the internal structure has been flattened (for example, a - multi-line report may list a name field on every third line). - - Roughly equivalent to:: - - def islice(iterable, *args): - # islice('ABCDEFG', 2) --> A B - # islice('ABCDEFG', 2, 4) --> C D - # islice('ABCDEFG', 2, None) --> C D E F G - # islice('ABCDEFG', 0, None, 2) --> A C E G - s = slice(*args) - start, stop, step = s.start or 0, s.stop or sys.maxsize, s.step or 1 - it = iter(range(start, stop, step)) - try: - nexti = next(it) - except StopIteration: - # Consume *iterable* up to the *start* position. - for i, element in zip(range(start), iterable): - pass - return - try: - for i, element in enumerate(iterable): - if i == nexti: - yield element - nexti = next(it) - except StopIteration: - # Consume to *stop*. - for i, element in zip(range(i + 1, stop), iterable): - pass - - -.. function:: pairwise(iterable) - - Return successive overlapping pairs taken from the input *iterable*. - - The number of 2-tuples in the output iterator will be one fewer than the - number of inputs. It will be empty if the input iterable has fewer than - two values. - - Roughly equivalent to:: - - def pairwise(iterable): - # pairwise('ABCDEFG') --> AB BC CD DE EF FG - a, b = tee(iterable) - next(b, None) - return zip(a, b) - - .. versionadded:: 3.10 - - -.. function:: permutations(iterable, r=None) - - Return successive *r* length permutations of elements in the *iterable*. - - If *r* is not specified or is ``None``, then *r* defaults to the length - of the *iterable* and all possible full-length permutations - are generated. - - The permutation tuples are emitted in lexicographic order according to - the order of the input *iterable*. So, if the input *iterable* is sorted, - the output tuples will be produced in sorted order. - - Elements are treated as unique based on their position, not on their - value. So if the input elements are unique, there will be no repeated - values within a permutation. - - Roughly equivalent to:: - - def permutations(iterable, r=None): - # permutations('ABCD', 2) --> AB AC AD BA BC BD CA CB CD DA DB DC - # permutations(range(3)) --> 012 021 102 120 201 210 - pool = tuple(iterable) - n = len(pool) - r = n if r is None else r - if r > n: - return - indices = list(range(n)) - cycles = list(range(n, n-r, -1)) - yield tuple(pool[i] for i in indices[:r]) - while n: - for i in reversed(range(r)): - cycles[i] -= 1 - if cycles[i] == 0: - indices[i:] = indices[i+1:] + indices[i:i+1] - cycles[i] = n - i - else: - j = cycles[i] - indices[i], indices[-j] = indices[-j], indices[i] - yield tuple(pool[i] for i in indices[:r]) - break - else: - return - - The code for :func:`permutations` can be also expressed as a subsequence of - :func:`product`, filtered to exclude entries with repeated elements (those - from the same position in the input pool):: - - def permutations(iterable, r=None): - pool = tuple(iterable) - n = len(pool) - r = n if r is None else r - for indices in product(range(n), repeat=r): - if len(set(indices)) == r: - yield tuple(pool[i] for i in indices) - - The number of items returned is ``n! / (n-r)!`` when ``0 <= r <= n`` - or zero when ``r > n``. - -.. function:: product(*iterables, repeat=1) - - Cartesian product of input iterables. - - Roughly equivalent to nested for-loops in a generator expression. For example, - ``product(A, B)`` returns the same as ``((x,y) for x in A for y in B)``. - - The nested loops cycle like an odometer with the rightmost element advancing - on every iteration. This pattern creates a lexicographic ordering so that if - the input's iterables are sorted, the product tuples are emitted in sorted - order. - - To compute the product of an iterable with itself, specify the number of - repetitions with the optional *repeat* keyword argument. For example, - ``product(A, repeat=4)`` means the same as ``product(A, A, A, A)``. - - This function is roughly equivalent to the following code, except that the - actual implementation does not build up intermediate results in memory:: - - def product(*args, repeat=1): - # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy - # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111 - pools = [tuple(pool) for pool in args] * repeat - result = [[]] - for pool in pools: - result = [x+[y] for x in result for y in pool] - for prod in result: - yield tuple(prod) - - Before :func:`product` runs, it completely consumes the input iterables, - keeping pools of values in memory to generate the products. Accordingly, - it is only useful with finite inputs. - -.. function:: repeat(object[, times]) - - Make an iterator that returns *object* over and over again. Runs indefinitely - unless the *times* argument is specified. - - Roughly equivalent to:: - - def repeat(object, times=None): - # repeat(10, 3) --> 10 10 10 - if times is None: - while True: - yield object - else: - for i in range(times): - yield object - - A common use for *repeat* is to supply a stream of constant values to *map* - or *zip*: - - .. doctest:: - - >>> list(map(pow, range(10), repeat(2))) - [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] - -.. function:: starmap(function, iterable) - - Make an iterator that computes the function using arguments obtained from - the iterable. Used instead of :func:`map` when argument parameters are already - grouped in tuples from a single iterable (when the data has been - "pre-zipped"). - - The difference between :func:`map` and :func:`starmap` parallels the - distinction between ``function(a,b)`` and ``function(*c)``. Roughly - equivalent to:: - - def starmap(function, iterable): - # starmap(pow, [(2,5), (3,2), (10,3)]) --> 32 9 1000 - for args in iterable: - yield function(*args) - - -.. function:: takewhile(predicate, iterable) - - Make an iterator that returns elements from the iterable as long as the - predicate is true. Roughly equivalent to:: - - def takewhile(predicate, iterable): - # takewhile(lambda x: x<5, [1,4,6,4,1]) --> 1 4 - for x in iterable: - if predicate(x): - yield x - else: - break - - -.. function:: tee(iterable, n=2) - - Return *n* independent iterators from a single iterable. - - The following Python code helps explain what *tee* does (although the actual - implementation is more complex and uses only a single underlying - :abbr:`FIFO (first-in, first-out)` queue):: - - def tee(iterable, n=2): - it = iter(iterable) - deques = [collections.deque() for i in range(n)] - def gen(mydeque): - while True: - if not mydeque: # when the local deque is empty - try: - newval = next(it) # fetch a new value and - except StopIteration: - return - for d in deques: # load it to all the deques - d.append(newval) - yield mydeque.popleft() - return tuple(gen(d) for d in deques) - - Once a :func:`tee` has been created, the original *iterable* should not be - used anywhere else; otherwise, the *iterable* could get advanced without - the tee objects being informed. - - ``tee`` iterators are not threadsafe. A :exc:`RuntimeError` may be - raised when using simultaneously iterators returned by the same :func:`tee` - call, even if the original *iterable* is threadsafe. - - This itertool may require significant auxiliary storage (depending on how - much temporary data needs to be stored). In general, if one iterator uses - most or all of the data before another iterator starts, it is faster to use - :func:`list` instead of :func:`tee`. - - -.. function:: zip_longest(*iterables, fillvalue=None) - - Make an iterator that aggregates elements from each of the iterables. If the - iterables are of uneven length, missing values are filled-in with *fillvalue*. - Iteration continues until the longest iterable is exhausted. Roughly equivalent to:: - - def zip_longest(*args, fillvalue=None): - # zip_longest('ABCD', 'xy', fillvalue='-') --> Ax By C- D- - iterators = [iter(it) for it in args] - num_active = len(iterators) - if not num_active: - return - while True: - values = [] - for i, it in enumerate(iterators): - try: - value = next(it) - except StopIteration: - num_active -= 1 - if not num_active: - return - iterators[i] = repeat(fillvalue) - value = fillvalue - values.append(value) - yield tuple(values) - - If one of the iterables is potentially infinite, then the :func:`zip_longest` - function should be wrapped with something that limits the number of calls - (for example :func:`islice` or :func:`takewhile`). If not specified, - *fillvalue* defaults to ``None``. - - -.. _itertools-recipes: - -Itertools Recipes ------------------ - -This section shows recipes for creating an extended toolset using the existing -itertools as building blocks. - -The primary purpose of the itertools recipes is educational. The recipes show -various ways of thinking about individual tools — for example, that -``chain.from_iterable`` is related to the concept of flattening. The recipes -also give ideas about ways that the tools can be combined — for example, how -``compress()`` and ``range()`` can work together. The recipes also show patterns -for using itertools with the :mod:`operator` and :mod:`collections` modules as -well as with the built-in itertools such as ``map()``, ``filter()``, -``reversed()``, and ``enumerate()``. - -A secondary purpose of the recipes is to serve as an incubator. The -``accumulate()``, ``compress()``, and ``pairwise()`` itertools started out as -recipes. Currently, the ``iter_index()`` recipe is being tested to see -whether it proves its worth. - -Substantially all of these recipes and many, many others can be installed from -the `more-itertools project `_ found -on the Python Package Index:: - - python -m pip install more-itertools - -Many of the recipes offer the same high performance as the underlying toolset. -Superior memory performance is kept by processing elements one at a time -rather than bringing the whole iterable into memory all at once. Code volume is -kept small by linking the tools together in a functional style which helps -eliminate temporary variables. High speed is retained by preferring -"vectorized" building blocks over the use of for-loops and :term:`generator`\s -which incur interpreter overhead. - -.. testcode:: - - import collections - import math - import operator - import random - - def take(n, iterable): - "Return first n items of the iterable as a list" - return list(islice(iterable, n)) - - def prepend(value, iterator): - "Prepend a single value in front of an iterator" - # prepend(1, [2, 3, 4]) --> 1 2 3 4 - return chain([value], iterator) - - def tabulate(function, start=0): - "Return function(0), function(1), ..." - return map(function, count(start)) - - def tail(n, iterable): - "Return an iterator over the last n items" - # tail(3, 'ABCDEFG') --> E F G - return iter(collections.deque(iterable, maxlen=n)) - - def consume(iterator, n=None): - "Advance the iterator n-steps ahead. If n is None, consume entirely." - # Use functions that consume iterators at C speed. - if n is None: - # feed the entire iterator into a zero-length deque - collections.deque(iterator, maxlen=0) - else: - # advance to the empty slice starting at position n - next(islice(iterator, n, n), None) - - def nth(iterable, n, default=None): - "Returns the nth item or a default value" - return next(islice(iterable, n, None), default) - - def all_equal(iterable): - "Returns True if all the elements are equal to each other" - g = groupby(iterable) - return next(g, True) and not next(g, False) - - def quantify(iterable, pred=bool): - "Count how many times the predicate is True" - return sum(map(pred, iterable)) - - def ncycles(iterable, n): - "Returns the sequence elements n times" - return chain.from_iterable(repeat(tuple(iterable), n)) - - def batched(iterable, n): - "Batch data into tuples of length n. The last batch may be shorter." - # batched('ABCDEFG', 3) --> ABC DEF G - if n < 1: - raise ValueError('n must be at least one') - it = iter(iterable) - while batch := tuple(islice(it, n)): - yield batch - - def grouper(iterable, n, *, incomplete='fill', fillvalue=None): - "Collect data into non-overlapping fixed-length chunks or blocks" - # grouper('ABCDEFG', 3, fillvalue='x') --> ABC DEF Gxx - # grouper('ABCDEFG', 3, incomplete='strict') --> ABC DEF ValueError - # grouper('ABCDEFG', 3, incomplete='ignore') --> ABC DEF - args = [iter(iterable)] * n - if incomplete == 'fill': - return zip_longest(*args, fillvalue=fillvalue) - if incomplete == 'strict': - return zip(*args, strict=True) - if incomplete == 'ignore': - return zip(*args) - else: - raise ValueError('Expected fill, strict, or ignore') - - def sumprod(vec1, vec2): - "Compute a sum of products." - return sum(starmap(operator.mul, zip(vec1, vec2, strict=True))) - - def sum_of_squares(it): - "Add up the squares of the input values." - # sum_of_squares([10, 20, 30]) -> 1400 - return sumprod(*tee(it)) - - def transpose(it): - "Swap the rows and columns of the input." - # transpose([(1, 2, 3), (11, 22, 33)]) --> (1, 11) (2, 22) (3, 33) - return zip(*it, strict=True) - - def matmul(m1, m2): - "Multiply two matrices." - # matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]]) --> (49, 80), (41, 60) - n = len(m2[0]) - return batched(starmap(sumprod, product(m1, transpose(m2))), n) - - def convolve(signal, kernel): - # See: https://betterexplained.com/articles/intuitive-convolution/ - # convolve(data, [0.25, 0.25, 0.25, 0.25]) --> Moving average (blur) - # convolve(data, [1, -1]) --> 1st finite difference (1st derivative) - # convolve(data, [1, -2, 1]) --> 2nd finite difference (2nd derivative) - kernel = tuple(kernel)[::-1] - n = len(kernel) - window = collections.deque([0], maxlen=n) * n - for x in chain(signal, repeat(0, n-1)): - window.append(x) - yield sumprod(kernel, window) - - def polynomial_from_roots(roots): - """Compute a polynomial's coefficients from its roots. - - (x - 5) (x + 4) (x - 3) expands to: x³ -4x² -17x + 60 - """ - # polynomial_from_roots([5, -4, 3]) --> [1, -4, -17, 60] - expansion = [1] - for r in roots: - expansion = convolve(expansion, (1, -r)) - return list(expansion) - - def polynomial_eval(coefficients, x): - """Evaluate a polynomial at a specific value. - - Computes with better numeric stability than Horner's method. - """ - # Evaluate x³ -4x² -17x + 60 at x = 2.5 - # polynomial_eval([1, -4, -17, 60], x=2.5) --> 8.125 - n = len(coefficients) - if n == 0: - return x * 0 # coerce zero to the type of x - powers = map(pow, repeat(x), reversed(range(n))) - return sumprod(coefficients, powers) - - def iter_index(iterable, value, start=0): - "Return indices where a value occurs in a sequence or iterable." - # iter_index('AABCADEAF', 'A') --> 0 1 4 7 - try: - seq_index = iterable.index - except AttributeError: - # Slow path for general iterables - it = islice(iterable, start, None) - i = start - 1 - try: - while True: - yield (i := i + operator.indexOf(it, value) + 1) - except ValueError: - pass - else: - # Fast path for sequences - i = start - 1 - try: - while True: - yield (i := seq_index(value, i+1)) - except ValueError: - pass - - def sieve(n): - "Primes less than n" - # sieve(30) --> 2 3 5 7 11 13 17 19 23 29 - data = bytearray((0, 1)) * (n // 2) - data[:3] = 0, 0, 0 - limit = math.isqrt(n) + 1 - for p in compress(range(limit), data): - data[p*p : n : p+p] = bytes(len(range(p*p, n, p+p))) - data[2] = 1 - return iter_index(data, 1) if n > 2 else iter([]) - - def factor(n): - "Prime factors of n." - # factor(99) --> 3 3 11 - for prime in sieve(math.isqrt(n) + 1): - while True: - quotient, remainder = divmod(n, prime) - if remainder: - break - yield prime - n = quotient - if n == 1: - return - if n > 1: - yield n - - def flatten(list_of_lists): - "Flatten one level of nesting" - return chain.from_iterable(list_of_lists) - - def repeatfunc(func, times=None, *args): - """Repeat calls to func with specified arguments. - - Example: repeatfunc(random.random) - """ - if times is None: - return starmap(func, repeat(args)) - return starmap(func, repeat(args, times)) - - def triplewise(iterable): - "Return overlapping triplets from an iterable" - # triplewise('ABCDEFG') --> ABC BCD CDE DEF EFG - for (a, _), (b, c) in pairwise(pairwise(iterable)): - yield a, b, c - - def sliding_window(iterable, n): - # sliding_window('ABCDEFG', 4) --> ABCD BCDE CDEF DEFG - it = iter(iterable) - window = collections.deque(islice(it, n), maxlen=n) - if len(window) == n: - yield tuple(window) - for x in it: - window.append(x) - yield tuple(window) - - def roundrobin(*iterables): - "roundrobin('ABC', 'D', 'EF') --> A D E B F C" - # Recipe credited to George Sakkis - num_active = len(iterables) - nexts = cycle(iter(it).__next__ for it in iterables) - while num_active: - try: - for next in nexts: - yield next() - except StopIteration: - # Remove the iterator we just exhausted from the cycle. - num_active -= 1 - nexts = cycle(islice(nexts, num_active)) - - def partition(pred, iterable): - "Use a predicate to partition entries into false entries and true entries" - # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 - t1, t2 = tee(iterable) - return filterfalse(pred, t1), filter(pred, t2) - - def before_and_after(predicate, it): - """ Variant of takewhile() that allows complete - access to the remainder of the iterator. - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) # takewhile() would lose the 'd' - 'dEfGhI' - - Note that the first iterator must be fully - consumed before the second iterator can - generate valid results. - """ - it = iter(it) - transition = [] - def true_iterator(): - for elem in it: - if predicate(elem): - yield elem - else: - transition.append(elem) - return - def remainder_iterator(): - yield from transition - yield from it - return true_iterator(), remainder_iterator() - - def subslices(seq): - "Return all contiguous non-empty subslices of a sequence" - # subslices('ABCD') --> A AB ABC ABCD B BC BCD C CD D - slices = starmap(slice, combinations(range(len(seq) + 1), 2)) - return map(operator.getitem, repeat(seq), slices) - - def powerset(iterable): - "powerset([1,2,3]) --> () (1,) (2,) (3,) (1,2) (1,3) (2,3) (1,2,3)" - s = list(iterable) - return chain.from_iterable(combinations(s, r) for r in range(len(s)+1)) - - def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBcCAD', str.lower) --> A B c D - seen = set() - if key is None: - for element in filterfalse(seen.__contains__, iterable): - seen.add(element) - yield element - # For order preserving deduplication, - # a faster but non-lazy solution is: - # yield from dict.fromkeys(iterable) - else: - for element in iterable: - k = key(element) - if k not in seen: - seen.add(k) - yield element - # For use cases that allow the last matching element to be returned, - # a faster but non-lazy solution is: - # t1, t2 = tee(iterable) - # yield from dict(zip(map(key, t1), t2)).values() - - def unique_justseen(iterable, key=None): - "List unique elements, preserving order. Remember only the element just seen." - # unique_justseen('AAAABBBCCDAABBB') --> A B C D A B - # unique_justseen('ABBcCAD', str.lower) --> A B c A D - return map(next, map(operator.itemgetter(1), groupby(iterable, key))) - - def iter_except(func, exception, first=None): - """ Call a function repeatedly until an exception is raised. - - Converts a call-until-exception interface to an iterator interface. - Like builtins.iter(func, sentinel) but uses an exception instead - of a sentinel to end the loop. - - Examples: - iter_except(functools.partial(heappop, h), IndexError) # priority queue iterator - iter_except(d.popitem, KeyError) # non-blocking dict iterator - iter_except(d.popleft, IndexError) # non-blocking deque iterator - iter_except(q.get_nowait, Queue.Empty) # loop over a producer Queue - iter_except(s.pop, KeyError) # non-blocking set iterator - - """ - try: - if first is not None: - yield first() # For database APIs needing an initial cast to db.first() - while True: - yield func() - except exception: - pass - - def first_true(iterable, default=False, pred=None): - """Returns the first true value in the iterable. - - If no true value is found, returns *default* - - If *pred* is not None, returns the first item - for which pred(item) is true. - - """ - # first_true([a,b,c], x) --> a or b or c or x - # first_true([a,b], x, f) --> a if f(a) else b if f(b) else x - return next(filter(pred, iterable), default) - - def nth_combination(iterable, r, index): - "Equivalent to list(combinations(iterable, r))[index]" - pool = tuple(iterable) - n = len(pool) - c = math.comb(n, r) - if index < 0: - index += c - if index < 0 or index >= c: - raise IndexError - result = [] - while r: - c, n, r = c*r//n, n-1, r-1 - while index >= c: - index -= c - c, n = c*(n-r)//n, n-1 - result.append(pool[-1-n]) - return tuple(result) - -.. doctest:: - :hide: - - These examples no longer appear in the docs but are guaranteed - to keep working. - - >>> amounts = [120.15, 764.05, 823.14] - >>> for checknum, amount in zip(count(1200), amounts): - ... print('Check %d is for $%.2f' % (checknum, amount)) - ... - Check 1200 is for $120.15 - Check 1201 is for $764.05 - Check 1202 is for $823.14 - - >>> import operator - >>> for cube in map(operator.pow, range(1,4), repeat(3)): - ... print(cube) - ... - 1 - 8 - 27 - - >>> reportlines = ['EuroPython', 'Roster', '', 'alex', '', 'laura', '', 'martin', '', 'walter', '', 'samuele'] - >>> for name in islice(reportlines, 3, None, 2): - ... print(name.title()) - ... - Alex - Laura - Martin - Walter - Samuele - - >>> from operator import itemgetter - >>> d = dict(a=1, b=2, c=1, d=2, e=1, f=2, g=3) - >>> di = sorted(sorted(d.items()), key=itemgetter(1)) - >>> for k, g in groupby(di, itemgetter(1)): - ... print(k, list(map(itemgetter(0), g))) - ... - 1 ['a', 'c', 'e'] - 2 ['b', 'd', 'f'] - 3 ['g'] - - # Find runs of consecutive numbers using groupby. The key to the solution - # is differencing with a range so that consecutive numbers all appear in - # same group. - >>> data = [ 1, 4,5,6, 10, 15,16,17,18, 22, 25,26,27,28] - >>> for k, g in groupby(enumerate(data), lambda t:t[0]-t[1]): - ... print(list(map(operator.itemgetter(1), g))) - ... - [1] - [4, 5, 6] - [10] - [15, 16, 17, 18] - [22] - [25, 26, 27, 28] - - Now, we test all of the itertool recipes - - >>> take(10, count()) - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - - >>> list(prepend(1, [2, 3, 4])) - [1, 2, 3, 4] - - >>> list(enumerate('abc')) - [(0, 'a'), (1, 'b'), (2, 'c')] - - >>> list(islice(tabulate(lambda x: 2*x), 4)) - [0, 2, 4, 6] - - >>> list(tail(3, 'ABCDEFG')) - ['E', 'F', 'G'] - - >>> it = iter(range(10)) - >>> consume(it, 3) - >>> next(it) - 3 - >>> consume(it) - >>> next(it, 'Done') - 'Done' - - >>> nth('abcde', 3) - 'd' - - >>> nth('abcde', 9) is None - True - - >>> [all_equal(s) for s in ('', 'A', 'AAAA', 'AAAB', 'AAABA')] - [True, True, True, False, False] - - >>> quantify(range(99), lambda x: x%2==0) - 50 - - >>> quantify([True, False, False, True, True]) - 3 - - >>> quantify(range(12), pred=lambda x: x%2==1) - 6 - - >>> a = [[1, 2, 3], [4, 5, 6]] - >>> list(flatten(a)) - [1, 2, 3, 4, 5, 6] - - >>> list(repeatfunc(pow, 5, 2, 3)) - [8, 8, 8, 8, 8] - - >>> take(5, map(int, repeatfunc(random.random))) - [0, 0, 0, 0, 0] - - >>> list(ncycles('abc', 3)) - ['a', 'b', 'c', 'a', 'b', 'c', 'a', 'b', 'c'] - - >>> sumprod([1,2,3], [4,5,6]) - 32 - - >>> sum_of_squares([10, 20, 30]) - 1400 - - >>> list(transpose([(1, 2, 3), (11, 22, 33)])) - [(1, 11), (2, 22), (3, 33)] - - >>> list(matmul([(7, 5), (3, 5)], [[2, 5], [7, 9]])) - [(49, 80), (41, 60)] - >>> list(matmul([[2, 5], [7, 9], [3, 4]], [[7, 11, 5, 4, 9], [3, 5, 2, 6, 3]])) - [(29, 47, 20, 38, 33), (76, 122, 53, 82, 90), (33, 53, 23, 36, 39)] - - >>> data = [20, 40, 24, 32, 20, 28, 16] - >>> list(convolve(data, [0.25, 0.25, 0.25, 0.25])) - [5.0, 15.0, 21.0, 29.0, 29.0, 26.0, 24.0, 16.0, 11.0, 4.0] - >>> list(convolve(data, [1, -1])) - [20, 20, -16, 8, -12, 8, -12, -16] - >>> list(convolve(data, [1, -2, 1])) - [20, 0, -36, 24, -20, 20, -20, -4, 16] - - >>> polynomial_from_roots([5, -4, 3]) - [1, -4, -17, 60] - >>> factored = lambda x: (x - 5) * (x + 4) * (x - 3) - >>> expanded = lambda x: x**3 -4*x**2 -17*x + 60 - >>> all(factored(x) == expanded(x) for x in range(-10, 11)) - True - - >>> list(iter_index('AABCADEAF', 'A')) - [0, 1, 4, 7] - >>> list(iter_index('AABCADEAF', 'B')) - [2] - >>> list(iter_index('AABCADEAF', 'X')) - [] - >>> list(iter_index('', 'X')) - [] - >>> list(iter_index('AABCADEAF', 'A', 1)) - [1, 4, 7] - >>> list(iter_index(iter('AABCADEAF'), 'A', 1)) - [1, 4, 7] - >>> list(iter_index('AABCADEAF', 'A', 2)) - [4, 7] - >>> list(iter_index(iter('AABCADEAF'), 'A', 2)) - [4, 7] - >>> list(iter_index('AABCADEAF', 'A', 10)) - [] - >>> list(iter_index(iter('AABCADEAF'), 'A', 10)) - [] - - >>> list(sieve(30)) - [2, 3, 5, 7, 11, 13, 17, 19, 23, 29] - >>> small_primes = [2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97] - >>> all(list(sieve(n)) == [p for p in small_primes if p < n] for n in range(101)) - True - >>> len(list(sieve(100))) - 25 - >>> len(list(sieve(1_000))) - 168 - >>> len(list(sieve(10_000))) - 1229 - >>> len(list(sieve(100_000))) - 9592 - >>> len(list(sieve(1_000_000))) - 78498 - >>> carmichael = {561, 1105, 1729, 2465, 2821, 6601, 8911} # https://oeis.org/A002997 - >>> set(sieve(10_000)).isdisjoint(carmichael) - True - - >>> list(factor(0)) - [] - >>> list(factor(1)) - [] - >>> list(factor(2)) - [2] - >>> list(factor(3)) - [3] - >>> list(factor(4)) - [2, 2] - >>> list(factor(5)) - [5] - >>> list(factor(6)) - [2, 3] - >>> list(factor(7)) - [7] - >>> list(factor(8)) - [2, 2, 2] - >>> list(factor(9)) - [3, 3] - >>> list(factor(10)) - [2, 5] - >>> list(factor(128_884_753_939)) # large prime - [128884753939] - >>> list(factor(999953 * 999983)) # large semiprime - [999953, 999983] - >>> list(factor(6 ** 20)) == [2] * 20 + [3] * 20 # large power - True - >>> list(factor(909_909_090_909)) # large multiterm composite - [3, 3, 7, 13, 13, 751, 113797] - >>> math.prod([3, 3, 7, 13, 13, 751, 113797]) - 909909090909 - >>> all(math.prod(factor(n)) == n for n in range(1, 2_000)) - True - >>> all(set(factor(n)) <= set(sieve(n+1)) for n in range(2_000)) - True - >>> all(list(factor(n)) == sorted(factor(n)) for n in range(2_000)) - True - - >>> list(flatten([('a', 'b'), (), ('c', 'd', 'e'), ('f',), ('g', 'h', 'i')])) - ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i'] - - >>> random.seed(85753098575309) - >>> list(repeatfunc(random.random, 3)) - [0.16370491282496968, 0.45889608687313455, 0.3747076837820118] - >>> list(repeatfunc(chr, 3, 65)) - ['A', 'A', 'A'] - >>> list(repeatfunc(pow, 3, 2, 5)) - [32, 32, 32] - - >>> list(grouper('abcdefg', 3, fillvalue='x')) - [('a', 'b', 'c'), ('d', 'e', 'f'), ('g', 'x', 'x')] - - >>> it = grouper('abcdefg', 3, incomplete='strict') - >>> next(it) - ('a', 'b', 'c') - >>> next(it) - ('d', 'e', 'f') - >>> next(it) - Traceback (most recent call last): - ... - ValueError: zip() argument 2 is shorter than argument 1 - - >>> list(grouper('abcdefg', n=3, incomplete='ignore')) - [('a', 'b', 'c'), ('d', 'e', 'f')] - - >>> list(batched('ABCDEFG', 3)) - [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)] - >>> list(batched('ABCDEF', 3)) - [('A', 'B', 'C'), ('D', 'E', 'F')] - >>> list(batched('ABCDE', 3)) - [('A', 'B', 'C'), ('D', 'E')] - >>> list(batched('ABCD', 3)) - [('A', 'B', 'C'), ('D',)] - >>> list(batched('ABC', 3)) - [('A', 'B', 'C')] - >>> list(batched('AB', 3)) - [('A', 'B')] - >>> list(batched('A', 3)) - [('A',)] - >>> list(batched('', 3)) - [] - >>> list(batched('ABCDEFG', 2)) - [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)] - >>> list(batched('ABCDEFG', 1)) - [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)] - >>> s = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' - >>> all(list(flatten(batched(s[:n], 5))) == list(s[:n]) for n in range(len(s))) - True - - >>> list(triplewise('ABCDEFG')) - [('A', 'B', 'C'), ('B', 'C', 'D'), ('C', 'D', 'E'), ('D', 'E', 'F'), ('E', 'F', 'G')] - - >>> list(sliding_window('ABCDEFG', 4)) - [('A', 'B', 'C', 'D'), ('B', 'C', 'D', 'E'), ('C', 'D', 'E', 'F'), ('D', 'E', 'F', 'G')] - - >>> list(roundrobin('abc', 'd', 'ef')) - ['a', 'd', 'e', 'b', 'f', 'c'] - - >>> def is_odd(x): - ... return x % 2 == 1 - - >>> evens, odds = partition(is_odd, range(10)) - >>> list(evens) - [0, 2, 4, 6, 8] - >>> list(odds) - [1, 3, 5, 7, 9] - - >>> it = iter('ABCdEfGhI') - >>> all_upper, remainder = before_and_after(str.isupper, it) - >>> ''.join(all_upper) - 'ABC' - >>> ''.join(remainder) - 'dEfGhI' - - >>> list(subslices('ABCD')) - ['A', 'AB', 'ABC', 'ABCD', 'B', 'BC', 'BCD', 'C', 'CD', 'D'] - - >>> list(powerset([1,2,3])) - [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] - - >>> all(len(list(powerset(range(n)))) == 2**n for n in range(18)) - True - - >>> list(powerset('abcde')) == sorted(sorted(set(powerset('abcde'))), key=len) - True - - >>> list(unique_everseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'D'] - >>> list(unique_everseen('ABBcCAD', str.lower)) - ['A', 'B', 'c', 'D'] - - >>> list(unique_justseen('AAAABBBCCDAABBB')) - ['A', 'B', 'C', 'D', 'A', 'B'] - >>> list(unique_justseen('ABBCcAD', str.lower)) - ['A', 'B', 'C', 'A', 'D'] - >>> list(unique_justseen('ABBcCAD', str.lower)) - ['A', 'B', 'c', 'A', 'D'] - - >>> d = dict(a=1, b=2, c=3) - >>> it = iter_except(d.popitem, KeyError) - >>> d['d'] = 4 - >>> next(it) - ('d', 4) - >>> next(it) - ('c', 3) - >>> next(it) - ('b', 2) - >>> d['e'] = 5 - >>> next(it) - ('e', 5) - >>> next(it) - ('a', 1) - >>> next(it, 'empty') - 'empty' - - >>> first_true('ABC0DEF1', '9', str.isdigit) - '0' - - >>> population = 'ABCDEFGH' - >>> for r in range(len(population) + 1): - ... seq = list(combinations(population, r)) - ... for i in range(len(seq)): - ... assert nth_combination(population, r, i) == seq[i] - ... for i in range(-len(seq), 0): - ... assert nth_combination(population, r, i) == seq[i] - - >>> iterable = 'abcde' - >>> r = 3 - >>> combos = list(combinations(iterable, r)) - >>> all(nth_combination(iterable, r, i) == comb for i, comb in enumerate(combos)) - True diff --git a/Doc/library/json.rst b/Doc/library/json.rst deleted file mode 100644 index e234fe92bc9995..00000000000000 --- a/Doc/library/json.rst +++ /dev/null @@ -1,766 +0,0 @@ -:mod:`json` --- JSON encoder and decoder -======================================== - -.. module:: json - :synopsis: Encode and decode the JSON format. - -.. moduleauthor:: Bob Ippolito -.. sectionauthor:: Bob Ippolito - -**Source code:** :source:`Lib/json/__init__.py` - --------------- - -`JSON (JavaScript Object Notation) `_, specified by -:rfc:`7159` (which obsoletes :rfc:`4627`) and by -`ECMA-404 `_, -is a lightweight data interchange format inspired by -`JavaScript `_ object literal syntax -(although it is not a strict subset of JavaScript [#rfc-errata]_ ). - -.. warning:: - Be cautious when parsing JSON data from untrusted sources. A malicious - JSON string may cause the decoder to consume considerable CPU and memory - resources. Limiting the size of data to be parsed is recommended. - -:mod:`json` exposes an API familiar to users of the standard library -:mod:`marshal` and :mod:`pickle` modules. - -Encoding basic Python object hierarchies:: - - >>> import json - >>> json.dumps(['foo', {'bar': ('baz', None, 1.0, 2)}]) - '["foo", {"bar": ["baz", null, 1.0, 2]}]' - >>> print(json.dumps("\"foo\bar")) - "\"foo\bar" - >>> print(json.dumps('\u1234')) - "\u1234" - >>> print(json.dumps('\\')) - "\\" - >>> print(json.dumps({"c": 0, "b": 0, "a": 0}, sort_keys=True)) - {"a": 0, "b": 0, "c": 0} - >>> from io import StringIO - >>> io = StringIO() - >>> json.dump(['streaming API'], io) - >>> io.getvalue() - '["streaming API"]' - -Compact encoding:: - - >>> import json - >>> json.dumps([1, 2, 3, {'4': 5, '6': 7}], separators=(',', ':')) - '[1,2,3,{"4":5,"6":7}]' - -Pretty printing:: - - >>> import json - >>> print(json.dumps({'4': 5, '6': 7}, sort_keys=True, indent=4)) - { - "4": 5, - "6": 7 - } - -Decoding JSON:: - - >>> import json - >>> json.loads('["foo", {"bar":["baz", null, 1.0, 2]}]') - ['foo', {'bar': ['baz', None, 1.0, 2]}] - >>> json.loads('"\\"foo\\bar"') - '"foo\x08ar' - >>> from io import StringIO - >>> io = StringIO('["streaming API"]') - >>> json.load(io) - ['streaming API'] - -Specializing JSON object decoding:: - - >>> import json - >>> def as_complex(dct): - ... if '__complex__' in dct: - ... return complex(dct['real'], dct['imag']) - ... return dct - ... - >>> json.loads('{"__complex__": true, "real": 1, "imag": 2}', - ... object_hook=as_complex) - (1+2j) - >>> import decimal - >>> json.loads('1.1', parse_float=decimal.Decimal) - Decimal('1.1') - -Extending :class:`JSONEncoder`:: - - >>> import json - >>> class ComplexEncoder(json.JSONEncoder): - ... def default(self, obj): - ... if isinstance(obj, complex): - ... return [obj.real, obj.imag] - ... # Let the base class default method raise the TypeError - ... return json.JSONEncoder.default(self, obj) - ... - >>> json.dumps(2 + 1j, cls=ComplexEncoder) - '[2.0, 1.0]' - >>> ComplexEncoder().encode(2 + 1j) - '[2.0, 1.0]' - >>> list(ComplexEncoder().iterencode(2 + 1j)) - ['[2.0', ', 1.0', ']'] - - -Using :mod:`json.tool` from the shell to validate and pretty-print: - -.. code-block:: shell-session - - $ echo '{"json":"obj"}' | python -m json.tool - { - "json": "obj" - } - $ echo '{1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 2 (char 1) - -See :ref:`json-commandline` for detailed documentation. - -.. note:: - - JSON is a subset of `YAML `_ 1.2. The JSON produced by - this module's default settings (in particular, the default *separators* - value) is also a subset of YAML 1.0 and 1.1. This module can thus also be - used as a YAML serializer. - -.. note:: - - This module's encoders and decoders preserve input and output order by - default. Order is only lost if the underlying containers are unordered. - - -Basic Usage ------------ - -.. function:: dump(obj, fp, *, skipkeys=False, ensure_ascii=True, \ - check_circular=True, allow_nan=True, cls=None, \ - indent=None, separators=None, default=None, \ - sort_keys=False, **kw) - - Serialize *obj* as a JSON formatted stream to *fp* (a ``.write()``-supporting - :term:`file-like object`) using this :ref:`conversion table - `. - - If *skipkeys* is true (default: ``False``), then dict keys that are not - of a basic type (:class:`str`, :class:`int`, :class:`float`, :class:`bool`, - ``None``) will be skipped instead of raising a :exc:`TypeError`. - - The :mod:`json` module always produces :class:`str` objects, not - :class:`bytes` objects. Therefore, ``fp.write()`` must support :class:`str` - input. - - If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. - - If *check_circular* is false (default: ``True``), then the circular - reference check for container types will be skipped and a circular reference - will result in a :exc:`RecursionError` (or worse). - - If *allow_nan* is false (default: ``True``), then it will be a - :exc:`ValueError` to serialize out of range :class:`float` values (``nan``, - ``inf``, ``-inf``) in strict compliance of the JSON specification. - If *allow_nan* is true, their JavaScript equivalents (``NaN``, - ``Infinity``, ``-Infinity``) will be used. - - If *indent* is a non-negative integer or string, then JSON array elements and - object members will be pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the most compact representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. - - .. versionchanged:: 3.2 - Allow strings for *indent* in addition to integers. - - If specified, *separators* should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. - - .. versionchanged:: 3.4 - Use ``(',', ': ')`` as default if *indent* is not ``None``. - - If specified, *default* should be a function that gets called for objects that - can't otherwise be serialized. It should return a JSON encodable version of - the object or raise a :exc:`TypeError`. If not specified, :exc:`TypeError` - is raised. - - If *sort_keys* is true (default: ``False``), then the output of - dictionaries will be sorted by key. - - To use a custom :class:`JSONEncoder` subclass (e.g. one that overrides the - :meth:`~JSONEncoder.default` method to serialize additional types), specify it with the - *cls* kwarg; otherwise :class:`JSONEncoder` is used. - - .. versionchanged:: 3.6 - All optional parameters are now :ref:`keyword-only `. - - .. note:: - - Unlike :mod:`pickle` and :mod:`marshal`, JSON is not a framed protocol, - so trying to serialize multiple objects with repeated calls to - :func:`dump` using the same *fp* will result in an invalid JSON file. - -.. function:: dumps(obj, *, skipkeys=False, ensure_ascii=True, \ - check_circular=True, allow_nan=True, cls=None, \ - indent=None, separators=None, default=None, \ - sort_keys=False, **kw) - - Serialize *obj* to a JSON formatted :class:`str` using this :ref:`conversion - table `. The arguments have the same meaning as in - :func:`dump`. - - .. note:: - - Keys in key/value pairs of JSON are always of the type :class:`str`. When - a dictionary is converted into JSON, all the keys of the dictionary are - coerced to strings. As a result of this, if a dictionary is converted - into JSON and then back into a dictionary, the dictionary may not equal - the original one. That is, ``loads(dumps(x)) != x`` if x has non-string - keys. - -.. function:: load(fp, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) - - Deserialize *fp* (a ``.read()``-supporting :term:`text file` or - :term:`binary file` containing a JSON document) to a Python object using - this :ref:`conversion table `. - - *object_hook* is an optional function that will be called with the result of - any object literal decoded (a :class:`dict`). The return value of - *object_hook* will be used instead of the :class:`dict`. This feature can be used - to implement custom decoders (e.g. `JSON-RPC `_ - class hinting). - - *object_pairs_hook* is an optional function that will be called with the - result of any object literal decoded with an ordered list of pairs. The - return value of *object_pairs_hook* will be used instead of the - :class:`dict`. This feature can be used to implement custom decoders. - If *object_hook* is also defined, the *object_pairs_hook* takes priority. - - .. versionchanged:: 3.1 - Added support for *object_pairs_hook*. - - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). - - .. versionchanged:: 3.11 - The default *parse_int* of :func:`int` now limits the maximum length of - the integer string via the interpreter's :ref:`integer string - conversion length limitation ` to help avoid denial - of service attacks. - - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. - - .. versionchanged:: 3.1 - *parse_constant* doesn't get called on 'null', 'true', 'false' anymore. - - To use a custom :class:`JSONDecoder` subclass, specify it with the ``cls`` - kwarg; otherwise :class:`JSONDecoder` is used. Additional keyword arguments - will be passed to the constructor of the class. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - - .. versionchanged:: 3.6 - All optional parameters are now :ref:`keyword-only `. - - .. versionchanged:: 3.6 - *fp* can now be a :term:`binary file`. The input encoding should be - UTF-8, UTF-16 or UTF-32. - -.. function:: loads(s, *, cls=None, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, object_pairs_hook=None, **kw) - - Deserialize *s* (a :class:`str`, :class:`bytes` or :class:`bytearray` - instance containing a JSON document) to a Python object using this - :ref:`conversion table `. - - The other arguments have the same meaning as in :func:`load`. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - - .. versionchanged:: 3.6 - *s* can now be of type :class:`bytes` or :class:`bytearray`. The - input encoding should be UTF-8, UTF-16 or UTF-32. - - .. versionchanged:: 3.9 - The keyword argument *encoding* has been removed. - - -Encoders and Decoders ---------------------- - -.. class:: JSONDecoder(*, object_hook=None, parse_float=None, parse_int=None, parse_constant=None, strict=True, object_pairs_hook=None) - - Simple JSON decoder. - - Performs the following translations in decoding by default: - - .. _json-to-py-table: - - +---------------+-------------------+ - | JSON | Python | - +===============+===================+ - | object | dict | - +---------------+-------------------+ - | array | list | - +---------------+-------------------+ - | string | str | - +---------------+-------------------+ - | number (int) | int | - +---------------+-------------------+ - | number (real) | float | - +---------------+-------------------+ - | true | True | - +---------------+-------------------+ - | false | False | - +---------------+-------------------+ - | null | None | - +---------------+-------------------+ - - It also understands ``NaN``, ``Infinity``, and ``-Infinity`` as their - corresponding ``float`` values, which is outside the JSON spec. - - *object_hook*, if specified, will be called with the result of every JSON - object decoded and its return value will be used in place of the given - :class:`dict`. This can be used to provide custom deserializations (e.g. to - support `JSON-RPC `_ class hinting). - - *object_pairs_hook*, if specified will be called with the result of every - JSON object decoded with an ordered list of pairs. The return value of - *object_pairs_hook* will be used instead of the :class:`dict`. This - feature can be used to implement custom decoders. If *object_hook* is also - defined, the *object_pairs_hook* takes priority. - - .. versionchanged:: 3.1 - Added support for *object_pairs_hook*. - - *parse_float*, if specified, will be called with the string of every JSON - float to be decoded. By default, this is equivalent to ``float(num_str)``. - This can be used to use another datatype or parser for JSON floats - (e.g. :class:`decimal.Decimal`). - - *parse_int*, if specified, will be called with the string of every JSON int - to be decoded. By default, this is equivalent to ``int(num_str)``. This can - be used to use another datatype or parser for JSON integers - (e.g. :class:`float`). - - *parse_constant*, if specified, will be called with one of the following - strings: ``'-Infinity'``, ``'Infinity'``, ``'NaN'``. - This can be used to raise an exception if invalid JSON numbers - are encountered. - - If *strict* is false (``True`` is the default), then control characters - will be allowed inside strings. Control characters in this context are - those with character codes in the 0--31 range, including ``'\t'`` (tab), - ``'\n'``, ``'\r'`` and ``'\0'``. - - If the data being deserialized is not a valid JSON document, a - :exc:`JSONDecodeError` will be raised. - - .. versionchanged:: 3.6 - All parameters are now :ref:`keyword-only `. - - .. method:: decode(s) - - Return the Python representation of *s* (a :class:`str` instance - containing a JSON document). - - :exc:`JSONDecodeError` will be raised if the given JSON document is not - valid. - - .. method:: raw_decode(s) - - Decode a JSON document from *s* (a :class:`str` beginning with a - JSON document) and return a 2-tuple of the Python representation - and the index in *s* where the document ended. - - This can be used to decode a JSON document from a string that may have - extraneous data at the end. - - -.. class:: JSONEncoder(*, skipkeys=False, ensure_ascii=True, check_circular=True, allow_nan=True, sort_keys=False, indent=None, separators=None, default=None) - - Extensible JSON encoder for Python data structures. - - Supports the following objects and types by default: - - .. _py-to-json-table: - - +----------------------------------------+---------------+ - | Python | JSON | - +========================================+===============+ - | dict | object | - +----------------------------------------+---------------+ - | list, tuple | array | - +----------------------------------------+---------------+ - | str | string | - +----------------------------------------+---------------+ - | int, float, int- & float-derived Enums | number | - +----------------------------------------+---------------+ - | True | true | - +----------------------------------------+---------------+ - | False | false | - +----------------------------------------+---------------+ - | None | null | - +----------------------------------------+---------------+ - - .. versionchanged:: 3.4 - Added support for int- and float-derived Enum classes. - - To extend this to recognize other objects, subclass and implement a - :meth:`~JSONEncoder.default` method with another method that returns a serializable object - for ``o`` if possible, otherwise it should call the superclass implementation - (to raise :exc:`TypeError`). - - If *skipkeys* is false (the default), a :exc:`TypeError` will be raised when - trying to encode keys that are not :class:`str`, :class:`int`, :class:`float` - or ``None``. If *skipkeys* is true, such items are simply skipped. - - If *ensure_ascii* is true (the default), the output is guaranteed to - have all incoming non-ASCII characters escaped. If *ensure_ascii* is - false, these characters will be output as-is. - - If *check_circular* is true (the default), then lists, dicts, and custom - encoded objects will be checked for circular references during encoding to - prevent an infinite recursion (which would cause a :exc:`RecursionError`). - Otherwise, no such check takes place. - - If *allow_nan* is true (the default), then ``NaN``, ``Infinity``, and - ``-Infinity`` will be encoded as such. This behavior is not JSON - specification compliant, but is consistent with most JavaScript based - encoders and decoders. Otherwise, it will be a :exc:`ValueError` to encode - such floats. - - If *sort_keys* is true (default: ``False``), then the output of dictionaries - will be sorted by key; this is useful for regression tests to ensure that - JSON serializations can be compared on a day-to-day basis. - - If *indent* is a non-negative integer or string, then JSON array elements and - object members will be pretty-printed with that indent level. An indent level - of 0, negative, or ``""`` will only insert newlines. ``None`` (the default) - selects the most compact representation. Using a positive integer indent - indents that many spaces per level. If *indent* is a string (such as ``"\t"``), - that string is used to indent each level. - - .. versionchanged:: 3.2 - Allow strings for *indent* in addition to integers. - - If specified, *separators* should be an ``(item_separator, key_separator)`` - tuple. The default is ``(', ', ': ')`` if *indent* is ``None`` and - ``(',', ': ')`` otherwise. To get the most compact JSON representation, - you should specify ``(',', ':')`` to eliminate whitespace. - - .. versionchanged:: 3.4 - Use ``(',', ': ')`` as default if *indent* is not ``None``. - - If specified, *default* should be a function that gets called for objects that - can't otherwise be serialized. It should return a JSON encodable version of - the object or raise a :exc:`TypeError`. If not specified, :exc:`TypeError` - is raised. - - .. versionchanged:: 3.6 - All parameters are now :ref:`keyword-only `. - - - .. method:: default(o) - - Implement this method in a subclass such that it returns a serializable - object for *o*, or calls the base implementation (to raise a - :exc:`TypeError`). - - For example, to support arbitrary iterators, you could implement - :meth:`~JSONEncoder.default` like this:: - - def default(self, o): - try: - iterable = iter(o) - except TypeError: - pass - else: - return list(iterable) - # Let the base class default method raise the TypeError - return json.JSONEncoder.default(self, o) - - - .. method:: encode(o) - - Return a JSON string representation of a Python data structure, *o*. For - example:: - - >>> json.JSONEncoder().encode({"foo": ["bar", "baz"]}) - '{"foo": ["bar", "baz"]}' - - - .. method:: iterencode(o) - - Encode the given object, *o*, and yield each string representation as - available. For example:: - - for chunk in json.JSONEncoder().iterencode(bigobject): - mysocket.write(chunk) - - -Exceptions ----------- - -.. exception:: JSONDecodeError(msg, doc, pos) - - Subclass of :exc:`ValueError` with the following additional attributes: - - .. attribute:: msg - - The unformatted error message. - - .. attribute:: doc - - The JSON document being parsed. - - .. attribute:: pos - - The start index of *doc* where parsing failed. - - .. attribute:: lineno - - The line corresponding to *pos*. - - .. attribute:: colno - - The column corresponding to *pos*. - - .. versionadded:: 3.5 - - -Standard Compliance and Interoperability ----------------------------------------- - -The JSON format is specified by :rfc:`7159` and by -`ECMA-404 `_. -This section details this module's level of compliance with the RFC. -For simplicity, :class:`JSONEncoder` and :class:`JSONDecoder` subclasses, and -parameters other than those explicitly mentioned, are not considered. - -This module does not comply with the RFC in a strict fashion, implementing some -extensions that are valid JavaScript but not valid JSON. In particular: - -- Infinite and NaN number values are accepted and output; -- Repeated names within an object are accepted, and only the value of the last - name-value pair is used. - -Since the RFC permits RFC-compliant parsers to accept input texts that are not -RFC-compliant, this module's deserializer is technically RFC-compliant under -default settings. - -Character Encodings -^^^^^^^^^^^^^^^^^^^ - -The RFC requires that JSON be represented using either UTF-8, UTF-16, or -UTF-32, with UTF-8 being the recommended default for maximum interoperability. - -As permitted, though not required, by the RFC, this module's serializer sets -*ensure_ascii=True* by default, thus escaping the output so that the resulting -strings only contain ASCII characters. - -Other than the *ensure_ascii* parameter, this module is defined strictly in -terms of conversion between Python objects and -:class:`Unicode strings `, and thus does not otherwise directly address -the issue of character encodings. - -The RFC prohibits adding a byte order mark (BOM) to the start of a JSON text, -and this module's serializer does not add a BOM to its output. -The RFC permits, but does not require, JSON deserializers to ignore an initial -BOM in their input. This module's deserializer raises a :exc:`ValueError` -when an initial BOM is present. - -The RFC does not explicitly forbid JSON strings which contain byte sequences -that don't correspond to valid Unicode characters (e.g. unpaired UTF-16 -surrogates), but it does note that they may cause interoperability problems. -By default, this module accepts and outputs (when present in the original -:class:`str`) code points for such sequences. - - -Infinite and NaN Number Values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The RFC does not permit the representation of infinite or NaN number values. -Despite that, by default, this module accepts and outputs ``Infinity``, -``-Infinity``, and ``NaN`` as if they were valid JSON number literal values:: - - >>> # Neither of these calls raises an exception, but the results are not valid JSON - >>> json.dumps(float('-inf')) - '-Infinity' - >>> json.dumps(float('nan')) - 'NaN' - >>> # Same when deserializing - >>> json.loads('-Infinity') - -inf - >>> json.loads('NaN') - nan - -In the serializer, the *allow_nan* parameter can be used to alter this -behavior. In the deserializer, the *parse_constant* parameter can be used to -alter this behavior. - - -Repeated Names Within an Object -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The RFC specifies that the names within a JSON object should be unique, but -does not mandate how repeated names in JSON objects should be handled. By -default, this module does not raise an exception; instead, it ignores all but -the last name-value pair for a given name:: - - >>> weird_json = '{"x": 1, "x": 2, "x": 3}' - >>> json.loads(weird_json) - {'x': 3} - -The *object_pairs_hook* parameter can be used to alter this behavior. - - -Top-level Non-Object, Non-Array Values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The old version of JSON specified by the obsolete :rfc:`4627` required that -the top-level value of a JSON text must be either a JSON object or array -(Python :class:`dict` or :class:`list`), and could not be a JSON null, -boolean, number, or string value. :rfc:`7159` removed that restriction, and -this module does not and has never implemented that restriction in either its -serializer or its deserializer. - -Regardless, for maximum interoperability, you may wish to voluntarily adhere -to the restriction yourself. - - -Implementation Limitations -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Some JSON deserializer implementations may set limits on: - -* the size of accepted JSON texts -* the maximum level of nesting of JSON objects and arrays -* the range and precision of JSON numbers -* the content and maximum length of JSON strings - -This module does not impose any such limits beyond those of the relevant -Python datatypes themselves or the Python interpreter itself. - -When serializing to JSON, beware any such limitations in applications that may -consume your JSON. In particular, it is common for JSON numbers to be -deserialized into IEEE 754 double precision numbers and thus subject to that -representation's range and precision limitations. This is especially relevant -when serializing Python :class:`int` values of extremely large magnitude, or -when serializing instances of "exotic" numerical types such as -:class:`decimal.Decimal`. - - -.. _json-commandline: -.. program:: json.tool - -Command Line Interface ----------------------- - -.. module:: json.tool - :synopsis: A command line to validate and pretty-print JSON. - -**Source code:** :source:`Lib/json/tool.py` - --------------- - -The :mod:`json.tool` module provides a simple command line interface to validate -and pretty-print JSON objects. - -If the optional ``infile`` and ``outfile`` arguments are not -specified, :data:`sys.stdin` and :data:`sys.stdout` will be used respectively: - -.. code-block:: shell-session - - $ echo '{"json": "obj"}' | python -m json.tool - { - "json": "obj" - } - $ echo '{1.2:3.4}' | python -m json.tool - Expecting property name enclosed in double quotes: line 1 column 2 (char 1) - -.. versionchanged:: 3.5 - The output is now in the same order as the input. Use the - :option:`--sort-keys` option to sort the output of dictionaries - alphabetically by key. - - -Command line options -^^^^^^^^^^^^^^^^^^^^ - -.. option:: infile - - The JSON file to be validated or pretty-printed: - - .. code-block:: shell-session - - $ python -m json.tool mp_films.json - [ - { - "title": "And Now for Something Completely Different", - "year": 1971 - }, - { - "title": "Monty Python and the Holy Grail", - "year": 1975 - } - ] - - If *infile* is not specified, read from :data:`sys.stdin`. - -.. option:: outfile - - Write the output of the *infile* to the given *outfile*. Otherwise, write it - to :data:`sys.stdout`. - -.. option:: --sort-keys - - Sort the output of dictionaries alphabetically by key. - - .. versionadded:: 3.5 - -.. option:: --no-ensure-ascii - - Disable escaping of non-ascii characters, see :func:`json.dumps` for more information. - - .. versionadded:: 3.9 - -.. option:: --json-lines - - Parse every input line as separate JSON object. - - .. versionadded:: 3.8 - -.. option:: --indent, --tab, --no-indent, --compact - - Mutually exclusive options for whitespace control. - - .. versionadded:: 3.9 - -.. option:: -h, --help - - Show the help message. - - -.. rubric:: Footnotes - -.. [#rfc-errata] As noted in `the errata for RFC 7159 - `_, - JSON permits literal U+2028 (LINE SEPARATOR) and - U+2029 (PARAGRAPH SEPARATOR) characters in strings, whereas JavaScript - (as of ECMAScript Edition 5.1) does not. diff --git a/Doc/library/keyword.rst b/Doc/library/keyword.rst deleted file mode 100644 index c3b4699cb05af6..00000000000000 --- a/Doc/library/keyword.rst +++ /dev/null @@ -1,40 +0,0 @@ -:mod:`keyword` --- Testing for Python keywords -============================================== - -.. module:: keyword - :synopsis: Test whether a string is a keyword in Python. - -**Source code:** :source:`Lib/keyword.py` - --------------- - -This module allows a Python program to determine if a string is a -:ref:`keyword ` or :ref:`soft keyword `. - - -.. function:: iskeyword(s) - - Return ``True`` if *s* is a Python :ref:`keyword `. - - -.. data:: kwlist - - Sequence containing all the :ref:`keywords ` defined for the - interpreter. If any keywords are defined to only be active when particular - :mod:`__future__` statements are in effect, these will be included as well. - - -.. function:: issoftkeyword(s) - - Return ``True`` if *s* is a Python :ref:`soft keyword `. - - .. versionadded:: 3.9 - - -.. data:: softkwlist - - Sequence containing all the :ref:`soft keywords ` defined for the - interpreter. If any soft keywords are defined to only be active when particular - :mod:`__future__` statements are in effect, these will be included as well. - - .. versionadded:: 3.9 diff --git a/Doc/library/language.rst b/Doc/library/language.rst deleted file mode 100644 index 510f064c76d795..00000000000000 --- a/Doc/library/language.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _language: - -************************ -Python Language Services -************************ - -Python provides a number of modules to assist in working with the Python -language. These modules support tokenizing, parsing, syntax analysis, bytecode -disassembly, and various other facilities. - -These modules include: - - -.. toctree:: - - ast.rst - symtable.rst - token.rst - keyword.rst - tokenize.rst - tabnanny.rst - pyclbr.rst - py_compile.rst - compileall.rst - dis.rst - pickletools.rst diff --git a/Doc/library/linecache.rst b/Doc/library/linecache.rst deleted file mode 100644 index dd9f4ee45ba82e..00000000000000 --- a/Doc/library/linecache.rst +++ /dev/null @@ -1,67 +0,0 @@ -:mod:`linecache` --- Random access to text lines -================================================ - -.. module:: linecache - :synopsis: Provides random access to individual lines from text files. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/linecache.py` - --------------- - -The :mod:`linecache` module allows one to get any line from a Python source file, while -attempting to optimize internally, using a cache, the common case where many -lines are read from a single file. This is used by the :mod:`traceback` module -to retrieve source lines for inclusion in the formatted traceback. - -The :func:`tokenize.open` function is used to open files. This -function uses :func:`tokenize.detect_encoding` to get the encoding of the -file; in the absence of an encoding token, the file encoding defaults to UTF-8. - -The :mod:`linecache` module defines the following functions: - - -.. function:: getline(filename, lineno, module_globals=None) - - Get line *lineno* from file named *filename*. This function will never raise an - exception --- it will return ``''`` on errors (the terminating newline character - will be included for lines that are found). - - .. index:: triple: module; search; path - - If a file named *filename* is not found, the function first checks - for a :pep:`302` ``__loader__`` in *module_globals*. - If there is such a loader and it defines a ``get_source`` method, - then that determines the source lines - (if ``get_source()`` returns ``None``, then ``''`` is returned). - Finally, if *filename* is a relative filename, - it is looked up relative to the entries in the module search path, ``sys.path``. - - -.. function:: clearcache() - - Clear the cache. Use this function if you no longer need lines from files - previously read using :func:`getline`. - - -.. function:: checkcache(filename=None) - - Check the cache for validity. Use this function if files in the cache may have - changed on disk, and you require the updated version. If *filename* is omitted, - it will check all the entries in the cache. - -.. function:: lazycache(filename, module_globals) - - Capture enough detail about a non-file-based module to permit getting its - lines later via :func:`getline` even if *module_globals* is ``None`` in the later - call. This avoids doing I/O until a line is actually needed, without having - to carry the module globals around indefinitely. - - .. versionadded:: 3.5 - -Example:: - - >>> import linecache - >>> linecache.getline(linecache.__file__, 8) - 'import sys\n' diff --git a/Doc/library/locale.rst b/Doc/library/locale.rst deleted file mode 100644 index 5ea5957eb143cd..00000000000000 --- a/Doc/library/locale.rst +++ /dev/null @@ -1,635 +0,0 @@ -:mod:`locale` --- Internationalization services -=============================================== - -.. module:: locale - :synopsis: Internationalization services. - -.. moduleauthor:: Martin von Löwis -.. sectionauthor:: Martin von Löwis - -**Source code:** :source:`Lib/locale.py` - --------------- - -The :mod:`locale` module opens access to the POSIX locale database and -functionality. The POSIX locale mechanism allows programmers to deal with -certain cultural issues in an application, without requiring the programmer to -know all the specifics of each country where the software is executed. - -.. index:: pair: module; _locale - -The :mod:`locale` module is implemented on top of the :mod:`_locale` module, -which in turn uses an ANSI C locale implementation if available. - -The :mod:`locale` module defines the following exception and functions: - - -.. exception:: Error - - Exception raised when the locale passed to :func:`setlocale` is not - recognized. - - -.. function:: setlocale(category, locale=None) - - If *locale* is given and not ``None``, :func:`setlocale` modifies the locale - setting for the *category*. The available categories are listed in the data - description below. *locale* may be a string, or an iterable of two strings - (language code and encoding). If it's an iterable, it's converted to a locale - name using the locale aliasing engine. An empty string specifies the user's - default settings. If the modification of the locale fails, the exception - :exc:`Error` is raised. If successful, the new locale setting is returned. - - If *locale* is omitted or ``None``, the current setting for *category* is - returned. - - :func:`setlocale` is not thread-safe on most systems. Applications typically - start with a call of :: - - import locale - locale.setlocale(locale.LC_ALL, '') - - This sets the locale for all categories to the user's default setting (typically - specified in the :envvar:`LANG` environment variable). If the locale is not - changed thereafter, using multithreading should not cause problems. - - -.. function:: localeconv() - - Returns the database of the local conventions as a dictionary. This dictionary - has the following strings as keys: - - .. tabularcolumns:: |l|l|L| - - +----------------------+-------------------------------------+--------------------------------+ - | Category | Key | Meaning | - +======================+=====================================+================================+ - | :const:`LC_NUMERIC` | ``'decimal_point'`` | Decimal point character. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'grouping'`` | Sequence of numbers specifying | - | | | which relative positions the | - | | | ``'thousands_sep'`` is | - | | | expected. If the sequence is | - | | | terminated with | - | | | :const:`CHAR_MAX`, no further | - | | | grouping is performed. If the | - | | | sequence terminates with a | - | | | ``0``, the last group size is | - | | | repeatedly used. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'thousands_sep'`` | Character used between groups. | - +----------------------+-------------------------------------+--------------------------------+ - | :const:`LC_MONETARY` | ``'int_curr_symbol'`` | International currency symbol. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'currency_symbol'`` | Local currency symbol. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'p_cs_precedes/n_cs_precedes'`` | Whether the currency symbol | - | | | precedes the value (for | - | | | positive resp. negative | - | | | values). | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'p_sep_by_space/n_sep_by_space'`` | Whether the currency symbol is | - | | | separated from the value by a | - | | | space (for positive resp. | - | | | negative values). | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'mon_decimal_point'`` | Decimal point used for | - | | | monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'frac_digits'`` | Number of fractional digits | - | | | used in local formatting of | - | | | monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'int_frac_digits'`` | Number of fractional digits | - | | | used in international | - | | | formatting of monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'mon_thousands_sep'`` | Group separator used for | - | | | monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'mon_grouping'`` | Equivalent to ``'grouping'``, | - | | | used for monetary values. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'positive_sign'`` | Symbol used to annotate a | - | | | positive monetary value. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'negative_sign'`` | Symbol used to annotate a | - | | | negative monetary value. | - +----------------------+-------------------------------------+--------------------------------+ - | | ``'p_sign_posn/n_sign_posn'`` | The position of the sign (for | - | | | positive resp. negative | - | | | values), see below. | - +----------------------+-------------------------------------+--------------------------------+ - - All numeric values can be set to :const:`CHAR_MAX` to indicate that there is no - value specified in this locale. - - The possible values for ``'p_sign_posn'`` and ``'n_sign_posn'`` are given below. - - +--------------+-----------------------------------------+ - | Value | Explanation | - +==============+=========================================+ - | ``0`` | Currency and value are surrounded by | - | | parentheses. | - +--------------+-----------------------------------------+ - | ``1`` | The sign should precede the value and | - | | currency symbol. | - +--------------+-----------------------------------------+ - | ``2`` | The sign should follow the value and | - | | currency symbol. | - +--------------+-----------------------------------------+ - | ``3`` | The sign should immediately precede the | - | | value. | - +--------------+-----------------------------------------+ - | ``4`` | The sign should immediately follow the | - | | value. | - +--------------+-----------------------------------------+ - | ``CHAR_MAX`` | Nothing is specified in this locale. | - +--------------+-----------------------------------------+ - - The function temporarily sets the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` - locale or the ``LC_MONETARY`` locale if locales are different and numeric or - monetary strings are non-ASCII. This temporary change affects other threads. - - .. versionchanged:: 3.7 - The function now temporarily sets the ``LC_CTYPE`` locale to the - ``LC_NUMERIC`` locale in some cases. - - -.. function:: nl_langinfo(option) - - Return some locale-specific information as a string. This function is not - available on all systems, and the set of possible options might also vary - across platforms. The possible argument values are numbers, for which - symbolic constants are available in the locale module. - - The :func:`nl_langinfo` function accepts one of the following keys. Most - descriptions are taken from the corresponding description in the GNU C - library. - - .. data:: CODESET - - Get a string with the name of the character encoding used in the - selected locale. - - .. data:: D_T_FMT - - Get a string that can be used as a format string for :func:`time.strftime` to - represent date and time in a locale-specific way. - - .. data:: D_FMT - - Get a string that can be used as a format string for :func:`time.strftime` to - represent a date in a locale-specific way. - - .. data:: T_FMT - - Get a string that can be used as a format string for :func:`time.strftime` to - represent a time in a locale-specific way. - - .. data:: T_FMT_AMPM - - Get a format string for :func:`time.strftime` to represent time in the am/pm - format. - - .. data:: DAY_1 ... DAY_7 - - Get the name of the n-th day of the week. - - .. note:: - - This follows the US convention of :const:`DAY_1` being Sunday, not the - international convention (ISO 8601) that Monday is the first day of the - week. - - .. data:: ABDAY_1 ... ABDAY_7 - - Get the abbreviated name of the n-th day of the week. - - .. data:: MON_1 ... MON_12 - - Get the name of the n-th month. - - .. data:: ABMON_1 ... ABMON_12 - - Get the abbreviated name of the n-th month. - - .. data:: RADIXCHAR - - Get the radix character (decimal dot, decimal comma, etc.). - - .. data:: THOUSEP - - Get the separator character for thousands (groups of three digits). - - .. data:: YESEXPR - - Get a regular expression that can be used with the regex function to - recognize a positive response to a yes/no question. - - .. data:: NOEXPR - - Get a regular expression that can be used with the regex(3) function to - recognize a negative response to a yes/no question. - - .. note:: - - The regular expressions for :const:`YESEXPR` and - :const:`NOEXPR` use syntax suitable for the - :c:func:`regex` function from the C library, which might - differ from the syntax used in :mod:`re`. - - .. data:: CRNCYSTR - - Get the currency symbol, preceded by "-" if the symbol should appear before - the value, "+" if the symbol should appear after the value, or "." if the - symbol should replace the radix character. - - .. data:: ERA - - Get a string that represents the era used in the current locale. - - Most locales do not define this value. An example of a locale which does - define this value is the Japanese one. In Japan, the traditional - representation of dates includes the name of the era corresponding to the - then-emperor's reign. - - Normally it should not be necessary to use this value directly. Specifying - the ``E`` modifier in their format strings causes the :func:`time.strftime` - function to use this information. The format of the returned string is not - specified, and therefore you should not assume knowledge of it on different - systems. - - .. data:: ERA_D_T_FMT - - Get a format string for :func:`time.strftime` to represent date and time in a - locale-specific era-based way. - - .. data:: ERA_D_FMT - - Get a format string for :func:`time.strftime` to represent a date in a - locale-specific era-based way. - - .. data:: ERA_T_FMT - - Get a format string for :func:`time.strftime` to represent a time in a - locale-specific era-based way. - - .. data:: ALT_DIGITS - - Get a representation of up to 100 values used to represent the values - 0 to 99. - - -.. function:: getdefaultlocale([envvars]) - - Tries to determine the default locale settings and returns them as a tuple of - the form ``(language code, encoding)``. - - According to POSIX, a program which has not called ``setlocale(LC_ALL, '')`` - runs using the portable ``'C'`` locale. Calling ``setlocale(LC_ALL, '')`` lets - it use the default locale as defined by the :envvar:`LANG` variable. Since we - do not want to interfere with the current locale setting we thus emulate the - behavior in the way described above. - - To maintain compatibility with other platforms, not only the :envvar:`LANG` - variable is tested, but a list of variables given as envvars parameter. The - first found to be defined will be used. *envvars* defaults to the search - path used in GNU gettext; it must always contain the variable name - ``'LANG'``. The GNU gettext search path contains ``'LC_ALL'``, - ``'LC_CTYPE'``, ``'LANG'`` and ``'LANGUAGE'``, in that order. - - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be - determined. - - .. deprecated-removed:: 3.11 3.15 - - -.. function:: getlocale(category=LC_CTYPE) - - Returns the current setting for the given locale category as sequence containing - *language code*, *encoding*. *category* may be one of the :const:`LC_\*` values - except :const:`LC_ALL`. It defaults to :const:`LC_CTYPE`. - - Except for the code ``'C'``, the language code corresponds to :rfc:`1766`. - *language code* and *encoding* may be ``None`` if their values cannot be - determined. - - -.. function:: getpreferredencoding(do_setlocale=True) - - Return the :term:`locale encoding` used for text data, according to user - preferences. User preferences are expressed differently on different - systems, and might not be available programmatically on some systems, so - this function only returns a guess. - - On some systems, it is necessary to invoke :func:`setlocale` to obtain the - user preferences, so this function is not thread-safe. If invoking setlocale - is not necessary or desired, *do_setlocale* should be set to ``False``. - - On Android or if the :ref:`Python UTF-8 Mode ` is enabled, always - return ``'utf-8'``, the :term:`locale encoding` and the *do_setlocale* - argument are ignored. - - The :ref:`Python preinitialization ` configures the LC_CTYPE - locale. See also the :term:`filesystem encoding and error handler`. - - .. versionchanged:: 3.7 - The function now always returns ``"utf-8"`` on Android or if the - :ref:`Python UTF-8 Mode ` is enabled. - - -.. function:: getencoding() - - Get the current :term:`locale encoding`: - - * On Android and VxWorks, return ``"utf-8"``. - * On Unix, return the encoding of the current :data:`LC_CTYPE` locale. - Return ``"utf-8"`` if ``nl_langinfo(CODESET)`` returns an empty string: - for example, if the current LC_CTYPE locale is not supported. - * On Windows, return the ANSI code page. - - The :ref:`Python preinitialization ` configures the LC_CTYPE - locale. See also the :term:`filesystem encoding and error handler`. - - This function is similar to - :func:`getpreferredencoding(False) ` except this - function ignores the :ref:`Python UTF-8 Mode `. - - .. versionadded:: 3.11 - - -.. function:: normalize(localename) - - Returns a normalized locale code for the given locale name. The returned locale - code is formatted for use with :func:`setlocale`. If normalization fails, the - original name is returned unchanged. - - If the given encoding is not known, the function defaults to the default - encoding for the locale code just like :func:`setlocale`. - - -.. function:: resetlocale(category=LC_ALL) - - Sets the locale for *category* to the default setting. - - The default setting is determined by calling :func:`getdefaultlocale`. - *category* defaults to :const:`LC_ALL`. - - .. deprecated-removed:: 3.11 3.13 - - -.. function:: strcoll(string1, string2) - - Compares two strings according to the current :const:`LC_COLLATE` setting. As - any other compare function, returns a negative, or a positive value, or ``0``, - depending on whether *string1* collates before or after *string2* or is equal to - it. - - -.. function:: strxfrm(string) - - Transforms a string to one that can be used in locale-aware - comparisons. For example, ``strxfrm(s1) < strxfrm(s2)`` is - equivalent to ``strcoll(s1, s2) < 0``. This function can be used - when the same string is compared repeatedly, e.g. when collating a - sequence of strings. - - -.. function:: format_string(format, val, grouping=False, monetary=False) - - Formats a number *val* according to the current :const:`LC_NUMERIC` setting. - The format follows the conventions of the ``%`` operator. For floating point - values, the decimal point is modified if appropriate. If *grouping* is ``True``, - also takes the grouping into account. - - If *monetary* is true, the conversion uses monetary thousands separator and - grouping strings. - - Processes formatting specifiers as in ``format % val``, but takes the current - locale settings into account. - - .. versionchanged:: 3.7 - The *monetary* keyword parameter was added. - - -.. function:: format(format, val, grouping=False, monetary=False) - - Please note that this function works like :meth:`format_string` but will - only work for exactly one ``%char`` specifier. For example, ``'%f'`` and - ``'%.0f'`` are both valid specifiers, but ``'%f KiB'`` is not. - - For whole format strings, use :func:`format_string`. - - .. deprecated:: 3.7 - Use :meth:`format_string` instead. - - -.. function:: currency(val, symbol=True, grouping=False, international=False) - - Formats a number *val* according to the current :const:`LC_MONETARY` settings. - - The returned string includes the currency symbol if *symbol* is true, which is - the default. If *grouping* is ``True`` (which is not the default), grouping is done - with the value. If *international* is ``True`` (which is not the default), the - international currency symbol is used. - - .. note:: - - This function will not work with the 'C' locale, so you have to set a - locale via :func:`setlocale` first. - - -.. function:: str(float) - - Formats a floating point number using the same format as the built-in function - ``str(float)``, but takes the decimal point into account. - - -.. function:: delocalize(string) - - Converts a string into a normalized number string, following the - :const:`LC_NUMERIC` settings. - - .. versionadded:: 3.5 - - -.. function:: localize(string, grouping=False, monetary=False) - - Converts a normalized number string into a formatted string following the - :const:`LC_NUMERIC` settings. - - .. versionadded:: 3.10 - - -.. function:: atof(string, func=float) - - Converts a string to a number, following the :const:`LC_NUMERIC` settings, - by calling *func* on the result of calling :func:`delocalize` on *string*. - - -.. function:: atoi(string) - - Converts a string to an integer, following the :const:`LC_NUMERIC` conventions. - - -.. data:: LC_CTYPE - - Locale category for the character type functions. Most importantly, this - category defines the text encoding, i.e. how bytes are interpreted as - Unicode codepoints. See :pep:`538` and :pep:`540` for how this variable - might be automatically coerced to ``C.UTF-8`` to avoid issues created by - invalid settings in containers or incompatible settings passed over remote - SSH connections. - - Python doesn't internally use locale-dependent character transformation functions - from ``ctype.h``. Instead, an internal ``pyctype.h`` provides locale-independent - equivalents like :c:macro:`!Py_TOLOWER`. - - -.. data:: LC_COLLATE - - Locale category for sorting strings. The functions :func:`strcoll` and - :func:`strxfrm` of the :mod:`locale` module are affected. - - -.. data:: LC_TIME - - Locale category for the formatting of time. The function :func:`time.strftime` - follows these conventions. - - -.. data:: LC_MONETARY - - Locale category for formatting of monetary values. The available options are - available from the :func:`localeconv` function. - - -.. data:: LC_MESSAGES - - Locale category for message display. Python currently does not support - application specific locale-aware messages. Messages displayed by the operating - system, like those returned by :func:`os.strerror` might be affected by this - category. - - This value may not be available on operating systems not conforming to the - POSIX standard, most notably Windows. - - -.. data:: LC_NUMERIC - - Locale category for formatting numbers. The functions :func:`.format`, - :func:`atoi`, :func:`atof` and :func:`.str` of the :mod:`locale` module are - affected by that category. All other numeric formatting operations are not - affected. - - -.. data:: LC_ALL - - Combination of all locale settings. If this flag is used when the locale is - changed, setting the locale for all categories is attempted. If that fails for - any category, no category is changed at all. When the locale is retrieved using - this flag, a string indicating the setting for all categories is returned. This - string can be later used to restore the settings. - - -.. data:: CHAR_MAX - - This is a symbolic constant used for different values returned by - :func:`localeconv`. - - -Example:: - - >>> import locale - >>> loc = locale.getlocale() # get current locale - # use German locale; name might vary with platform - >>> locale.setlocale(locale.LC_ALL, 'de_DE') - >>> locale.strcoll('f\xe4n', 'foo') # compare a string containing an umlaut - >>> locale.setlocale(locale.LC_ALL, '') # use user's preferred locale - >>> locale.setlocale(locale.LC_ALL, 'C') # use default (C) locale - >>> locale.setlocale(locale.LC_ALL, loc) # restore saved locale - - -Background, details, hints, tips and caveats --------------------------------------------- - -The C standard defines the locale as a program-wide property that may be -relatively expensive to change. On top of that, some implementations are broken -in such a way that frequent locale changes may cause core dumps. This makes the -locale somewhat painful to use correctly. - -Initially, when a program is started, the locale is the ``C`` locale, no matter -what the user's preferred locale is. There is one exception: the -:data:`LC_CTYPE` category is changed at startup to set the current locale -encoding to the user's preferred locale encoding. The program must explicitly -say that it wants the user's preferred locale settings for other categories by -calling ``setlocale(LC_ALL, '')``. - -It is generally a bad idea to call :func:`setlocale` in some library routine, -since as a side effect it affects the entire program. Saving and restoring it -is almost as bad: it is expensive and affects other threads that happen to run -before the settings have been restored. - -If, when coding a module for general use, you need a locale independent version -of an operation that is affected by the locale (such as -certain formats used with :func:`time.strftime`), you will have to find a way to -do it without using the standard library routine. Even better is convincing -yourself that using locale settings is okay. Only as a last resort should you -document that your module is not compatible with non-\ ``C`` locale settings. - -The only way to perform numeric operations according to the locale is to use the -special functions defined by this module: :func:`atof`, :func:`atoi`, -:func:`.format`, :func:`.str`. - -There is no way to perform case conversions and character classifications -according to the locale. For (Unicode) text strings these are done according -to the character value only, while for byte strings, the conversions and -classifications are done according to the ASCII value of the byte, and bytes -whose high bit is set (i.e., non-ASCII bytes) are never converted or considered -part of a character class such as letter or whitespace. - - -.. _embedding-locale: - -For extension writers and programs that embed Python ----------------------------------------------------- - -Extension modules should never call :func:`setlocale`, except to find out what -the current locale is. But since the return value can only be used portably to -restore it, that is not very useful (except perhaps to find out whether or not -the locale is ``C``). - -When Python code uses the :mod:`locale` module to change the locale, this also -affects the embedding application. If the embedding application doesn't want -this to happen, it should remove the :mod:`_locale` extension module (which does -all the work) from the table of built-in modules in the :file:`config.c` file, -and make sure that the :mod:`_locale` module is not accessible as a shared -library. - - -.. _locale-gettext: - -Access to message catalogs --------------------------- - -.. function:: gettext(msg) -.. function:: dgettext(domain, msg) -.. function:: dcgettext(domain, msg, category) -.. function:: textdomain(domain) -.. function:: bindtextdomain(domain, dir) - -The locale module exposes the C library's gettext interface on systems that -provide this interface. It consists of the functions :func:`!gettext`, -:func:`!dgettext`, :func:`!dcgettext`, :func:`!textdomain`, :func:`!bindtextdomain`, -and :func:`!bind_textdomain_codeset`. These are similar to the same functions in -the :mod:`gettext` module, but use the C library's binary format for message -catalogs, and the C library's search algorithms for locating message catalogs. - -Python applications should normally find no need to invoke these functions, and -should use :mod:`gettext` instead. A known exception to this rule are -applications that link with additional C libraries which internally invoke -:c:func:`gettext` or :c:func:`dcgettext`. For these applications, it may be -necessary to bind the text domain, so that the libraries can properly locate -their message catalogs. diff --git a/Doc/library/logging.config.rst b/Doc/library/logging.config.rst deleted file mode 100644 index 45aa67a2e49f62..00000000000000 --- a/Doc/library/logging.config.rst +++ /dev/null @@ -1,918 +0,0 @@ -:mod:`logging.config` --- Logging configuration -=============================================== - -.. module:: logging.config - :synopsis: Configuration of the logging module. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -**Source code:** :source:`Lib/logging/config.py` - -.. sidebar:: Important - - This page contains only reference information. For tutorials, - please see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Logging Cookbook ` - --------------- - -This section describes the API for configuring the logging module. - -.. _logging-config-api: - -Configuration functions -^^^^^^^^^^^^^^^^^^^^^^^ - -The following functions configure the logging module. They are located in the -:mod:`logging.config` module. Their use is optional --- you can configure the -logging module using these functions or by making calls to the main API (defined -in :mod:`logging` itself) and defining handlers which are declared either in -:mod:`logging` or :mod:`logging.handlers`. - -.. function:: dictConfig(config) - - Takes the logging configuration from a dictionary. The contents of - this dictionary are described in :ref:`logging-config-dictschema` - below. - - If an error is encountered during configuration, this function will - raise a :exc:`ValueError`, :exc:`TypeError`, :exc:`AttributeError` - or :exc:`ImportError` with a suitably descriptive message. The - following is a (possibly incomplete) list of conditions which will - raise an error: - - * A ``level`` which is not a string or which is a string not - corresponding to an actual logging level. - * A ``propagate`` value which is not a boolean. - * An id which does not have a corresponding destination. - * A non-existent handler id found during an incremental call. - * An invalid logger name. - * Inability to resolve to an internal or external object. - - Parsing is performed by the :class:`DictConfigurator` class, whose - constructor is passed the dictionary used for configuration, and - has a :meth:`configure` method. The :mod:`logging.config` module - has a callable attribute :attr:`dictConfigClass` - which is initially set to :class:`DictConfigurator`. - You can replace the value of :attr:`dictConfigClass` with a - suitable implementation of your own. - - :func:`dictConfig` calls :attr:`dictConfigClass` passing - the specified dictionary, and then calls the :meth:`configure` method on - the returned object to put the configuration into effect:: - - def dictConfig(config): - dictConfigClass(config).configure() - - For example, a subclass of :class:`DictConfigurator` could call - ``DictConfigurator.__init__()`` in its own :meth:`__init__()`, then - set up custom prefixes which would be usable in the subsequent - :meth:`configure` call. :attr:`dictConfigClass` would be bound to - this new subclass, and then :func:`dictConfig` could be called exactly as - in the default, uncustomized state. - - .. versionadded:: 3.2 - -.. function:: fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=None) - - Reads the logging configuration from a :mod:`configparser`\-format file. The - format of the file should be as described in - :ref:`logging-config-fileformat`. - This function can be called several times from an application, allowing an - end user to select from various pre-canned configurations (if the developer - provides a mechanism to present the choices and load the chosen - configuration). - - It will raise :exc:`FileNotFoundError` if the file - doesn't exist and :exc:`RuntimeError` if the file is invalid or - empty. - - :param fname: A filename, or a file-like object, or an instance derived - from :class:`~configparser.RawConfigParser`. If a - ``RawConfigParser``-derived instance is passed, it is used as - is. Otherwise, a :class:`~configparser.Configparser` is - instantiated, and the configuration read by it from the - object passed in ``fname``. If that has a :meth:`readline` - method, it is assumed to be a file-like object and read using - :meth:`~configparser.ConfigParser.read_file`; otherwise, - it is assumed to be a filename and passed to - :meth:`~configparser.ConfigParser.read`. - - - :param defaults: Defaults to be passed to the ConfigParser can be specified - in this argument. - - :param disable_existing_loggers: If specified as ``False``, loggers which - exist when this call is made are left - enabled. The default is ``True`` because this - enables old behaviour in a - backward-compatible way. This behaviour is to - disable any existing non-root loggers unless - they or their ancestors are explicitly named - in the logging configuration. - - :param encoding: The encoding used to open file when *fname* is filename. - - .. versionchanged:: 3.4 - An instance of a subclass of :class:`~configparser.RawConfigParser` is - now accepted as a value for ``fname``. This facilitates: - - * Use of a configuration file where logging configuration is just part - of the overall application configuration. - * Use of a configuration read from a file, and then modified by the using - application (e.g. based on command-line parameters or other aspects - of the runtime environment) before being passed to ``fileConfig``. - - .. versionadded:: 3.10 - The *encoding* parameter is added. - - .. versionchanged:: 3.11.4 - An exception will be thrown if the provided file - doesn't exist or is invalid or empty. - -.. function:: listen(port=DEFAULT_LOGGING_CONFIG_PORT, verify=None) - - Starts up a socket server on the specified port, and listens for new - configurations. If no port is specified, the module's default - :const:`DEFAULT_LOGGING_CONFIG_PORT` is used. Logging configurations will be - sent as a file suitable for processing by :func:`dictConfig` or - :func:`fileConfig`. Returns a :class:`~threading.Thread` instance on which - you can call :meth:`~threading.Thread.start` to start the server, and which - you can :meth:`~threading.Thread.join` when appropriate. To stop the server, - call :func:`stopListening`. - - The ``verify`` argument, if specified, should be a callable which should - verify whether bytes received across the socket are valid and should be - processed. This could be done by encrypting and/or signing what is sent - across the socket, such that the ``verify`` callable can perform - signature verification and/or decryption. The ``verify`` callable is called - with a single argument - the bytes received across the socket - and should - return the bytes to be processed, or ``None`` to indicate that the bytes should - be discarded. The returned bytes could be the same as the passed in bytes - (e.g. when only verification is done), or they could be completely different - (perhaps if decryption were performed). - - To send a configuration to the socket, read in the configuration file and - send it to the socket as a sequence of bytes preceded by a four-byte length - string packed in binary using ``struct.pack('>L', n)``. - - .. _logging-eval-security: - - .. note:: - - Because portions of the configuration are passed through - :func:`eval`, use of this function may open its users to a security risk. - While the function only binds to a socket on ``localhost``, and so does - not accept connections from remote machines, there are scenarios where - untrusted code could be run under the account of the process which calls - :func:`listen`. Specifically, if the process calling :func:`listen` runs - on a multi-user machine where users cannot trust each other, then a - malicious user could arrange to run essentially arbitrary code in a - victim user's process, simply by connecting to the victim's - :func:`listen` socket and sending a configuration which runs whatever - code the attacker wants to have executed in the victim's process. This is - especially easy to do if the default port is used, but not hard even if a - different port is used. To avoid the risk of this happening, use the - ``verify`` argument to :func:`listen` to prevent unrecognised - configurations from being applied. - - .. versionchanged:: 3.4 - The ``verify`` argument was added. - - .. note:: - - If you want to send configurations to the listener which don't - disable existing loggers, you will need to use a JSON format for - the configuration, which will use :func:`dictConfig` for configuration. - This method allows you to specify ``disable_existing_loggers`` as - ``False`` in the configuration you send. - - -.. function:: stopListening() - - Stops the listening server which was created with a call to :func:`listen`. - This is typically called before calling :meth:`join` on the return value from - :func:`listen`. - - -Security considerations -^^^^^^^^^^^^^^^^^^^^^^^ - -The logging configuration functionality tries to offer convenience, and in part this -is done by offering the ability to convert text in configuration files into Python -objects used in logging configuration - for example, as described in -:ref:`logging-config-dict-userdef`. However, these same mechanisms (importing -callables from user-defined modules and calling them with parameters from the -configuration) could be used to invoke any code you like, and for this reason you -should treat configuration files from untrusted sources with *extreme caution* and -satisfy yourself that nothing bad can happen if you load them, before actually loading -them. - - -.. _logging-config-dictschema: - -Configuration dictionary schema -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Describing a logging configuration requires listing the various -objects to create and the connections between them; for example, you -may create a handler named 'console' and then say that the logger -named 'startup' will send its messages to the 'console' handler. -These objects aren't limited to those provided by the :mod:`logging` -module because you might write your own formatter or handler class. -The parameters to these classes may also need to include external -objects such as ``sys.stderr``. The syntax for describing these -objects and connections is defined in :ref:`logging-config-dict-connections` -below. - -Dictionary Schema Details -""""""""""""""""""""""""" - -The dictionary passed to :func:`dictConfig` must contain the following -keys: - -* *version* - to be set to an integer value representing the schema - version. The only valid value at present is 1, but having this key - allows the schema to evolve while still preserving backwards - compatibility. - -All other keys are optional, but if present they will be interpreted -as described below. In all cases below where a 'configuring dict' is -mentioned, it will be checked for the special ``'()'`` key to see if a -custom instantiation is required. If so, the mechanism described in -:ref:`logging-config-dict-userdef` below is used to create an instance; -otherwise, the context is used to determine what to instantiate. - -.. _logging-config-dictschema-formatters: - -* *formatters* - the corresponding value will be a dict in which each - key is a formatter id and each value is a dict describing how to - configure the corresponding :class:`~logging.Formatter` instance. - - The configuring dict is searched for the following optional keys - which correspond to the arguments passed to create a - :class:`~logging.Formatter` object: - - * ``format`` - * ``datefmt`` - * ``style`` - * ``validate`` (since version >=3.8) - - An optional ``class`` key indicates the name of the formatter's - class (as a dotted module and class name). The instantiation - arguments are as for :class:`~logging.Formatter`, thus this key is - most useful for instantiating a customised subclass of - :class:`~logging.Formatter`. For example, the alternative class - might present exception tracebacks in an expanded or condensed - format. If your formatter requires different or extra configuration - keys, you should use :ref:`logging-config-dict-userdef`. - -* *filters* - the corresponding value will be a dict in which each key - is a filter id and each value is a dict describing how to configure - the corresponding Filter instance. - - The configuring dict is searched for the key ``name`` (defaulting to the - empty string) and this is used to construct a :class:`logging.Filter` - instance. - -* *handlers* - the corresponding value will be a dict in which each - key is a handler id and each value is a dict describing how to - configure the corresponding Handler instance. - - The configuring dict is searched for the following keys: - - * ``class`` (mandatory). This is the fully qualified name of the - handler class. - - * ``level`` (optional). The level of the handler. - - * ``formatter`` (optional). The id of the formatter for this - handler. - - * ``filters`` (optional). A list of ids of the filters for this - handler. - - .. versionchanged:: 3.11 - ``filters`` can take filter instances in addition to ids. - - All *other* keys are passed through as keyword arguments to the - handler's constructor. For example, given the snippet: - - .. code-block:: yaml - - handlers: - console: - class : logging.StreamHandler - formatter: brief - level : INFO - filters: [allow_foo] - stream : ext://sys.stdout - file: - class : logging.handlers.RotatingFileHandler - formatter: precise - filename: logconfig.log - maxBytes: 1024 - backupCount: 3 - - the handler with id ``console`` is instantiated as a - :class:`logging.StreamHandler`, using ``sys.stdout`` as the underlying - stream. The handler with id ``file`` is instantiated as a - :class:`logging.handlers.RotatingFileHandler` with the keyword arguments - ``filename='logconfig.log', maxBytes=1024, backupCount=3``. - -* *loggers* - the corresponding value will be a dict in which each key - is a logger name and each value is a dict describing how to - configure the corresponding Logger instance. - - The configuring dict is searched for the following keys: - - * ``level`` (optional). The level of the logger. - - * ``propagate`` (optional). The propagation setting of the logger. - - * ``filters`` (optional). A list of ids of the filters for this - logger. - - .. versionchanged:: 3.11 - ``filters`` can take filter instances in addition to ids. - - * ``handlers`` (optional). A list of ids of the handlers for this - logger. - - The specified loggers will be configured according to the level, - propagation, filters and handlers specified. - -* *root* - this will be the configuration for the root logger. - Processing of the configuration will be as for any logger, except - that the ``propagate`` setting will not be applicable. - -* *incremental* - whether the configuration is to be interpreted as - incremental to the existing configuration. This value defaults to - ``False``, which means that the specified configuration replaces the - existing configuration with the same semantics as used by the - existing :func:`fileConfig` API. - - If the specified value is ``True``, the configuration is processed - as described in the section on :ref:`logging-config-dict-incremental`. - -* *disable_existing_loggers* - whether any existing non-root loggers are - to be disabled. This setting mirrors the parameter of the same name in - :func:`fileConfig`. If absent, this parameter defaults to ``True``. - This value is ignored if *incremental* is ``True``. - -.. _logging-config-dict-incremental: - -Incremental Configuration -""""""""""""""""""""""""" - -It is difficult to provide complete flexibility for incremental -configuration. For example, because objects such as filters -and formatters are anonymous, once a configuration is set up, it is -not possible to refer to such anonymous objects when augmenting a -configuration. - -Furthermore, there is not a compelling case for arbitrarily altering -the object graph of loggers, handlers, filters, formatters at -run-time, once a configuration is set up; the verbosity of loggers and -handlers can be controlled just by setting levels (and, in the case of -loggers, propagation flags). Changing the object graph arbitrarily in -a safe way is problematic in a multi-threaded environment; while not -impossible, the benefits are not worth the complexity it adds to the -implementation. - -Thus, when the ``incremental`` key of a configuration dict is present -and is ``True``, the system will completely ignore any ``formatters`` and -``filters`` entries, and process only the ``level`` -settings in the ``handlers`` entries, and the ``level`` and -``propagate`` settings in the ``loggers`` and ``root`` entries. - -Using a value in the configuration dict lets configurations to be sent -over the wire as pickled dicts to a socket listener. Thus, the logging -verbosity of a long-running application can be altered over time with -no need to stop and restart the application. - -.. _logging-config-dict-connections: - -Object connections -"""""""""""""""""" - -The schema describes a set of logging objects - loggers, -handlers, formatters, filters - which are connected to each other in -an object graph. Thus, the schema needs to represent connections -between the objects. For example, say that, once configured, a -particular logger has attached to it a particular handler. For the -purposes of this discussion, we can say that the logger represents the -source, and the handler the destination, of a connection between the -two. Of course in the configured objects this is represented by the -logger holding a reference to the handler. In the configuration dict, -this is done by giving each destination object an id which identifies -it unambiguously, and then using the id in the source object's -configuration to indicate that a connection exists between the source -and the destination object with that id. - -So, for example, consider the following YAML snippet: - -.. code-block:: yaml - - formatters: - brief: - # configuration for formatter with id 'brief' goes here - precise: - # configuration for formatter with id 'precise' goes here - handlers: - h1: #This is an id - # configuration of handler with id 'h1' goes here - formatter: brief - h2: #This is another id - # configuration of handler with id 'h2' goes here - formatter: precise - loggers: - foo.bar.baz: - # other configuration for logger 'foo.bar.baz' - handlers: [h1, h2] - -(Note: YAML used here because it's a little more readable than the -equivalent Python source form for the dictionary.) - -The ids for loggers are the logger names which would be used -programmatically to obtain a reference to those loggers, e.g. -``foo.bar.baz``. The ids for Formatters and Filters can be any string -value (such as ``brief``, ``precise`` above) and they are transient, -in that they are only meaningful for processing the configuration -dictionary and used to determine connections between objects, and are -not persisted anywhere when the configuration call is complete. - -The above snippet indicates that logger named ``foo.bar.baz`` should -have two handlers attached to it, which are described by the handler -ids ``h1`` and ``h2``. The formatter for ``h1`` is that described by id -``brief``, and the formatter for ``h2`` is that described by id -``precise``. - - -.. _logging-config-dict-userdef: - -User-defined objects -"""""""""""""""""""" - -The schema supports user-defined objects for handlers, filters and -formatters. (Loggers do not need to have different types for -different instances, so there is no support in this configuration -schema for user-defined logger classes.) - -Objects to be configured are described by dictionaries -which detail their configuration. In some places, the logging system -will be able to infer from the context how an object is to be -instantiated, but when a user-defined object is to be instantiated, -the system will not know how to do this. In order to provide complete -flexibility for user-defined object instantiation, the user needs -to provide a 'factory' - a callable which is called with a -configuration dictionary and which returns the instantiated object. -This is signalled by an absolute import path to the factory being -made available under the special key ``'()'``. Here's a concrete -example: - -.. code-block:: yaml - - formatters: - brief: - format: '%(message)s' - default: - format: '%(asctime)s %(levelname)-8s %(name)-15s %(message)s' - datefmt: '%Y-%m-%d %H:%M:%S' - custom: - (): my.package.customFormatterFactory - bar: baz - spam: 99.9 - answer: 42 - -The above YAML snippet defines three formatters. The first, with id -``brief``, is a standard :class:`logging.Formatter` instance with the -specified format string. The second, with id ``default``, has a -longer format and also defines the time format explicitly, and will -result in a :class:`logging.Formatter` initialized with those two format -strings. Shown in Python source form, the ``brief`` and ``default`` -formatters have configuration sub-dictionaries:: - - { - 'format' : '%(message)s' - } - -and:: - - { - 'format' : '%(asctime)s %(levelname)-8s %(name)-15s %(message)s', - 'datefmt' : '%Y-%m-%d %H:%M:%S' - } - -respectively, and as these dictionaries do not contain the special key -``'()'``, the instantiation is inferred from the context: as a result, -standard :class:`logging.Formatter` instances are created. The -configuration sub-dictionary for the third formatter, with id -``custom``, is:: - - { - '()' : 'my.package.customFormatterFactory', - 'bar' : 'baz', - 'spam' : 99.9, - 'answer' : 42 - } - -and this contains the special key ``'()'``, which means that -user-defined instantiation is wanted. In this case, the specified -factory callable will be used. If it is an actual callable it will be -used directly - otherwise, if you specify a string (as in the example) -the actual callable will be located using normal import mechanisms. -The callable will be called with the **remaining** items in the -configuration sub-dictionary as keyword arguments. In the above -example, the formatter with id ``custom`` will be assumed to be -returned by the call:: - - my.package.customFormatterFactory(bar='baz', spam=99.9, answer=42) - -.. warning:: The values for keys such as ``bar``, ``spam`` and ``answer`` in - the above example should not be configuration dictionaries or references such - as ``cfg://foo`` or ``ext://bar``, because they will not be processed by the - configuration machinery, but passed to the callable as-is. - -The key ``'()'`` has been used as the special key because it is not a -valid keyword parameter name, and so will not clash with the names of -the keyword arguments used in the call. The ``'()'`` also serves as a -mnemonic that the corresponding value is a callable. - -.. versionchanged:: 3.11 - The ``filters`` member of ``handlers`` and ``loggers`` can take - filter instances in addition to ids. - -You can also specify a special key ``'.'`` whose value is a dictionary is a -mapping of attribute names to values. If found, the specified attributes will -be set on the user-defined object before it is returned. Thus, with the -following configuration:: - - { - '()' : 'my.package.customFormatterFactory', - 'bar' : 'baz', - 'spam' : 99.9, - 'answer' : 42, - '.' { - 'foo': 'bar', - 'baz': 'bozz' - } - } - -the returned formatter will have attribute ``foo`` set to ``'bar'`` and -attribute ``baz`` set to ``'bozz'``. - -.. warning:: The values for attributes such as ``foo`` and ``baz`` in - the above example should not be configuration dictionaries or references such - as ``cfg://foo`` or ``ext://bar``, because they will not be processed by the - configuration machinery, but set as attribute values as-is. - - -.. _handler-config-dict-order: - -Handler configuration order -""""""""""""""""""""""""""" - -Handlers are configured in alphabetical order of their keys, and a configured -handler replaces the configuration dictionary in (a working copy of) the -``handlers`` dictionary in the schema. If you use a construct such as -``cfg://handlers.foo``, then initially ``handlers['foo']`` points to the -configuration dictionary for the handler named ``foo``, and later (once that -handler has been configured) it points to the configured handler instance. -Thus, ``cfg://handlers.foo`` could resolve to either a dictionary or a handler -instance. In general, it is wise to name handlers in a way such that dependent -handlers are configured _after_ any handlers they depend on; that allows -something like ``cfg://handlers.foo`` to be used in configuring a handler that -depends on handler ``foo``. If that dependent handler were named ``bar``, -problems would result, because the configuration of ``bar`` would be attempted -before that of ``foo``, and ``foo`` would not yet have been configured. -However, if the dependent handler were named ``foobar``, it would be configured -after ``foo``, with the result that ``cfg://handlers.foo`` would resolve to -configured handler ``foo``, and not its configuration dictionary. - - -.. _logging-config-dict-externalobj: - -Access to external objects -"""""""""""""""""""""""""" - -There are times where a configuration needs to refer to objects -external to the configuration, for example ``sys.stderr``. If the -configuration dict is constructed using Python code, this is -straightforward, but a problem arises when the configuration is -provided via a text file (e.g. JSON, YAML). In a text file, there is -no standard way to distinguish ``sys.stderr`` from the literal string -``'sys.stderr'``. To facilitate this distinction, the configuration -system looks for certain special prefixes in string values and -treat them specially. For example, if the literal string -``'ext://sys.stderr'`` is provided as a value in the configuration, -then the ``ext://`` will be stripped off and the remainder of the -value processed using normal import mechanisms. - -The handling of such prefixes is done in a way analogous to protocol -handling: there is a generic mechanism to look for prefixes which -match the regular expression ``^(?P[a-z]+)://(?P.*)$`` -whereby, if the ``prefix`` is recognised, the ``suffix`` is processed -in a prefix-dependent manner and the result of the processing replaces -the string value. If the prefix is not recognised, then the string -value will be left as-is. - - -.. _logging-config-dict-internalobj: - -Access to internal objects -"""""""""""""""""""""""""" - -As well as external objects, there is sometimes also a need to refer -to objects in the configuration. This will be done implicitly by the -configuration system for things that it knows about. For example, the -string value ``'DEBUG'`` for a ``level`` in a logger or handler will -automatically be converted to the value ``logging.DEBUG``, and the -``handlers``, ``filters`` and ``formatter`` entries will take an -object id and resolve to the appropriate destination object. - -However, a more generic mechanism is needed for user-defined -objects which are not known to the :mod:`logging` module. For -example, consider :class:`logging.handlers.MemoryHandler`, which takes -a ``target`` argument which is another handler to delegate to. Since -the system already knows about this class, then in the configuration, -the given ``target`` just needs to be the object id of the relevant -target handler, and the system will resolve to the handler from the -id. If, however, a user defines a ``my.package.MyHandler`` which has -an ``alternate`` handler, the configuration system would not know that -the ``alternate`` referred to a handler. To cater for this, a generic -resolution system allows the user to specify: - -.. code-block:: yaml - - handlers: - file: - # configuration of file handler goes here - - custom: - (): my.package.MyHandler - alternate: cfg://handlers.file - -The literal string ``'cfg://handlers.file'`` will be resolved in an -analogous way to strings with the ``ext://`` prefix, but looking -in the configuration itself rather than the import namespace. The -mechanism allows access by dot or by index, in a similar way to -that provided by ``str.format``. Thus, given the following snippet: - -.. code-block:: yaml - - handlers: - email: - class: logging.handlers.SMTPHandler - mailhost: localhost - fromaddr: my_app@domain.tld - toaddrs: - - support_team@domain.tld - - dev_team@domain.tld - subject: Houston, we have a problem. - -in the configuration, the string ``'cfg://handlers'`` would resolve to -the dict with key ``handlers``, the string ``'cfg://handlers.email`` -would resolve to the dict with key ``email`` in the ``handlers`` dict, -and so on. The string ``'cfg://handlers.email.toaddrs[1]`` would -resolve to ``'dev_team@domain.tld'`` and the string -``'cfg://handlers.email.toaddrs[0]'`` would resolve to the value -``'support_team@domain.tld'``. The ``subject`` value could be accessed -using either ``'cfg://handlers.email.subject'`` or, equivalently, -``'cfg://handlers.email[subject]'``. The latter form only needs to be -used if the key contains spaces or non-alphanumeric characters. If an -index value consists only of decimal digits, access will be attempted -using the corresponding integer value, falling back to the string -value if needed. - -Given a string ``cfg://handlers.myhandler.mykey.123``, this will -resolve to ``config_dict['handlers']['myhandler']['mykey']['123']``. -If the string is specified as ``cfg://handlers.myhandler.mykey[123]``, -the system will attempt to retrieve the value from -``config_dict['handlers']['myhandler']['mykey'][123]``, and fall back -to ``config_dict['handlers']['myhandler']['mykey']['123']`` if that -fails. - - -.. _logging-import-resolution: - -Import resolution and custom importers -"""""""""""""""""""""""""""""""""""""" - -Import resolution, by default, uses the builtin :func:`__import__` function -to do its importing. You may want to replace this with your own importing -mechanism: if so, you can replace the :attr:`importer` attribute of the -:class:`DictConfigurator` or its superclass, the -:class:`BaseConfigurator` class. However, you need to be -careful because of the way functions are accessed from classes via -descriptors. If you are using a Python callable to do your imports, and you -want to define it at class level rather than instance level, you need to wrap -it with :func:`staticmethod`. For example:: - - from importlib import import_module - from logging.config import BaseConfigurator - - BaseConfigurator.importer = staticmethod(import_module) - -You don't need to wrap with :func:`staticmethod` if you're setting the import -callable on a configurator *instance*. - - -.. _logging-config-fileformat: - -Configuration file format -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The configuration file format understood by :func:`fileConfig` is based on -:mod:`configparser` functionality. The file must contain sections called -``[loggers]``, ``[handlers]`` and ``[formatters]`` which identify by name the -entities of each type which are defined in the file. For each such entity, there -is a separate section which identifies how that entity is configured. Thus, for -a logger named ``log01`` in the ``[loggers]`` section, the relevant -configuration details are held in a section ``[logger_log01]``. Similarly, a -handler called ``hand01`` in the ``[handlers]`` section will have its -configuration held in a section called ``[handler_hand01]``, while a formatter -called ``form01`` in the ``[formatters]`` section will have its configuration -specified in a section called ``[formatter_form01]``. The root logger -configuration must be specified in a section called ``[logger_root]``. - -.. note:: - - The :func:`fileConfig` API is older than the :func:`dictConfig` API and does - not provide functionality to cover certain aspects of logging. For example, - you cannot configure :class:`~logging.Filter` objects, which provide for - filtering of messages beyond simple integer levels, using :func:`fileConfig`. - If you need to have instances of :class:`~logging.Filter` in your logging - configuration, you will need to use :func:`dictConfig`. Note that future - enhancements to configuration functionality will be added to - :func:`dictConfig`, so it's worth considering transitioning to this newer - API when it's convenient to do so. - -Examples of these sections in the file are given below. - -.. code-block:: ini - - [loggers] - keys=root,log02,log03,log04,log05,log06,log07 - - [handlers] - keys=hand01,hand02,hand03,hand04,hand05,hand06,hand07,hand08,hand09 - - [formatters] - keys=form01,form02,form03,form04,form05,form06,form07,form08,form09 - -The root logger must specify a level and a list of handlers. An example of a -root logger section is given below. - -.. code-block:: ini - - [logger_root] - level=NOTSET - handlers=hand01 - -The ``level`` entry can be one of ``DEBUG, INFO, WARNING, ERROR, CRITICAL`` or -``NOTSET``. For the root logger only, ``NOTSET`` means that all messages will be -logged. Level values are :ref:`evaluated ` in the context of the ``logging`` -package's namespace. - -The ``handlers`` entry is a comma-separated list of handler names, which must -appear in the ``[handlers]`` section. These names must appear in the -``[handlers]`` section and have corresponding sections in the configuration -file. - -For loggers other than the root logger, some additional information is required. -This is illustrated by the following example. - -.. code-block:: ini - - [logger_parser] - level=DEBUG - handlers=hand01 - propagate=1 - qualname=compiler.parser - -The ``level`` and ``handlers`` entries are interpreted as for the root logger, -except that if a non-root logger's level is specified as ``NOTSET``, the system -consults loggers higher up the hierarchy to determine the effective level of the -logger. The ``propagate`` entry is set to 1 to indicate that messages must -propagate to handlers higher up the logger hierarchy from this logger, or 0 to -indicate that messages are **not** propagated to handlers up the hierarchy. The -``qualname`` entry is the hierarchical channel name of the logger, that is to -say the name used by the application to get the logger. - -Sections which specify handler configuration are exemplified by the following. - -.. code-block:: ini - - [handler_hand01] - class=StreamHandler - level=NOTSET - formatter=form01 - args=(sys.stdout,) - -The ``class`` entry indicates the handler's class (as determined by :func:`eval` -in the ``logging`` package's namespace). The ``level`` is interpreted as for -loggers, and ``NOTSET`` is taken to mean 'log everything'. - -The ``formatter`` entry indicates the key name of the formatter for this -handler. If blank, a default formatter (``logging._defaultFormatter``) is used. -If a name is specified, it must appear in the ``[formatters]`` section and have -a corresponding section in the configuration file. - -The ``args`` entry, when :ref:`evaluated ` in the context of the ``logging`` -package's namespace, is the list of arguments to the constructor for the handler -class. Refer to the constructors for the relevant handlers, or to the examples -below, to see how typical entries are constructed. If not provided, it defaults -to ``()``. - -The optional ``kwargs`` entry, when :ref:`evaluated ` in the context of the -``logging`` package's namespace, is the keyword argument dict to the constructor -for the handler class. If not provided, it defaults to ``{}``. - -.. code-block:: ini - - [handler_hand02] - class=FileHandler - level=DEBUG - formatter=form02 - args=('python.log', 'w') - - [handler_hand03] - class=handlers.SocketHandler - level=INFO - formatter=form03 - args=('localhost', handlers.DEFAULT_TCP_LOGGING_PORT) - - [handler_hand04] - class=handlers.DatagramHandler - level=WARN - formatter=form04 - args=('localhost', handlers.DEFAULT_UDP_LOGGING_PORT) - - [handler_hand05] - class=handlers.SysLogHandler - level=ERROR - formatter=form05 - args=(('localhost', handlers.SYSLOG_UDP_PORT), handlers.SysLogHandler.LOG_USER) - - [handler_hand06] - class=handlers.NTEventLogHandler - level=CRITICAL - formatter=form06 - args=('Python Application', '', 'Application') - - [handler_hand07] - class=handlers.SMTPHandler - level=WARN - formatter=form07 - args=('localhost', 'from@abc', ['user1@abc', 'user2@xyz'], 'Logger Subject') - kwargs={'timeout': 10.0} - - [handler_hand08] - class=handlers.MemoryHandler - level=NOTSET - formatter=form08 - target= - args=(10, ERROR) - - [handler_hand09] - class=handlers.HTTPHandler - level=NOTSET - formatter=form09 - args=('localhost:9022', '/log', 'GET') - kwargs={'secure': True} - -Sections which specify formatter configuration are typified by the following. - -.. code-block:: ini - - [formatter_form01] - format=F1 %(asctime)s %(levelname)s %(message)s - datefmt= - style=% - validate=True - class=logging.Formatter - -The arguments for the formatter configuration are the same as the keys -in the dictionary schema :ref:`formatters section -`. - -.. note:: - - Due to the use of :func:`eval` as described above, there are - potential security risks which result from using the :func:`listen` to send - and receive configurations via sockets. The risks are limited to where - multiple users with no mutual trust run code on the same machine; see the - :func:`listen` documentation for more information. - -.. seealso:: - - Module :mod:`logging` - API reference for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. diff --git a/Doc/library/logging.handlers.rst b/Doc/library/logging.handlers.rst deleted file mode 100644 index bb30f716dd3d39..00000000000000 --- a/Doc/library/logging.handlers.rst +++ /dev/null @@ -1,1195 +0,0 @@ -:mod:`logging.handlers` --- Logging handlers -============================================ - -.. module:: logging.handlers - :synopsis: Handlers for the logging module. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -**Source code:** :source:`Lib/logging/handlers.py` - -.. sidebar:: Important - - This page contains only reference information. For tutorials, - please see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Logging Cookbook ` - --------------- - -.. currentmodule:: logging - -The following useful handlers are provided in the package. Note that three of -the handlers (:class:`StreamHandler`, :class:`FileHandler` and -:class:`NullHandler`) are actually defined in the :mod:`logging` module itself, -but have been documented here along with the other handlers. - -.. _stream-handler: - -StreamHandler -^^^^^^^^^^^^^ - -The :class:`StreamHandler` class, located in the core :mod:`logging` package, -sends logging output to streams such as *sys.stdout*, *sys.stderr* or any -file-like object (or, more precisely, any object which supports :meth:`write` -and :meth:`flush` methods). - - -.. class:: StreamHandler(stream=None) - - Returns a new instance of the :class:`StreamHandler` class. If *stream* is - specified, the instance will use it for logging output; otherwise, *sys.stderr* - will be used. - - - .. method:: emit(record) - - If a formatter is specified, it is used to format the record. The record - is then written to the stream followed by :attr:`terminator`. If exception information - is present, it is formatted using :func:`traceback.print_exception` and - appended to the stream. - - - .. method:: flush() - - Flushes the stream by calling its :meth:`flush` method. Note that the - :meth:`close` method is inherited from :class:`~logging.Handler` and so - does no output, so an explicit :meth:`flush` call may be needed at times. - - .. method:: setStream(stream) - - Sets the instance's stream to the specified value, if it is different. - The old stream is flushed before the new stream is set. - - :param stream: The stream that the handler should use. - - :return: the old stream, if the stream was changed, or *None* if it wasn't. - - .. versionadded:: 3.7 - - .. attribute:: terminator - - String used as the terminator when writing a formatted record to a stream. - Default value is ``'\n'``. - - If you don't want a newline termination, you can set the handler instance's - ``terminator`` attribute to the empty string. - - In earlier versions, the terminator was hardcoded as ``'\n'``. - - .. versionadded:: 3.2 - - -.. _file-handler: - -FileHandler -^^^^^^^^^^^ - -The :class:`FileHandler` class, located in the core :mod:`logging` package, -sends logging output to a disk file. It inherits the output functionality from -:class:`StreamHandler`. - - -.. class:: FileHandler(filename, mode='a', encoding=None, delay=False, errors=None) - - Returns a new instance of the :class:`FileHandler` class. The specified file is - opened and used as the stream for logging. If *mode* is not specified, - ``'a'`` is used. If *encoding* is not ``None``, it is used to open the file - with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. If - *errors* is specified, it's used to determine how encoding errors are handled. - - .. versionchanged:: 3.6 - As well as string values, :class:`~pathlib.Path` objects are also accepted - for the *filename* argument. - - .. versionchanged:: 3.9 - The *errors* parameter was added. - - .. method:: close() - - Closes the file. - - .. method:: emit(record) - - Outputs the record to the file. - - Note that if the file was closed due to logging shutdown at exit and the file - mode is 'w', the record will not be emitted (see :issue:`42378`). - - -.. _null-handler: - -NullHandler -^^^^^^^^^^^ - -.. versionadded:: 3.1 - -The :class:`NullHandler` class, located in the core :mod:`logging` package, -does not do any formatting or output. It is essentially a 'no-op' handler -for use by library developers. - -.. class:: NullHandler() - - Returns a new instance of the :class:`NullHandler` class. - - .. method:: emit(record) - - This method does nothing. - - .. method:: handle(record) - - This method does nothing. - - .. method:: createLock() - - This method returns ``None`` for the lock, since there is no - underlying I/O to which access needs to be serialized. - - -See :ref:`library-config` for more information on how to use -:class:`NullHandler`. - -.. _watched-file-handler: - -WatchedFileHandler -^^^^^^^^^^^^^^^^^^ - -.. currentmodule:: logging.handlers - -The :class:`WatchedFileHandler` class, located in the :mod:`logging.handlers` -module, is a :class:`FileHandler` which watches the file it is logging to. If -the file changes, it is closed and reopened using the file name. - -A file change can happen because of usage of programs such as *newsyslog* and -*logrotate* which perform log file rotation. This handler, intended for use -under Unix/Linux, watches the file to see if it has changed since the last emit. -(A file is deemed to have changed if its device or inode have changed.) If the -file has changed, the old file stream is closed, and the file opened to get a -new stream. - -This handler is not appropriate for use under Windows, because under Windows -open log files cannot be moved or renamed - logging opens the files with -exclusive locks - and so there is no need for such a handler. Furthermore, -*ST_INO* is not supported under Windows; :func:`~os.stat` always returns zero -for this value. - - -.. class:: WatchedFileHandler(filename, mode='a', encoding=None, delay=False, errors=None) - - Returns a new instance of the :class:`WatchedFileHandler` class. The specified - file is opened and used as the stream for logging. If *mode* is not specified, - ``'a'`` is used. If *encoding* is not ``None``, it is used to open the file - with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. If - *errors* is provided, it determines how encoding errors are handled. - - .. versionchanged:: 3.6 - As well as string values, :class:`~pathlib.Path` objects are also accepted - for the *filename* argument. - - .. versionchanged:: 3.9 - The *errors* parameter was added. - - .. method:: reopenIfNeeded() - - Checks to see if the file has changed. If it has, the existing stream is - flushed and closed and the file opened again, typically as a precursor to - outputting the record to the file. - - .. versionadded:: 3.6 - - - .. method:: emit(record) - - Outputs the record to the file, but first calls :meth:`reopenIfNeeded` to - reopen the file if it has changed. - -.. _base-rotating-handler: - -BaseRotatingHandler -^^^^^^^^^^^^^^^^^^^ - -The :class:`BaseRotatingHandler` class, located in the :mod:`logging.handlers` -module, is the base class for the rotating file handlers, -:class:`RotatingFileHandler` and :class:`TimedRotatingFileHandler`. You should -not need to instantiate this class, but it has attributes and methods you may -need to override. - -.. class:: BaseRotatingHandler(filename, mode, encoding=None, delay=False, errors=None) - - The parameters are as for :class:`FileHandler`. The attributes are: - - .. attribute:: namer - - If this attribute is set to a callable, the :meth:`rotation_filename` - method delegates to this callable. The parameters passed to the callable - are those passed to :meth:`rotation_filename`. - - .. note:: The namer function is called quite a few times during rollover, - so it should be as simple and as fast as possible. It should also - return the same output every time for a given input, otherwise the - rollover behaviour may not work as expected. - - It's also worth noting that care should be taken when using a namer to - preserve certain attributes in the filename which are used during rotation. - For example, :class:`RotatingFileHandler` expects to have a set of log files - whose names contain successive integers, so that rotation works as expected, - and :class:`TimedRotatingFileHandler` deletes old log files (based on the - ``backupCount`` parameter passed to the handler's initializer) by determining - the oldest files to delete. For this to happen, the filenames should be - sortable using the date/time portion of the filename, and a namer needs to - respect this. (If a namer is wanted that doesn't respect this scheme, it will - need to be used in a subclass of :class:`TimedRotatingFileHandler` which - overrides the :meth:`~TimedRotatingFileHandler.getFilesToDelete` method to - fit in with the custom naming scheme.) - - .. versionadded:: 3.3 - - - .. attribute:: BaseRotatingHandler.rotator - - If this attribute is set to a callable, the :meth:`rotate` method - delegates to this callable. The parameters passed to the callable are - those passed to :meth:`rotate`. - - .. versionadded:: 3.3 - - .. method:: BaseRotatingHandler.rotation_filename(default_name) - - Modify the filename of a log file when rotating. - - This is provided so that a custom filename can be provided. - - The default implementation calls the 'namer' attribute of the handler, - if it's callable, passing the default name to it. If the attribute isn't - callable (the default is ``None``), the name is returned unchanged. - - :param default_name: The default name for the log file. - - .. versionadded:: 3.3 - - - .. method:: BaseRotatingHandler.rotate(source, dest) - - When rotating, rotate the current log. - - The default implementation calls the 'rotator' attribute of the handler, - if it's callable, passing the source and dest arguments to it. If the - attribute isn't callable (the default is ``None``), the source is simply - renamed to the destination. - - :param source: The source filename. This is normally the base - filename, e.g. 'test.log'. - :param dest: The destination filename. This is normally - what the source is rotated to, e.g. 'test.log.1'. - - .. versionadded:: 3.3 - -The reason the attributes exist is to save you having to subclass - you can use -the same callables for instances of :class:`RotatingFileHandler` and -:class:`TimedRotatingFileHandler`. If either the namer or rotator callable -raises an exception, this will be handled in the same way as any other -exception during an :meth:`emit` call, i.e. via the :meth:`handleError` method -of the handler. - -If you need to make more significant changes to rotation processing, you can -override the methods. - -For an example, see :ref:`cookbook-rotator-namer`. - - -.. _rotating-file-handler: - -RotatingFileHandler -^^^^^^^^^^^^^^^^^^^ - -The :class:`RotatingFileHandler` class, located in the :mod:`logging.handlers` -module, supports rotation of disk log files. - - -.. class:: RotatingFileHandler(filename, mode='a', maxBytes=0, backupCount=0, encoding=None, delay=False, errors=None) - - Returns a new instance of the :class:`RotatingFileHandler` class. The specified - file is opened and used as the stream for logging. If *mode* is not specified, - ``'a'`` is used. If *encoding* is not ``None``, it is used to open the file - with that encoding. If *delay* is true, then file opening is deferred until the - first call to :meth:`emit`. By default, the file grows indefinitely. If - *errors* is provided, it determines how encoding errors are handled. - - You can use the *maxBytes* and *backupCount* values to allow the file to - :dfn:`rollover` at a predetermined size. When the size is about to be exceeded, - the file is closed and a new file is silently opened for output. Rollover occurs - whenever the current log file is nearly *maxBytes* in length; but if either of - *maxBytes* or *backupCount* is zero, rollover never occurs, so you generally want - to set *backupCount* to at least 1, and have a non-zero *maxBytes*. - When *backupCount* is non-zero, the system will save old log files by appending - the extensions '.1', '.2' etc., to the filename. For example, with a *backupCount* - of 5 and a base file name of :file:`app.log`, you would get :file:`app.log`, - :file:`app.log.1`, :file:`app.log.2`, up to :file:`app.log.5`. The file being - written to is always :file:`app.log`. When this file is filled, it is closed - and renamed to :file:`app.log.1`, and if files :file:`app.log.1`, - :file:`app.log.2`, etc. exist, then they are renamed to :file:`app.log.2`, - :file:`app.log.3` etc. respectively. - - .. versionchanged:: 3.6 - As well as string values, :class:`~pathlib.Path` objects are also accepted - for the *filename* argument. - - .. versionchanged:: 3.9 - The *errors* parameter was added. - - .. method:: doRollover() - - Does a rollover, as described above. - - - .. method:: emit(record) - - Outputs the record to the file, catering for rollover as described - previously. - -.. _timed-rotating-file-handler: - -TimedRotatingFileHandler -^^^^^^^^^^^^^^^^^^^^^^^^ - -The :class:`TimedRotatingFileHandler` class, located in the -:mod:`logging.handlers` module, supports rotation of disk log files at certain -timed intervals. - - -.. class:: TimedRotatingFileHandler(filename, when='h', interval=1, backupCount=0, encoding=None, delay=False, utc=False, atTime=None, errors=None) - - Returns a new instance of the :class:`TimedRotatingFileHandler` class. The - specified file is opened and used as the stream for logging. On rotating it also - sets the filename suffix. Rotating happens based on the product of *when* and - *interval*. - - You can use the *when* to specify the type of *interval*. The list of possible - values is below. Note that they are not case sensitive. - - +----------------+----------------------------+-------------------------+ - | Value | Type of interval | If/how *atTime* is used | - +================+============================+=========================+ - | ``'S'`` | Seconds | Ignored | - +----------------+----------------------------+-------------------------+ - | ``'M'`` | Minutes | Ignored | - +----------------+----------------------------+-------------------------+ - | ``'H'`` | Hours | Ignored | - +----------------+----------------------------+-------------------------+ - | ``'D'`` | Days | Ignored | - +----------------+----------------------------+-------------------------+ - | ``'W0'-'W6'`` | Weekday (0=Monday) | Used to compute initial | - | | | rollover time | - +----------------+----------------------------+-------------------------+ - | ``'midnight'`` | Roll over at midnight, if | Used to compute initial | - | | *atTime* not specified, | rollover time | - | | else at time *atTime* | | - +----------------+----------------------------+-------------------------+ - - When using weekday-based rotation, specify 'W0' for Monday, 'W1' for - Tuesday, and so on up to 'W6' for Sunday. In this case, the value passed for - *interval* isn't used. - - The system will save old log files by appending extensions to the filename. - The extensions are date-and-time based, using the strftime format - ``%Y-%m-%d_%H-%M-%S`` or a leading portion thereof, depending on the - rollover interval. - - When computing the next rollover time for the first time (when the handler - is created), the last modification time of an existing log file, or else - the current time, is used to compute when the next rotation will occur. - - If the *utc* argument is true, times in UTC will be used; otherwise - local time is used. - - If *backupCount* is nonzero, at most *backupCount* files - will be kept, and if more would be created when rollover occurs, the oldest - one is deleted. The deletion logic uses the interval to determine which - files to delete, so changing the interval may leave old files lying around. - - If *delay* is true, then file opening is deferred until the first call to - :meth:`emit`. - - If *atTime* is not ``None``, it must be a ``datetime.time`` instance which - specifies the time of day when rollover occurs, for the cases where rollover - is set to happen "at midnight" or "on a particular weekday". Note that in - these cases, the *atTime* value is effectively used to compute the *initial* - rollover, and subsequent rollovers would be calculated via the normal - interval calculation. - - If *errors* is specified, it's used to determine how encoding errors are - handled. - - .. note:: Calculation of the initial rollover time is done when the handler - is initialised. Calculation of subsequent rollover times is done only - when rollover occurs, and rollover occurs only when emitting output. If - this is not kept in mind, it might lead to some confusion. For example, - if an interval of "every minute" is set, that does not mean you will - always see log files with times (in the filename) separated by a minute; - if, during application execution, logging output is generated more - frequently than once a minute, *then* you can expect to see log files - with times separated by a minute. If, on the other hand, logging messages - are only output once every five minutes (say), then there will be gaps in - the file times corresponding to the minutes where no output (and hence no - rollover) occurred. - - .. versionchanged:: 3.4 - *atTime* parameter was added. - - .. versionchanged:: 3.6 - As well as string values, :class:`~pathlib.Path` objects are also accepted - for the *filename* argument. - - .. versionchanged:: 3.9 - The *errors* parameter was added. - - .. method:: doRollover() - - Does a rollover, as described above. - - .. method:: emit(record) - - Outputs the record to the file, catering for rollover as described above. - - .. method:: getFilesToDelete() - - Returns a list of filenames which should be deleted as part of rollover. These - are the absolute paths of the oldest backup log files written by the handler. - -.. _socket-handler: - -SocketHandler -^^^^^^^^^^^^^ - -The :class:`SocketHandler` class, located in the :mod:`logging.handlers` module, -sends logging output to a network socket. The base class uses a TCP socket. - - -.. class:: SocketHandler(host, port) - - Returns a new instance of the :class:`SocketHandler` class intended to - communicate with a remote machine whose address is given by *host* and *port*. - - .. versionchanged:: 3.4 - If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a TCP socket is created. - - .. method:: close() - - Closes the socket. - - - .. method:: emit() - - Pickles the record's attribute dictionary and writes it to the socket in - binary format. If there is an error with the socket, silently drops the - packet. If the connection was previously lost, re-establishes the - connection. To unpickle the record at the receiving end into a - :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` - function. - - - .. method:: handleError() - - Handles an error which has occurred during :meth:`emit`. The most likely - cause is a lost connection. Closes the socket so that we can retry on the - next event. - - - .. method:: makeSocket() - - This is a factory method which allows subclasses to define the precise - type of socket they want. The default implementation creates a TCP socket - (:const:`socket.SOCK_STREAM`). - - - .. method:: makePickle(record) - - Pickles the record's attribute dictionary in binary format with a length - prefix, and returns it ready for transmission across the socket. The - details of this operation are equivalent to:: - - data = pickle.dumps(record_attr_dict, 1) - datalen = struct.pack('>L', len(data)) - return datalen + data - - Note that pickles aren't completely secure. If you are concerned about - security, you may want to override this method to implement a more secure - mechanism. For example, you can sign pickles using HMAC and then verify - them on the receiving end, or alternatively you can disable unpickling of - global objects on the receiving end. - - - .. method:: send(packet) - - Send a pickled byte-string *packet* to the socket. The format of the sent - byte-string is as described in the documentation for - :meth:`~SocketHandler.makePickle`. - - This function allows for partial sends, which can happen when the network - is busy. - - - .. method:: createSocket() - - Tries to create a socket; on failure, uses an exponential back-off - algorithm. On initial failure, the handler will drop the message it was - trying to send. When subsequent messages are handled by the same - instance, it will not try connecting until some time has passed. The - default parameters are such that the initial delay is one second, and if - after that delay the connection still can't be made, the handler will - double the delay each time up to a maximum of 30 seconds. - - This behaviour is controlled by the following handler attributes: - - * ``retryStart`` (initial delay, defaulting to 1.0 seconds). - * ``retryFactor`` (multiplier, defaulting to 2.0). - * ``retryMax`` (maximum delay, defaulting to 30.0 seconds). - - This means that if the remote listener starts up *after* the handler has - been used, you could lose messages (since the handler won't even attempt - a connection until the delay has elapsed, but just silently drop messages - during the delay period). - - -.. _datagram-handler: - -DatagramHandler -^^^^^^^^^^^^^^^ - -The :class:`DatagramHandler` class, located in the :mod:`logging.handlers` -module, inherits from :class:`SocketHandler` to support sending logging messages -over UDP sockets. - - -.. class:: DatagramHandler(host, port) - - Returns a new instance of the :class:`DatagramHandler` class intended to - communicate with a remote machine whose address is given by *host* and *port*. - - .. note:: As UDP is not a streaming protocol, there is no persistent connection - between an instance of this handler and *host*. For this reason, when using a - network socket, a DNS lookup might have to be made each time an event is - logged, which can introduce some latency into the system. If this affects you, - you can do a lookup yourself and initialize this handler using the looked-up IP - address rather than the hostname. - - .. versionchanged:: 3.4 - If ``port`` is specified as ``None``, a Unix domain socket is created - using the value in ``host`` - otherwise, a UDP socket is created. - - .. method:: emit() - - Pickles the record's attribute dictionary and writes it to the socket in - binary format. If there is an error with the socket, silently drops the - packet. To unpickle the record at the receiving end into a - :class:`~logging.LogRecord`, use the :func:`~logging.makeLogRecord` - function. - - - .. method:: makeSocket() - - The factory method of :class:`SocketHandler` is here overridden to create - a UDP socket (:const:`socket.SOCK_DGRAM`). - - - .. method:: send(s) - - Send a pickled byte-string to a socket. The format of the sent byte-string - is as described in the documentation for :meth:`SocketHandler.makePickle`. - - -.. _syslog-handler: - -SysLogHandler -^^^^^^^^^^^^^ - -The :class:`SysLogHandler` class, located in the :mod:`logging.handlers` module, -supports sending logging messages to a remote or local Unix syslog. - - -.. class:: SysLogHandler(address=('localhost', SYSLOG_UDP_PORT), facility=LOG_USER, socktype=socket.SOCK_DGRAM) - - Returns a new instance of the :class:`SysLogHandler` class intended to - communicate with a remote Unix machine whose address is given by *address* in - the form of a ``(host, port)`` tuple. If *address* is not specified, - ``('localhost', 514)`` is used. The address is used to open a socket. An - alternative to providing a ``(host, port)`` tuple is providing an address as a - string, for example '/dev/log'. In this case, a Unix domain socket is used to - send the message to the syslog. If *facility* is not specified, - :const:`LOG_USER` is used. The type of socket opened depends on the - *socktype* argument, which defaults to :const:`socket.SOCK_DGRAM` and thus - opens a UDP socket. To open a TCP socket (for use with the newer syslog - daemons such as rsyslog), specify a value of :const:`socket.SOCK_STREAM`. - - Note that if your server is not listening on UDP port 514, - :class:`SysLogHandler` may appear not to work. In that case, check what - address you should be using for a domain socket - it's system dependent. - For example, on Linux it's usually '/dev/log' but on OS/X it's - '/var/run/syslog'. You'll need to check your platform and use the - appropriate address (you may need to do this check at runtime if your - application needs to run on several platforms). On Windows, you pretty - much have to use the UDP option. - - .. note:: On macOS 12.x (Monterey), Apple has changed the behaviour of their - syslog daemon - it no longer listens on a domain socket. Therefore, you cannot - expect :class:`SysLogHandler` to work on this system. - - See :gh:`91070` for more information. - - .. versionchanged:: 3.2 - *socktype* was added. - - - .. method:: close() - - Closes the socket to the remote host. - - .. method:: createSocket() - - Tries to create a socket and, if it's not a datagram socket, connect it - to the other end. This method is called during handler initialization, - but it's not regarded as an error if the other end isn't listening at - this point - the method will be called again when emitting an event, if - there is no socket at that point. - - .. versionadded:: 3.11 - - .. method:: emit(record) - - The record is formatted, and then sent to the syslog server. If exception - information is present, it is *not* sent to the server. - - .. versionchanged:: 3.2.1 - (See: :issue:`12168`.) In earlier versions, the message sent to the - syslog daemons was always terminated with a NUL byte, because early - versions of these daemons expected a NUL terminated message - even - though it's not in the relevant specification (:rfc:`5424`). More recent - versions of these daemons don't expect the NUL byte but strip it off - if it's there, and even more recent daemons (which adhere more closely - to RFC 5424) pass the NUL byte on as part of the message. - - To enable easier handling of syslog messages in the face of all these - differing daemon behaviours, the appending of the NUL byte has been - made configurable, through the use of a class-level attribute, - ``append_nul``. This defaults to ``True`` (preserving the existing - behaviour) but can be set to ``False`` on a ``SysLogHandler`` instance - in order for that instance to *not* append the NUL terminator. - - .. versionchanged:: 3.3 - (See: :issue:`12419`.) In earlier versions, there was no facility for - an "ident" or "tag" prefix to identify the source of the message. This - can now be specified using a class-level attribute, defaulting to - ``""`` to preserve existing behaviour, but which can be overridden on - a ``SysLogHandler`` instance in order for that instance to prepend - the ident to every message handled. Note that the provided ident must - be text, not bytes, and is prepended to the message exactly as is. - - .. method:: encodePriority(facility, priority) - - Encodes the facility and priority into an integer. You can pass in strings - or integers - if strings are passed, internal mapping dictionaries are - used to convert them to integers. - - The symbolic ``LOG_`` values are defined in :class:`SysLogHandler` and - mirror the values defined in the ``sys/syslog.h`` header file. - - **Priorities** - - +--------------------------+---------------+ - | Name (string) | Symbolic value| - +==========================+===============+ - | ``alert`` | LOG_ALERT | - +--------------------------+---------------+ - | ``crit`` or ``critical`` | LOG_CRIT | - +--------------------------+---------------+ - | ``debug`` | LOG_DEBUG | - +--------------------------+---------------+ - | ``emerg`` or ``panic`` | LOG_EMERG | - +--------------------------+---------------+ - | ``err`` or ``error`` | LOG_ERR | - +--------------------------+---------------+ - | ``info`` | LOG_INFO | - +--------------------------+---------------+ - | ``notice`` | LOG_NOTICE | - +--------------------------+---------------+ - | ``warn`` or ``warning`` | LOG_WARNING | - +--------------------------+---------------+ - - **Facilities** - - +---------------+---------------+ - | Name (string) | Symbolic value| - +===============+===============+ - | ``auth`` | LOG_AUTH | - +---------------+---------------+ - | ``authpriv`` | LOG_AUTHPRIV | - +---------------+---------------+ - | ``cron`` | LOG_CRON | - +---------------+---------------+ - | ``daemon`` | LOG_DAEMON | - +---------------+---------------+ - | ``ftp`` | LOG_FTP | - +---------------+---------------+ - | ``kern`` | LOG_KERN | - +---------------+---------------+ - | ``lpr`` | LOG_LPR | - +---------------+---------------+ - | ``mail`` | LOG_MAIL | - +---------------+---------------+ - | ``news`` | LOG_NEWS | - +---------------+---------------+ - | ``syslog`` | LOG_SYSLOG | - +---------------+---------------+ - | ``user`` | LOG_USER | - +---------------+---------------+ - | ``uucp`` | LOG_UUCP | - +---------------+---------------+ - | ``local0`` | LOG_LOCAL0 | - +---------------+---------------+ - | ``local1`` | LOG_LOCAL1 | - +---------------+---------------+ - | ``local2`` | LOG_LOCAL2 | - +---------------+---------------+ - | ``local3`` | LOG_LOCAL3 | - +---------------+---------------+ - | ``local4`` | LOG_LOCAL4 | - +---------------+---------------+ - | ``local5`` | LOG_LOCAL5 | - +---------------+---------------+ - | ``local6`` | LOG_LOCAL6 | - +---------------+---------------+ - | ``local7`` | LOG_LOCAL7 | - +---------------+---------------+ - - .. method:: mapPriority(levelname) - - Maps a logging level name to a syslog priority name. - You may need to override this if you are using custom levels, or - if the default algorithm is not suitable for your needs. The - default algorithm maps ``DEBUG``, ``INFO``, ``WARNING``, ``ERROR`` and - ``CRITICAL`` to the equivalent syslog names, and all other level - names to 'warning'. - -.. _nt-eventlog-handler: - -NTEventLogHandler -^^^^^^^^^^^^^^^^^ - -The :class:`NTEventLogHandler` class, located in the :mod:`logging.handlers` -module, supports sending logging messages to a local Windows NT, Windows 2000 or -Windows XP event log. Before you can use it, you need Mark Hammond's Win32 -extensions for Python installed. - - -.. class:: NTEventLogHandler(appname, dllname=None, logtype='Application') - - Returns a new instance of the :class:`NTEventLogHandler` class. The *appname* is - used to define the application name as it appears in the event log. An - appropriate registry entry is created using this name. The *dllname* should give - the fully qualified pathname of a .dll or .exe which contains message - definitions to hold in the log (if not specified, ``'win32service.pyd'`` is used - - this is installed with the Win32 extensions and contains some basic - placeholder message definitions. Note that use of these placeholders will make - your event logs big, as the entire message source is held in the log. If you - want slimmer logs, you have to pass in the name of your own .dll or .exe which - contains the message definitions you want to use in the event log). The - *logtype* is one of ``'Application'``, ``'System'`` or ``'Security'``, and - defaults to ``'Application'``. - - - .. method:: close() - - At this point, you can remove the application name from the registry as a - source of event log entries. However, if you do this, you will not be able - to see the events as you intended in the Event Log Viewer - it needs to be - able to access the registry to get the .dll name. The current version does - not do this. - - - .. method:: emit(record) - - Determines the message ID, event category and event type, and then logs - the message in the NT event log. - - - .. method:: getEventCategory(record) - - Returns the event category for the record. Override this if you want to - specify your own categories. This version returns 0. - - - .. method:: getEventType(record) - - Returns the event type for the record. Override this if you want to - specify your own types. This version does a mapping using the handler's - typemap attribute, which is set up in :meth:`__init__` to a dictionary - which contains mappings for :const:`DEBUG`, :const:`INFO`, - :const:`WARNING`, :const:`ERROR` and :const:`CRITICAL`. If you are using - your own levels, you will either need to override this method or place a - suitable dictionary in the handler's *typemap* attribute. - - - .. method:: getMessageID(record) - - Returns the message ID for the record. If you are using your own messages, - you could do this by having the *msg* passed to the logger being an ID - rather than a format string. Then, in here, you could use a dictionary - lookup to get the message ID. This version returns 1, which is the base - message ID in :file:`win32service.pyd`. - -.. _smtp-handler: - -SMTPHandler -^^^^^^^^^^^ - -The :class:`SMTPHandler` class, located in the :mod:`logging.handlers` module, -supports sending logging messages to an email address via SMTP. - - -.. class:: SMTPHandler(mailhost, fromaddr, toaddrs, subject, credentials=None, secure=None, timeout=1.0) - - Returns a new instance of the :class:`SMTPHandler` class. The instance is - initialized with the from and to addresses and subject line of the email. The - *toaddrs* should be a list of strings. To specify a non-standard SMTP port, use - the (host, port) tuple format for the *mailhost* argument. If you use a string, - the standard SMTP port is used. If your SMTP server requires authentication, you - can specify a (username, password) tuple for the *credentials* argument. - - To specify the use of a secure protocol (TLS), pass in a tuple to the - *secure* argument. This will only be used when authentication credentials are - supplied. The tuple should be either an empty tuple, or a single-value tuple - with the name of a keyfile, or a 2-value tuple with the names of the keyfile - and certificate file. (This tuple is passed to the - :meth:`smtplib.SMTP.starttls` method.) - - A timeout can be specified for communication with the SMTP server using the - *timeout* argument. - - .. versionadded:: 3.3 - The *timeout* argument was added. - - .. method:: emit(record) - - Formats the record and sends it to the specified addressees. - - - .. method:: getSubject(record) - - If you want to specify a subject line which is record-dependent, override - this method. - -.. _memory-handler: - -MemoryHandler -^^^^^^^^^^^^^ - -The :class:`MemoryHandler` class, located in the :mod:`logging.handlers` module, -supports buffering of logging records in memory, periodically flushing them to a -:dfn:`target` handler. Flushing occurs whenever the buffer is full, or when an -event of a certain severity or greater is seen. - -:class:`MemoryHandler` is a subclass of the more general -:class:`BufferingHandler`, which is an abstract class. This buffers logging -records in memory. Whenever each record is added to the buffer, a check is made -by calling :meth:`shouldFlush` to see if the buffer should be flushed. If it -should, then :meth:`flush` is expected to do the flushing. - - -.. class:: BufferingHandler(capacity) - - Initializes the handler with a buffer of the specified capacity. Here, - *capacity* means the number of logging records buffered. - - - .. method:: emit(record) - - Append the record to the buffer. If :meth:`shouldFlush` returns true, - call :meth:`flush` to process the buffer. - - - .. method:: flush() - - For a :class:`BufferingHandler` instance, flushing means that it sets the - buffer to an empty list. This method can be overwritten to implement more useful - flushing behavior. - - - .. method:: shouldFlush(record) - - Return ``True`` if the buffer is up to capacity. This method can be - overridden to implement custom flushing strategies. - - -.. class:: MemoryHandler(capacity, flushLevel=ERROR, target=None, flushOnClose=True) - - Returns a new instance of the :class:`MemoryHandler` class. The instance is - initialized with a buffer size of *capacity* (number of records buffered). - If *flushLevel* is not specified, :const:`ERROR` is used. If no *target* is - specified, the target will need to be set using :meth:`setTarget` before this - handler does anything useful. If *flushOnClose* is specified as ``False``, - then the buffer is *not* flushed when the handler is closed. If not specified - or specified as ``True``, the previous behaviour of flushing the buffer will - occur when the handler is closed. - - .. versionchanged:: 3.6 - The *flushOnClose* parameter was added. - - - .. method:: close() - - Calls :meth:`flush`, sets the target to ``None`` and clears the - buffer. - - - .. method:: flush() - - For a :class:`MemoryHandler` instance, flushing means just sending the buffered - records to the target, if there is one. The buffer is also cleared when - buffered records are sent to the target. Override if you want different behavior. - - - .. method:: setTarget(target) - - Sets the target handler for this handler. - - - .. method:: shouldFlush(record) - - Checks for buffer full or a record at the *flushLevel* or higher. - - -.. _http-handler: - -HTTPHandler -^^^^^^^^^^^ - -The :class:`HTTPHandler` class, located in the :mod:`logging.handlers` module, -supports sending logging messages to a web server, using either ``GET`` or -``POST`` semantics. - - -.. class:: HTTPHandler(host, url, method='GET', secure=False, credentials=None, context=None) - - Returns a new instance of the :class:`HTTPHandler` class. The *host* can be - of the form ``host:port``, should you need to use a specific port number. If - no *method* is specified, ``GET`` is used. If *secure* is true, a HTTPS - connection will be used. The *context* parameter may be set to a - :class:`ssl.SSLContext` instance to configure the SSL settings used for the - HTTPS connection. If *credentials* is specified, it should be a 2-tuple - consisting of userid and password, which will be placed in a HTTP - 'Authorization' header using Basic authentication. If you specify - credentials, you should also specify secure=True so that your userid and - password are not passed in cleartext across the wire. - - .. versionchanged:: 3.5 - The *context* parameter was added. - - .. method:: mapLogRecord(record) - - Provides a dictionary, based on ``record``, which is to be URL-encoded - and sent to the web server. The default implementation just returns - ``record.__dict__``. This method can be overridden if e.g. only a - subset of :class:`~logging.LogRecord` is to be sent to the web server, or - if more specific customization of what's sent to the server is required. - - .. method:: emit(record) - - Sends the record to the web server as a URL-encoded dictionary. The - :meth:`mapLogRecord` method is used to convert the record to the - dictionary to be sent. - - .. note:: Since preparing a record for sending it to a web server is not - the same as a generic formatting operation, using - :meth:`~logging.Handler.setFormatter` to specify a - :class:`~logging.Formatter` for a :class:`HTTPHandler` has no effect. - Instead of calling :meth:`~logging.Handler.format`, this handler calls - :meth:`mapLogRecord` and then :func:`urllib.parse.urlencode` to encode the - dictionary in a form suitable for sending to a web server. - - -.. _queue-handler: - - -QueueHandler -^^^^^^^^^^^^ - -.. versionadded:: 3.2 - -The :class:`QueueHandler` class, located in the :mod:`logging.handlers` module, -supports sending logging messages to a queue, such as those implemented in the -:mod:`queue` or :mod:`multiprocessing` modules. - -Along with the :class:`QueueListener` class, :class:`QueueHandler` can be used -to let handlers do their work on a separate thread from the one which does the -logging. This is important in web applications and also other service -applications where threads servicing clients need to respond as quickly as -possible, while any potentially slow operations (such as sending an email via -:class:`SMTPHandler`) are done on a separate thread. - -.. class:: QueueHandler(queue) - - Returns a new instance of the :class:`QueueHandler` class. The instance is - initialized with the queue to send messages to. The *queue* can be any - queue-like object; it's used as-is by the :meth:`enqueue` method, which - needs to know how to send messages to it. The queue is not *required* to - have the task tracking API, which means that you can use - :class:`~queue.SimpleQueue` instances for *queue*. - - .. note:: If you are using :mod:`multiprocessing`, you should avoid using - :class:`~queue.SimpleQueue` and instead use :class:`multiprocessing.Queue`. - - .. method:: emit(record) - - Enqueues the result of preparing the LogRecord. Should an exception - occur (e.g. because a bounded queue has filled up), the - :meth:`~logging.Handler.handleError` method is called to handle the - error. This can result in the record silently being dropped (if - :data:`logging.raiseExceptions` is ``False``) or a message printed to - ``sys.stderr`` (if :data:`logging.raiseExceptions` is ``True``). - - .. method:: prepare(record) - - Prepares a record for queuing. The object returned by this - method is enqueued. - - The base implementation formats the record to merge the message, - arguments, exception and stack information, if present. It also removes - unpickleable items from the record in-place. Specifically, it overwrites - the record's :attr:`msg` and :attr:`message` attributes with the merged - message (obtained by calling the handler's :meth:`format` method), and - sets the :attr:`args`, :attr:`exc_info` and :attr:`exc_text` attributes - to ``None``. - - You might want to override this method if you want to convert - the record to a dict or JSON string, or send a modified copy - of the record while leaving the original intact. - - .. note:: The base implementation formats the message with arguments, sets - the ``message`` and ``msg`` attributes to the formatted message and - sets the ``args`` and ``exc_text`` attributes to ``None`` to allow - pickling and to prevent further attempts at formatting. This means - that a handler on the :class:`QueueListener` side won't have the - information to do custom formatting, e.g. of exceptions. You may wish - to subclass ``QueueHandler`` and override this method to e.g. avoid - setting ``exc_text`` to ``None``. Note that the ``message`` / ``msg`` - / ``args`` changes are related to ensuring the record is pickleable, - and you might or might not be able to avoid doing that depending on - whether your ``args`` are pickleable. (Note that you may have to - consider not only your own code but also code in any libraries that - you use.) - - .. method:: enqueue(record) - - Enqueues the record on the queue using ``put_nowait()``; you may - want to override this if you want to use blocking behaviour, or a - timeout, or a customized queue implementation. - - - -.. _queue-listener: - -QueueListener -^^^^^^^^^^^^^ - -.. versionadded:: 3.2 - -The :class:`QueueListener` class, located in the :mod:`logging.handlers` -module, supports receiving logging messages from a queue, such as those -implemented in the :mod:`queue` or :mod:`multiprocessing` modules. The -messages are received from a queue in an internal thread and passed, on -the same thread, to one or more handlers for processing. While -:class:`QueueListener` is not itself a handler, it is documented here -because it works hand-in-hand with :class:`QueueHandler`. - -Along with the :class:`QueueHandler` class, :class:`QueueListener` can be used -to let handlers do their work on a separate thread from the one which does the -logging. This is important in web applications and also other service -applications where threads servicing clients need to respond as quickly as -possible, while any potentially slow operations (such as sending an email via -:class:`SMTPHandler`) are done on a separate thread. - -.. class:: QueueListener(queue, *handlers, respect_handler_level=False) - - Returns a new instance of the :class:`QueueListener` class. The instance is - initialized with the queue to send messages to and a list of handlers which - will handle entries placed on the queue. The queue can be any queue-like - object; it's passed as-is to the :meth:`dequeue` method, which needs - to know how to get messages from it. The queue is not *required* to have the - task tracking API (though it's used if available), which means that you can - use :class:`~queue.SimpleQueue` instances for *queue*. - - .. note:: If you are using :mod:`multiprocessing`, you should avoid using - :class:`~queue.SimpleQueue` and instead use :class:`multiprocessing.Queue`. - - If ``respect_handler_level`` is ``True``, a handler's level is respected - (compared with the level for the message) when deciding whether to pass - messages to that handler; otherwise, the behaviour is as in previous Python - versions - to always pass each message to each handler. - - .. versionchanged:: 3.5 - The ``respect_handler_level`` argument was added. - - .. method:: dequeue(block) - - Dequeues a record and return it, optionally blocking. - - The base implementation uses ``get()``. You may want to override this - method if you want to use timeouts or work with custom queue - implementations. - - .. method:: prepare(record) - - Prepare a record for handling. - - This implementation just returns the passed-in record. You may want to - override this method if you need to do any custom marshalling or - manipulation of the record before passing it to the handlers. - - .. method:: handle(record) - - Handle a record. - - This just loops through the handlers offering them the record - to handle. The actual object passed to the handlers is that which - is returned from :meth:`prepare`. - - .. method:: start() - - Starts the listener. - - This starts up a background thread to monitor the queue for - LogRecords to process. - - .. method:: stop() - - Stops the listener. - - This asks the thread to terminate, and then waits for it to do so. - Note that if you don't call this before your application exits, there - may be some records still left on the queue, which won't be processed. - - .. method:: enqueue_sentinel() - - Writes a sentinel to the queue to tell the listener to quit. This - implementation uses ``put_nowait()``. You may want to override this - method if you want to use timeouts or work with custom queue - implementations. - - .. versionadded:: 3.3 - - -.. seealso:: - - Module :mod:`logging` - API reference for the logging module. - - Module :mod:`logging.config` - Configuration API for the logging module. - - diff --git a/Doc/library/logging.rst b/Doc/library/logging.rst deleted file mode 100644 index 8741d64a1dbdeb..00000000000000 --- a/Doc/library/logging.rst +++ /dev/null @@ -1,1483 +0,0 @@ -:mod:`logging` --- Logging facility for Python -============================================== - -.. module:: logging - :synopsis: Flexible event logging system for applications. - -.. moduleauthor:: Vinay Sajip -.. sectionauthor:: Vinay Sajip - -**Source code:** :source:`Lib/logging/__init__.py` - -.. index:: pair: Errors; logging - -.. sidebar:: Important - - This page contains the API reference information. For tutorial - information and discussion of more advanced topics, see - - * :ref:`Basic Tutorial ` - * :ref:`Advanced Tutorial ` - * :ref:`Logging Cookbook ` - --------------- - -This module defines functions and classes which implement a flexible event -logging system for applications and libraries. - -The key benefit of having the logging API provided by a standard library module -is that all Python modules can participate in logging, so your application log -can include your own messages integrated with messages from third-party -modules. - -The simplest example: - -.. code-block:: none - - >>> import logging - >>> logging.warning('Watch out!') - WARNING:root:Watch out! - -The module provides a lot of functionality and flexibility. If you are -unfamiliar with logging, the best way to get to grips with it is to view the -tutorials (**see the links above and on the right**). - -The basic classes defined by the module, together with their functions, are -listed below. - -* Loggers expose the interface that application code directly uses. -* Handlers send the log records (created by loggers) to the appropriate - destination. -* Filters provide a finer grained facility for determining which log records - to output. -* Formatters specify the layout of log records in the final output. - - -.. _logger: - -Logger Objects --------------- - -Loggers have the following attributes and methods. Note that Loggers should -*NEVER* be instantiated directly, but always through the module-level function -``logging.getLogger(name)``. Multiple calls to :func:`getLogger` with the same -name will always return a reference to the same Logger object. - -The ``name`` is potentially a period-separated hierarchical value, like -``foo.bar.baz`` (though it could also be just plain ``foo``, for example). -Loggers that are further down in the hierarchical list are children of loggers -higher up in the list. For example, given a logger with a name of ``foo``, -loggers with names of ``foo.bar``, ``foo.bar.baz``, and ``foo.bam`` are all -descendants of ``foo``. The logger name hierarchy is analogous to the Python -package hierarchy, and identical to it if you organise your loggers on a -per-module basis using the recommended construction -``logging.getLogger(__name__)``. That's because in a module, ``__name__`` -is the module's name in the Python package namespace. - - -.. class:: Logger - - .. attribute:: Logger.propagate - - If this attribute evaluates to true, events logged to this logger will be - passed to the handlers of higher level (ancestor) loggers, in addition to - any handlers attached to this logger. Messages are passed directly to the - ancestor loggers' handlers - neither the level nor filters of the ancestor - loggers in question are considered. - - If this evaluates to false, logging messages are not passed to the handlers - of ancestor loggers. - - Spelling it out with an example: If the propagate attribute of the logger named - ``A.B.C`` evaluates to true, any event logged to ``A.B.C`` via a method call such as - ``logging.getLogger('A.B.C').error(...)`` will [subject to passing that logger's - level and filter settings] be passed in turn to any handlers attached to loggers - named ``A.B``, ``A`` and the root logger, after first being passed to any handlers - attached to ``A.B.C``. If any logger in the chain ``A.B.C``, ``A.B``, ``A`` has its - ``propagate`` attribute set to false, then that is the last logger whose handlers - are offered the event to handle, and propagation stops at that point. - - The constructor sets this attribute to ``True``. - - .. note:: If you attach a handler to a logger *and* one or more of its - ancestors, it may emit the same record multiple times. In general, you - should not need to attach a handler to more than one logger - if you just - attach it to the appropriate logger which is highest in the logger - hierarchy, then it will see all events logged by all descendant loggers, - provided that their propagate setting is left set to ``True``. A common - scenario is to attach handlers only to the root logger, and to let - propagation take care of the rest. - - .. method:: Logger.setLevel(level) - - Sets the threshold for this logger to *level*. Logging messages which are less - severe than *level* will be ignored; logging messages which have severity *level* - or higher will be emitted by whichever handler or handlers service this logger, - unless a handler's level has been set to a higher severity level than *level*. - - When a logger is created, the level is set to :const:`NOTSET` (which causes - all messages to be processed when the logger is the root logger, or delegation - to the parent when the logger is a non-root logger). Note that the root logger - is created with level :const:`WARNING`. - - The term 'delegation to the parent' means that if a logger has a level of - NOTSET, its chain of ancestor loggers is traversed until either an ancestor with - a level other than NOTSET is found, or the root is reached. - - If an ancestor is found with a level other than NOTSET, then that ancestor's - level is treated as the effective level of the logger where the ancestor search - began, and is used to determine how a logging event is handled. - - If the root is reached, and it has a level of NOTSET, then all messages will be - processed. Otherwise, the root's level will be used as the effective level. - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. Note, however, that levels are internally stored - as integers, and methods such as e.g. :meth:`getEffectiveLevel` and - :meth:`isEnabledFor` will return/expect to be passed integers. - - - .. method:: Logger.isEnabledFor(level) - - Indicates if a message of severity *level* would be processed by this logger. - This method checks first the module-level level set by - ``logging.disable(level)`` and then the logger's effective level as determined - by :meth:`getEffectiveLevel`. - - - .. method:: Logger.getEffectiveLevel() - - Indicates the effective level for this logger. If a value other than - :const:`NOTSET` has been set using :meth:`setLevel`, it is returned. Otherwise, - the hierarchy is traversed towards the root until a value other than - :const:`NOTSET` is found, and that value is returned. The value returned is - an integer, typically one of :const:`logging.DEBUG`, :const:`logging.INFO` - etc. - - - .. method:: Logger.getChild(suffix) - - Returns a logger which is a descendant to this logger, as determined by the suffix. - Thus, ``logging.getLogger('abc').getChild('def.ghi')`` would return the same - logger as would be returned by ``logging.getLogger('abc.def.ghi')``. This is a - convenience method, useful when the parent logger is named using e.g. ``__name__`` - rather than a literal string. - - .. versionadded:: 3.2 - - - .. method:: Logger.debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on this logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - No % formatting operation is performed on *msg* when no *args* are supplied. - - There are four keyword arguments in *kwargs* which are inspected: - *exc_info*, *stack_info*, *stacklevel* and *extra*. - - If *exc_info* does not evaluate as false, it causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *stacklevel*, which defaults to ``1``. - If greater than 1, the corresponding number of stack frames are skipped - when computing the line number and function name set in the :class:`LogRecord` - created for the logging event. This can be used in logging helpers so that - the function name, filename and line number recorded are not the information - for the helper function/method, but rather its caller. The name of this - parameter mirrors the equivalent one in the :mod:`warnings` module. - - The fourth keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the :class:`LogRecord` - created for the logging event with user-defined attributes. These custom - attributes can then be used as you like. For example, they could be - incorporated into logged messages. For example:: - - FORMAT = '%(asctime)s %(clientip)-15s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logger = logging.getLogger('tcpserver') - logger.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the section on :ref:`logrecord-attributes` for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the :class:`LogRecord`. If these are missing, the message will - not be logged because a string formatting exception will occur. So in this case, - you always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - If no handler is attached to this logger (or any of its ancestors, - taking into account the relevant :attr:`Logger.propagate` attributes), - the message will be sent to the handler set on :attr:`lastResort`. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. - - .. versionchanged:: 3.5 - The *exc_info* parameter can now accept exception instances. - - .. versionchanged:: 3.8 - The *stacklevel* parameter was added. - - - .. method:: Logger.info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on this logger. The arguments are - interpreted as for :meth:`debug`. - - .. note:: There is an obsolete method ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - .. method:: Logger.error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on this logger. The arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.log(level, msg, *args, **kwargs) - - Logs a message with integer level *level* on this logger. The other arguments are - interpreted as for :meth:`debug`. - - - .. method:: Logger.exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on this logger. The arguments are - interpreted as for :meth:`debug`. Exception info is added to the logging - message. This method should only be called from an exception handler. - - - .. method:: Logger.addFilter(filter) - - Adds the specified filter *filter* to this logger. - - - .. method:: Logger.removeFilter(filter) - - Removes the specified filter *filter* from this logger. - - - .. method:: Logger.filter(record) - - Apply this logger's filters to the record and return ``True`` if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be processed (passed to handlers). If one returns a false value, no - further processing of the record occurs. - - - .. method:: Logger.addHandler(hdlr) - - Adds the specified handler *hdlr* to this logger. - - - .. method:: Logger.removeHandler(hdlr) - - Removes the specified handler *hdlr* from this logger. - - - .. method:: Logger.findCaller(stack_info=False, stacklevel=1) - - Finds the caller's source filename and line number. Returns the filename, line - number, function name and stack information as a 4-element tuple. The stack - information is returned as ``None`` unless *stack_info* is ``True``. - - The *stacklevel* parameter is passed from code calling the :meth:`debug` - and other APIs. If greater than 1, the excess is used to skip stack frames - before determining the values to be returned. This will generally be useful - when calling logging APIs from helper/wrapper code, so that the information - in the event log refers not to the helper/wrapper code, but to the code that - calls it. - - - .. method:: Logger.handle(record) - - Handles a record by passing it to all handlers associated with this logger and - its ancestors (until a false value of *propagate* is found). This method is used - for unpickled records received from a socket, as well as those created locally. - Logger-level filtering is applied using :meth:`~Logger.filter`. - - - .. method:: Logger.makeRecord(name, level, fn, lno, msg, args, exc_info, func=None, extra=None, sinfo=None) - - This is a factory method which can be overridden in subclasses to create - specialized :class:`LogRecord` instances. - - .. method:: Logger.hasHandlers() - - Checks to see if this logger has any handlers configured. This is done by - looking for handlers in this logger and its parents in the logger hierarchy. - Returns ``True`` if a handler was found, else ``False``. The method stops searching - up the hierarchy whenever a logger with the 'propagate' attribute set to - false is found - that will be the last logger which is checked for the - existence of handlers. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.7 - Loggers can now be pickled and unpickled. - -.. _levels: - -Logging Levels --------------- - -The numeric values of logging levels are given in the following table. These are -primarily of interest if you want to define your own levels, and need them to -have specific values relative to the predefined levels. If you define a level -with the same numeric value, it overwrites the predefined value; the predefined -name is lost. - -+-----------------------+---------------+-------------------------------------+ -| Level | Numeric value | What it means / When to use it | -+=======================+===============+=====================================+ -| .. py:data:: NOTSET | 0 | When set on a logger, indicates that| -| | | ancestor loggers are to be consulted| -| | | to determine the effective level. | -| | | If that still resolves to | -| | | :const:`!NOTSET`, then all events | -| | | are logged. When set on a handler, | -| | | all events are handled. | -+-----------------------+---------------+-------------------------------------+ -| .. py:data:: DEBUG | 10 | Detailed information, typically only| -| | | of interest to a developer trying to| -| | | diagnose a problem. | -+-----------------------+---------------+-------------------------------------+ -| .. py:data:: INFO | 20 | Confirmation that things are working| -| | | as expected. | -+-----------------------+---------------+-------------------------------------+ -| .. py:data:: WARNING | 30 | An indication that something | -| | | unexpected happened, or that a | -| | | problem might occur in the near | -| | | future (e.g. 'disk space low'). The | -| | | software is still working as | -| | | expected. | -+-----------------------+---------------+-------------------------------------+ -| .. py:data:: ERROR | 40 | Due to a more serious problem, the | -| | | software has not been able to | -| | | perform some function. | -+-----------------------+---------------+-------------------------------------+ -| .. py:data:: CRITICAL | 50 | A serious error, indicating that the| -| | | program itself may be unable to | -| | | continue running. | -+-----------------------+---------------+-------------------------------------+ - - -.. _handler: - -Handler Objects ---------------- - -Handlers have the following attributes and methods. Note that :class:`Handler` -is never instantiated directly; this class acts as a base for more useful -subclasses. However, the :meth:`!__init__` method in subclasses needs to call -:meth:`Handler.__init__`. - -.. class:: Handler - - .. method:: Handler.__init__(level=NOTSET) - - Initializes the :class:`Handler` instance by setting its level, setting the list - of filters to the empty list and creating a lock (using :meth:`createLock`) for - serializing access to an I/O mechanism. - - - .. method:: Handler.createLock() - - Initializes a thread lock which can be used to serialize access to underlying - I/O functionality which may not be threadsafe. - - - .. method:: Handler.acquire() - - Acquires the thread lock created with :meth:`createLock`. - - - .. method:: Handler.release() - - Releases the thread lock acquired with :meth:`acquire`. - - - .. method:: Handler.setLevel(level) - - Sets the threshold for this handler to *level*. Logging messages which are - less severe than *level* will be ignored. When a handler is created, the - level is set to :const:`NOTSET` (which causes all messages to be - processed). - - See :ref:`levels` for a list of levels. - - .. versionchanged:: 3.2 - The *level* parameter now accepts a string representation of the - level such as 'INFO' as an alternative to the integer constants - such as :const:`INFO`. - - - .. method:: Handler.setFormatter(fmt) - - Sets the :class:`Formatter` for this handler to *fmt*. - - - .. method:: Handler.addFilter(filter) - - Adds the specified filter *filter* to this handler. - - - .. method:: Handler.removeFilter(filter) - - Removes the specified filter *filter* from this handler. - - - .. method:: Handler.filter(record) - - Apply this handler's filters to the record and return ``True`` if the - record is to be processed. The filters are consulted in turn, until one of - them returns a false value. If none of them return a false value, the record - will be emitted. If one returns a false value, the handler will not emit the - record. - - - .. method:: Handler.flush() - - Ensure all logging output has been flushed. This version does nothing and is - intended to be implemented by subclasses. - - - .. method:: Handler.close() - - Tidy up any resources used by the handler. This version does no output but - removes the handler from an internal list of handlers which is closed when - :func:`shutdown` is called. Subclasses should ensure that this gets called - from overridden :meth:`close` methods. - - - .. method:: Handler.handle(record) - - Conditionally emits the specified logging record, depending on filters which may - have been added to the handler. Wraps the actual emission of the record with - acquisition/release of the I/O thread lock. - - - .. method:: Handler.handleError(record) - - This method should be called from handlers when an exception is encountered - during an :meth:`emit` call. If the module-level attribute - ``raiseExceptions`` is ``False``, exceptions get silently ignored. This is - what is mostly wanted for a logging system - most users will not care about - errors in the logging system, they are more interested in application - errors. You could, however, replace this with a custom handler if you wish. - The specified record is the one which was being processed when the exception - occurred. (The default value of ``raiseExceptions`` is ``True``, as that is - more useful during development). - - - .. method:: Handler.format(record) - - Do formatting for a record - if a formatter is set, use it. Otherwise, use the - default formatter for the module. - - - .. method:: Handler.emit(record) - - Do whatever it takes to actually log the specified logging record. This version - is intended to be implemented by subclasses and so raises a - :exc:`NotImplementedError`. - - .. warning:: This method is called after a handler-level lock is acquired, which - is released after this method returns. When you override this method, note - that you should be careful when calling anything that invokes other parts of - the logging API which might do locking, because that might result in a - deadlock. Specifically: - - * Logging configuration APIs acquire the module-level lock, and then - individual handler-level locks as those handlers are configured. - - * Many logging APIs lock the module-level lock. If such an API is called - from this method, it could cause a deadlock if a configuration call is - made on another thread, because that thread will try to acquire the - module-level lock *before* the handler-level lock, whereas this thread - tries to acquire the module-level lock *after* the handler-level lock - (because in this method, the handler-level lock has already been acquired). - -For a list of handlers included as standard, see :mod:`logging.handlers`. - -.. _formatter-objects: - -Formatter Objects ------------------ - -.. currentmodule:: logging - -:class:`Formatter` objects have the following attributes and methods. They are -responsible for converting a :class:`LogRecord` to (usually) a string which can -be interpreted by either a human or an external system. The base -:class:`Formatter` allows a formatting string to be specified. If none is -supplied, the default value of ``'%(message)s'`` is used, which just includes -the message in the logging call. To have additional items of information in the -formatted output (such as a timestamp), keep reading. - -A Formatter can be initialized with a format string which makes use of knowledge -of the :class:`LogRecord` attributes - such as the default value mentioned above -making use of the fact that the user's message and arguments are pre-formatted -into a :class:`LogRecord`'s *message* attribute. This format string contains -standard Python %-style mapping keys. See section :ref:`old-string-formatting` -for more information on string formatting. - -The useful mapping keys in a :class:`LogRecord` are given in the section on -:ref:`logrecord-attributes`. - - -.. class:: Formatter(fmt=None, datefmt=None, style='%', validate=True, *, defaults=None) - - Returns a new instance of the :class:`Formatter` class. The instance is - initialized with a format string for the message as a whole, as well as a - format string for the date/time portion of a message. If no *fmt* is - specified, ``'%(message)s'`` is used. If no *datefmt* is specified, a format - is used which is described in the :meth:`formatTime` documentation. - - The *style* parameter can be one of '%', '{' or '$' and determines how - the format string will be merged with its data: using one of %-formatting, - :meth:`str.format` or :class:`string.Template`. This only applies to the - format string *fmt* (e.g. ``'%(message)s'`` or ``{message}``), not to the - actual log messages passed to ``Logger.debug`` etc; see - :ref:`formatting-styles` for more information on using {- and $-formatting - for log messages. - - The *defaults* parameter can be a dictionary with default values to use in - custom fields. For example: - ``logging.Formatter('%(ip)s %(message)s', defaults={"ip": None})`` - - .. versionchanged:: 3.2 - The *style* parameter was added. - - .. versionchanged:: 3.8 - The *validate* parameter was added. Incorrect or mismatched style and fmt - will raise a ``ValueError``. - For example: ``logging.Formatter('%(asctime)s - %(message)s', style='{')``. - - .. versionchanged:: 3.10 - The *defaults* parameter was added. - - .. method:: format(record) - - The record's attribute dictionary is used as the operand to a string - formatting operation. Returns the resulting string. Before formatting the - dictionary, a couple of preparatory steps are carried out. The *message* - attribute of the record is computed using *msg* % *args*. If the - formatting string contains ``'(asctime)'``, :meth:`formatTime` is called - to format the event time. If there is exception information, it is - formatted using :meth:`formatException` and appended to the message. Note - that the formatted exception information is cached in attribute - *exc_text*. This is useful because the exception information can be - pickled and sent across the wire, but you should be careful if you have - more than one :class:`Formatter` subclass which customizes the formatting - of exception information. In this case, you will have to clear the cached - value (by setting the *exc_text* attribute to ``None``) after a formatter - has done its formatting, so that the next formatter to handle the event - doesn't use the cached value, but recalculates it afresh. - - If stack information is available, it's appended after the exception - information, using :meth:`formatStack` to transform it if necessary. - - - .. method:: formatTime(record, datefmt=None) - - This method should be called from :meth:`format` by a formatter which - wants to make use of a formatted time. This method can be overridden in - formatters to provide for any specific requirement, but the basic behavior - is as follows: if *datefmt* (a string) is specified, it is used with - :func:`time.strftime` to format the creation time of the - record. Otherwise, the format '%Y-%m-%d %H:%M:%S,uuu' is used, where the - uuu part is a millisecond value and the other letters are as per the - :func:`time.strftime` documentation. An example time in this format is - ``2003-01-23 00:29:50,411``. The resulting string is returned. - - This function uses a user-configurable function to convert the creation - time to a tuple. By default, :func:`time.localtime` is used; to change - this for a particular formatter instance, set the ``converter`` attribute - to a function with the same signature as :func:`time.localtime` or - :func:`time.gmtime`. To change it for all formatters, for example if you - want all logging times to be shown in GMT, set the ``converter`` - attribute in the ``Formatter`` class. - - .. versionchanged:: 3.3 - Previously, the default format was hard-coded as in this example: - ``2010-09-06 22:38:15,292`` where the part before the comma is - handled by a strptime format string (``'%Y-%m-%d %H:%M:%S'``), and the - part after the comma is a millisecond value. Because strptime does not - have a format placeholder for milliseconds, the millisecond value is - appended using another format string, ``'%s,%03d'`` --- and both of these - format strings have been hardcoded into this method. With the change, - these strings are defined as class-level attributes which can be - overridden at the instance level when desired. The names of the - attributes are ``default_time_format`` (for the strptime format string) - and ``default_msec_format`` (for appending the millisecond value). - - .. versionchanged:: 3.9 - The ``default_msec_format`` can be ``None``. - - .. method:: formatException(exc_info) - - Formats the specified exception information (a standard exception tuple as - returned by :func:`sys.exc_info`) as a string. This default implementation - just uses :func:`traceback.print_exception`. The resulting string is - returned. - - .. method:: formatStack(stack_info) - - Formats the specified stack information (a string as returned by - :func:`traceback.print_stack`, but with the last newline removed) as a - string. This default implementation just returns the input value. - -.. class:: BufferingFormatter(linefmt=None) - - A base formatter class suitable for subclassing when you want to format a - number of records. You can pass a :class:`Formatter` instance which you want - to use to format each line (that corresponds to a single record). If not - specified, the default formatter (which just outputs the event message) is - used as the line formatter. - - .. method:: formatHeader(records) - - Return a header for a list of *records*. The base implementation just - returns the empty string. You will need to override this method if you - want specific behaviour, e.g. to show the count of records, a title or a - separator line. - - .. method:: formatFooter(records) - - Return a footer for a list of *records*. The base implementation just - returns the empty string. You will need to override this method if you - want specific behaviour, e.g. to show the count of records or a separator - line. - - .. method:: format(records) - - Return formatted text for a list of *records*. The base implementation - just returns the empty string if there are no records; otherwise, it - returns the concatenation of the header, each record formatted with the - line formatter, and the footer. - -.. _filter: - -Filter Objects --------------- - -``Filters`` can be used by ``Handlers`` and ``Loggers`` for more sophisticated -filtering than is provided by levels. The base filter class only allows events -which are below a certain point in the logger hierarchy. For example, a filter -initialized with 'A.B' will allow events logged by loggers 'A.B', 'A.B.C', -'A.B.C.D', 'A.B.D' etc. but not 'A.BB', 'B.A.B' etc. If initialized with the -empty string, all events are passed. - - -.. class:: Filter(name='') - - Returns an instance of the :class:`Filter` class. If *name* is specified, it - names a logger which, together with its children, will have its events allowed - through the filter. If *name* is the empty string, allows every event. - - - .. method:: filter(record) - - Is the specified record to be logged? Returns zero for no, nonzero for - yes. If deemed appropriate, the record may be modified in-place by this - method. - -Note that filters attached to handlers are consulted before an event is -emitted by the handler, whereas filters attached to loggers are consulted -whenever an event is logged (using :meth:`debug`, :meth:`info`, -etc.), before sending an event to handlers. This means that events which have -been generated by descendant loggers will not be filtered by a logger's filter -setting, unless the filter has also been applied to those descendant loggers. - -You don't actually need to subclass ``Filter``: you can pass any instance -which has a ``filter`` method with the same semantics. - -.. versionchanged:: 3.2 - You don't need to create specialized ``Filter`` classes, or use other - classes with a ``filter`` method: you can use a function (or other - callable) as a filter. The filtering logic will check to see if the filter - object has a ``filter`` attribute: if it does, it's assumed to be a - ``Filter`` and its :meth:`~Filter.filter` method is called. Otherwise, it's - assumed to be a callable and called with the record as the single - parameter. The returned value should conform to that returned by - :meth:`~Filter.filter`. - -Although filters are used primarily to filter records based on more -sophisticated criteria than levels, they get to see every record which is -processed by the handler or logger they're attached to: this can be useful if -you want to do things like counting how many records were processed by a -particular logger or handler, or adding, changing or removing attributes in -the :class:`LogRecord` being processed. Obviously changing the LogRecord needs -to be done with some care, but it does allow the injection of contextual -information into logs (see :ref:`filters-contextual`). - - -.. _log-record: - -LogRecord Objects ------------------ - -:class:`LogRecord` instances are created automatically by the :class:`Logger` -every time something is logged, and can be created manually via -:func:`makeLogRecord` (for example, from a pickled event received over the -wire). - - -.. class:: LogRecord(name, level, pathname, lineno, msg, args, exc_info, func=None, sinfo=None) - - Contains all the information pertinent to the event being logged. - - The primary information is passed in *msg* and *args*, - which are combined using ``msg % args`` to create - the :attr:`!message` attribute of the record. - - :param name: The name of the logger used to log the event - represented by this :class:`!LogRecord`. - Note that the logger name in the :class:`!LogRecord` - will always have this value, - even though it may be emitted by a handler - attached to a different (ancestor) logger. - :type name: str - - :param level: The :ref:`numeric level ` of the logging event - (such as ``10`` for ``DEBUG``, ``20`` for ``INFO``, etc). - Note that this is converted to *two* attributes of the LogRecord: - :attr:`!levelno` for the numeric value - and :attr:`!levelname` for the corresponding level name. - :type level: int - - :param pathname: The full string path of the source file - where the logging call was made. - :type pathname: str - - :param lineno: The line number in the source file - where the logging call was made. - :type lineno: int - - :param msg: The event description message, - which can be a %-format string with placeholders for variable data, - or an arbitrary object (see :ref:`arbitrary-object-messages`). - :type msg: typing.Any - - :param args: Variable data to merge into the *msg* argument - to obtain the event description. - :type args: tuple | dict[str, typing.Any] - - :param exc_info: An exception tuple with the current exception information, - as returned by :func:`sys.exc_info`, - or ``None`` if no exception information is available. - :type exc_info: tuple[type[BaseException], BaseException, types.TracebackType] | None - - :param func: The name of the function or method - from which the logging call was invoked. - :type func: str | None - - :param sinfo: A text string representing stack information - from the base of the stack in the current thread, - up to the logging call. - :type sinfo: str | None - - .. method:: getMessage() - - Returns the message for this :class:`LogRecord` instance after merging any - user-supplied arguments with the message. If the user-supplied message - argument to the logging call is not a string, :func:`str` is called on it to - convert it to a string. This allows use of user-defined classes as - messages, whose ``__str__`` method can return the actual format string to - be used. - - .. versionchanged:: 3.2 - The creation of a :class:`LogRecord` has been made more configurable by - providing a factory which is used to create the record. The factory can be - set using :func:`getLogRecordFactory` and :func:`setLogRecordFactory` - (see this for the factory's signature). - - This functionality can be used to inject your own values into a - :class:`LogRecord` at creation time. You can use the following pattern:: - - old_factory = logging.getLogRecordFactory() - - def record_factory(*args, **kwargs): - record = old_factory(*args, **kwargs) - record.custom_attribute = 0xdecafbad - return record - - logging.setLogRecordFactory(record_factory) - - With this pattern, multiple factories could be chained, and as long - as they don't overwrite each other's attributes or unintentionally - overwrite the standard attributes listed above, there should be no - surprises. - - -.. _logrecord-attributes: - -LogRecord attributes --------------------- - -The LogRecord has a number of attributes, most of which are derived from the -parameters to the constructor. (Note that the names do not always correspond -exactly between the LogRecord constructor parameters and the LogRecord -attributes.) These attributes can be used to merge data from the record into -the format string. The following table lists (in alphabetical order) the -attribute names, their meanings and the corresponding placeholder in a %-style -format string. - -If you are using {}-formatting (:func:`str.format`), you can use -``{attrname}`` as the placeholder in the format string. If you are using -$-formatting (:class:`string.Template`), use the form ``${attrname}``. In -both cases, of course, replace ``attrname`` with the actual attribute name -you want to use. - -In the case of {}-formatting, you can specify formatting flags by placing them -after the attribute name, separated from it with a colon. For example: a -placeholder of ``{msecs:03.0f}`` would format a millisecond value of ``4`` as -``004``. Refer to the :meth:`str.format` documentation for full details on -the options available to you. - -+----------------+-------------------------+-----------------------------------------------+ -| Attribute name | Format | Description | -+================+=========================+===============================================+ -| args | You shouldn't need to | The tuple of arguments merged into ``msg`` to | -| | format this yourself. | produce ``message``, or a dict whose values | -| | | are used for the merge (when there is only one| -| | | argument, and it is a dictionary). | -+----------------+-------------------------+-----------------------------------------------+ -| asctime | ``%(asctime)s`` | Human-readable time when the | -| | | :class:`LogRecord` was created. By default | -| | | this is of the form '2003-07-08 16:49:45,896' | -| | | (the numbers after the comma are millisecond | -| | | portion of the time). | -+----------------+-------------------------+-----------------------------------------------+ -| created | ``%(created)f`` | Time when the :class:`LogRecord` was created | -| | | (as returned by :func:`time.time`). | -+----------------+-------------------------+-----------------------------------------------+ -| exc_info | You shouldn't need to | Exception tuple (à la ``sys.exc_info``) or, | -| | format this yourself. | if no exception has occurred, ``None``. | -+----------------+-------------------------+-----------------------------------------------+ -| filename | ``%(filename)s`` | Filename portion of ``pathname``. | -+----------------+-------------------------+-----------------------------------------------+ -| funcName | ``%(funcName)s`` | Name of function containing the logging call. | -+----------------+-------------------------+-----------------------------------------------+ -| levelname | ``%(levelname)s`` | Text logging level for the message | -| | | (``'DEBUG'``, ``'INFO'``, ``'WARNING'``, | -| | | ``'ERROR'``, ``'CRITICAL'``). | -+----------------+-------------------------+-----------------------------------------------+ -| levelno | ``%(levelno)s`` | Numeric logging level for the message | -| | | (:const:`DEBUG`, :const:`INFO`, | -| | | :const:`WARNING`, :const:`ERROR`, | -| | | :const:`CRITICAL`). | -+----------------+-------------------------+-----------------------------------------------+ -| lineno | ``%(lineno)d`` | Source line number where the logging call was | -| | | issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| message | ``%(message)s`` | The logged message, computed as ``msg % | -| | | args``. This is set when | -| | | :meth:`Formatter.format` is invoked. | -+----------------+-------------------------+-----------------------------------------------+ -| module | ``%(module)s`` | Module (name portion of ``filename``). | -+----------------+-------------------------+-----------------------------------------------+ -| msecs | ``%(msecs)d`` | Millisecond portion of the time when the | -| | | :class:`LogRecord` was created. | -+----------------+-------------------------+-----------------------------------------------+ -| msg | You shouldn't need to | The format string passed in the original | -| | format this yourself. | logging call. Merged with ``args`` to | -| | | produce ``message``, or an arbitrary object | -| | | (see :ref:`arbitrary-object-messages`). | -+----------------+-------------------------+-----------------------------------------------+ -| name | ``%(name)s`` | Name of the logger used to log the call. | -+----------------+-------------------------+-----------------------------------------------+ -| pathname | ``%(pathname)s`` | Full pathname of the source file where the | -| | | logging call was issued (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| process | ``%(process)d`` | Process ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| processName | ``%(processName)s`` | Process name (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| relativeCreated| ``%(relativeCreated)d`` | Time in milliseconds when the LogRecord was | -| | | created, relative to the time the logging | -| | | module was loaded. | -+----------------+-------------------------+-----------------------------------------------+ -| stack_info | You shouldn't need to | Stack frame information (where available) | -| | format this yourself. | from the bottom of the stack in the current | -| | | thread, up to and including the stack frame | -| | | of the logging call which resulted in the | -| | | creation of this record. | -+----------------+-------------------------+-----------------------------------------------+ -| thread | ``%(thread)d`` | Thread ID (if available). | -+----------------+-------------------------+-----------------------------------------------+ -| threadName | ``%(threadName)s`` | Thread name (if available). | -+----------------+-------------------------+-----------------------------------------------+ - -.. versionchanged:: 3.1 - *processName* was added. - - -.. _logger-adapter: - -LoggerAdapter Objects ---------------------- - -:class:`LoggerAdapter` instances are used to conveniently pass contextual -information into logging calls. For a usage example, see the section on -:ref:`adding contextual information to your logging output `. - -.. class:: LoggerAdapter(logger, extra) - - Returns an instance of :class:`LoggerAdapter` initialized with an - underlying :class:`Logger` instance and a dict-like object. - - .. method:: process(msg, kwargs) - - Modifies the message and/or keyword arguments passed to a logging call in - order to insert contextual information. This implementation takes the object - passed as *extra* to the constructor and adds it to *kwargs* using key - 'extra'. The return value is a (*msg*, *kwargs*) tuple which has the - (possibly modified) versions of the arguments passed in. - - .. attribute:: manager - - Delegates to the underlying :attr:`!manager`` on *logger*. - - .. attribute:: _log - - Delegates to the underlying :meth:`!_log`` method on *logger*. - - In addition to the above, :class:`LoggerAdapter` supports the following - methods of :class:`Logger`: :meth:`~Logger.debug`, :meth:`~Logger.info`, - :meth:`~Logger.warning`, :meth:`~Logger.error`, :meth:`~Logger.exception`, - :meth:`~Logger.critical`, :meth:`~Logger.log`, :meth:`~Logger.isEnabledFor`, - :meth:`~Logger.getEffectiveLevel`, :meth:`~Logger.setLevel` and - :meth:`~Logger.hasHandlers`. These methods have the same signatures as their - counterparts in :class:`Logger`, so you can use the two types of instances - interchangeably. - - .. versionchanged:: 3.2 - - The :meth:`~Logger.isEnabledFor`, :meth:`~Logger.getEffectiveLevel`, - :meth:`~Logger.setLevel` and :meth:`~Logger.hasHandlers` methods were added - to :class:`LoggerAdapter`. These methods delegate to the underlying logger. - - .. versionchanged:: 3.6 - - Attribute :attr:`!manager` and method :meth:`!_log` were added, which - delegate to the underlying logger and allow adapters to be nested. - - -Thread Safety -------------- - -The logging module is intended to be thread-safe without any special work -needing to be done by its clients. It achieves this though using threading -locks; there is one lock to serialize access to the module's shared data, and -each handler also creates a lock to serialize access to its underlying I/O. - -If you are implementing asynchronous signal handlers using the :mod:`signal` -module, you may not be able to use logging from within such handlers. This is -because lock implementations in the :mod:`threading` module are not always -re-entrant, and so cannot be invoked from such signal handlers. - - -Module-Level Functions ----------------------- - -In addition to the classes described above, there are a number of module-level -functions. - - -.. function:: getLogger(name=None) - - Return a logger with the specified name or, if name is ``None``, return a - logger which is the root logger of the hierarchy. If specified, the name is - typically a dot-separated hierarchical name like *'a'*, *'a.b'* or *'a.b.c.d'*. - Choice of these names is entirely up to the developer who is using logging. - - All calls to this function with a given name return the same logger instance. - This means that logger instances never need to be passed between different parts - of an application. - - -.. function:: getLoggerClass() - - Return either the standard :class:`Logger` class, or the last class passed to - :func:`setLoggerClass`. This function may be called from within a new class - definition, to ensure that installing a customized :class:`Logger` class will - not undo customizations already applied by other code. For example:: - - class MyLogger(logging.getLoggerClass()): - # ... override behaviour here - - -.. function:: getLogRecordFactory() - - Return a callable which is used to create a :class:`LogRecord`. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`setLogRecordFactory`, - to allow developers more control over how the :class:`LogRecord` - representing a logging event is constructed. - - See :func:`setLogRecordFactory` for more information about the how the - factory is called. - -.. function:: debug(msg, *args, **kwargs) - - Logs a message with level :const:`DEBUG` on the root logger. The *msg* is the - message format string, and the *args* are the arguments which are merged into - *msg* using the string formatting operator. (Note that this means that you can - use keywords in the format string, together with a single dictionary argument.) - - There are three keyword arguments in *kwargs* which are inspected: *exc_info* - which, if it does not evaluate as false, causes exception information to be - added to the logging message. If an exception tuple (in the format returned by - :func:`sys.exc_info`) or an exception instance is provided, it is used; - otherwise, :func:`sys.exc_info` is called to get the exception information. - - The second optional keyword argument is *stack_info*, which defaults to - ``False``. If true, stack information is added to the logging - message, including the actual logging call. Note that this is not the same - stack information as that displayed through specifying *exc_info*: The - former is stack frames from the bottom of the stack up to the logging call - in the current thread, whereas the latter is information about stack frames - which have been unwound, following an exception, while searching for - exception handlers. - - You can specify *stack_info* independently of *exc_info*, e.g. to just show - how you got to a certain point in your code, even when no exceptions were - raised. The stack frames are printed following a header line which says: - - .. code-block:: none - - Stack (most recent call last): - - This mimics the ``Traceback (most recent call last):`` which is used when - displaying exception frames. - - The third optional keyword argument is *extra* which can be used to pass a - dictionary which is used to populate the __dict__ of the LogRecord created for - the logging event with user-defined attributes. These custom attributes can then - be used as you like. For example, they could be incorporated into logged - messages. For example:: - - FORMAT = '%(asctime)s %(clientip)-15s %(user)-8s %(message)s' - logging.basicConfig(format=FORMAT) - d = {'clientip': '192.168.0.1', 'user': 'fbloggs'} - logging.warning('Protocol problem: %s', 'connection reset', extra=d) - - would print something like: - - .. code-block:: none - - 2006-02-08 22:20:02,165 192.168.0.1 fbloggs Protocol problem: connection reset - - The keys in the dictionary passed in *extra* should not clash with the keys used - by the logging system. (See the :class:`Formatter` documentation for more - information on which keys are used by the logging system.) - - If you choose to use these attributes in logged messages, you need to exercise - some care. In the above example, for instance, the :class:`Formatter` has been - set up with a format string which expects 'clientip' and 'user' in the attribute - dictionary of the LogRecord. If these are missing, the message will not be - logged because a string formatting exception will occur. So in this case, you - always need to pass the *extra* dictionary with these keys. - - While this might be annoying, this feature is intended for use in specialized - circumstances, such as multi-threaded servers where the same code executes in - many contexts, and interesting conditions which arise are dependent on this - context (such as remote client IP address and authenticated user name, in the - above example). In such circumstances, it is likely that specialized - :class:`Formatter`\ s would be used with particular :class:`Handler`\ s. - - This function (as well as :func:`info`, :func:`warning`, :func:`error` and - :func:`critical`) will call :func:`basicConfig` if the root logger doesn't - have any handler attached. - - .. versionchanged:: 3.2 - The *stack_info* parameter was added. - -.. function:: info(msg, *args, **kwargs) - - Logs a message with level :const:`INFO` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: warning(msg, *args, **kwargs) - - Logs a message with level :const:`WARNING` on the root logger. The arguments - are interpreted as for :func:`debug`. - - .. note:: There is an obsolete function ``warn`` which is functionally - identical to ``warning``. As ``warn`` is deprecated, please do not use - it - use ``warning`` instead. - - -.. function:: error(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. - - -.. function:: critical(msg, *args, **kwargs) - - Logs a message with level :const:`CRITICAL` on the root logger. The arguments - are interpreted as for :func:`debug`. - - -.. function:: exception(msg, *args, **kwargs) - - Logs a message with level :const:`ERROR` on the root logger. The arguments are - interpreted as for :func:`debug`. Exception info is added to the logging - message. This function should only be called from an exception handler. - -.. function:: log(level, msg, *args, **kwargs) - - Logs a message with level *level* on the root logger. The other arguments are - interpreted as for :func:`debug`. - -.. function:: disable(level=CRITICAL) - - Provides an overriding level *level* for all loggers which takes precedence over - the logger's own level. When the need arises to temporarily throttle logging - output down across the whole application, this function can be useful. Its - effect is to disable all logging calls of severity *level* and below, so that - if you call it with a value of INFO, then all INFO and DEBUG events would be - discarded, whereas those of severity WARNING and above would be processed - according to the logger's effective level. If - ``logging.disable(logging.NOTSET)`` is called, it effectively removes this - overriding level, so that logging output again depends on the effective - levels of individual loggers. - - Note that if you have defined any custom logging level higher than - ``CRITICAL`` (this is not recommended), you won't be able to rely on the - default value for the *level* parameter, but will have to explicitly supply a - suitable value. - - .. versionchanged:: 3.7 - The *level* parameter was defaulted to level ``CRITICAL``. See - :issue:`28524` for more information about this change. - -.. function:: addLevelName(level, levelName) - - Associates level *level* with text *levelName* in an internal dictionary, which is - used to map numeric levels to a textual representation, for example when a - :class:`Formatter` formats a message. This function can also be used to define - your own levels. The only constraints are that all levels used must be - registered using this function, levels should be positive integers and they - should increase in increasing order of severity. - - .. note:: If you are thinking of defining your own levels, please see the - section on :ref:`custom-levels`. - -.. function:: getLevelNamesMapping() - - Returns a mapping from level names to their corresponding logging levels. For example, the - string "CRITICAL" maps to :const:`CRITICAL`. The returned mapping is copied from an internal - mapping on each call to this function. - - .. versionadded:: 3.11 - -.. function:: getLevelName(level) - - Returns the textual or numeric representation of logging level *level*. - - If *level* is one of the predefined levels :const:`CRITICAL`, :const:`ERROR`, - :const:`WARNING`, :const:`INFO` or :const:`DEBUG` then you get the - corresponding string. If you have associated levels with names using - :func:`addLevelName` then the name you have associated with *level* is - returned. If a numeric value corresponding to one of the defined levels is - passed in, the corresponding string representation is returned. - - The *level* parameter also accepts a string representation of the level such - as 'INFO'. In such cases, this functions returns the corresponding numeric - value of the level. - - If no matching numeric or string value is passed in, the string - 'Level %s' % level is returned. - - .. note:: Levels are internally integers (as they need to be compared in the - logging logic). This function is used to convert between an integer level - and the level name displayed in the formatted log output by means of the - ``%(levelname)s`` format specifier (see :ref:`logrecord-attributes`), and - vice versa. - - .. versionchanged:: 3.4 - In Python versions earlier than 3.4, this function could also be passed a - text level, and would return the corresponding numeric value of the level. - This undocumented behaviour was considered a mistake, and was removed in - Python 3.4, but reinstated in 3.4.2 due to retain backward compatibility. - -.. function:: makeLogRecord(attrdict) - - Creates and returns a new :class:`LogRecord` instance whose attributes are - defined by *attrdict*. This function is useful for taking a pickled - :class:`LogRecord` attribute dictionary, sent over a socket, and reconstituting - it as a :class:`LogRecord` instance at the receiving end. - - -.. function:: basicConfig(**kwargs) - - Does basic configuration for the logging system by creating a - :class:`StreamHandler` with a default :class:`Formatter` and adding it to the - root logger. The functions :func:`debug`, :func:`info`, :func:`warning`, - :func:`error` and :func:`critical` will call :func:`basicConfig` automatically - if no handlers are defined for the root logger. - - This function does nothing if the root logger already has handlers - configured, unless the keyword argument *force* is set to ``True``. - - .. note:: This function should be called from the main thread - before other threads are started. In versions of Python prior to - 2.7.1 and 3.2, if this function is called from multiple threads, - it is possible (in rare circumstances) that a handler will be added - to the root logger more than once, leading to unexpected results - such as messages being duplicated in the log. - - The following keyword arguments are supported. - - .. tabularcolumns:: |l|L| - - +--------------+---------------------------------------------+ - | Format | Description | - +==============+=============================================+ - | *filename* | Specifies that a :class:`FileHandler` be | - | | created, using the specified filename, | - | | rather than a :class:`StreamHandler`. | - +--------------+---------------------------------------------+ - | *filemode* | If *filename* is specified, open the file | - | | in this :ref:`mode `. Defaults | - | | to ``'a'``. | - +--------------+---------------------------------------------+ - | *format* | Use the specified format string for the | - | | handler. Defaults to attributes | - | | ``levelname``, ``name`` and ``message`` | - | | separated by colons. | - +--------------+---------------------------------------------+ - | *datefmt* | Use the specified date/time format, as | - | | accepted by :func:`time.strftime`. | - +--------------+---------------------------------------------+ - | *style* | If *format* is specified, use this style | - | | for the format string. One of ``'%'``, | - | | ``'{'`` or ``'$'`` for :ref:`printf-style | - | | `, | - | | :meth:`str.format` or | - | | :class:`string.Template` respectively. | - | | Defaults to ``'%'``. | - +--------------+---------------------------------------------+ - | *level* | Set the root logger level to the specified | - | | :ref:`level `. | - +--------------+---------------------------------------------+ - | *stream* | Use the specified stream to initialize the | - | | :class:`StreamHandler`. Note that this | - | | argument is incompatible with *filename* - | - | | if both are present, a ``ValueError`` is | - | | raised. | - +--------------+---------------------------------------------+ - | *handlers* | If specified, this should be an iterable of | - | | already created handlers to add to the root | - | | logger. Any handlers which don't already | - | | have a formatter set will be assigned the | - | | default formatter created in this function. | - | | Note that this argument is incompatible | - | | with *filename* or *stream* - if both | - | | are present, a ``ValueError`` is raised. | - +--------------+---------------------------------------------+ - | *force* | If this keyword argument is specified as | - | | true, any existing handlers attached to the | - | | root logger are removed and closed, before | - | | carrying out the configuration as specified | - | | by the other arguments. | - +--------------+---------------------------------------------+ - | *encoding* | If this keyword argument is specified along | - | | with *filename*, its value is used when the | - | | :class:`FileHandler` is created, and thus | - | | used when opening the output file. | - +--------------+---------------------------------------------+ - | *errors* | If this keyword argument is specified along | - | | with *filename*, its value is used when the | - | | :class:`FileHandler` is created, and thus | - | | used when opening the output file. If not | - | | specified, the value 'backslashreplace' is | - | | used. Note that if ``None`` is specified, | - | | it will be passed as such to :func:`open`, | - | | which means that it will be treated the | - | | same as passing 'errors'. | - +--------------+---------------------------------------------+ - - .. versionchanged:: 3.2 - The *style* argument was added. - - .. versionchanged:: 3.3 - The *handlers* argument was added. Additional checks were added to - catch situations where incompatible arguments are specified (e.g. - *handlers* together with *stream* or *filename*, or *stream* - together with *filename*). - - .. versionchanged:: 3.8 - The *force* argument was added. - - .. versionchanged:: 3.9 - The *encoding* and *errors* arguments were added. - -.. function:: shutdown() - - Informs the logging system to perform an orderly shutdown by flushing and - closing all handlers. This should be called at application exit and no - further use of the logging system should be made after this call. - - When the logging module is imported, it registers this function as an exit - handler (see :mod:`atexit`), so normally there's no need to do that - manually. - - -.. function:: setLoggerClass(klass) - - Tells the logging system to use the class *klass* when instantiating a logger. - The class should define :meth:`!__init__` such that only a name argument is - required, and the :meth:`!__init__` should call :meth:`!Logger.__init__`. This - function is typically called before any loggers are instantiated by applications - which need to use custom logger behavior. After this call, as at any other - time, do not instantiate loggers directly using the subclass: continue to use - the :func:`logging.getLogger` API to get your loggers. - - -.. function:: setLogRecordFactory(factory) - - Set a callable which is used to create a :class:`LogRecord`. - - :param factory: The factory callable to be used to instantiate a log record. - - .. versionadded:: 3.2 - This function has been provided, along with :func:`getLogRecordFactory`, to - allow developers more control over how the :class:`LogRecord` representing - a logging event is constructed. - - The factory has the following signature: - - ``factory(name, level, fn, lno, msg, args, exc_info, func=None, sinfo=None, **kwargs)`` - - :name: The logger name. - :level: The logging level (numeric). - :fn: The full pathname of the file where the logging call was made. - :lno: The line number in the file where the logging call was made. - :msg: The logging message. - :args: The arguments for the logging message. - :exc_info: An exception tuple, or ``None``. - :func: The name of the function or method which invoked the logging - call. - :sinfo: A stack traceback such as is provided by - :func:`traceback.print_stack`, showing the call hierarchy. - :kwargs: Additional keyword arguments. - - -Module-Level Attributes ------------------------ - -.. attribute:: lastResort - - A "handler of last resort" is available through this attribute. This - is a :class:`StreamHandler` writing to ``sys.stderr`` with a level of - ``WARNING``, and is used to handle logging events in the absence of any - logging configuration. The end result is to just print the message to - ``sys.stderr``. This replaces the earlier error message saying that - "no handlers could be found for logger XYZ". If you need the earlier - behaviour for some reason, ``lastResort`` can be set to ``None``. - - .. versionadded:: 3.2 - -Integration with the warnings module ------------------------------------- - -The :func:`captureWarnings` function can be used to integrate :mod:`logging` -with the :mod:`warnings` module. - -.. function:: captureWarnings(capture) - - This function is used to turn the capture of warnings by logging on and - off. - - If *capture* is ``True``, warnings issued by the :mod:`warnings` module will - be redirected to the logging system. Specifically, a warning will be - formatted using :func:`warnings.formatwarning` and the resulting string - logged to a logger named ``'py.warnings'`` with a severity of :const:`WARNING`. - - If *capture* is ``False``, the redirection of warnings to the logging system - will stop, and warnings will be redirected to their original destinations - (i.e. those in effect before ``captureWarnings(True)`` was called). - - -.. seealso:: - - Module :mod:`logging.config` - Configuration API for the logging module. - - Module :mod:`logging.handlers` - Useful handlers included with the logging module. - - :pep:`282` - A Logging System - The proposal which described this feature for inclusion in the Python standard - library. - - `Original Python logging package `_ - This is the original source for the :mod:`logging` package. The version of the - package available from this site is suitable for use with Python 1.5.2, 2.1.x - and 2.2.x, which do not include the :mod:`logging` package in the standard - library. diff --git a/Doc/library/lzma.rst b/Doc/library/lzma.rst deleted file mode 100644 index 0d69c3bc01d1e2..00000000000000 --- a/Doc/library/lzma.rst +++ /dev/null @@ -1,438 +0,0 @@ -:mod:`lzma` --- Compression using the LZMA algorithm -==================================================== - -.. module:: lzma - :synopsis: A Python wrapper for the liblzma compression library. - -.. moduleauthor:: Nadeem Vawda -.. sectionauthor:: Nadeem Vawda - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/lzma.py` - --------------- - -This module provides classes and convenience functions for compressing and -decompressing data using the LZMA compression algorithm. Also included is a file -interface supporting the ``.xz`` and legacy ``.lzma`` file formats used by the -:program:`xz` utility, as well as raw compressed streams. - -The interface provided by this module is very similar to that of the :mod:`bz2` -module. Note that :class:`LZMAFile` and :class:`bz2.BZ2File` are *not* -thread-safe, so if you need to use a single :class:`LZMAFile` instance -from multiple threads, it is necessary to protect it with a lock. - - -.. exception:: LZMAError - - This exception is raised when an error occurs during compression or - decompression, or while initializing the compressor/decompressor state. - - -Reading and writing compressed files ------------------------------------- - -.. function:: open(filename, mode="rb", *, format=None, check=-1, preset=None, filters=None, encoding=None, errors=None, newline=None) - - Open an LZMA-compressed file in binary or text mode, returning a :term:`file - object`. - - The *filename* argument can be either an actual file name (given as a - :class:`str`, :class:`bytes` or :term:`path-like ` object), in - which case the named file is opened, or it can be an existing file object - to read from or write to. - - The *mode* argument can be any of ``"r"``, ``"rb"``, ``"w"``, ``"wb"``, - ``"x"``, ``"xb"``, ``"a"`` or ``"ab"`` for binary mode, or ``"rt"``, - ``"wt"``, ``"xt"``, or ``"at"`` for text mode. The default is ``"rb"``. - - When opening a file for reading, the *format* and *filters* arguments have - the same meanings as for :class:`LZMADecompressor`. In this case, the *check* - and *preset* arguments should not be used. - - When opening a file for writing, the *format*, *check*, *preset* and - *filters* arguments have the same meanings as for :class:`LZMACompressor`. - - For binary mode, this function is equivalent to the :class:`LZMAFile` - constructor: ``LZMAFile(filename, mode, ...)``. In this case, the *encoding*, - *errors* and *newline* arguments must not be provided. - - For text mode, a :class:`LZMAFile` object is created, and wrapped in an - :class:`io.TextIOWrapper` instance with the specified encoding, error - handling behavior, and line ending(s). - - .. versionchanged:: 3.4 - Added support for the ``"x"``, ``"xb"`` and ``"xt"`` modes. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. class:: LZMAFile(filename=None, mode="r", *, format=None, check=-1, preset=None, filters=None) - - Open an LZMA-compressed file in binary mode. - - An :class:`LZMAFile` can wrap an already-open :term:`file object`, or operate - directly on a named file. The *filename* argument specifies either the file - object to wrap, or the name of the file to open (as a :class:`str`, - :class:`bytes` or :term:`path-like ` object). When wrapping an - existing file object, the wrapped file will not be closed when the - :class:`LZMAFile` is closed. - - The *mode* argument can be either ``"r"`` for reading (default), ``"w"`` for - overwriting, ``"x"`` for exclusive creation, or ``"a"`` for appending. These - can equivalently be given as ``"rb"``, ``"wb"``, ``"xb"`` and ``"ab"`` - respectively. - - If *filename* is a file object (rather than an actual file name), a mode of - ``"w"`` does not truncate the file, and is instead equivalent to ``"a"``. - - When opening a file for reading, the input file may be the concatenation of - multiple separate compressed streams. These are transparently decoded as a - single logical stream. - - When opening a file for reading, the *format* and *filters* arguments have - the same meanings as for :class:`LZMADecompressor`. In this case, the *check* - and *preset* arguments should not be used. - - When opening a file for writing, the *format*, *check*, *preset* and - *filters* arguments have the same meanings as for :class:`LZMACompressor`. - - :class:`LZMAFile` supports all the members specified by - :class:`io.BufferedIOBase`, except for :meth:`~io.BufferedIOBase.detach` - and :meth:`~io.IOBase.truncate`. - Iteration and the :keyword:`with` statement are supported. - - The following method is also provided: - - .. method:: peek(size=-1) - - Return buffered data without advancing the file position. At least one - byte of data will be returned, unless EOF has been reached. The exact - number of bytes returned is unspecified (the *size* argument is ignored). - - .. note:: While calling :meth:`peek` does not change the file position of - the :class:`LZMAFile`, it may change the position of the underlying - file object (e.g. if the :class:`LZMAFile` was constructed by passing a - file object for *filename*). - - .. versionchanged:: 3.4 - Added support for the ``"x"`` and ``"xb"`` modes. - - .. versionchanged:: 3.5 - The :meth:`~io.BufferedIOBase.read` method now accepts an argument of - ``None``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -Compressing and decompressing data in memory --------------------------------------------- - -.. class:: LZMACompressor(format=FORMAT_XZ, check=-1, preset=None, filters=None) - - Create a compressor object, which can be used to compress data incrementally. - - For a more convenient way of compressing a single chunk of data, see - :func:`compress`. - - The *format* argument specifies what container format should be used. - Possible values are: - - * :const:`FORMAT_XZ`: The ``.xz`` container format. - This is the default format. - - * :const:`FORMAT_ALONE`: The legacy ``.lzma`` container format. - This format is more limited than ``.xz`` -- it does not support integrity - checks or multiple filters. - - * :const:`FORMAT_RAW`: A raw data stream, not using any container format. - This format specifier does not support integrity checks, and requires that - you always specify a custom filter chain (for both compression and - decompression). Additionally, data compressed in this manner cannot be - decompressed using :const:`FORMAT_AUTO` (see :class:`LZMADecompressor`). - - The *check* argument specifies the type of integrity check to include in the - compressed data. This check is used when decompressing, to ensure that the - data has not been corrupted. Possible values are: - - * :const:`CHECK_NONE`: No integrity check. - This is the default (and the only acceptable value) for - :const:`FORMAT_ALONE` and :const:`FORMAT_RAW`. - - * :const:`CHECK_CRC32`: 32-bit Cyclic Redundancy Check. - - * :const:`CHECK_CRC64`: 64-bit Cyclic Redundancy Check. - This is the default for :const:`FORMAT_XZ`. - - * :const:`CHECK_SHA256`: 256-bit Secure Hash Algorithm. - - If the specified check is not supported, an :class:`LZMAError` is raised. - - The compression settings can be specified either as a preset compression - level (with the *preset* argument), or in detail as a custom filter chain - (with the *filters* argument). - - The *preset* argument (if provided) should be an integer between ``0`` and - ``9`` (inclusive), optionally OR-ed with the constant - :const:`PRESET_EXTREME`. If neither *preset* nor *filters* are given, the - default behavior is to use :const:`PRESET_DEFAULT` (preset level ``6``). - Higher presets produce smaller output, but make the compression process - slower. - - .. note:: - - In addition to being more CPU-intensive, compression with higher presets - also requires much more memory (and produces output that needs more memory - to decompress). With preset ``9`` for example, the overhead for an - :class:`LZMACompressor` object can be as high as 800 MiB. For this reason, - it is generally best to stick with the default preset. - - The *filters* argument (if provided) should be a filter chain specifier. - See :ref:`filter-chain-specs` for details. - - .. method:: compress(data) - - Compress *data* (a :class:`bytes` object), returning a :class:`bytes` - object containing compressed data for at least part of the input. Some of - *data* may be buffered internally, for use in later calls to - :meth:`compress` and :meth:`flush`. The returned data should be - concatenated with the output of any previous calls to :meth:`compress`. - - .. method:: flush() - - Finish the compression process, returning a :class:`bytes` object - containing any data stored in the compressor's internal buffers. - - The compressor cannot be used after this method has been called. - - -.. class:: LZMADecompressor(format=FORMAT_AUTO, memlimit=None, filters=None) - - Create a decompressor object, which can be used to decompress data - incrementally. - - For a more convenient way of decompressing an entire compressed stream at - once, see :func:`decompress`. - - The *format* argument specifies the container format that should be used. The - default is :const:`FORMAT_AUTO`, which can decompress both ``.xz`` and - ``.lzma`` files. Other possible values are :const:`FORMAT_XZ`, - :const:`FORMAT_ALONE`, and :const:`FORMAT_RAW`. - - The *memlimit* argument specifies a limit (in bytes) on the amount of memory - that the decompressor can use. When this argument is used, decompression will - fail with an :class:`LZMAError` if it is not possible to decompress the input - within the given memory limit. - - The *filters* argument specifies the filter chain that was used to create - the stream being decompressed. This argument is required if *format* is - :const:`FORMAT_RAW`, but should not be used for other formats. - See :ref:`filter-chain-specs` for more information about filter chains. - - .. note:: - This class does not transparently handle inputs containing multiple - compressed streams, unlike :func:`decompress` and :class:`LZMAFile`. To - decompress a multi-stream input with :class:`LZMADecompressor`, you must - create a new decompressor for each stream. - - .. method:: decompress(data, max_length=-1) - - Decompress *data* (a :term:`bytes-like object`), returning - uncompressed data as bytes. Some of *data* may be buffered - internally, for use in later calls to :meth:`decompress`. The - returned data should be concatenated with the output of any - previous calls to :meth:`decompress`. - - If *max_length* is nonnegative, returns at most *max_length* - bytes of decompressed data. If this limit is reached and further - output can be produced, the :attr:`~.needs_input` attribute will - be set to ``False``. In this case, the next call to - :meth:`~.decompress` may provide *data* as ``b''`` to obtain - more of the output. - - If all of the input data was decompressed and returned (either - because this was less than *max_length* bytes, or because - *max_length* was negative), the :attr:`~.needs_input` attribute - will be set to ``True``. - - Attempting to decompress data after the end of stream is reached - raises an :exc:`EOFError`. Any data found after the end of the - stream is ignored and saved in the :attr:`~.unused_data` attribute. - - .. versionchanged:: 3.5 - Added the *max_length* parameter. - - .. attribute:: check - - The ID of the integrity check used by the input stream. This may be - :const:`CHECK_UNKNOWN` until enough of the input has been decoded to - determine what integrity check it uses. - - .. attribute:: eof - - ``True`` if the end-of-stream marker has been reached. - - .. attribute:: unused_data - - Data found after the end of the compressed stream. - - Before the end of the stream is reached, this will be ``b""``. - - .. attribute:: needs_input - - ``False`` if the :meth:`.decompress` method can provide more - decompressed data before requiring new uncompressed input. - - .. versionadded:: 3.5 - -.. function:: compress(data, format=FORMAT_XZ, check=-1, preset=None, filters=None) - - Compress *data* (a :class:`bytes` object), returning the compressed data as a - :class:`bytes` object. - - See :class:`LZMACompressor` above for a description of the *format*, *check*, - *preset* and *filters* arguments. - - -.. function:: decompress(data, format=FORMAT_AUTO, memlimit=None, filters=None) - - Decompress *data* (a :class:`bytes` object), returning the uncompressed data - as a :class:`bytes` object. - - If *data* is the concatenation of multiple distinct compressed streams, - decompress all of these streams, and return the concatenation of the results. - - See :class:`LZMADecompressor` above for a description of the *format*, - *memlimit* and *filters* arguments. - - -Miscellaneous -------------- - -.. function:: is_check_supported(check) - - Return ``True`` if the given integrity check is supported on this system. - - :const:`CHECK_NONE` and :const:`CHECK_CRC32` are always supported. - :const:`CHECK_CRC64` and :const:`CHECK_SHA256` may be unavailable if you are - using a version of :program:`liblzma` that was compiled with a limited - feature set. - - -.. _filter-chain-specs: - -Specifying custom filter chains -------------------------------- - -A filter chain specifier is a sequence of dictionaries, where each dictionary -contains the ID and options for a single filter. Each dictionary must contain -the key ``"id"``, and may contain additional keys to specify filter-dependent -options. Valid filter IDs are as follows: - -* Compression filters: - - * :const:`FILTER_LZMA1` (for use with :const:`FORMAT_ALONE`) - * :const:`FILTER_LZMA2` (for use with :const:`FORMAT_XZ` and :const:`FORMAT_RAW`) - -* Delta filter: - - * :const:`FILTER_DELTA` - -* Branch-Call-Jump (BCJ) filters: - - * :const:`FILTER_X86` - * :const:`FILTER_IA64` - * :const:`FILTER_ARM` - * :const:`FILTER_ARMTHUMB` - * :const:`FILTER_POWERPC` - * :const:`FILTER_SPARC` - -A filter chain can consist of up to 4 filters, and cannot be empty. The last -filter in the chain must be a compression filter, and any other filters must be -delta or BCJ filters. - -Compression filters support the following options (specified as additional -entries in the dictionary representing the filter): - -* ``preset``: A compression preset to use as a source of default values for - options that are not specified explicitly. -* ``dict_size``: Dictionary size in bytes. This should be between 4 KiB and - 1.5 GiB (inclusive). -* ``lc``: Number of literal context bits. -* ``lp``: Number of literal position bits. The sum ``lc + lp`` must be at - most 4. -* ``pb``: Number of position bits; must be at most 4. -* ``mode``: :const:`MODE_FAST` or :const:`MODE_NORMAL`. -* ``nice_len``: What should be considered a "nice length" for a match. - This should be 273 or less. -* ``mf``: What match finder to use -- :const:`MF_HC3`, :const:`MF_HC4`, - :const:`MF_BT2`, :const:`MF_BT3`, or :const:`MF_BT4`. -* ``depth``: Maximum search depth used by match finder. 0 (default) means to - select automatically based on other filter options. - -The delta filter stores the differences between bytes, producing more repetitive -input for the compressor in certain circumstances. It supports one option, -``dist``. This indicates the distance between bytes to be subtracted. The -default is 1, i.e. take the differences between adjacent bytes. - -The BCJ filters are intended to be applied to machine code. They convert -relative branches, calls and jumps in the code to use absolute addressing, with -the aim of increasing the redundancy that can be exploited by the compressor. -These filters support one option, ``start_offset``. This specifies the address -that should be mapped to the beginning of the input data. The default is 0. - - -Examples --------- - -Reading in a compressed file:: - - import lzma - with lzma.open("file.xz") as f: - file_content = f.read() - -Creating a compressed file:: - - import lzma - data = b"Insert Data Here" - with lzma.open("file.xz", "w") as f: - f.write(data) - -Compressing data in memory:: - - import lzma - data_in = b"Insert Data Here" - data_out = lzma.compress(data_in) - -Incremental compression:: - - import lzma - lzc = lzma.LZMACompressor() - out1 = lzc.compress(b"Some data\n") - out2 = lzc.compress(b"Another piece of data\n") - out3 = lzc.compress(b"Even more data\n") - out4 = lzc.flush() - # Concatenate all the partial results: - result = b"".join([out1, out2, out3, out4]) - -Writing compressed data to an already-open file:: - - import lzma - with open("file.xz", "wb") as f: - f.write(b"This data will not be compressed\n") - with lzma.open(f, "w") as lzf: - lzf.write(b"This *will* be compressed\n") - f.write(b"Not compressed\n") - -Creating a compressed file using a custom filter chain:: - - import lzma - my_filters = [ - {"id": lzma.FILTER_DELTA, "dist": 5}, - {"id": lzma.FILTER_LZMA2, "preset": 7 | lzma.PRESET_EXTREME}, - ] - with lzma.open("file.xz", "w", filters=my_filters) as f: - f.write(b"blah blah blah") diff --git a/Doc/library/mailbox.rst b/Doc/library/mailbox.rst deleted file mode 100644 index b27deb20f13236..00000000000000 --- a/Doc/library/mailbox.rst +++ /dev/null @@ -1,1590 +0,0 @@ -:mod:`mailbox` --- Manipulate mailboxes in various formats -========================================================== - -.. module:: mailbox - :synopsis: Manipulate mailboxes in various formats - -.. moduleauthor:: Gregory K. Johnson -.. sectionauthor:: Gregory K. Johnson - -**Source code:** :source:`Lib/mailbox.py` - --------------- - -This module defines two classes, :class:`Mailbox` and :class:`Message`, for -accessing and manipulating on-disk mailboxes and the messages they contain. -:class:`Mailbox` offers a dictionary-like mapping from keys to messages. -:class:`Message` extends the :mod:`email.message` module's -:class:`~email.message.Message` class with format-specific state and behavior. -Supported mailbox formats are Maildir, mbox, MH, Babyl, and MMDF. - - -.. seealso:: - - Module :mod:`email` - Represent and manipulate messages. - - -.. _mailbox-objects: - -:class:`Mailbox` objects ------------------------- - -.. class:: Mailbox - - A mailbox, which may be inspected and modified. - - The :class:`Mailbox` class defines an interface and is not intended to be - instantiated. Instead, format-specific subclasses should inherit from - :class:`Mailbox` and your code should instantiate a particular subclass. - - The :class:`Mailbox` interface is dictionary-like, with small keys - corresponding to messages. Keys are issued by the :class:`Mailbox` instance - with which they will be used and are only meaningful to that :class:`Mailbox` - instance. A key continues to identify a message even if the corresponding - message is modified, such as by replacing it with another message. - - Messages may be added to a :class:`Mailbox` instance using the set-like - method :meth:`add` and removed using a ``del`` statement or the set-like - methods :meth:`remove` and :meth:`discard`. - - :class:`Mailbox` interface semantics differ from dictionary semantics in some - noteworthy ways. Each time a message is requested, a new representation - (typically a :class:`Message` instance) is generated based upon the current - state of the mailbox. Similarly, when a message is added to a - :class:`Mailbox` instance, the provided message representation's contents are - copied. In neither case is a reference to the message representation kept by - the :class:`Mailbox` instance. - - The default :class:`Mailbox` iterator iterates over message representations, - not keys as the default dictionary iterator does. Moreover, modification of a - mailbox during iteration is safe and well-defined. Messages added to the - mailbox after an iterator is created will not be seen by the - iterator. Messages removed from the mailbox before the iterator yields them - will be silently skipped, though using a key from an iterator may result in a - :exc:`KeyError` exception if the corresponding message is subsequently - removed. - - .. warning:: - - Be very cautious when modifying mailboxes that might be simultaneously - changed by some other process. The safest mailbox format to use for such - tasks is Maildir; try to avoid using single-file formats such as mbox for - concurrent writing. If you're modifying a mailbox, you *must* lock it by - calling the :meth:`lock` and :meth:`unlock` methods *before* reading any - messages in the file or making any changes by adding or deleting a - message. Failing to lock the mailbox runs the risk of losing messages or - corrupting the entire mailbox. - - :class:`Mailbox` instances have the following methods: - - - .. method:: add(message) - - Add *message* to the mailbox and return the key that has been assigned to - it. - - Parameter *message* may be a :class:`Message` instance, an - :class:`email.message.Message` instance, a string, a byte string, or a - file-like object (which should be open in binary mode). If *message* is - an instance of the - appropriate format-specific :class:`Message` subclass (e.g., if it's an - :class:`mboxMessage` instance and this is an :class:`mbox` instance), its - format-specific information is used. Otherwise, reasonable defaults for - format-specific information are used. - - .. versionchanged:: 3.2 - Support for binary input was added. - - - .. method:: remove(key) - __delitem__(key) - discard(key) - - Delete the message corresponding to *key* from the mailbox. - - If no such message exists, a :exc:`KeyError` exception is raised if the - method was called as :meth:`remove` or :meth:`__delitem__` but no - exception is raised if the method was called as :meth:`discard`. The - behavior of :meth:`discard` may be preferred if the underlying mailbox - format supports concurrent modification by other processes. - - - .. method:: __setitem__(key, message) - - Replace the message corresponding to *key* with *message*. Raise a - :exc:`KeyError` exception if no message already corresponds to *key*. - - As with :meth:`add`, parameter *message* may be a :class:`Message` - instance, an :class:`email.message.Message` instance, a string, a byte - string, or a file-like object (which should be open in binary mode). If - *message* is an - instance of the appropriate format-specific :class:`Message` subclass - (e.g., if it's an :class:`mboxMessage` instance and this is an - :class:`mbox` instance), its format-specific information is - used. Otherwise, the format-specific information of the message that - currently corresponds to *key* is left unchanged. - - - .. method:: iterkeys() - keys() - - Return an iterator over all keys if called as :meth:`iterkeys` or return a - list of keys if called as :meth:`keys`. - - - .. method:: itervalues() - __iter__() - values() - - Return an iterator over representations of all messages if called as - :meth:`itervalues` or :meth:`__iter__` or return a list of such - representations if called as :meth:`values`. The messages are represented - as instances of the appropriate format-specific :class:`Message` subclass - unless a custom message factory was specified when the :class:`Mailbox` - instance was initialized. - - .. note:: - - The behavior of :meth:`__iter__` is unlike that of dictionaries, which - iterate over keys. - - - .. method:: iteritems() - items() - - Return an iterator over (*key*, *message*) pairs, where *key* is a key and - *message* is a message representation, if called as :meth:`iteritems` or - return a list of such pairs if called as :meth:`items`. The messages are - represented as instances of the appropriate format-specific - :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. - - - .. method:: get(key, default=None) - __getitem__(key) - - Return a representation of the message corresponding to *key*. If no such - message exists, *default* is returned if the method was called as - :meth:`get` and a :exc:`KeyError` exception is raised if the method was - called as :meth:`~object.__getitem__`. The message is represented as an instance - of the appropriate format-specific :class:`Message` subclass unless a - custom message factory was specified when the :class:`Mailbox` instance - was initialized. - - - .. method:: get_message(key) - - Return a representation of the message corresponding to *key* as an - instance of the appropriate format-specific :class:`Message` subclass, or - raise a :exc:`KeyError` exception if no such message exists. - - - .. method:: get_bytes(key) - - Return a byte representation of the message corresponding to *key*, or - raise a :exc:`KeyError` exception if no such message exists. - - .. versionadded:: 3.2 - - - .. method:: get_string(key) - - Return a string representation of the message corresponding to *key*, or - raise a :exc:`KeyError` exception if no such message exists. The - message is processed through :class:`email.message.Message` to - convert it to a 7bit clean representation. - - - .. method:: get_file(key) - - Return a file-like representation of the message corresponding to *key*, - or raise a :exc:`KeyError` exception if no such message exists. The - file-like object behaves as if open in binary mode. This file should be - closed once it is no longer needed. - - .. versionchanged:: 3.2 - The file object really is a binary file; previously it was incorrectly - returned in text mode. Also, the file-like object now supports the - context management protocol: you can use a :keyword:`with` statement to - automatically close it. - - .. note:: - - Unlike other representations of messages, file-like representations are - not necessarily independent of the :class:`Mailbox` instance that - created them or of the underlying mailbox. More specific documentation - is provided by each subclass. - - - .. method:: __contains__(key) - - Return ``True`` if *key* corresponds to a message, ``False`` otherwise. - - - .. method:: __len__() - - Return a count of messages in the mailbox. - - - .. method:: clear() - - Delete all messages from the mailbox. - - - .. method:: pop(key, default=None) - - Return a representation of the message corresponding to *key* and delete - the message. If no such message exists, return *default*. The message is - represented as an instance of the appropriate format-specific - :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. - - - .. method:: popitem() - - Return an arbitrary (*key*, *message*) pair, where *key* is a key and - *message* is a message representation, and delete the corresponding - message. If the mailbox is empty, raise a :exc:`KeyError` exception. The - message is represented as an instance of the appropriate format-specific - :class:`Message` subclass unless a custom message factory was specified - when the :class:`Mailbox` instance was initialized. - - - .. method:: update(arg) - - Parameter *arg* should be a *key*-to-*message* mapping or an iterable of - (*key*, *message*) pairs. Updates the mailbox so that, for each given - *key* and *message*, the message corresponding to *key* is set to - *message* as if by using :meth:`__setitem__`. As with :meth:`__setitem__`, - each *key* must already correspond to a message in the mailbox or else a - :exc:`KeyError` exception will be raised, so in general it is incorrect - for *arg* to be a :class:`Mailbox` instance. - - .. note:: - - Unlike with dictionaries, keyword arguments are not supported. - - - .. method:: flush() - - Write any pending changes to the filesystem. For some :class:`Mailbox` - subclasses, changes are always written immediately and :meth:`flush` does - nothing, but you should still make a habit of calling this method. - - - .. method:: lock() - - Acquire an exclusive advisory lock on the mailbox so that other processes - know not to modify it. An :exc:`ExternalClashError` is raised if the lock - is not available. The particular locking mechanisms used depend upon the - mailbox format. You should *always* lock the mailbox before making any - modifications to its contents. - - - .. method:: unlock() - - Release the lock on the mailbox, if any. - - - .. method:: close() - - Flush the mailbox, unlock it if necessary, and close any open files. For - some :class:`Mailbox` subclasses, this method does nothing. - - -.. _mailbox-maildir: - -:class:`Maildir` -^^^^^^^^^^^^^^^^ - - -.. class:: Maildir(dirname, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in Maildir format. Parameter - *factory* is a callable object that accepts a file-like message representation - (which behaves as if opened in binary mode) and returns a custom representation. - If *factory* is ``None``, :class:`MaildirMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - If *create* is ``True`` and the *dirname* path exists, it will be treated as - an existing maildir without attempting to verify its directory layout. - - It is for historical reasons that *dirname* is named as such rather than *path*. - - Maildir is a directory-based mailbox format invented for the qmail mail - transfer agent and now widely supported by other programs. Messages in a - Maildir mailbox are stored in separate files within a common directory - structure. This design allows Maildir mailboxes to be accessed and modified - by multiple unrelated programs without data corruption, so file locking is - unnecessary. - - Maildir mailboxes contain three subdirectories, namely: :file:`tmp`, - :file:`new`, and :file:`cur`. Messages are created momentarily in the - :file:`tmp` subdirectory and then moved to the :file:`new` subdirectory to - finalize delivery. A mail user agent may subsequently move the message to the - :file:`cur` subdirectory and store information about the state of the message - in a special "info" section appended to its file name. - - Folders of the style introduced by the Courier mail transfer agent are also - supported. Any subdirectory of the main mailbox is considered a folder if - ``'.'`` is the first character in its name. Folder names are represented by - :class:`Maildir` without the leading ``'.'``. Each folder is itself a Maildir - mailbox but should not contain other folders. Instead, a logical nesting is - indicated using ``'.'`` to delimit levels, e.g., "Archived.2005.07". - - .. note:: - - The Maildir specification requires the use of a colon (``':'``) in certain - message file names. However, some operating systems do not permit this - character in file names, If you wish to use a Maildir-like format on such - an operating system, you should specify another character to use - instead. The exclamation point (``'!'``) is a popular choice. For - example:: - - import mailbox - mailbox.Maildir.colon = '!' - - The :attr:`colon` attribute may also be set on a per-instance basis. - - :class:`Maildir` instances have all of the methods of :class:`Mailbox` in - addition to the following: - - - .. method:: list_folders() - - Return a list of the names of all folders. - - - .. method:: get_folder(folder) - - Return a :class:`Maildir` instance representing the folder whose name is - *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder - does not exist. - - - .. method:: add_folder(folder) - - Create a folder whose name is *folder* and return a :class:`Maildir` - instance representing it. - - - .. method:: remove_folder(folder) - - Delete the folder whose name is *folder*. If the folder contains any - messages, a :exc:`NotEmptyError` exception will be raised and the folder - will not be deleted. - - - .. method:: clean() - - Delete temporary files from the mailbox that have not been accessed in the - last 36 hours. The Maildir specification says that mail-reading programs - should do this occasionally. - - Some :class:`Mailbox` methods implemented by :class:`Maildir` deserve special - remarks: - - - .. method:: add(message) - __setitem__(key, message) - update(arg) - - .. warning:: - - These methods generate unique file names based upon the current process - ID. When using multiple threads, undetected name clashes may occur and - cause corruption of the mailbox unless threads are coordinated to avoid - using these methods to manipulate the same mailbox simultaneously. - - - .. method:: flush() - - All changes to Maildir mailboxes are immediately applied, so this method - does nothing. - - - .. method:: lock() - unlock() - - Maildir mailboxes do not support (or require) locking, so these methods do - nothing. - - - .. method:: close() - - :class:`Maildir` instances do not keep any open files and the underlying - mailboxes do not support locking, so this method does nothing. - - - .. method:: get_file(key) - - Depending upon the host platform, it may not be possible to modify or - remove the underlying message while the returned file remains open. - - -.. seealso:: - - `maildir man page from Courier `_ - A specification of the format. Describes a common extension for - supporting folders. - - `Using maildir format `_ - Notes on Maildir by its inventor. Includes an updated name-creation scheme and - details on "info" semantics. - - -.. _mailbox-mbox: - -:class:`mbox` -^^^^^^^^^^^^^ - - -.. class:: mbox(path, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in mbox format. Parameter *factory* - is a callable object that accepts a file-like message representation (which - behaves as if opened in binary mode) and returns a custom representation. If - *factory* is ``None``, :class:`mboxMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - The mbox format is the classic format for storing mail on Unix systems. All - messages in an mbox mailbox are stored in a single file with the beginning of - each message indicated by a line whose first five characters are "From ". - - Several variations of the mbox format exist to address perceived shortcomings in - the original. In the interest of compatibility, :class:`mbox` implements the - original format, which is sometimes referred to as :dfn:`mboxo`. This means that - the :mailheader:`Content-Length` header, if present, is ignored and that any - occurrences of "From " at the beginning of a line in a message body are - transformed to ">From " when storing the message, although occurrences of ">From - " are not transformed to "From " when reading the message. - - Some :class:`Mailbox` methods implemented by :class:`mbox` deserve special - remarks: - - - .. method:: get_file(key) - - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`mbox` instance may yield unpredictable results or raise an - exception. - - - .. method:: lock() - unlock() - - Three locking mechanisms are used---dot locking and, if available, the - :c:func:`!flock` and :c:func:`!lockf` system calls. - - -.. seealso:: - - `mbox man page from tin `_ - A specification of the format, with details on locking. - - `Configuring Netscape Mail on Unix: Why The Content-Length Format is Bad `_ - An argument for using the original mbox format rather than a variation. - - `"mbox" is a family of several mutually incompatible mailbox formats `_ - A history of mbox variations. - - -.. _mailbox-mh: - -:class:`MH` -^^^^^^^^^^^ - - -.. class:: MH(path, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in MH format. Parameter *factory* - is a callable object that accepts a file-like message representation (which - behaves as if opened in binary mode) and returns a custom representation. If - *factory* is ``None``, :class:`MHMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - MH is a directory-based mailbox format invented for the MH Message Handling - System, a mail user agent. Each message in an MH mailbox resides in its own - file. An MH mailbox may contain other MH mailboxes (called :dfn:`folders`) in - addition to messages. Folders may be nested indefinitely. MH mailboxes also - support :dfn:`sequences`, which are named lists used to logically group - messages without moving them to sub-folders. Sequences are defined in a file - called :file:`.mh_sequences` in each folder. - - The :class:`MH` class manipulates MH mailboxes, but it does not attempt to - emulate all of :program:`mh`'s behaviors. In particular, it does not modify - and is not affected by the :file:`context` or :file:`.mh_profile` files that - are used by :program:`mh` to store its state and configuration. - - :class:`MH` instances have all of the methods of :class:`Mailbox` in addition - to the following: - - - .. method:: list_folders() - - Return a list of the names of all folders. - - - .. method:: get_folder(folder) - - Return an :class:`MH` instance representing the folder whose name is - *folder*. A :exc:`NoSuchMailboxError` exception is raised if the folder - does not exist. - - - .. method:: add_folder(folder) - - Create a folder whose name is *folder* and return an :class:`MH` instance - representing it. - - - .. method:: remove_folder(folder) - - Delete the folder whose name is *folder*. If the folder contains any - messages, a :exc:`NotEmptyError` exception will be raised and the folder - will not be deleted. - - - .. method:: get_sequences() - - Return a dictionary of sequence names mapped to key lists. If there are no - sequences, the empty dictionary is returned. - - - .. method:: set_sequences(sequences) - - Re-define the sequences that exist in the mailbox based upon *sequences*, - a dictionary of names mapped to key lists, like returned by - :meth:`get_sequences`. - - - .. method:: pack() - - Rename messages in the mailbox as necessary to eliminate gaps in - numbering. Entries in the sequences list are updated correspondingly. - - .. note:: - - Already-issued keys are invalidated by this operation and should not be - subsequently used. - - Some :class:`Mailbox` methods implemented by :class:`MH` deserve special - remarks: - - - .. method:: remove(key) - __delitem__(key) - discard(key) - - These methods immediately delete the message. The MH convention of marking - a message for deletion by prepending a comma to its name is not used. - - - .. method:: lock() - unlock() - - Three locking mechanisms are used---dot locking and, if available, the - :c:func:`!flock` and :c:func:`!lockf` system calls. For MH mailboxes, locking - the mailbox means locking the :file:`.mh_sequences` file and, only for the - duration of any operations that affect them, locking individual message - files. - - - .. method:: get_file(key) - - Depending upon the host platform, it may not be possible to remove the - underlying message while the returned file remains open. - - - .. method:: flush() - - All changes to MH mailboxes are immediately applied, so this method does - nothing. - - - .. method:: close() - - :class:`MH` instances do not keep any open files, so this method is - equivalent to :meth:`unlock`. - - -.. seealso:: - - `nmh - Message Handling System `_ - Home page of :program:`nmh`, an updated version of the original :program:`mh`. - - `MH & nmh: Email for Users & Programmers `_ - A GPL-licensed book on :program:`mh` and :program:`nmh`, with some information - on the mailbox format. - - -.. _mailbox-babyl: - -:class:`Babyl` -^^^^^^^^^^^^^^ - - -.. class:: Babyl(path, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in Babyl format. Parameter - *factory* is a callable object that accepts a file-like message representation - (which behaves as if opened in binary mode) and returns a custom representation. - If *factory* is ``None``, :class:`BabylMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - Babyl is a single-file mailbox format used by the Rmail mail user agent - included with Emacs. The beginning of a message is indicated by a line - containing the two characters Control-Underscore (``'\037'``) and Control-L - (``'\014'``). The end of a message is indicated by the start of the next - message or, in the case of the last message, a line containing a - Control-Underscore (``'\037'``) character. - - Messages in a Babyl mailbox have two sets of headers, original headers and - so-called visible headers. Visible headers are typically a subset of the - original headers that have been reformatted or abridged to be more - attractive. Each message in a Babyl mailbox also has an accompanying list of - :dfn:`labels`, or short strings that record extra information about the - message, and a list of all user-defined labels found in the mailbox is kept - in the Babyl options section. - - :class:`Babyl` instances have all of the methods of :class:`Mailbox` in - addition to the following: - - - .. method:: get_labels() - - Return a list of the names of all user-defined labels used in the mailbox. - - .. note:: - - The actual messages are inspected to determine which labels exist in - the mailbox rather than consulting the list of labels in the Babyl - options section, but the Babyl section is updated whenever the mailbox - is modified. - - Some :class:`Mailbox` methods implemented by :class:`Babyl` deserve special - remarks: - - - .. method:: get_file(key) - - In Babyl mailboxes, the headers of a message are not stored contiguously - with the body of the message. To generate a file-like representation, the - headers and body are copied together into an :class:`io.BytesIO` instance, - which has an API identical to that of a - file. As a result, the file-like object is truly independent of the - underlying mailbox but does not save memory compared to a string - representation. - - - .. method:: lock() - unlock() - - Three locking mechanisms are used---dot locking and, if available, the - :c:func:`!flock` and :c:func:`!lockf` system calls. - - -.. seealso:: - - `Format of Version 5 Babyl Files `_ - A specification of the Babyl format. - - `Reading Mail with Rmail `_ - The Rmail manual, with some information on Babyl semantics. - - -.. _mailbox-mmdf: - -:class:`MMDF` -^^^^^^^^^^^^^ - - -.. class:: MMDF(path, factory=None, create=True) - - A subclass of :class:`Mailbox` for mailboxes in MMDF format. Parameter *factory* - is a callable object that accepts a file-like message representation (which - behaves as if opened in binary mode) and returns a custom representation. If - *factory* is ``None``, :class:`MMDFMessage` is used as the default message - representation. If *create* is ``True``, the mailbox is created if it does not - exist. - - MMDF is a single-file mailbox format invented for the Multichannel Memorandum - Distribution Facility, a mail transfer agent. Each message is in the same - form as an mbox message but is bracketed before and after by lines containing - four Control-A (``'\001'``) characters. As with the mbox format, the - beginning of each message is indicated by a line whose first five characters - are "From ", but additional occurrences of "From " are not transformed to - ">From " when storing messages because the extra message separator lines - prevent mistaking such occurrences for the starts of subsequent messages. - - Some :class:`Mailbox` methods implemented by :class:`MMDF` deserve special - remarks: - - - .. method:: get_file(key) - - Using the file after calling :meth:`flush` or :meth:`close` on the - :class:`MMDF` instance may yield unpredictable results or raise an - exception. - - - .. method:: lock() - unlock() - - Three locking mechanisms are used---dot locking and, if available, the - :c:func:`!flock` and :c:func:`!lockf` system calls. - - -.. seealso:: - - `mmdf man page from tin `_ - A specification of MMDF format from the documentation of tin, a newsreader. - - `MMDF `_ - A Wikipedia article describing the Multichannel Memorandum Distribution - Facility. - - -.. _mailbox-message-objects: - -:class:`Message` objects ------------------------- - - -.. class:: Message(message=None) - - A subclass of the :mod:`email.message` module's - :class:`~email.message.Message`. Subclasses of :class:`mailbox.Message` add - mailbox-format-specific state and behavior. - - If *message* is omitted, the new instance is created in a default, empty state. - If *message* is an :class:`email.message.Message` instance, its contents are - copied; furthermore, any format-specific information is converted insofar as - possible if *message* is a :class:`Message` instance. If *message* is a string, - a byte string, - or a file, it should contain an :rfc:`2822`\ -compliant message, which is read - and parsed. Files should be open in binary mode, but text mode files - are accepted for backward compatibility. - - The format-specific state and behaviors offered by subclasses vary, but in - general it is only the properties that are not specific to a particular - mailbox that are supported (although presumably the properties are specific - to a particular mailbox format). For example, file offsets for single-file - mailbox formats and file names for directory-based mailbox formats are not - retained, because they are only applicable to the original mailbox. But state - such as whether a message has been read by the user or marked as important is - retained, because it applies to the message itself. - - There is no requirement that :class:`Message` instances be used to represent - messages retrieved using :class:`Mailbox` instances. In some situations, the - time and memory required to generate :class:`Message` representations might - not be acceptable. For such situations, :class:`Mailbox` instances also - offer string and file-like representations, and a custom message factory may - be specified when a :class:`Mailbox` instance is initialized. - - -.. _mailbox-maildirmessage: - -:class:`MaildirMessage` -^^^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: MaildirMessage(message=None) - - A message with Maildir-specific behaviors. Parameter *message* has the same - meaning as with the :class:`Message` constructor. - - Typically, a mail user agent application moves all of the messages in the - :file:`new` subdirectory to the :file:`cur` subdirectory after the first time - the user opens and closes the mailbox, recording that the messages are old - whether or not they've actually been read. Each message in :file:`cur` has an - "info" section added to its file name to store information about its state. - (Some mail readers may also add an "info" section to messages in - :file:`new`.) The "info" section may take one of two forms: it may contain - "2," followed by a list of standardized flags (e.g., "2,FR") or it may - contain "1," followed by so-called experimental information. Standard flags - for Maildir messages are as follows: - - +------+---------+--------------------------------+ - | Flag | Meaning | Explanation | - +======+=========+================================+ - | D | Draft | Under composition | - +------+---------+--------------------------------+ - | F | Flagged | Marked as important | - +------+---------+--------------------------------+ - | P | Passed | Forwarded, resent, or bounced | - +------+---------+--------------------------------+ - | R | Replied | Replied to | - +------+---------+--------------------------------+ - | S | Seen | Read | - +------+---------+--------------------------------+ - | T | Trashed | Marked for subsequent deletion | - +------+---------+--------------------------------+ - - :class:`MaildirMessage` instances offer the following methods: - - - .. method:: get_subdir() - - Return either "new" (if the message should be stored in the :file:`new` - subdirectory) or "cur" (if the message should be stored in the :file:`cur` - subdirectory). - - .. note:: - - A message is typically moved from :file:`new` to :file:`cur` after its - mailbox has been accessed, whether or not the message is has been - read. A message ``msg`` has been read if ``"S" in msg.get_flags()`` is - ``True``. - - - .. method:: set_subdir(subdir) - - Set the subdirectory the message should be stored in. Parameter *subdir* - must be either "new" or "cur". - - - .. method:: get_flags() - - Return a string specifying the flags that are currently set. If the - message complies with the standard Maildir format, the result is the - concatenation in alphabetical order of zero or one occurrence of each of - ``'D'``, ``'F'``, ``'P'``, ``'R'``, ``'S'``, and ``'T'``. The empty string - is returned if no flags are set or if "info" contains experimental - semantics. - - - .. method:: set_flags(flags) - - Set the flags specified by *flags* and unset all others. - - - .. method:: add_flag(flag) - - Set the flag(s) specified by *flag* without changing other flags. To add - more than one flag at a time, *flag* may be a string of more than one - character. The current "info" is overwritten whether or not it contains - experimental information rather than flags. - - - .. method:: remove_flag(flag) - - Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than - one character. If "info" contains experimental information rather than - flags, the current "info" is not modified. - - - .. method:: get_date() - - Return the delivery date of the message as a floating-point number - representing seconds since the epoch. - - - .. method:: set_date(date) - - Set the delivery date of the message to *date*, a floating-point number - representing seconds since the epoch. - - - .. method:: get_info() - - Return a string containing the "info" for a message. This is useful for - accessing and modifying "info" that is experimental (i.e., not a list of - flags). - - - .. method:: set_info(info) - - Set "info" to *info*, which should be a string. - -When a :class:`MaildirMessage` instance is created based upon an -:class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` -and :mailheader:`X-Status` headers are omitted and the following conversions -take place: - -+--------------------+----------------------------------------------+ -| Resulting state | :class:`mboxMessage` or :class:`MMDFMessage` | -| | state | -+====================+==============================================+ -| "cur" subdirectory | O flag | -+--------------------+----------------------------------------------+ -| F flag | F flag | -+--------------------+----------------------------------------------+ -| R flag | A flag | -+--------------------+----------------------------------------------+ -| S flag | R flag | -+--------------------+----------------------------------------------+ -| T flag | D flag | -+--------------------+----------------------------------------------+ - -When a :class:`MaildirMessage` instance is created based upon an -:class:`MHMessage` instance, the following conversions take place: - -+-------------------------------+--------------------------+ -| Resulting state | :class:`MHMessage` state | -+===============================+==========================+ -| "cur" subdirectory | "unseen" sequence | -+-------------------------------+--------------------------+ -| "cur" subdirectory and S flag | no "unseen" sequence | -+-------------------------------+--------------------------+ -| F flag | "flagged" sequence | -+-------------------------------+--------------------------+ -| R flag | "replied" sequence | -+-------------------------------+--------------------------+ - -When a :class:`MaildirMessage` instance is created based upon a -:class:`BabylMessage` instance, the following conversions take place: - -+-------------------------------+-------------------------------+ -| Resulting state | :class:`BabylMessage` state | -+===============================+===============================+ -| "cur" subdirectory | "unseen" label | -+-------------------------------+-------------------------------+ -| "cur" subdirectory and S flag | no "unseen" label | -+-------------------------------+-------------------------------+ -| P flag | "forwarded" or "resent" label | -+-------------------------------+-------------------------------+ -| R flag | "answered" label | -+-------------------------------+-------------------------------+ -| T flag | "deleted" label | -+-------------------------------+-------------------------------+ - - -.. _mailbox-mboxmessage: - -:class:`mboxMessage` -^^^^^^^^^^^^^^^^^^^^ - - -.. class:: mboxMessage(message=None) - - A message with mbox-specific behaviors. Parameter *message* has the same meaning - as with the :class:`Message` constructor. - - Messages in an mbox mailbox are stored together in a single file. The - sender's envelope address and the time of delivery are typically stored in a - line beginning with "From " that is used to indicate the start of a message, - though there is considerable variation in the exact format of this data among - mbox implementations. Flags that indicate the state of the message, such as - whether it has been read or marked as important, are typically stored in - :mailheader:`Status` and :mailheader:`X-Status` headers. - - Conventional flags for mbox messages are as follows: - - +------+----------+--------------------------------+ - | Flag | Meaning | Explanation | - +======+==========+================================+ - | R | Read | Read | - +------+----------+--------------------------------+ - | O | Old | Previously detected by MUA | - +------+----------+--------------------------------+ - | D | Deleted | Marked for subsequent deletion | - +------+----------+--------------------------------+ - | F | Flagged | Marked as important | - +------+----------+--------------------------------+ - | A | Answered | Replied to | - +------+----------+--------------------------------+ - - The "R" and "O" flags are stored in the :mailheader:`Status` header, and the - "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The - flags and headers typically appear in the order mentioned. - - :class:`mboxMessage` instances offer the following methods: - - - .. method:: get_from() - - Return a string representing the "From " line that marks the start of the - message in an mbox mailbox. The leading "From " and the trailing newline - are excluded. - - - .. method:: set_from(from_, time_=None) - - Set the "From " line to *from_*, which should be specified without a - leading "From " or trailing newline. For convenience, *time_* may be - specified and will be formatted appropriately and appended to *from_*. If - *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). - - - .. method:: get_flags() - - Return a string specifying the flags that are currently set. If the - message complies with the conventional format, the result is the - concatenation in the following order of zero or one occurrence of each of - ``'R'``, ``'O'``, ``'D'``, ``'F'``, and ``'A'``. - - - .. method:: set_flags(flags) - - Set the flags specified by *flags* and unset all others. Parameter *flags* - should be the concatenation in any order of zero or more occurrences of - each of ``'R'``, ``'O'``, ``'D'``, ``'F'``, and ``'A'``. - - - .. method:: add_flag(flag) - - Set the flag(s) specified by *flag* without changing other flags. To add - more than one flag at a time, *flag* may be a string of more than one - character. - - - .. method:: remove_flag(flag) - - Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than - one character. - -When an :class:`mboxMessage` instance is created based upon a -:class:`MaildirMessage` instance, a "From " line is generated based upon the -:class:`MaildirMessage` instance's delivery date, and the following conversions -take place: - -+-----------------+-------------------------------+ -| Resulting state | :class:`MaildirMessage` state | -+=================+===============================+ -| R flag | S flag | -+-----------------+-------------------------------+ -| O flag | "cur" subdirectory | -+-----------------+-------------------------------+ -| D flag | T flag | -+-----------------+-------------------------------+ -| F flag | F flag | -+-----------------+-------------------------------+ -| A flag | R flag | -+-----------------+-------------------------------+ - -When an :class:`mboxMessage` instance is created based upon an -:class:`MHMessage` instance, the following conversions take place: - -+-------------------+--------------------------+ -| Resulting state | :class:`MHMessage` state | -+===================+==========================+ -| R flag and O flag | no "unseen" sequence | -+-------------------+--------------------------+ -| O flag | "unseen" sequence | -+-------------------+--------------------------+ -| F flag | "flagged" sequence | -+-------------------+--------------------------+ -| A flag | "replied" sequence | -+-------------------+--------------------------+ - -When an :class:`mboxMessage` instance is created based upon a -:class:`BabylMessage` instance, the following conversions take place: - -+-------------------+-----------------------------+ -| Resulting state | :class:`BabylMessage` state | -+===================+=============================+ -| R flag and O flag | no "unseen" label | -+-------------------+-----------------------------+ -| O flag | "unseen" label | -+-------------------+-----------------------------+ -| D flag | "deleted" label | -+-------------------+-----------------------------+ -| A flag | "answered" label | -+-------------------+-----------------------------+ - -When a :class:`Message` instance is created based upon an :class:`MMDFMessage` -instance, the "From " line is copied and all flags directly correspond: - -+-----------------+----------------------------+ -| Resulting state | :class:`MMDFMessage` state | -+=================+============================+ -| R flag | R flag | -+-----------------+----------------------------+ -| O flag | O flag | -+-----------------+----------------------------+ -| D flag | D flag | -+-----------------+----------------------------+ -| F flag | F flag | -+-----------------+----------------------------+ -| A flag | A flag | -+-----------------+----------------------------+ - - -.. _mailbox-mhmessage: - -:class:`MHMessage` -^^^^^^^^^^^^^^^^^^ - - -.. class:: MHMessage(message=None) - - A message with MH-specific behaviors. Parameter *message* has the same meaning - as with the :class:`Message` constructor. - - MH messages do not support marks or flags in the traditional sense, but they - do support sequences, which are logical groupings of arbitrary messages. Some - mail reading programs (although not the standard :program:`mh` and - :program:`nmh`) use sequences in much the same way flags are used with other - formats, as follows: - - +----------+------------------------------------------+ - | Sequence | Explanation | - +==========+==========================================+ - | unseen | Not read, but previously detected by MUA | - +----------+------------------------------------------+ - | replied | Replied to | - +----------+------------------------------------------+ - | flagged | Marked as important | - +----------+------------------------------------------+ - - :class:`MHMessage` instances offer the following methods: - - - .. method:: get_sequences() - - Return a list of the names of sequences that include this message. - - - .. method:: set_sequences(sequences) - - Set the list of sequences that include this message. - - - .. method:: add_sequence(sequence) - - Add *sequence* to the list of sequences that include this message. - - - .. method:: remove_sequence(sequence) - - Remove *sequence* from the list of sequences that include this message. - -When an :class:`MHMessage` instance is created based upon a -:class:`MaildirMessage` instance, the following conversions take place: - -+--------------------+-------------------------------+ -| Resulting state | :class:`MaildirMessage` state | -+====================+===============================+ -| "unseen" sequence | no S flag | -+--------------------+-------------------------------+ -| "replied" sequence | R flag | -+--------------------+-------------------------------+ -| "flagged" sequence | F flag | -+--------------------+-------------------------------+ - -When an :class:`MHMessage` instance is created based upon an -:class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` -and :mailheader:`X-Status` headers are omitted and the following conversions -take place: - -+--------------------+----------------------------------------------+ -| Resulting state | :class:`mboxMessage` or :class:`MMDFMessage` | -| | state | -+====================+==============================================+ -| "unseen" sequence | no R flag | -+--------------------+----------------------------------------------+ -| "replied" sequence | A flag | -+--------------------+----------------------------------------------+ -| "flagged" sequence | F flag | -+--------------------+----------------------------------------------+ - -When an :class:`MHMessage` instance is created based upon a -:class:`BabylMessage` instance, the following conversions take place: - -+--------------------+-----------------------------+ -| Resulting state | :class:`BabylMessage` state | -+====================+=============================+ -| "unseen" sequence | "unseen" label | -+--------------------+-----------------------------+ -| "replied" sequence | "answered" label | -+--------------------+-----------------------------+ - - -.. _mailbox-babylmessage: - -:class:`BabylMessage` -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: BabylMessage(message=None) - - A message with Babyl-specific behaviors. Parameter *message* has the same - meaning as with the :class:`Message` constructor. - - Certain message labels, called :dfn:`attributes`, are defined by convention - to have special meanings. The attributes are as follows: - - +-----------+------------------------------------------+ - | Label | Explanation | - +===========+==========================================+ - | unseen | Not read, but previously detected by MUA | - +-----------+------------------------------------------+ - | deleted | Marked for subsequent deletion | - +-----------+------------------------------------------+ - | filed | Copied to another file or mailbox | - +-----------+------------------------------------------+ - | answered | Replied to | - +-----------+------------------------------------------+ - | forwarded | Forwarded | - +-----------+------------------------------------------+ - | edited | Modified by the user | - +-----------+------------------------------------------+ - | resent | Resent | - +-----------+------------------------------------------+ - - By default, Rmail displays only visible headers. The :class:`BabylMessage` - class, though, uses the original headers because they are more - complete. Visible headers may be accessed explicitly if desired. - - :class:`BabylMessage` instances offer the following methods: - - - .. method:: get_labels() - - Return a list of labels on the message. - - - .. method:: set_labels(labels) - - Set the list of labels on the message to *labels*. - - - .. method:: add_label(label) - - Add *label* to the list of labels on the message. - - - .. method:: remove_label(label) - - Remove *label* from the list of labels on the message. - - - .. method:: get_visible() - - Return an :class:`Message` instance whose headers are the message's - visible headers and whose body is empty. - - - .. method:: set_visible(visible) - - Set the message's visible headers to be the same as the headers in - *message*. Parameter *visible* should be a :class:`Message` instance, an - :class:`email.message.Message` instance, a string, or a file-like object - (which should be open in text mode). - - - .. method:: update_visible() - - When a :class:`BabylMessage` instance's original headers are modified, the - visible headers are not automatically modified to correspond. This method - updates the visible headers as follows: each visible header with a - corresponding original header is set to the value of the original header, - each visible header without a corresponding original header is removed, - and any of :mailheader:`Date`, :mailheader:`From`, :mailheader:`Reply-To`, - :mailheader:`To`, :mailheader:`CC`, and :mailheader:`Subject` that are - present in the original headers but not the visible headers are added to - the visible headers. - -When a :class:`BabylMessage` instance is created based upon a -:class:`MaildirMessage` instance, the following conversions take place: - -+-------------------+-------------------------------+ -| Resulting state | :class:`MaildirMessage` state | -+===================+===============================+ -| "unseen" label | no S flag | -+-------------------+-------------------------------+ -| "deleted" label | T flag | -+-------------------+-------------------------------+ -| "answered" label | R flag | -+-------------------+-------------------------------+ -| "forwarded" label | P flag | -+-------------------+-------------------------------+ - -When a :class:`BabylMessage` instance is created based upon an -:class:`mboxMessage` or :class:`MMDFMessage` instance, the :mailheader:`Status` -and :mailheader:`X-Status` headers are omitted and the following conversions -take place: - -+------------------+----------------------------------------------+ -| Resulting state | :class:`mboxMessage` or :class:`MMDFMessage` | -| | state | -+==================+==============================================+ -| "unseen" label | no R flag | -+------------------+----------------------------------------------+ -| "deleted" label | D flag | -+------------------+----------------------------------------------+ -| "answered" label | A flag | -+------------------+----------------------------------------------+ - -When a :class:`BabylMessage` instance is created based upon an -:class:`MHMessage` instance, the following conversions take place: - -+------------------+--------------------------+ -| Resulting state | :class:`MHMessage` state | -+==================+==========================+ -| "unseen" label | "unseen" sequence | -+------------------+--------------------------+ -| "answered" label | "replied" sequence | -+------------------+--------------------------+ - - -.. _mailbox-mmdfmessage: - -:class:`MMDFMessage` -^^^^^^^^^^^^^^^^^^^^ - - -.. class:: MMDFMessage(message=None) - - A message with MMDF-specific behaviors. Parameter *message* has the same meaning - as with the :class:`Message` constructor. - - As with message in an mbox mailbox, MMDF messages are stored with the - sender's address and the delivery date in an initial line beginning with - "From ". Likewise, flags that indicate the state of the message are - typically stored in :mailheader:`Status` and :mailheader:`X-Status` headers. - - Conventional flags for MMDF messages are identical to those of mbox message - and are as follows: - - +------+----------+--------------------------------+ - | Flag | Meaning | Explanation | - +======+==========+================================+ - | R | Read | Read | - +------+----------+--------------------------------+ - | O | Old | Previously detected by MUA | - +------+----------+--------------------------------+ - | D | Deleted | Marked for subsequent deletion | - +------+----------+--------------------------------+ - | F | Flagged | Marked as important | - +------+----------+--------------------------------+ - | A | Answered | Replied to | - +------+----------+--------------------------------+ - - The "R" and "O" flags are stored in the :mailheader:`Status` header, and the - "D", "F", and "A" flags are stored in the :mailheader:`X-Status` header. The - flags and headers typically appear in the order mentioned. - - :class:`MMDFMessage` instances offer the following methods, which are - identical to those offered by :class:`mboxMessage`: - - - .. method:: get_from() - - Return a string representing the "From " line that marks the start of the - message in an mbox mailbox. The leading "From " and the trailing newline - are excluded. - - - .. method:: set_from(from_, time_=None) - - Set the "From " line to *from_*, which should be specified without a - leading "From " or trailing newline. For convenience, *time_* may be - specified and will be formatted appropriately and appended to *from_*. If - *time_* is specified, it should be a :class:`time.struct_time` instance, a - tuple suitable for passing to :meth:`time.strftime`, or ``True`` (to use - :meth:`time.gmtime`). - - - .. method:: get_flags() - - Return a string specifying the flags that are currently set. If the - message complies with the conventional format, the result is the - concatenation in the following order of zero or one occurrence of each of - ``'R'``, ``'O'``, ``'D'``, ``'F'``, and ``'A'``. - - - .. method:: set_flags(flags) - - Set the flags specified by *flags* and unset all others. Parameter *flags* - should be the concatenation in any order of zero or more occurrences of - each of ``'R'``, ``'O'``, ``'D'``, ``'F'``, and ``'A'``. - - - .. method:: add_flag(flag) - - Set the flag(s) specified by *flag* without changing other flags. To add - more than one flag at a time, *flag* may be a string of more than one - character. - - - .. method:: remove_flag(flag) - - Unset the flag(s) specified by *flag* without changing other flags. To - remove more than one flag at a time, *flag* maybe a string of more than - one character. - -When an :class:`MMDFMessage` instance is created based upon a -:class:`MaildirMessage` instance, a "From " line is generated based upon the -:class:`MaildirMessage` instance's delivery date, and the following conversions -take place: - -+-----------------+-------------------------------+ -| Resulting state | :class:`MaildirMessage` state | -+=================+===============================+ -| R flag | S flag | -+-----------------+-------------------------------+ -| O flag | "cur" subdirectory | -+-----------------+-------------------------------+ -| D flag | T flag | -+-----------------+-------------------------------+ -| F flag | F flag | -+-----------------+-------------------------------+ -| A flag | R flag | -+-----------------+-------------------------------+ - -When an :class:`MMDFMessage` instance is created based upon an -:class:`MHMessage` instance, the following conversions take place: - -+-------------------+--------------------------+ -| Resulting state | :class:`MHMessage` state | -+===================+==========================+ -| R flag and O flag | no "unseen" sequence | -+-------------------+--------------------------+ -| O flag | "unseen" sequence | -+-------------------+--------------------------+ -| F flag | "flagged" sequence | -+-------------------+--------------------------+ -| A flag | "replied" sequence | -+-------------------+--------------------------+ - -When an :class:`MMDFMessage` instance is created based upon a -:class:`BabylMessage` instance, the following conversions take place: - -+-------------------+-----------------------------+ -| Resulting state | :class:`BabylMessage` state | -+===================+=============================+ -| R flag and O flag | no "unseen" label | -+-------------------+-----------------------------+ -| O flag | "unseen" label | -+-------------------+-----------------------------+ -| D flag | "deleted" label | -+-------------------+-----------------------------+ -| A flag | "answered" label | -+-------------------+-----------------------------+ - -When an :class:`MMDFMessage` instance is created based upon an -:class:`mboxMessage` instance, the "From " line is copied and all flags directly -correspond: - -+-----------------+----------------------------+ -| Resulting state | :class:`mboxMessage` state | -+=================+============================+ -| R flag | R flag | -+-----------------+----------------------------+ -| O flag | O flag | -+-----------------+----------------------------+ -| D flag | D flag | -+-----------------+----------------------------+ -| F flag | F flag | -+-----------------+----------------------------+ -| A flag | A flag | -+-----------------+----------------------------+ - - -Exceptions ----------- - -The following exception classes are defined in the :mod:`mailbox` module: - - -.. exception:: Error() - - The based class for all other module-specific exceptions. - - -.. exception:: NoSuchMailboxError() - - Raised when a mailbox is expected but is not found, such as when instantiating a - :class:`Mailbox` subclass with a path that does not exist (and with the *create* - parameter set to ``False``), or when opening a folder that does not exist. - - -.. exception:: NotEmptyError() - - Raised when a mailbox is not empty but is expected to be, such as when deleting - a folder that contains messages. - - -.. exception:: ExternalClashError() - - Raised when some mailbox-related condition beyond the control of the program - causes it to be unable to proceed, such as when failing to acquire a lock that - another program already holds a lock, or when a uniquely generated file name - already exists. - - -.. exception:: FormatError() - - Raised when the data in a file cannot be parsed, such as when an :class:`MH` - instance attempts to read a corrupted :file:`.mh_sequences` file. - - -.. _mailbox-examples: - -Examples --------- - -A simple example of printing the subjects of all messages in a mailbox that seem -interesting:: - - import mailbox - for message in mailbox.mbox('~/mbox'): - subject = message['subject'] # Could possibly be None. - if subject and 'python' in subject.lower(): - print(subject) - -To copy all mail from a Babyl mailbox to an MH mailbox, converting all of the -format-specific information that can be converted:: - - import mailbox - destination = mailbox.MH('~/Mail') - destination.lock() - for message in mailbox.Babyl('~/RMAIL'): - destination.add(mailbox.MHMessage(message)) - destination.flush() - destination.unlock() - -This example sorts mail from several mailing lists into different mailboxes, -being careful to avoid mail corruption due to concurrent modification by other -programs, mail loss due to interruption of the program, or premature termination -due to malformed messages in the mailbox:: - - import mailbox - import email.errors - - list_names = ('python-list', 'python-dev', 'python-bugs') - - boxes = {name: mailbox.mbox('~/email/%s' % name) for name in list_names} - inbox = mailbox.Maildir('~/Maildir', factory=None) - - for key in inbox.iterkeys(): - try: - message = inbox[key] - except email.errors.MessageParseError: - continue # The message is malformed. Just leave it. - - for name in list_names: - list_id = message['list-id'] - if list_id and name in list_id: - # Get mailbox to use - box = boxes[name] - - # Write copy to disk before removing original. - # If there's a crash, you might duplicate a message, but - # that's better than losing a message completely. - box.lock() - box.add(message) - box.flush() - box.unlock() - - # Remove original message - inbox.lock() - inbox.discard(key) - inbox.flush() - inbox.unlock() - break # Found destination, so stop looking. - - for box in boxes.itervalues(): - box.close() - diff --git a/Doc/library/mailcap.rst b/Doc/library/mailcap.rst deleted file mode 100644 index bfaedb46091991..00000000000000 --- a/Doc/library/mailcap.rst +++ /dev/null @@ -1,94 +0,0 @@ -:mod:`mailcap` --- Mailcap file handling -======================================== - -.. module:: mailcap - :synopsis: Mailcap file handling. - :deprecated: - -**Source code:** :source:`Lib/mailcap.py` - -.. deprecated-removed:: 3.11 3.13 - The :mod:`mailcap` module is deprecated - (see :pep:`PEP 594 <594#mailcap>` for details). - The :mod:`mimetypes` module provides an alternative. - --------------- - -Mailcap files are used to configure how MIME-aware applications such as mail -readers and web browsers react to files with different MIME types. (The name -"mailcap" is derived from the phrase "mail capability".) For example, a mailcap -file might contain a line like ``video/mpeg; xmpeg %s``. Then, if the user -encounters an email message or web document with the MIME type -:mimetype:`video/mpeg`, ``%s`` will be replaced by a filename (usually one -belonging to a temporary file) and the :program:`xmpeg` program can be -automatically started to view the file. - -The mailcap format is documented in :rfc:`1524`, "A User Agent Configuration -Mechanism For Multimedia Mail Format Information", but is not an internet -standard. However, mailcap files are supported on most Unix systems. - - -.. function:: findmatch(caps, MIMEtype, key='view', filename='/dev/null', plist=[]) - - Return a 2-tuple; the first element is a string containing the command line to - be executed (which can be passed to :func:`os.system`), and the second element - is the mailcap entry for a given MIME type. If no matching MIME type can be - found, ``(None, None)`` is returned. - - *key* is the name of the field desired, which represents the type of activity to - be performed; the default value is 'view', since in the most common case you - simply want to view the body of the MIME-typed data. Other possible values - might be 'compose' and 'edit', if you wanted to create a new body of the given - MIME type or alter the existing body data. See :rfc:`1524` for a complete list - of these fields. - - *filename* is the filename to be substituted for ``%s`` in the command line; the - default value is ``'/dev/null'`` which is almost certainly not what you want, so - usually you'll override it by specifying a filename. - - *plist* can be a list containing named parameters; the default value is simply - an empty list. Each entry in the list must be a string containing the parameter - name, an equals sign (``'='``), and the parameter's value. Mailcap entries can - contain named parameters like ``%{foo}``, which will be replaced by the value - of the parameter named 'foo'. For example, if the command line ``showpartial - %{id} %{number} %{total}`` was in a mailcap file, and *plist* was set to - ``['id=1', 'number=2', 'total=3']``, the resulting command line would be - ``'showpartial 1 2 3'``. - - In a mailcap file, the "test" field can optionally be specified to test some - external condition (such as the machine architecture, or the window system in - use) to determine whether or not the mailcap line applies. :func:`findmatch` - will automatically check such conditions and skip the entry if the check fails. - - .. versionchanged:: 3.11 - - To prevent security issues with shell metacharacters (symbols that have - special effects in a shell command line), ``findmatch`` will refuse - to inject ASCII characters other than alphanumerics and ``@+=:,./-_`` - into the returned command line. - - If a disallowed character appears in *filename*, ``findmatch`` will always - return ``(None, None)`` as if no entry was found. - If such a character appears elsewhere (a value in *plist* or in *MIMEtype*), - ``findmatch`` will ignore all mailcap entries which use that value. - A :mod:`warning ` will be raised in either case. - -.. function:: getcaps() - - Returns a dictionary mapping MIME types to a list of mailcap file entries. This - dictionary must be passed to the :func:`findmatch` function. An entry is stored - as a list of dictionaries, but it shouldn't be necessary to know the details of - this representation. - - The information is derived from all of the mailcap files found on the system. - Settings in the user's mailcap file :file:`$HOME/.mailcap` will override - settings in the system mailcap files :file:`/etc/mailcap`, - :file:`/usr/etc/mailcap`, and :file:`/usr/local/etc/mailcap`. - -An example usage:: - - >>> import mailcap - >>> d = mailcap.getcaps() - >>> mailcap.findmatch(d, 'video/mpeg', filename='tmp1223') - ('xmpeg tmp1223', {'view': 'xmpeg %s'}) - diff --git a/Doc/library/markup.rst b/Doc/library/markup.rst deleted file mode 100644 index 1588aa8a6aee70..00000000000000 --- a/Doc/library/markup.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _markup: - -********************************** -Structured Markup Processing Tools -********************************** - -Python supports a variety of modules to work with various forms of structured -data markup. This includes modules to work with the Standard Generalized Markup -Language (SGML) and the Hypertext Markup Language (HTML), and several interfaces -for working with the Extensible Markup Language (XML). - - -.. toctree:: - - html.rst - html.parser.rst - html.entities.rst - xml.rst - xml.etree.elementtree.rst - xml.dom.rst - xml.dom.minidom.rst - xml.dom.pulldom.rst - xml.sax.rst - xml.sax.handler.rst - xml.sax.utils.rst - xml.sax.reader.rst - pyexpat.rst diff --git a/Doc/library/marshal.rst b/Doc/library/marshal.rst deleted file mode 100644 index 0556f19699dc15..00000000000000 --- a/Doc/library/marshal.rst +++ /dev/null @@ -1,136 +0,0 @@ -:mod:`marshal` --- Internal Python object serialization -======================================================= - -.. module:: marshal - :synopsis: Convert Python objects to streams of bytes and back (with different - constraints). - --------------- - -This module contains functions that can read and write Python values in a binary -format. The format is specific to Python, but independent of machine -architecture issues (e.g., you can write a Python value to a file on a PC, -transport the file to a Sun, and read it back there). Details of the format are -undocumented on purpose; it may change between Python versions (although it -rarely does). [#]_ - -.. index:: - pair: module; pickle - pair: module; shelve - -This is not a general "persistence" module. For general persistence and -transfer of Python objects through RPC calls, see the modules :mod:`pickle` and -:mod:`shelve`. The :mod:`marshal` module exists mainly to support reading and -writing the "pseudo-compiled" code for Python modules of :file:`.pyc` files. -Therefore, the Python maintainers reserve the right to modify the marshal format -in backward incompatible ways should the need arise. If you're serializing and -de-serializing Python objects, use the :mod:`pickle` module instead -- the -performance is comparable, version independence is guaranteed, and pickle -supports a substantially wider range of objects than marshal. - -.. warning:: - - The :mod:`marshal` module is not intended to be secure against erroneous or - maliciously constructed data. Never unmarshal data received from an - untrusted or unauthenticated source. - -.. index:: object; code, code object - -Not all Python object types are supported; in general, only objects whose value -is independent from a particular invocation of Python can be written and read by -this module. The following types are supported: booleans, integers, floating -point numbers, complex numbers, strings, bytes, bytearrays, tuples, lists, sets, -frozensets, dictionaries, and code objects, where it should be understood that -tuples, lists, sets, frozensets and dictionaries are only supported as long as -the values contained therein are themselves supported. The -singletons :const:`None`, :const:`Ellipsis` and :exc:`StopIteration` can also be -marshalled and unmarshalled. -For format *version* lower than 3, recursive lists, sets and dictionaries cannot -be written (see below). - -There are functions that read/write files as well as functions operating on -bytes-like objects. - -The module defines these functions: - - -.. function:: dump(value, file[, version]) - - Write the value on the open file. The value must be a supported type. The - file must be a writeable :term:`binary file`. - - If the value has (or contains an object that has) an unsupported type, a - :exc:`ValueError` exception is raised --- but garbage data will also be written - to the file. The object will not be properly read back by :func:`load`. - - The *version* argument indicates the data format that ``dump`` should use - (see below). - - .. audit-event:: marshal.dumps value,version marshal.dump - - -.. function:: load(file) - - Read one value from the open file and return it. If no valid value is read - (e.g. because the data has a different Python version's incompatible marshal - format), raise :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. The - file must be a readable :term:`binary file`. - - .. audit-event:: marshal.load "" marshal.load - - .. note:: - - If an object containing an unsupported type was marshalled with :func:`dump`, - :func:`load` will substitute ``None`` for the unmarshallable type. - - .. versionchanged:: 3.10 - - This call used to raise a ``code.__new__`` audit event for each code object. Now - it raises a single ``marshal.load`` event for the entire load operation. - - -.. function:: dumps(value[, version]) - - Return the bytes object that would be written to a file by ``dump(value, file)``. The - value must be a supported type. Raise a :exc:`ValueError` exception if value - has (or contains an object that has) an unsupported type. - - The *version* argument indicates the data format that ``dumps`` should use - (see below). - - .. audit-event:: marshal.dumps value,version marshal.dump - - -.. function:: loads(bytes) - - Convert the :term:`bytes-like object` to a value. If no valid value is found, raise - :exc:`EOFError`, :exc:`ValueError` or :exc:`TypeError`. Extra bytes in the - input are ignored. - - .. audit-event:: marshal.loads bytes marshal.load - - .. versionchanged:: 3.10 - - This call used to raise a ``code.__new__`` audit event for each code object. Now - it raises a single ``marshal.loads`` event for the entire load operation. - - -In addition, the following constants are defined: - -.. data:: version - - Indicates the format that the module uses. Version 0 is the historical - format, version 1 shares interned strings and version 2 uses a binary format - for floating point numbers. - Version 3 adds support for object instancing and recursion. - The current version is 4. - - -.. rubric:: Footnotes - -.. [#] The name of this module stems from a bit of terminology used by the designers of - Modula-3 (amongst others), who use the term "marshalling" for shipping of data - around in a self-contained form. Strictly speaking, "to marshal" means to - convert some data from internal to external form (in an RPC buffer for instance) - and "unmarshalling" for the reverse process. - diff --git a/Doc/library/math.rst b/Doc/library/math.rst deleted file mode 100644 index 8d48656d41eb0a..00000000000000 --- a/Doc/library/math.rst +++ /dev/null @@ -1,700 +0,0 @@ -:mod:`math` --- Mathematical functions -====================================== - -.. module:: math - :synopsis: Mathematical functions (sin() etc.). - -.. testsetup:: - - from math import fsum - --------------- - -This module provides access to the mathematical functions defined by the C -standard. - -These functions cannot be used with complex numbers; use the functions of the -same name from the :mod:`cmath` module if you require support for complex -numbers. The distinction between functions which support complex numbers and -those which don't is made since most users do not want to learn quite as much -mathematics as required to understand complex numbers. Receiving an exception -instead of a complex result allows earlier detection of the unexpected complex -number used as a parameter, so that the programmer can determine how and why it -was generated in the first place. - -The following functions are provided by this module. Except when explicitly -noted otherwise, all return values are floats. - - -Number-theoretic and representation functions ---------------------------------------------- - -.. function:: ceil(x) - - Return the ceiling of *x*, the smallest integer greater than or equal to *x*. - If *x* is not a float, delegates to :meth:`x.__ceil__ `, - which should return an :class:`~numbers.Integral` value. - - -.. function:: comb(n, k) - - Return the number of ways to choose *k* items from *n* items without repetition - and without order. - - Evaluates to ``n! / (k! * (n - k)!)`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - Also called the binomial coefficient because it is equivalent - to the coefficient of k-th term in polynomial expansion of - ``(1 + x)ⁿ``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - -.. function:: copysign(x, y) - - Return a float with the magnitude (absolute value) of *x* but the sign of - *y*. On platforms that support signed zeros, ``copysign(1.0, -0.0)`` - returns *-1.0*. - - -.. function:: fabs(x) - - Return the absolute value of *x*. - - -.. function:: factorial(n) - - Return *n* factorial as an integer. Raises :exc:`ValueError` if *n* is not integral or - is negative. - - .. deprecated:: 3.9 - Accepting floats with integral values (like ``5.0``) is deprecated. - - -.. function:: floor(x) - - Return the floor of *x*, the largest integer less than or equal to *x*. If - *x* is not a float, delegates to :meth:`x.__floor__ `, which - should return an :class:`~numbers.Integral` value. - - -.. function:: fmod(x, y) - - Return ``fmod(x, y)``, as defined by the platform C library. Note that the - Python expression ``x % y`` may not return the same result. The intent of the C - standard is that ``fmod(x, y)`` be exactly (mathematically; to infinite - precision) equal to ``x - n*y`` for some integer *n* such that the result has - the same sign as *x* and magnitude less than ``abs(y)``. Python's ``x % y`` - returns a result with the sign of *y* instead, and may not be exactly computable - for float arguments. For example, ``fmod(-1e-100, 1e100)`` is ``-1e-100``, but - the result of Python's ``-1e-100 % 1e100`` is ``1e100-1e-100``, which cannot be - represented exactly as a float, and rounds to the surprising ``1e100``. For - this reason, function :func:`fmod` is generally preferred when working with - floats, while Python's ``x % y`` is preferred when working with integers. - - -.. function:: frexp(x) - - Return the mantissa and exponent of *x* as the pair ``(m, e)``. *m* is a float - and *e* is an integer such that ``x == m * 2**e`` exactly. If *x* is zero, - returns ``(0.0, 0)``, otherwise ``0.5 <= abs(m) < 1``. This is used to "pick - apart" the internal representation of a float in a portable way. - - -.. function:: fsum(iterable) - - Return an accurate floating point sum of values in the iterable. Avoids - loss of precision by tracking multiple intermediate partial sums:: - - >>> sum([.1, .1, .1, .1, .1, .1, .1, .1, .1, .1]) - 0.9999999999999999 - >>> fsum([.1, .1, .1, .1, .1, .1, .1, .1, .1, .1]) - 1.0 - - The algorithm's accuracy depends on IEEE-754 arithmetic guarantees and the - typical case where the rounding mode is half-even. On some non-Windows - builds, the underlying C library uses extended precision addition and may - occasionally double-round an intermediate sum causing it to be off in its - least significant bit. - - For further discussion and two alternative approaches, see the `ASPN cookbook - recipes for accurate floating point summation - `_\. - - -.. function:: gcd(*integers) - - Return the greatest common divisor of the specified integer arguments. - If any of the arguments is nonzero, then the returned value is the largest - positive integer that is a divisor of all arguments. If all arguments - are zero, then the returned value is ``0``. ``gcd()`` without arguments - returns ``0``. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.9 - Added support for an arbitrary number of arguments. Formerly, only two - arguments were supported. - - -.. function:: isclose(a, b, *, rel_tol=1e-09, abs_tol=0.0) - - Return ``True`` if the values *a* and *b* are close to each other and - ``False`` otherwise. - - Whether or not two values are considered close is determined according to - given absolute and relative tolerances. - - *rel_tol* is the relative tolerance -- it is the maximum allowed difference - between *a* and *b*, relative to the larger absolute value of *a* or *b*. - For example, to set a tolerance of 5%, pass ``rel_tol=0.05``. The default - tolerance is ``1e-09``, which assures that the two values are the same - within about 9 decimal digits. *rel_tol* must be greater than zero. - - *abs_tol* is the minimum absolute tolerance -- useful for comparisons near - zero. *abs_tol* must be at least zero. - - If no errors occur, the result will be: - ``abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)``. - - The IEEE 754 special values of ``NaN``, ``inf``, and ``-inf`` will be - handled according to IEEE rules. Specifically, ``NaN`` is not considered - close to any other value, including ``NaN``. ``inf`` and ``-inf`` are only - considered close to themselves. - - .. versionadded:: 3.5 - - .. seealso:: - - :pep:`485` -- A function for testing approximate equality - - -.. function:: isfinite(x) - - Return ``True`` if *x* is neither an infinity nor a NaN, and - ``False`` otherwise. (Note that ``0.0`` *is* considered finite.) - - .. versionadded:: 3.2 - - -.. function:: isinf(x) - - Return ``True`` if *x* is a positive or negative infinity, and - ``False`` otherwise. - - -.. function:: isnan(x) - - Return ``True`` if *x* is a NaN (not a number), and ``False`` otherwise. - - -.. function:: isqrt(n) - - Return the integer square root of the nonnegative integer *n*. This is the - floor of the exact square root of *n*, or equivalently the greatest integer - *a* such that *a*\ ² |nbsp| ≤ |nbsp| *n*. - - For some applications, it may be more convenient to have the least integer - *a* such that *n* |nbsp| ≤ |nbsp| *a*\ ², or in other words the ceiling of - the exact square root of *n*. For positive *n*, this can be computed using - ``a = 1 + isqrt(n - 1)``. - - .. versionadded:: 3.8 - - -.. function:: lcm(*integers) - - Return the least common multiple of the specified integer arguments. - If all arguments are nonzero, then the returned value is the smallest - positive integer that is a multiple of all arguments. If any of the arguments - is zero, then the returned value is ``0``. ``lcm()`` without arguments - returns ``1``. - - .. versionadded:: 3.9 - - -.. function:: ldexp(x, i) - - Return ``x * (2**i)``. This is essentially the inverse of function - :func:`frexp`. - - -.. function:: modf(x) - - Return the fractional and integer parts of *x*. Both results carry the sign - of *x* and are floats. - - -.. function:: nextafter(x, y) - - Return the next floating-point value after *x* towards *y*. - - If *x* is equal to *y*, return *y*. - - Examples: - - * ``math.nextafter(x, math.inf)`` goes up: towards positive infinity. - * ``math.nextafter(x, -math.inf)`` goes down: towards minus infinity. - * ``math.nextafter(x, 0.0)`` goes towards zero. - * ``math.nextafter(x, math.copysign(math.inf, x))`` goes away from zero. - - See also :func:`math.ulp`. - - .. versionadded:: 3.9 - -.. function:: perm(n, k=None) - - Return the number of ways to choose *k* items from *n* items - without repetition and with order. - - Evaluates to ``n! / (n - k)!`` when ``k <= n`` and evaluates - to zero when ``k > n``. - - If *k* is not specified or is None, then *k* defaults to *n* - and the function returns ``n!``. - - Raises :exc:`TypeError` if either of the arguments are not integers. - Raises :exc:`ValueError` if either of the arguments are negative. - - .. versionadded:: 3.8 - - -.. function:: prod(iterable, *, start=1) - - Calculate the product of all the elements in the input *iterable*. - The default *start* value for the product is ``1``. - - When the iterable is empty, return the start value. This function is - intended specifically for use with numeric values and may reject - non-numeric types. - - .. versionadded:: 3.8 - - -.. function:: remainder(x, y) - - Return the IEEE 754-style remainder of *x* with respect to *y*. For - finite *x* and finite nonzero *y*, this is the difference ``x - n*y``, - where ``n`` is the closest integer to the exact value of the quotient ``x / - y``. If ``x / y`` is exactly halfway between two consecutive integers, the - nearest *even* integer is used for ``n``. The remainder ``r = remainder(x, - y)`` thus always satisfies ``abs(r) <= 0.5 * abs(y)``. - - Special cases follow IEEE 754: in particular, ``remainder(x, math.inf)`` is - *x* for any finite *x*, and ``remainder(x, 0)`` and - ``remainder(math.inf, x)`` raise :exc:`ValueError` for any non-NaN *x*. - If the result of the remainder operation is zero, that zero will have - the same sign as *x*. - - On platforms using IEEE 754 binary floating-point, the result of this - operation is always exactly representable: no rounding error is introduced. - - .. versionadded:: 3.7 - - -.. function:: trunc(x) - - Return *x* with the fractional part - removed, leaving the integer part. This rounds toward 0: ``trunc()`` is - equivalent to :func:`floor` for positive *x*, and equivalent to :func:`ceil` - for negative *x*. If *x* is not a float, delegates to :meth:`x.__trunc__ - `, which should return an :class:`~numbers.Integral` value. - -.. function:: ulp(x) - - Return the value of the least significant bit of the float *x*: - - * If *x* is a NaN (not a number), return *x*. - * If *x* is negative, return ``ulp(-x)``. - * If *x* is a positive infinity, return *x*. - * If *x* is equal to zero, return the smallest positive - *denormalized* representable float (smaller than the minimum positive - *normalized* float, :data:`sys.float_info.min `). - * If *x* is equal to the largest positive representable float, - return the value of the least significant bit of *x*, such that the first - float smaller than *x* is ``x - ulp(x)``. - * Otherwise (*x* is a positive finite number), return the value of the least - significant bit of *x*, such that the first float bigger than *x* - is ``x + ulp(x)``. - - ULP stands for "Unit in the Last Place". - - See also :func:`math.nextafter` and :data:`sys.float_info.epsilon - `. - - .. versionadded:: 3.9 - - -Note that :func:`frexp` and :func:`modf` have a different call/return pattern -than their C equivalents: they take a single argument and return a pair of -values, rather than returning their second return value through an 'output -parameter' (there is no such thing in Python). - -For the :func:`ceil`, :func:`floor`, and :func:`modf` functions, note that *all* -floating-point numbers of sufficiently large magnitude are exact integers. -Python floats typically carry no more than 53 bits of precision (the same as the -platform C double type), in which case any float *x* with ``abs(x) >= 2**52`` -necessarily has no fractional bits. - - -Power and logarithmic functions -------------------------------- - -.. function:: cbrt(x) - - Return the cube root of *x*. - - .. versionadded:: 3.11 - - -.. function:: exp(x) - - Return *e* raised to the power *x*, where *e* = 2.718281... is the base - of natural logarithms. This is usually more accurate than ``math.e ** x`` - or ``pow(math.e, x)``. - - -.. function:: exp2(x) - - Return *2* raised to the power *x*. - - .. versionadded:: 3.11 - - -.. function:: expm1(x) - - Return *e* raised to the power *x*, minus 1. Here *e* is the base of natural - logarithms. For small floats *x*, the subtraction in ``exp(x) - 1`` - can result in a `significant loss of precision - `_\; the :func:`expm1` - function provides a way to compute this quantity to full precision:: - - >>> from math import exp, expm1 - >>> exp(1e-5) - 1 # gives result accurate to 11 places - 1.0000050000069649e-05 - >>> expm1(1e-5) # result accurate to full precision - 1.0000050000166668e-05 - - .. versionadded:: 3.2 - - -.. function:: log(x[, base]) - - With one argument, return the natural logarithm of *x* (to base *e*). - - With two arguments, return the logarithm of *x* to the given *base*, - calculated as ``log(x)/log(base)``. - - -.. function:: log1p(x) - - Return the natural logarithm of *1+x* (base *e*). The - result is calculated in a way which is accurate for *x* near zero. - - -.. function:: log2(x) - - Return the base-2 logarithm of *x*. This is usually more accurate than - ``log(x, 2)``. - - .. versionadded:: 3.3 - - .. seealso:: - - :meth:`int.bit_length` returns the number of bits necessary to represent - an integer in binary, excluding the sign and leading zeros. - - -.. function:: log10(x) - - Return the base-10 logarithm of *x*. This is usually more accurate - than ``log(x, 10)``. - - -.. function:: pow(x, y) - - Return ``x`` raised to the power ``y``. Exceptional cases follow - the IEEE 754 standard as far as possible. In particular, - ``pow(1.0, x)`` and ``pow(x, 0.0)`` always return ``1.0``, even - when ``x`` is a zero or a NaN. If both ``x`` and ``y`` are finite, - ``x`` is negative, and ``y`` is not an integer then ``pow(x, y)`` - is undefined, and raises :exc:`ValueError`. - - Unlike the built-in ``**`` operator, :func:`math.pow` converts both - its arguments to type :class:`float`. Use ``**`` or the built-in - :func:`pow` function for computing exact integer powers. - - .. versionchanged:: 3.11 - The special cases ``pow(0.0, -inf)`` and ``pow(-0.0, -inf)`` were - changed to return ``inf`` instead of raising :exc:`ValueError`, - for consistency with IEEE 754. - - -.. function:: sqrt(x) - - Return the square root of *x*. - - -Trigonometric functions ------------------------ - -.. function:: acos(x) - - Return the arc cosine of *x*, in radians. The result is between ``0`` and - ``pi``. - - -.. function:: asin(x) - - Return the arc sine of *x*, in radians. The result is between ``-pi/2`` and - ``pi/2``. - - -.. function:: atan(x) - - Return the arc tangent of *x*, in radians. The result is between ``-pi/2`` and - ``pi/2``. - - -.. function:: atan2(y, x) - - Return ``atan(y / x)``, in radians. The result is between ``-pi`` and ``pi``. - The vector in the plane from the origin to point ``(x, y)`` makes this angle - with the positive X axis. The point of :func:`atan2` is that the signs of both - inputs are known to it, so it can compute the correct quadrant for the angle. - For example, ``atan(1)`` and ``atan2(1, 1)`` are both ``pi/4``, but ``atan2(-1, - -1)`` is ``-3*pi/4``. - - -.. function:: cos(x) - - Return the cosine of *x* radians. - - -.. function:: dist(p, q) - - Return the Euclidean distance between two points *p* and *q*, each - given as a sequence (or iterable) of coordinates. The two points - must have the same dimension. - - Roughly equivalent to:: - - sqrt(sum((px - qx) ** 2.0 for px, qx in zip(p, q))) - - .. versionadded:: 3.8 - - -.. function:: hypot(*coordinates) - - Return the Euclidean norm, ``sqrt(sum(x**2 for x in coordinates))``. - This is the length of the vector from the origin to the point - given by the coordinates. - - For a two dimensional point ``(x, y)``, this is equivalent to computing - the hypotenuse of a right triangle using the Pythagorean theorem, - ``sqrt(x*x + y*y)``. - - .. versionchanged:: 3.8 - Added support for n-dimensional points. Formerly, only the two - dimensional case was supported. - - .. versionchanged:: 3.10 - Improved the algorithm's accuracy so that the maximum error is - under 1 ulp (unit in the last place). More typically, the result - is almost always correctly rounded to within 1/2 ulp. - - -.. function:: sin(x) - - Return the sine of *x* radians. - - -.. function:: tan(x) - - Return the tangent of *x* radians. - - -Angular conversion ------------------- - -.. function:: degrees(x) - - Convert angle *x* from radians to degrees. - - -.. function:: radians(x) - - Convert angle *x* from degrees to radians. - - -Hyperbolic functions --------------------- - -`Hyperbolic functions `_ -are analogs of trigonometric functions that are based on hyperbolas -instead of circles. - -.. function:: acosh(x) - - Return the inverse hyperbolic cosine of *x*. - - -.. function:: asinh(x) - - Return the inverse hyperbolic sine of *x*. - - -.. function:: atanh(x) - - Return the inverse hyperbolic tangent of *x*. - - -.. function:: cosh(x) - - Return the hyperbolic cosine of *x*. - - -.. function:: sinh(x) - - Return the hyperbolic sine of *x*. - - -.. function:: tanh(x) - - Return the hyperbolic tangent of *x*. - - -Special functions ------------------ - -.. function:: erf(x) - - Return the `error function `_ at - *x*. - - The :func:`erf` function can be used to compute traditional statistical - functions such as the `cumulative standard normal distribution - `_:: - - def phi(x): - 'Cumulative distribution function for the standard normal distribution' - return (1.0 + erf(x / sqrt(2.0))) / 2.0 - - .. versionadded:: 3.2 - - -.. function:: erfc(x) - - Return the complementary error function at *x*. The `complementary error - function `_ is defined as - ``1.0 - erf(x)``. It is used for large values of *x* where a subtraction - from one would cause a `loss of significance - `_\. - - .. versionadded:: 3.2 - - -.. function:: gamma(x) - - Return the `Gamma function `_ at - *x*. - - .. versionadded:: 3.2 - - -.. function:: lgamma(x) - - Return the natural logarithm of the absolute value of the Gamma - function at *x*. - - .. versionadded:: 3.2 - - -Constants ---------- - -.. data:: pi - - The mathematical constant *π* = 3.141592..., to available precision. - - -.. data:: e - - The mathematical constant *e* = 2.718281..., to available precision. - - -.. data:: tau - - The mathematical constant *τ* = 6.283185..., to available precision. - Tau is a circle constant equal to 2\ *π*, the ratio of a circle's circumference to - its radius. To learn more about Tau, check out Vi Hart's video `Pi is (still) - Wrong `_, and start celebrating - `Tau day `_ by eating twice as much pie! - - .. versionadded:: 3.6 - - -.. data:: inf - - A floating-point positive infinity. (For negative infinity, use - ``-math.inf``.) Equivalent to the output of ``float('inf')``. - - .. versionadded:: 3.5 - - -.. data:: nan - - A floating-point "not a number" (NaN) value. Equivalent to the output of - ``float('nan')``. Due to the requirements of the `IEEE-754 standard - `_, ``math.nan`` and ``float('nan')`` are - not considered to equal to any other numeric value, including themselves. To check - whether a number is a NaN, use the :func:`isnan` function to test - for NaNs instead of ``is`` or ``==``. - Example:: - - >>> import math - >>> math.nan == math.nan - False - >>> float('nan') == float('nan') - False - >>> math.isnan(math.nan) - True - >>> math.isnan(float('nan')) - True - - .. versionchanged:: 3.11 - It is now always available. - - .. versionadded:: 3.5 - - -.. impl-detail:: - - The :mod:`math` module consists mostly of thin wrappers around the platform C - math library functions. Behavior in exceptional cases follows Annex F of - the C99 standard where appropriate. The current implementation will raise - :exc:`ValueError` for invalid operations like ``sqrt(-1.0)`` or ``log(0.0)`` - (where C99 Annex F recommends signaling invalid operation or divide-by-zero), - and :exc:`OverflowError` for results that overflow (for example, - ``exp(1000.0)``). A NaN will not be returned from any of the functions - above unless one or more of the input arguments was a NaN; in that case, - most functions will return a NaN, but (again following C99 Annex F) there - are some exceptions to this rule, for example ``pow(float('nan'), 0.0)`` or - ``hypot(float('nan'), float('inf'))``. - - Note that Python makes no effort to distinguish signaling NaNs from - quiet NaNs, and behavior for signaling NaNs remains unspecified. - Typical behavior is to treat all NaNs as though they were quiet. - - -.. seealso:: - - Module :mod:`cmath` - Complex number versions of many of these functions. - -.. |nbsp| unicode:: 0xA0 - :trim: diff --git a/Doc/library/mimetypes.rst b/Doc/library/mimetypes.rst deleted file mode 100644 index f610032acbe417..00000000000000 --- a/Doc/library/mimetypes.rst +++ /dev/null @@ -1,274 +0,0 @@ -:mod:`mimetypes` --- Map filenames to MIME types -================================================ - -.. module:: mimetypes - :synopsis: Mapping of filename extensions to MIME types. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/mimetypes.py` - -.. index:: pair: MIME; content type - --------------- - -The :mod:`mimetypes` module converts between a filename or URL and the MIME type -associated with the filename extension. Conversions are provided from filename -to MIME type and from MIME type to filename extension; encodings are not -supported for the latter conversion. - -The module provides one class and a number of convenience functions. The -functions are the normal interface to this module, but some applications may be -interested in the class as well. - -The functions described below provide the primary interface for this module. If -the module has not been initialized, they will call :func:`init` if they rely on -the information :func:`init` sets up. - - -.. function:: guess_type(url, strict=True) - - .. index:: pair: MIME; headers - - Guess the type of a file based on its filename, path or URL, given by *url*. - URL can be a string or a :term:`path-like object`. - - The return value is a tuple ``(type, encoding)`` where *type* is ``None`` if the - type can't be guessed (missing or unknown suffix) or a string of the form - ``'type/subtype'``, usable for a MIME :mailheader:`content-type` header. - - *encoding* is ``None`` for no encoding or the name of the program used to encode - (e.g. :program:`compress` or :program:`gzip`). The encoding is suitable for use - as a :mailheader:`Content-Encoding` header, **not** as a - :mailheader:`Content-Transfer-Encoding` header. The mappings are table driven. - Encoding suffixes are case sensitive; type suffixes are first tried case - sensitively, then case insensitively. - - The optional *strict* argument is a flag specifying whether the list of known MIME types - is limited to only the official types `registered with IANA - `_. - When *strict* is ``True`` (the default), only the IANA types are supported; when - *strict* is ``False``, some additional non-standard but commonly used MIME types - are also recognized. - - .. versionchanged:: 3.8 - Added support for url being a :term:`path-like object`. - - -.. function:: guess_all_extensions(type, strict=True) - - Guess the extensions for a file based on its MIME type, given by *type*. The - return value is a list of strings giving all possible filename extensions, - including the leading dot (``'.'``). The extensions are not guaranteed to have - been associated with any particular data stream, but would be mapped to the MIME - type *type* by :func:`guess_type`. - - The optional *strict* argument has the same meaning as with the :func:`guess_type` function. - - -.. function:: guess_extension(type, strict=True) - - Guess the extension for a file based on its MIME type, given by *type*. The - return value is a string giving a filename extension, including the leading dot - (``'.'``). The extension is not guaranteed to have been associated with any - particular data stream, but would be mapped to the MIME type *type* by - :func:`guess_type`. If no extension can be guessed for *type*, ``None`` is - returned. - - The optional *strict* argument has the same meaning as with the :func:`guess_type` function. - -Some additional functions and data items are available for controlling the -behavior of the module. - - -.. function:: init(files=None) - - Initialize the internal data structures. If given, *files* must be a sequence - of file names which should be used to augment the default type map. If omitted, - the file names to use are taken from :const:`knownfiles`; on Windows, the - current registry settings are loaded. Each file named in *files* or - :const:`knownfiles` takes precedence over those named before it. Calling - :func:`init` repeatedly is allowed. - - Specifying an empty list for *files* will prevent the system defaults from - being applied: only the well-known values will be present from a built-in list. - - If *files* is ``None`` the internal data structure is completely rebuilt to its - initial default value. This is a stable operation and will produce the same results - when called multiple times. - - .. versionchanged:: 3.2 - Previously, Windows registry settings were ignored. - - -.. function:: read_mime_types(filename) - - Load the type map given in the file *filename*, if it exists. The type map is - returned as a dictionary mapping filename extensions, including the leading dot - (``'.'``), to strings of the form ``'type/subtype'``. If the file *filename* - does not exist or cannot be read, ``None`` is returned. - - -.. function:: add_type(type, ext, strict=True) - - Add a mapping from the MIME type *type* to the extension *ext*. When the - extension is already known, the new type will replace the old one. When the type - is already known the extension will be added to the list of known extensions. - - When *strict* is ``True`` (the default), the mapping will be added to the - official MIME types, otherwise to the non-standard ones. - - -.. data:: inited - - Flag indicating whether or not the global data structures have been initialized. - This is set to ``True`` by :func:`init`. - - -.. data:: knownfiles - - .. index:: single: file; mime.types - - List of type map file names commonly installed. These files are typically named - :file:`mime.types` and are installed in different locations by different - packages. - - -.. data:: suffix_map - - Dictionary mapping suffixes to suffixes. This is used to allow recognition of - encoded files for which the encoding and the type are indicated by the same - extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. - - -.. data:: encodings_map - - Dictionary mapping filename extensions to encoding types. - - -.. data:: types_map - - Dictionary mapping filename extensions to MIME types. - - -.. data:: common_types - - Dictionary mapping filename extensions to non-standard, but commonly found MIME - types. - - -An example usage of the module:: - - >>> import mimetypes - >>> mimetypes.init() - >>> mimetypes.knownfiles - ['/etc/mime.types', '/etc/httpd/mime.types', ... ] - >>> mimetypes.suffix_map['.tgz'] - '.tar.gz' - >>> mimetypes.encodings_map['.gz'] - 'gzip' - >>> mimetypes.types_map['.tgz'] - 'application/x-tar-gz' - - -.. _mimetypes-objects: - -MimeTypes Objects ------------------ - -The :class:`MimeTypes` class may be useful for applications which may want more -than one MIME-type database; it provides an interface similar to the one of the -:mod:`mimetypes` module. - - -.. class:: MimeTypes(filenames=(), strict=True) - - This class represents a MIME-types database. By default, it provides access to - the same database as the rest of this module. The initial database is a copy of - that provided by the module, and may be extended by loading additional - :file:`mime.types`\ -style files into the database using the :meth:`read` or - :meth:`readfp` methods. The mapping dictionaries may also be cleared before - loading additional data if the default data is not desired. - - The optional *filenames* parameter can be used to cause additional files to be - loaded "on top" of the default database. - - - .. attribute:: MimeTypes.suffix_map - - Dictionary mapping suffixes to suffixes. This is used to allow recognition of - encoded files for which the encoding and the type are indicated by the same - extension. For example, the :file:`.tgz` extension is mapped to :file:`.tar.gz` - to allow the encoding and type to be recognized separately. This is initially a - copy of the global :data:`suffix_map` defined in the module. - - - .. attribute:: MimeTypes.encodings_map - - Dictionary mapping filename extensions to encoding types. This is initially a - copy of the global :data:`encodings_map` defined in the module. - - - .. attribute:: MimeTypes.types_map - - Tuple containing two dictionaries, mapping filename extensions to MIME types: - the first dictionary is for the non-standards types and the second one is for - the standard types. They are initialized by :data:`common_types` and - :data:`types_map`. - - - .. attribute:: MimeTypes.types_map_inv - - Tuple containing two dictionaries, mapping MIME types to a list of filename - extensions: the first dictionary is for the non-standards types and the - second one is for the standard types. They are initialized by - :data:`common_types` and :data:`types_map`. - - - .. method:: MimeTypes.guess_extension(type, strict=True) - - Similar to the :func:`guess_extension` function, using the tables stored as part - of the object. - - - .. method:: MimeTypes.guess_type(url, strict=True) - - Similar to the :func:`guess_type` function, using the tables stored as part of - the object. - - - .. method:: MimeTypes.guess_all_extensions(type, strict=True) - - Similar to the :func:`guess_all_extensions` function, using the tables stored - as part of the object. - - - .. method:: MimeTypes.read(filename, strict=True) - - Load MIME information from a file named *filename*. This uses :meth:`readfp` to - parse the file. - - If *strict* is ``True``, information will be added to list of standard types, - else to the list of non-standard types. - - - .. method:: MimeTypes.readfp(fp, strict=True) - - Load MIME type information from an open file *fp*. The file must have the format of - the standard :file:`mime.types` files. - - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. - - - .. method:: MimeTypes.read_windows_registry(strict=True) - - Load MIME type information from the Windows registry. - - .. availability:: Windows. - - If *strict* is ``True``, information will be added to the list of standard - types, else to the list of non-standard types. - - .. versionadded:: 3.2 diff --git a/Doc/library/mm.rst b/Doc/library/mm.rst deleted file mode 100644 index cd06e9385a2612..00000000000000 --- a/Doc/library/mm.rst +++ /dev/null @@ -1,15 +0,0 @@ -.. _mmedia: - -******************* -Multimedia Services -******************* - -The modules described in this chapter implement various algorithms or interfaces -that are mainly useful for multimedia applications. They are available at the -discretion of the installation. Here's an overview: - - -.. toctree:: - - wave.rst - colorsys.rst diff --git a/Doc/library/mmap.rst b/Doc/library/mmap.rst deleted file mode 100644 index 21fa97a2a3cef8..00000000000000 --- a/Doc/library/mmap.rst +++ /dev/null @@ -1,380 +0,0 @@ -:mod:`mmap` --- Memory-mapped file support -========================================== - -.. module:: mmap - :synopsis: Interface to memory-mapped files for Unix and Windows. - --------------- - -.. include:: ../includes/wasm-notavail.rst - -Memory-mapped file objects behave like both :class:`bytearray` and like -:term:`file objects `. You can use mmap objects in most places -where :class:`bytearray` are expected; for example, you can use the :mod:`re` -module to search through a memory-mapped file. You can also change a single -byte by doing ``obj[index] = 97``, or change a subsequence by assigning to a -slice: ``obj[i1:i2] = b'...'``. You can also read and write data starting at -the current file position, and :meth:`seek` through the file to different positions. - -A memory-mapped file is created by the :class:`~mmap.mmap` constructor, which is -different on Unix and on Windows. In either case you must provide a file -descriptor for a file opened for update. If you wish to map an existing Python -file object, use its :meth:`~io.IOBase.fileno` method to obtain the correct value for the -*fileno* parameter. Otherwise, you can open the file using the -:func:`os.open` function, which returns a file descriptor directly (the file -still needs to be closed when done). - -.. note:: - If you want to create a memory-mapping for a writable, buffered file, you - should :func:`~io.IOBase.flush` the file first. This is necessary to ensure - that local modifications to the buffers are actually available to the - mapping. - -For both the Unix and Windows versions of the constructor, *access* may be -specified as an optional keyword parameter. *access* accepts one of four -values: :const:`ACCESS_READ`, :const:`ACCESS_WRITE`, or :const:`ACCESS_COPY` to -specify read-only, write-through or copy-on-write memory respectively, or -:const:`ACCESS_DEFAULT` to defer to *prot*. *access* can be used on both Unix -and Windows. If *access* is not specified, Windows mmap returns a -write-through mapping. The initial memory values for all three access types -are taken from the specified file. Assignment to an :const:`ACCESS_READ` -memory map raises a :exc:`TypeError` exception. Assignment to an -:const:`ACCESS_WRITE` memory map affects both memory and the underlying file. -Assignment to an :const:`ACCESS_COPY` memory map affects memory but does not -update the underlying file. - -.. versionchanged:: 3.7 - Added :const:`ACCESS_DEFAULT` constant. - -To map anonymous memory, -1 should be passed as the fileno along with the length. - -.. class:: mmap(fileno, length, tagname=None, access=ACCESS_DEFAULT[, offset]) - - **(Windows version)** Maps *length* bytes from the file specified by the - file handle *fileno*, and creates a mmap object. If *length* is larger - than the current size of the file, the file is extended to contain *length* - bytes. If *length* is ``0``, the maximum length of the map is the current - size of the file, except that if the file is empty Windows raises an - exception (you cannot create an empty mapping on Windows). - - *tagname*, if specified and not ``None``, is a string giving a tag name for - the mapping. Windows allows you to have many different mappings against - the same file. If you specify the name of an existing tag, that tag is - opened, otherwise a new tag of this name is created. If this parameter is - omitted or ``None``, the mapping is created without a name. Avoiding the - use of the tag parameter will assist in keeping your code portable between - Unix and Windows. - - *offset* may be specified as a non-negative integer offset. mmap references - will be relative to the offset from the beginning of the file. *offset* - defaults to 0. *offset* must be a multiple of the :const:`ALLOCATIONGRANULARITY`. - - .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap - -.. class:: mmap(fileno, length, flags=MAP_SHARED, prot=PROT_WRITE|PROT_READ, access=ACCESS_DEFAULT[, offset]) - :noindex: - - **(Unix version)** Maps *length* bytes from the file specified by the file - descriptor *fileno*, and returns a mmap object. If *length* is ``0``, the - maximum length of the map will be the current size of the file when - :class:`~mmap.mmap` is called. - - *flags* specifies the nature of the mapping. :const:`MAP_PRIVATE` creates a - private copy-on-write mapping, so changes to the contents of the mmap - object will be private to this process, and :const:`MAP_SHARED` creates a - mapping that's shared with all other processes mapping the same areas of - the file. The default value is :const:`MAP_SHARED`. Some systems have - additional possible flags with the full list specified in - :ref:`MAP_* constants `. - - *prot*, if specified, gives the desired memory protection; the two most - useful values are :const:`PROT_READ` and :const:`PROT_WRITE`, to specify - that the pages may be read or written. *prot* defaults to - :const:`PROT_READ \| PROT_WRITE`. - - *access* may be specified in lieu of *flags* and *prot* as an optional - keyword parameter. It is an error to specify both *flags*, *prot* and - *access*. See the description of *access* above for information on how to - use this parameter. - - *offset* may be specified as a non-negative integer offset. mmap references - will be relative to the offset from the beginning of the file. *offset* - defaults to 0. *offset* must be a multiple of :const:`ALLOCATIONGRANULARITY` - which is equal to :const:`PAGESIZE` on Unix systems. - - To ensure validity of the created memory mapping the file specified - by the descriptor *fileno* is internally automatically synchronized - with the physical backing store on macOS. - - This example shows a simple way of using :class:`~mmap.mmap`:: - - import mmap - - # write a simple example file - with open("hello.txt", "wb") as f: - f.write(b"Hello Python!\n") - - with open("hello.txt", "r+b") as f: - # memory-map the file, size 0 means whole file - mm = mmap.mmap(f.fileno(), 0) - # read content via standard file methods - print(mm.readline()) # prints b"Hello Python!\n" - # read content via slice notation - print(mm[:5]) # prints b"Hello" - # update content using slice notation; - # note that new content must have same size - mm[6:] = b" world!\n" - # ... and read again using standard file methods - mm.seek(0) - print(mm.readline()) # prints b"Hello world!\n" - # close the map - mm.close() - - - :class:`~mmap.mmap` can also be used as a context manager in a :keyword:`with` - statement:: - - import mmap - - with mmap.mmap(-1, 13) as mm: - mm.write(b"Hello world!") - - .. versionadded:: 3.2 - Context manager support. - - - The next example demonstrates how to create an anonymous map and exchange - data between the parent and child processes:: - - import mmap - import os - - mm = mmap.mmap(-1, 13) - mm.write(b"Hello world!") - - pid = os.fork() - - if pid == 0: # In a child process - mm.seek(0) - print(mm.readline()) - - mm.close() - - .. audit-event:: mmap.__new__ fileno,length,access,offset mmap.mmap - - Memory-mapped file objects support the following methods: - - .. method:: close() - - Closes the mmap. Subsequent calls to other methods of the object will - result in a ValueError exception being raised. This will not close - the open file. - - - .. attribute:: closed - - ``True`` if the file is closed. - - .. versionadded:: 3.2 - - - .. method:: find(sub[, start[, end]]) - - Returns the lowest index in the object where the subsequence *sub* is - found, such that *sub* is contained in the range [*start*, *end*]. - Optional arguments *start* and *end* are interpreted as in slice notation. - Returns ``-1`` on failure. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - - .. method:: flush([offset[, size]]) - - Flushes changes made to the in-memory copy of a file back to disk. Without - use of this call there is no guarantee that changes are written back before - the object is destroyed. If *offset* and *size* are specified, only - changes to the given range of bytes will be flushed to disk; otherwise, the - whole extent of the mapping is flushed. *offset* must be a multiple of the - :const:`PAGESIZE` or :const:`ALLOCATIONGRANULARITY`. - - ``None`` is returned to indicate success. An exception is raised when the - call failed. - - .. versionchanged:: 3.8 - Previously, a nonzero value was returned on success; zero was returned - on error under Windows. A zero value was returned on success; an - exception was raised on error under Unix. - - - .. method:: madvise(option[, start[, length]]) - - Send advice *option* to the kernel about the memory region beginning at - *start* and extending *length* bytes. *option* must be one of the - :ref:`MADV_* constants ` available on the system. If - *start* and *length* are omitted, the entire mapping is spanned. On - some systems (including Linux), *start* must be a multiple of the - :const:`PAGESIZE`. - - Availability: Systems with the ``madvise()`` system call. - - .. versionadded:: 3.8 - - - .. method:: move(dest, src, count) - - Copy the *count* bytes starting at offset *src* to the destination index - *dest*. If the mmap was created with :const:`ACCESS_READ`, then calls to - move will raise a :exc:`TypeError` exception. - - - .. method:: read([n]) - - Return a :class:`bytes` containing up to *n* bytes starting from the - current file position. If the argument is omitted, ``None`` or negative, - return all bytes from the current file position to the end of the - mapping. The file position is updated to point after the bytes that were - returned. - - .. versionchanged:: 3.3 - Argument can be omitted or ``None``. - - .. method:: read_byte() - - Returns a byte at the current file position as an integer, and advances - the file position by 1. - - - .. method:: readline() - - Returns a single line, starting at the current file position and up to the - next newline. The file position is updated to point after the bytes that were - returned. - - - .. method:: resize(newsize) - - Resizes the map and the underlying file, if any. If the mmap was created - with :const:`ACCESS_READ` or :const:`ACCESS_COPY`, resizing the map will - raise a :exc:`TypeError` exception. - - **On Windows**: Resizing the map will raise an :exc:`OSError` if there are other - maps against the same named file. Resizing an anonymous map (ie against the - pagefile) will silently create a new map with the original data copied over - up to the length of the new size. - - .. versionchanged:: 3.11 - Correctly fails if attempting to resize when another map is held - Allows resize against an anonymous map on Windows - - .. method:: rfind(sub[, start[, end]]) - - Returns the highest index in the object where the subsequence *sub* is - found, such that *sub* is contained in the range [*start*, *end*]. - Optional arguments *start* and *end* are interpreted as in slice notation. - Returns ``-1`` on failure. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - - .. method:: seek(pos[, whence]) - - Set the file's current position. *whence* argument is optional and - defaults to ``os.SEEK_SET`` or ``0`` (absolute file positioning); other - values are ``os.SEEK_CUR`` or ``1`` (seek relative to the current - position) and ``os.SEEK_END`` or ``2`` (seek relative to the file's end). - - - .. method:: size() - - Return the length of the file, which can be larger than the size of the - memory-mapped area. - - - .. method:: tell() - - Returns the current position of the file pointer. - - - .. method:: write(bytes) - - Write the bytes in *bytes* into memory at the current position of the - file pointer and return the number of bytes written (never less than - ``len(bytes)``, since if the write fails, a :exc:`ValueError` will be - raised). The file position is updated to point after the bytes that - were written. If the mmap was created with :const:`ACCESS_READ`, then - writing to it will raise a :exc:`TypeError` exception. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - .. versionchanged:: 3.6 - The number of bytes written is now returned. - - - .. method:: write_byte(byte) - - Write the integer *byte* into memory at the current - position of the file pointer; the file position is advanced by ``1``. If - the mmap was created with :const:`ACCESS_READ`, then writing to it will - raise a :exc:`TypeError` exception. - -.. _madvise-constants: - -MADV_* Constants -++++++++++++++++ - -.. data:: MADV_NORMAL - MADV_RANDOM - MADV_SEQUENTIAL - MADV_WILLNEED - MADV_DONTNEED - MADV_REMOVE - MADV_DONTFORK - MADV_DOFORK - MADV_HWPOISON - MADV_MERGEABLE - MADV_UNMERGEABLE - MADV_SOFT_OFFLINE - MADV_HUGEPAGE - MADV_NOHUGEPAGE - MADV_DONTDUMP - MADV_DODUMP - MADV_FREE - MADV_NOSYNC - MADV_AUTOSYNC - MADV_NOCORE - MADV_CORE - MADV_PROTECT - MADV_FREE_REUSABLE - MADV_FREE_REUSE - - These options can be passed to :meth:`mmap.madvise`. Not every option will - be present on every system. - - Availability: Systems with the madvise() system call. - - .. versionadded:: 3.8 - -.. _map-constants: - -MAP_* Constants -+++++++++++++++ - -.. data:: MAP_SHARED - MAP_PRIVATE - MAP_DENYWRITE - MAP_EXECUTABLE - MAP_ANON - MAP_ANONYMOUS - MAP_POPULATE - MAP_STACK - - These are the various flags that can be passed to :meth:`mmap.mmap`. Note that some options might not be present on some systems. - - .. versionchanged:: 3.10 - Added MAP_POPULATE constant. - - .. versionadded:: 3.11 - Added MAP_STACK constant. diff --git a/Doc/library/modulefinder.rst b/Doc/library/modulefinder.rst deleted file mode 100644 index 526f0ff868c2b7..00000000000000 --- a/Doc/library/modulefinder.rst +++ /dev/null @@ -1,114 +0,0 @@ -:mod:`modulefinder` --- Find modules used by a script -===================================================== - -.. module:: modulefinder - :synopsis: Find modules used by a script. - -.. sectionauthor:: A.M. Kuchling - -**Source code:** :source:`Lib/modulefinder.py` - --------------- - -This module provides a :class:`ModuleFinder` class that can be used to determine -the set of modules imported by a script. ``modulefinder.py`` can also be run as -a script, giving the filename of a Python script as its argument, after which a -report of the imported modules will be printed. - - -.. function:: AddPackagePath(pkg_name, path) - - Record that the package named *pkg_name* can be found in the specified *path*. - - -.. function:: ReplacePackage(oldname, newname) - - Allows specifying that the module named *oldname* is in fact the package named - *newname*. - - -.. class:: ModuleFinder(path=None, debug=0, excludes=[], replace_paths=[]) - - This class provides :meth:`run_script` and :meth:`report` methods to determine - the set of modules imported by a script. *path* can be a list of directories to - search for modules; if not specified, ``sys.path`` is used. *debug* sets the - debugging level; higher values make the class print debugging messages about - what it's doing. *excludes* is a list of module names to exclude from the - analysis. *replace_paths* is a list of ``(oldpath, newpath)`` tuples that will - be replaced in module paths. - - - .. method:: report() - - Print a report to standard output that lists the modules imported by the - script and their paths, as well as modules that are missing or seem to be - missing. - - .. method:: run_script(pathname) - - Analyze the contents of the *pathname* file, which must contain Python - code. - - .. attribute:: modules - - A dictionary mapping module names to modules. See - :ref:`modulefinder-example`. - - -.. _modulefinder-example: - -Example usage of :class:`ModuleFinder` --------------------------------------- - -The script that is going to get analyzed later on (bacon.py):: - - import re, itertools - - try: - import baconhameggs - except ImportError: - pass - - try: - import guido.python.ham - except ImportError: - pass - - -The script that will output the report of bacon.py:: - - from modulefinder import ModuleFinder - - finder = ModuleFinder() - finder.run_script('bacon.py') - - print('Loaded modules:') - for name, mod in finder.modules.items(): - print('%s: ' % name, end='') - print(','.join(list(mod.globalnames.keys())[:3])) - - print('-'*50) - print('Modules not imported:') - print('\n'.join(finder.badmodules.keys())) - -Sample output (may vary depending on the architecture):: - - Loaded modules: - _types: - copyreg: _inverted_registry,_slotnames,__all__ - re._compiler: isstring,_sre,_optimize_unicode - _sre: - re._constants: REPEAT_ONE,makedict,AT_END_LINE - sys: - re: __module__,finditer,_expand - itertools: - __main__: re,itertools,baconhameggs - re._parser: _PATTERNENDERS,SRE_FLAG_UNICODE - array: - types: __module__,IntType,TypeType - --------------------------------------------------- - Modules not imported: - guido.python.ham - baconhameggs - - diff --git a/Doc/library/modules.rst b/Doc/library/modules.rst deleted file mode 100644 index 8c5936a4d8de2b..00000000000000 --- a/Doc/library/modules.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. _modules: - -***************** -Importing Modules -***************** - -The modules described in this chapter provide new ways to import other Python -modules and hooks for customizing the import process. - -The full list of modules described in this chapter is: - - -.. toctree:: - - zipimport.rst - pkgutil.rst - modulefinder.rst - runpy.rst - importlib.rst - importlib.resources.rst - importlib.resources.abc.rst - importlib.metadata.rst - sys_path_init.rst diff --git a/Doc/library/msilib.rst b/Doc/library/msilib.rst deleted file mode 100644 index fbe55db9372300..00000000000000 --- a/Doc/library/msilib.rst +++ /dev/null @@ -1,565 +0,0 @@ -:mod:`msilib` --- Read and write Microsoft Installer files -========================================================== - -.. module:: msilib - :platform: Windows - :synopsis: Creation of Microsoft Installer files, and CAB files. - :deprecated: - -.. moduleauthor:: Martin v. Löwis -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/msilib/__init__.py` - -.. index:: single: msi - -.. deprecated-removed:: 3.11 3.13 - The :mod:`msilib` module is deprecated - (see :pep:`PEP 594 <594#msilib>` for details). - --------------- - -The :mod:`msilib` supports the creation of Microsoft Installer (``.msi``) files. -Because these files often contain an embedded "cabinet" file (``.cab``), it also -exposes an API to create CAB files. Support for reading ``.cab`` files is -currently not implemented; read support for the ``.msi`` database is possible. - -This package aims to provide complete access to all tables in an ``.msi`` file, -therefore, it is a fairly low-level API. One primary application of this -package is the creation of Python installer package itself (although that currently -uses a different version of ``msilib``). - -The package contents can be roughly split into four parts: low-level CAB -routines, low-level MSI routines, higher-level MSI routines, and standard table -structures. - - -.. function:: FCICreate(cabname, files) - - Create a new CAB file named *cabname*. *files* must be a list of tuples, each - containing the name of the file on disk, and the name of the file inside the CAB - file. - - The files are added to the CAB file in the order they appear in the list. All - files are added into a single CAB file, using the MSZIP compression algorithm. - - Callbacks to Python for the various steps of MSI creation are currently not - exposed. - - -.. function:: UuidCreate() - - Return the string representation of a new unique identifier. This wraps the - Windows API functions :c:func:`UuidCreate` and :c:func:`UuidToString`. - - -.. function:: OpenDatabase(path, persist) - - Return a new database object by calling MsiOpenDatabase. *path* is the file - name of the MSI file; *persist* can be one of the constants - ``MSIDBOPEN_CREATEDIRECT``, ``MSIDBOPEN_CREATE``, ``MSIDBOPEN_DIRECT``, - ``MSIDBOPEN_READONLY``, or ``MSIDBOPEN_TRANSACT``, and may include the flag - ``MSIDBOPEN_PATCHFILE``. See the Microsoft documentation for the meaning of - these flags; depending on the flags, an existing database is opened, or a new - one created. - - -.. function:: CreateRecord(count) - - Return a new record object by calling :c:func:`MSICreateRecord`. *count* is the - number of fields of the record. - - -.. function:: init_database(name, schema, ProductName, ProductCode, ProductVersion, Manufacturer) - - Create and return a new database *name*, initialize it with *schema*, and set - the properties *ProductName*, *ProductCode*, *ProductVersion*, and - *Manufacturer*. - - *schema* must be a module object containing ``tables`` and - ``_Validation_records`` attributes; typically, :mod:`msilib.schema` should be - used. - - The database will contain just the schema and the validation records when this - function returns. - - -.. function:: add_data(database, table, records) - - Add all *records* to the table named *table* in *database*. - - The *table* argument must be one of the predefined tables in the MSI schema, - e.g. ``'Feature'``, ``'File'``, ``'Component'``, ``'Dialog'``, ``'Control'``, - etc. - - *records* should be a list of tuples, each one containing all fields of a - record according to the schema of the table. For optional fields, - ``None`` can be passed. - - Field values can be ints, strings, or instances of the Binary class. - - -.. class:: Binary(filename) - - Represents entries in the Binary table; inserting such an object using - :func:`add_data` reads the file named *filename* into the table. - - -.. function:: add_tables(database, module) - - Add all table content from *module* to *database*. *module* must contain an - attribute *tables* listing all tables for which content should be added, and one - attribute per table that has the actual content. - - This is typically used to install the sequence tables. - - -.. function:: add_stream(database, name, path) - - Add the file *path* into the ``_Stream`` table of *database*, with the stream - name *name*. - - -.. function:: gen_uuid() - - Return a new UUID, in the format that MSI typically requires (i.e. in curly - braces, and with all hexdigits in uppercase). - - -.. seealso:: - - `FCICreate `_ - `UuidCreate `_ - `UuidToString `_ - -.. _database-objects: - -Database Objects ----------------- - - -.. method:: Database.OpenView(sql) - - Return a view object, by calling :c:func:`MSIDatabaseOpenView`. *sql* is the SQL - statement to execute. - - -.. method:: Database.Commit() - - Commit the changes pending in the current transaction, by calling - :c:func:`MSIDatabaseCommit`. - - -.. method:: Database.GetSummaryInformation(count) - - Return a new summary information object, by calling - :c:func:`MsiGetSummaryInformation`. *count* is the maximum number of updated - values. - -.. method:: Database.Close() - - Close the database object, through :c:func:`MsiCloseHandle`. - - .. versionadded:: 3.7 - -.. seealso:: - - `MSIDatabaseOpenView `_ - `MSIDatabaseCommit `_ - `MSIGetSummaryInformation `_ - `MsiCloseHandle `_ - -.. _view-objects: - -View Objects ------------- - - -.. method:: View.Execute(params) - - Execute the SQL query of the view, through :c:func:`MSIViewExecute`. If - *params* is not ``None``, it is a record describing actual values of the - parameter tokens in the query. - - -.. method:: View.GetColumnInfo(kind) - - Return a record describing the columns of the view, through calling - :c:func:`MsiViewGetColumnInfo`. *kind* can be either ``MSICOLINFO_NAMES`` or - ``MSICOLINFO_TYPES``. - - -.. method:: View.Fetch() - - Return a result record of the query, through calling :c:func:`MsiViewFetch`. - - -.. method:: View.Modify(kind, data) - - Modify the view, by calling :c:func:`MsiViewModify`. *kind* can be one of - ``MSIMODIFY_SEEK``, ``MSIMODIFY_REFRESH``, ``MSIMODIFY_INSERT``, - ``MSIMODIFY_UPDATE``, ``MSIMODIFY_ASSIGN``, ``MSIMODIFY_REPLACE``, - ``MSIMODIFY_MERGE``, ``MSIMODIFY_DELETE``, ``MSIMODIFY_INSERT_TEMPORARY``, - ``MSIMODIFY_VALIDATE``, ``MSIMODIFY_VALIDATE_NEW``, - ``MSIMODIFY_VALIDATE_FIELD``, or ``MSIMODIFY_VALIDATE_DELETE``. - - *data* must be a record describing the new data. - - -.. method:: View.Close() - - Close the view, through :c:func:`MsiViewClose`. - - -.. seealso:: - - `MsiViewExecute `_ - `MSIViewGetColumnInfo `_ - `MsiViewFetch `_ - `MsiViewModify `_ - `MsiViewClose `_ - -.. _summary-objects: - -Summary Information Objects ---------------------------- - - -.. method:: SummaryInformation.GetProperty(field) - - Return a property of the summary, through :c:func:`MsiSummaryInfoGetProperty`. - *field* is the name of the property, and can be one of the constants - ``PID_CODEPAGE``, ``PID_TITLE``, ``PID_SUBJECT``, ``PID_AUTHOR``, - ``PID_KEYWORDS``, ``PID_COMMENTS``, ``PID_TEMPLATE``, ``PID_LASTAUTHOR``, - ``PID_REVNUMBER``, ``PID_LASTPRINTED``, ``PID_CREATE_DTM``, - ``PID_LASTSAVE_DTM``, ``PID_PAGECOUNT``, ``PID_WORDCOUNT``, ``PID_CHARCOUNT``, - ``PID_APPNAME``, or ``PID_SECURITY``. - - -.. method:: SummaryInformation.GetPropertyCount() - - Return the number of summary properties, through - :c:func:`MsiSummaryInfoGetPropertyCount`. - - -.. method:: SummaryInformation.SetProperty(field, value) - - Set a property through :c:func:`MsiSummaryInfoSetProperty`. *field* can have the - same values as in :meth:`GetProperty`, *value* is the new value of the property. - Possible value types are integer and string. - - -.. method:: SummaryInformation.Persist() - - Write the modified properties to the summary information stream, using - :c:func:`MsiSummaryInfoPersist`. - - -.. seealso:: - - `MsiSummaryInfoGetProperty `_ - `MsiSummaryInfoGetPropertyCount `_ - `MsiSummaryInfoSetProperty `_ - `MsiSummaryInfoPersist `_ - -.. _record-objects: - -Record Objects --------------- - - -.. method:: Record.GetFieldCount() - - Return the number of fields of the record, through - :c:func:`MsiRecordGetFieldCount`. - - -.. method:: Record.GetInteger(field) - - Return the value of *field* as an integer where possible. *field* must - be an integer. - - -.. method:: Record.GetString(field) - - Return the value of *field* as a string where possible. *field* must - be an integer. - - -.. method:: Record.SetString(field, value) - - Set *field* to *value* through :c:func:`MsiRecordSetString`. *field* must be an - integer; *value* a string. - - -.. method:: Record.SetStream(field, value) - - Set *field* to the contents of the file named *value*, through - :c:func:`MsiRecordSetStream`. *field* must be an integer; *value* a string. - - -.. method:: Record.SetInteger(field, value) - - Set *field* to *value* through :c:func:`MsiRecordSetInteger`. Both *field* and - *value* must be an integer. - - -.. method:: Record.ClearData() - - Set all fields of the record to 0, through :c:func:`MsiRecordClearData`. - - -.. seealso:: - - `MsiRecordGetFieldCount `_ - `MsiRecordSetString `_ - `MsiRecordSetStream `_ - `MsiRecordSetInteger `_ - `MsiRecordClearData `_ - -.. _msi-errors: - -Errors ------- - -All wrappers around MSI functions raise :exc:`MSIError`; the string inside the -exception will contain more detail. - - -.. _cab: - -CAB Objects ------------ - - -.. class:: CAB(name) - - The class :class:`CAB` represents a CAB file. During MSI construction, files - will be added simultaneously to the ``Files`` table, and to a CAB file. Then, - when all files have been added, the CAB file can be written, then added to the - MSI file. - - *name* is the name of the CAB file in the MSI file. - - - .. method:: append(full, file, logical) - - Add the file with the pathname *full* to the CAB file, under the name - *logical*. If there is already a file named *logical*, a new file name is - created. - - Return the index of the file in the CAB file, and the new name of the file - inside the CAB file. - - - .. method:: commit(database) - - Generate a CAB file, add it as a stream to the MSI file, put it into the - ``Media`` table, and remove the generated file from the disk. - - -.. _msi-directory: - -Directory Objects ------------------ - - -.. class:: Directory(database, cab, basedir, physical, logical, default, [componentflags]) - - Create a new directory in the Directory table. There is a current component at - each point in time for the directory, which is either explicitly created through - :meth:`start_component`, or implicitly when files are added for the first time. - Files are added into the current component, and into the cab file. To create a - directory, a base directory object needs to be specified (can be ``None``), the - path to the physical directory, and a logical directory name. *default* - specifies the DefaultDir slot in the directory table. *componentflags* specifies - the default flags that new components get. - - - .. method:: start_component(component=None, feature=None, flags=None, keyfile=None, uuid=None) - - Add an entry to the Component table, and make this component the current - component for this directory. If no component name is given, the directory - name is used. If no *feature* is given, the current feature is used. If no - *flags* are given, the directory's default flags are used. If no *keyfile* - is given, the KeyPath is left null in the Component table. - - - .. method:: add_file(file, src=None, version=None, language=None) - - Add a file to the current component of the directory, starting a new one - if there is no current component. By default, the file name in the source - and the file table will be identical. If the *src* file is specified, it - is interpreted relative to the current directory. Optionally, a *version* - and a *language* can be specified for the entry in the File table. - - - .. method:: glob(pattern, exclude=None) - - Add a list of files to the current component as specified in the glob - pattern. Individual files can be excluded in the *exclude* list. - - - .. method:: remove_pyc() - - Remove ``.pyc`` files on uninstall. - - -.. seealso:: - - `Directory Table `_ - `File Table `_ - `Component Table `_ - `FeatureComponents Table `_ - -.. _features: - -Features --------- - - -.. class:: Feature(db, id, title, desc, display, level=1, parent=None, directory=None, attributes=0) - - Add a new record to the ``Feature`` table, using the values *id*, *parent.id*, - *title*, *desc*, *display*, *level*, *directory*, and *attributes*. The - resulting feature object can be passed to the :meth:`start_component` method of - :class:`Directory`. - - - .. method:: set_current() - - Make this feature the current feature of :mod:`msilib`. New components are - automatically added to the default feature, unless a feature is explicitly - specified. - - -.. seealso:: - - `Feature Table `_ - -.. _msi-gui: - -GUI classes ------------ - -:mod:`msilib` provides several classes that wrap the GUI tables in an MSI -database. However, no standard user interface is provided. - - -.. class:: Control(dlg, name) - - Base class of the dialog controls. *dlg* is the dialog object the control - belongs to, and *name* is the control's name. - - - .. method:: event(event, argument, condition=1, ordering=None) - - Make an entry into the ``ControlEvent`` table for this control. - - - .. method:: mapping(event, attribute) - - Make an entry into the ``EventMapping`` table for this control. - - - .. method:: condition(action, condition) - - Make an entry into the ``ControlCondition`` table for this control. - - -.. class:: RadioButtonGroup(dlg, name, property) - - Create a radio button control named *name*. *property* is the installer property - that gets set when a radio button is selected. - - - .. method:: add(name, x, y, width, height, text, value=None) - - Add a radio button named *name* to the group, at the coordinates *x*, *y*, - *width*, *height*, and with the label *text*. If *value* is ``None``, it - defaults to *name*. - - -.. class:: Dialog(db, name, x, y, w, h, attr, title, first, default, cancel) - - Return a new :class:`Dialog` object. An entry in the ``Dialog`` table is made, - with the specified coordinates, dialog attributes, title, name of the first, - default, and cancel controls. - - - .. method:: control(name, type, x, y, width, height, attributes, property, text, control_next, help) - - Return a new :class:`Control` object. An entry in the ``Control`` table is - made with the specified parameters. - - This is a generic method; for specific types, specialized methods are - provided. - - - .. method:: text(name, x, y, width, height, attributes, text) - - Add and return a ``Text`` control. - - - .. method:: bitmap(name, x, y, width, height, text) - - Add and return a ``Bitmap`` control. - - - .. method:: line(name, x, y, width, height) - - Add and return a ``Line`` control. - - - .. method:: pushbutton(name, x, y, width, height, attributes, text, next_control) - - Add and return a ``PushButton`` control. - - - .. method:: radiogroup(name, x, y, width, height, attributes, property, text, next_control) - - Add and return a ``RadioButtonGroup`` control. - - - .. method:: checkbox(name, x, y, width, height, attributes, property, text, next_control) - - Add and return a ``CheckBox`` control. - - -.. seealso:: - - `Dialog Table `_ - `Control Table `_ - `Control Types `_ - `ControlCondition Table `_ - `ControlEvent Table `_ - `EventMapping Table `_ - `RadioButton Table `_ - -.. _msi-tables: - -Precomputed tables ------------------- - -:mod:`msilib` provides a few subpackages that contain only schema and table -definitions. Currently, these definitions are based on MSI version 2.0. - - -.. data:: schema - - This is the standard MSI schema for MSI 2.0, with the *tables* variable - providing a list of table definitions, and *_Validation_records* providing the - data for MSI validation. - - -.. data:: sequence - - This module contains table contents for the standard sequence tables: - *AdminExecuteSequence*, *AdminUISequence*, *AdvtExecuteSequence*, - *InstallExecuteSequence*, and *InstallUISequence*. - - -.. data:: text - - This module contains definitions for the UIText and ActionText tables, for the - standard installer actions. diff --git a/Doc/library/msvcrt.rst b/Doc/library/msvcrt.rst deleted file mode 100644 index 32693e3d007c05..00000000000000 --- a/Doc/library/msvcrt.rst +++ /dev/null @@ -1,160 +0,0 @@ -:mod:`msvcrt` --- Useful routines from the MS VC++ runtime -========================================================== - -.. module:: msvcrt - :platform: Windows - :synopsis: Miscellaneous useful routines from the MS VC++ runtime. - -.. sectionauthor:: Fred L. Drake, Jr. - --------------- - -These functions provide access to some useful capabilities on Windows platforms. -Some higher-level modules use these functions to build the Windows -implementations of their services. For example, the :mod:`getpass` module uses -this in the implementation of the :func:`getpass` function. - -Further documentation on these functions can be found in the Platform API -documentation. - -The module implements both the normal and wide char variants of the console I/O -api. The normal API deals only with ASCII characters and is of limited use -for internationalized applications. The wide char API should be used where -ever possible. - -.. versionchanged:: 3.3 - Operations in this module now raise :exc:`OSError` where :exc:`IOError` - was raised. - - -.. _msvcrt-files: - -File Operations ---------------- - - -.. function:: locking(fd, mode, nbytes) - - Lock part of a file based on file descriptor *fd* from the C runtime. Raises - :exc:`OSError` on failure. The locked region of the file extends from the - current file position for *nbytes* bytes, and may continue beyond the end of the - file. *mode* must be one of the :const:`!LK_\*` constants listed below. Multiple - regions in a file may be locked at the same time, but may not overlap. Adjacent - regions are not merged; they must be unlocked individually. - - .. audit-event:: msvcrt.locking fd,mode,nbytes msvcrt.locking - - -.. data:: LK_LOCK - LK_RLCK - - Locks the specified bytes. If the bytes cannot be locked, the program - immediately tries again after 1 second. If, after 10 attempts, the bytes cannot - be locked, :exc:`OSError` is raised. - - -.. data:: LK_NBLCK - LK_NBRLCK - - Locks the specified bytes. If the bytes cannot be locked, :exc:`OSError` is - raised. - - -.. data:: LK_UNLCK - - Unlocks the specified bytes, which must have been previously locked. - - -.. function:: setmode(fd, flags) - - Set the line-end translation mode for the file descriptor *fd*. To set it to - text mode, *flags* should be :const:`os.O_TEXT`; for binary, it should be - :const:`os.O_BINARY`. - - -.. function:: open_osfhandle(handle, flags) - - Create a C runtime file descriptor from the file handle *handle*. The *flags* - parameter should be a bitwise OR of :const:`os.O_APPEND`, :const:`os.O_RDONLY`, - and :const:`os.O_TEXT`. The returned file descriptor may be used as a parameter - to :func:`os.fdopen` to create a file object. - - .. audit-event:: msvcrt.open_osfhandle handle,flags msvcrt.open_osfhandle - - -.. function:: get_osfhandle(fd) - - Return the file handle for the file descriptor *fd*. Raises :exc:`OSError` if - *fd* is not recognized. - - .. audit-event:: msvcrt.get_osfhandle fd msvcrt.get_osfhandle - - -.. _msvcrt-console: - -Console I/O ------------ - - -.. function:: kbhit() - - Return ``True`` if a keypress is waiting to be read. - - -.. function:: getch() - - Read a keypress and return the resulting character as a byte string. - Nothing is echoed to the console. This call will block if a keypress - is not already available, but will not wait for :kbd:`Enter` to be - pressed. If the pressed key was a special function key, this will - return ``'\000'`` or ``'\xe0'``; the next call will return the keycode. - The :kbd:`Control-C` keypress cannot be read with this function. - - -.. function:: getwch() - - Wide char variant of :func:`getch`, returning a Unicode value. - - -.. function:: getche() - - Similar to :func:`getch`, but the keypress will be echoed if it represents a - printable character. - - -.. function:: getwche() - - Wide char variant of :func:`getche`, returning a Unicode value. - - -.. function:: putch(char) - - Print the byte string *char* to the console without buffering. - - -.. function:: putwch(unicode_char) - - Wide char variant of :func:`putch`, accepting a Unicode value. - - -.. function:: ungetch(char) - - Cause the byte string *char* to be "pushed back" into the console buffer; - it will be the next character read by :func:`getch` or :func:`getche`. - - -.. function:: ungetwch(unicode_char) - - Wide char variant of :func:`ungetch`, accepting a Unicode value. - - -.. _msvcrt-other: - -Other Functions ---------------- - - -.. function:: heapmin() - - Force the :c:func:`malloc` heap to clean itself up and return unused blocks to - the operating system. On failure, this raises :exc:`OSError`. diff --git a/Doc/library/multiprocessing.rst b/Doc/library/multiprocessing.rst deleted file mode 100644 index 99dda5b5768fbd..00000000000000 --- a/Doc/library/multiprocessing.rst +++ /dev/null @@ -1,3025 +0,0 @@ -:mod:`multiprocessing` --- Process-based parallelism -==================================================== - -.. module:: multiprocessing - :synopsis: Process-based parallelism. - -**Source code:** :source:`Lib/multiprocessing/` - --------------- - -.. include:: ../includes/wasm-notavail.rst - -Introduction ------------- - -:mod:`multiprocessing` is a package that supports spawning processes using an -API similar to the :mod:`threading` module. The :mod:`multiprocessing` package -offers both local and remote concurrency, effectively side-stepping the -:term:`Global Interpreter Lock ` by using -subprocesses instead of threads. Due -to this, the :mod:`multiprocessing` module allows the programmer to fully -leverage multiple processors on a given machine. It runs on both Unix and -Windows. - -The :mod:`multiprocessing` module also introduces APIs which do not have -analogs in the :mod:`threading` module. A prime example of this is the -:class:`~multiprocessing.pool.Pool` object which offers a convenient means of -parallelizing the execution of a function across multiple input values, -distributing the input data across processes (data parallelism). The following -example demonstrates the common practice of defining such functions in a module -so that child processes can successfully import that module. This basic example -of data parallelism using :class:`~multiprocessing.pool.Pool`, :: - - from multiprocessing import Pool - - def f(x): - return x*x - - if __name__ == '__main__': - with Pool(5) as p: - print(p.map(f, [1, 2, 3])) - -will print to standard output :: - - [1, 4, 9] - - -.. seealso:: - - :class:`concurrent.futures.ProcessPoolExecutor` offers a higher level interface - to push tasks to a background process without blocking execution of the - calling process. Compared to using the :class:`~multiprocessing.pool.Pool` - interface directly, the :mod:`concurrent.futures` API more readily allows - the submission of work to the underlying process pool to be separated from - waiting for the results. - - -The :class:`Process` class -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In :mod:`multiprocessing`, processes are spawned by creating a :class:`Process` -object and then calling its :meth:`~Process.start` method. :class:`Process` -follows the API of :class:`threading.Thread`. A trivial example of a -multiprocess program is :: - - from multiprocessing import Process - - def f(name): - print('hello', name) - - if __name__ == '__main__': - p = Process(target=f, args=('bob',)) - p.start() - p.join() - -To show the individual process IDs involved, here is an expanded example:: - - from multiprocessing import Process - import os - - def info(title): - print(title) - print('module name:', __name__) - print('parent process:', os.getppid()) - print('process id:', os.getpid()) - - def f(name): - info('function f') - print('hello', name) - - if __name__ == '__main__': - info('main line') - p = Process(target=f, args=('bob',)) - p.start() - p.join() - -For an explanation of why the ``if __name__ == '__main__'`` part is -necessary, see :ref:`multiprocessing-programming`. - - - -Contexts and start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. _multiprocessing-start-methods: - -Depending on the platform, :mod:`multiprocessing` supports three ways -to start a process. These *start methods* are - - *spawn* - The parent process starts a fresh Python interpreter process. The - child process will only inherit those resources necessary to run - the process object's :meth:`~Process.run` method. In particular, - unnecessary file descriptors and handles from the parent process - will not be inherited. Starting a process using this method is - rather slow compared to using *fork* or *forkserver*. - - Available on Unix and Windows. The default on Windows and macOS. - - *fork* - The parent process uses :func:`os.fork` to fork the Python - interpreter. The child process, when it begins, is effectively - identical to the parent process. All resources of the parent are - inherited by the child process. Note that safely forking a - multithreaded process is problematic. - - Available on Unix only. The default on Unix. - - *forkserver* - When the program starts and selects the *forkserver* start method, - a server process is started. From then on, whenever a new process - is needed, the parent process connects to the server and requests - that it fork a new process. The fork server process is single - threaded so it is safe for it to use :func:`os.fork`. No - unnecessary resources are inherited. - - Available on Unix platforms which support passing file descriptors - over Unix pipes. - -.. versionchanged:: 3.8 - - On macOS, the *spawn* start method is now the default. The *fork* start - method should be considered unsafe as it can lead to crashes of the - subprocess. See :issue:`33725`. - -.. versionchanged:: 3.4 - *spawn* added on all Unix platforms, and *forkserver* added for - some Unix platforms. - Child processes no longer inherit all of the parents inheritable - handles on Windows. - -On Unix using the *spawn* or *forkserver* start methods will also -start a *resource tracker* process which tracks the unlinked named -system resources (such as named semaphores or -:class:`~multiprocessing.shared_memory.SharedMemory` objects) created -by processes of the program. When all processes -have exited the resource tracker unlinks any remaining tracked object. -Usually there should be none, but if a process was killed by a signal -there may be some "leaked" resources. (Neither leaked semaphores nor shared -memory segments will be automatically unlinked until the next reboot. This is -problematic for both objects because the system allows only a limited number of -named semaphores, and shared memory segments occupy some space in the main -memory.) - -To select a start method you use the :func:`set_start_method` in -the ``if __name__ == '__main__'`` clause of the main module. For -example:: - - import multiprocessing as mp - - def foo(q): - q.put('hello') - - if __name__ == '__main__': - mp.set_start_method('spawn') - q = mp.Queue() - p = mp.Process(target=foo, args=(q,)) - p.start() - print(q.get()) - p.join() - -:func:`set_start_method` should not be used more than once in the -program. - -Alternatively, you can use :func:`get_context` to obtain a context -object. Context objects have the same API as the multiprocessing -module, and allow one to use multiple start methods in the same -program. :: - - import multiprocessing as mp - - def foo(q): - q.put('hello') - - if __name__ == '__main__': - ctx = mp.get_context('spawn') - q = ctx.Queue() - p = ctx.Process(target=foo, args=(q,)) - p.start() - print(q.get()) - p.join() - -Note that objects related to one context may not be compatible with -processes for a different context. In particular, locks created using -the *fork* context cannot be passed to processes started using the -*spawn* or *forkserver* start methods. - -A library which wants to use a particular start method should probably -use :func:`get_context` to avoid interfering with the choice of the -library user. - -.. warning:: - - The ``'spawn'`` and ``'forkserver'`` start methods cannot currently - be used with "frozen" executables (i.e., binaries produced by - packages like **PyInstaller** and **cx_Freeze**) on Unix. - The ``'fork'`` start method does work. - - -Exchanging objects between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:mod:`multiprocessing` supports two types of communication channel between -processes: - -**Queues** - - The :class:`Queue` class is a near clone of :class:`queue.Queue`. For - example:: - - from multiprocessing import Process, Queue - - def f(q): - q.put([42, None, 'hello']) - - if __name__ == '__main__': - q = Queue() - p = Process(target=f, args=(q,)) - p.start() - print(q.get()) # prints "[42, None, 'hello']" - p.join() - - Queues are thread and process safe. - -**Pipes** - - The :func:`Pipe` function returns a pair of connection objects connected by a - pipe which by default is duplex (two-way). For example:: - - from multiprocessing import Process, Pipe - - def f(conn): - conn.send([42, None, 'hello']) - conn.close() - - if __name__ == '__main__': - parent_conn, child_conn = Pipe() - p = Process(target=f, args=(child_conn,)) - p.start() - print(parent_conn.recv()) # prints "[42, None, 'hello']" - p.join() - - The two connection objects returned by :func:`Pipe` represent the two ends of - the pipe. Each connection object has :meth:`~Connection.send` and - :meth:`~Connection.recv` methods (among others). Note that data in a pipe - may become corrupted if two processes (or threads) try to read from or write - to the *same* end of the pipe at the same time. Of course there is no risk - of corruption from processes using different ends of the pipe at the same - time. - - -Synchronization between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:mod:`multiprocessing` contains equivalents of all the synchronization -primitives from :mod:`threading`. For instance one can use a lock to ensure -that only one process prints to standard output at a time:: - - from multiprocessing import Process, Lock - - def f(l, i): - l.acquire() - try: - print('hello world', i) - finally: - l.release() - - if __name__ == '__main__': - lock = Lock() - - for num in range(10): - Process(target=f, args=(lock, num)).start() - -Without using the lock output from the different processes is liable to get all -mixed up. - - -Sharing state between processes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -As mentioned above, when doing concurrent programming it is usually best to -avoid using shared state as far as possible. This is particularly true when -using multiple processes. - -However, if you really do need to use some shared data then -:mod:`multiprocessing` provides a couple of ways of doing so. - -**Shared memory** - - Data can be stored in a shared memory map using :class:`Value` or - :class:`Array`. For example, the following code :: - - from multiprocessing import Process, Value, Array - - def f(n, a): - n.value = 3.1415927 - for i in range(len(a)): - a[i] = -a[i] - - if __name__ == '__main__': - num = Value('d', 0.0) - arr = Array('i', range(10)) - - p = Process(target=f, args=(num, arr)) - p.start() - p.join() - - print(num.value) - print(arr[:]) - - will print :: - - 3.1415927 - [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] - - The ``'d'`` and ``'i'`` arguments used when creating ``num`` and ``arr`` are - typecodes of the kind used by the :mod:`array` module: ``'d'`` indicates a - double precision float and ``'i'`` indicates a signed integer. These shared - objects will be process and thread-safe. - - For more flexibility in using shared memory one can use the - :mod:`multiprocessing.sharedctypes` module which supports the creation of - arbitrary ctypes objects allocated from shared memory. - -**Server process** - - A manager object returned by :func:`Manager` controls a server process which - holds Python objects and allows other processes to manipulate them using - proxies. - - A manager returned by :func:`Manager` will support types - :class:`list`, :class:`dict`, :class:`~managers.Namespace`, :class:`Lock`, - :class:`RLock`, :class:`Semaphore`, :class:`BoundedSemaphore`, - :class:`Condition`, :class:`Event`, :class:`Barrier`, - :class:`Queue`, :class:`Value` and :class:`Array`. For example, :: - - from multiprocessing import Process, Manager - - def f(d, l): - d[1] = '1' - d['2'] = 2 - d[0.25] = None - l.reverse() - - if __name__ == '__main__': - with Manager() as manager: - d = manager.dict() - l = manager.list(range(10)) - - p = Process(target=f, args=(d, l)) - p.start() - p.join() - - print(d) - print(l) - - will print :: - - {0.25: None, 1: '1', '2': 2} - [9, 8, 7, 6, 5, 4, 3, 2, 1, 0] - - Server process managers are more flexible than using shared memory objects - because they can be made to support arbitrary object types. Also, a single - manager can be shared by processes on different computers over a network. - They are, however, slower than using shared memory. - - -Using a pool of workers -~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`~multiprocessing.pool.Pool` class represents a pool of worker -processes. It has methods which allows tasks to be offloaded to the worker -processes in a few different ways. - -For example:: - - from multiprocessing import Pool, TimeoutError - import time - import os - - def f(x): - return x*x - - if __name__ == '__main__': - # start 4 worker processes - with Pool(processes=4) as pool: - - # print "[0, 1, 4,..., 81]" - print(pool.map(f, range(10))) - - # print same numbers in arbitrary order - for i in pool.imap_unordered(f, range(10)): - print(i) - - # evaluate "f(20)" asynchronously - res = pool.apply_async(f, (20,)) # runs in *only* one process - print(res.get(timeout=1)) # prints "400" - - # evaluate "os.getpid()" asynchronously - res = pool.apply_async(os.getpid, ()) # runs in *only* one process - print(res.get(timeout=1)) # prints the PID of that process - - # launching multiple evaluations asynchronously *may* use more processes - multiple_results = [pool.apply_async(os.getpid, ()) for i in range(4)] - print([res.get(timeout=1) for res in multiple_results]) - - # make a single worker sleep for 10 seconds - res = pool.apply_async(time.sleep, (10,)) - try: - print(res.get(timeout=1)) - except TimeoutError: - print("We lacked patience and got a multiprocessing.TimeoutError") - - print("For the moment, the pool remains available for more work") - - # exiting the 'with'-block has stopped the pool - print("Now the pool is closed and no longer available") - -Note that the methods of a pool should only ever be used by the -process which created it. - -.. note:: - - Functionality within this package requires that the ``__main__`` module be - importable by the children. This is covered in :ref:`multiprocessing-programming` - however it is worth pointing out here. This means that some examples, such - as the :class:`multiprocessing.pool.Pool` examples will not work in the - interactive interpreter. For example:: - - >>> from multiprocessing import Pool - >>> p = Pool(5) - >>> def f(x): - ... return x*x - ... - >>> with p: - ... p.map(f, [1,2,3]) - Process PoolWorker-1: - Process PoolWorker-2: - Process PoolWorker-3: - Traceback (most recent call last): - Traceback (most recent call last): - Traceback (most recent call last): - AttributeError: Can't get attribute 'f' on - AttributeError: Can't get attribute 'f' on - AttributeError: Can't get attribute 'f' on - - (If you try this it will actually output three full tracebacks - interleaved in a semi-random fashion, and then you may have to - stop the parent process somehow.) - - -Reference ---------- - -The :mod:`multiprocessing` package mostly replicates the API of the -:mod:`threading` module. - - -:class:`Process` and exceptions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. class:: Process(group=None, target=None, name=None, args=(), kwargs={}, \ - *, daemon=None) - - Process objects represent activity that is run in a separate process. The - :class:`Process` class has equivalents of all the methods of - :class:`threading.Thread`. - - The constructor should always be called with keyword arguments. *group* - should always be ``None``; it exists solely for compatibility with - :class:`threading.Thread`. *target* is the callable object to be invoked by - the :meth:`run()` method. It defaults to ``None``, meaning nothing is - called. *name* is the process name (see :attr:`name` for more details). - *args* is the argument tuple for the target invocation. *kwargs* is a - dictionary of keyword arguments for the target invocation. If provided, - the keyword-only *daemon* argument sets the process :attr:`daemon` flag - to ``True`` or ``False``. If ``None`` (the default), this flag will be - inherited from the creating process. - - By default, no arguments are passed to *target*. The *args* argument, - which defaults to ``()``, can be used to specify a list or tuple of the arguments - to pass to *target*. - - If a subclass overrides the constructor, it must make sure it invokes the - base class constructor (:meth:`Process.__init__`) before doing anything else - to the process. - - .. versionchanged:: 3.3 - Added the *daemon* argument. - - .. method:: run() - - Method representing the process's activity. - - You may override this method in a subclass. The standard :meth:`run` - method invokes the callable object passed to the object's constructor as - the target argument, if any, with sequential and keyword arguments taken - from the *args* and *kwargs* arguments, respectively. - - Using a list or tuple as the *args* argument passed to :class:`Process` - achieves the same effect. - - Example:: - - >>> from multiprocessing import Process - >>> p = Process(target=print, args=[1]) - >>> p.run() - 1 - >>> p = Process(target=print, args=(1,)) - >>> p.run() - 1 - - .. method:: start() - - Start the process's activity. - - This must be called at most once per process object. It arranges for the - object's :meth:`run` method to be invoked in a separate process. - - .. method:: join([timeout]) - - If the optional argument *timeout* is ``None`` (the default), the method - blocks until the process whose :meth:`join` method is called terminates. - If *timeout* is a positive number, it blocks at most *timeout* seconds. - Note that the method returns ``None`` if its process terminates or if the - method times out. Check the process's :attr:`exitcode` to determine if - it terminated. - - A process can be joined many times. - - A process cannot join itself because this would cause a deadlock. It is - an error to attempt to join a process before it has been started. - - .. attribute:: name - - The process's name. The name is a string used for identification purposes - only. It has no semantics. Multiple processes may be given the same - name. - - The initial name is set by the constructor. If no explicit name is - provided to the constructor, a name of the form - 'Process-N\ :sub:`1`:N\ :sub:`2`:...:N\ :sub:`k`' is constructed, where - each N\ :sub:`k` is the N-th child of its parent. - - .. method:: is_alive - - Return whether the process is alive. - - Roughly, a process object is alive from the moment the :meth:`start` - method returns until the child process terminates. - - .. attribute:: daemon - - The process's daemon flag, a Boolean value. This must be set before - :meth:`start` is called. - - The initial value is inherited from the creating process. - - When a process exits, it attempts to terminate all of its daemonic child - processes. - - Note that a daemonic process is not allowed to create child processes. - Otherwise a daemonic process would leave its children orphaned if it gets - terminated when its parent process exits. Additionally, these are **not** - Unix daemons or services, they are normal processes that will be - terminated (and not joined) if non-daemonic processes have exited. - - In addition to the :class:`threading.Thread` API, :class:`Process` objects - also support the following attributes and methods: - - .. attribute:: pid - - Return the process ID. Before the process is spawned, this will be - ``None``. - - .. attribute:: exitcode - - The child's exit code. This will be ``None`` if the process has not yet - terminated. - - If the child's :meth:`run` method returned normally, the exit code - will be 0. If it terminated via :func:`sys.exit` with an integer - argument *N*, the exit code will be *N*. - - If the child terminated due to an exception not caught within - :meth:`run`, the exit code will be 1. If it was terminated by - signal *N*, the exit code will be the negative value *-N*. - - .. attribute:: authkey - - The process's authentication key (a byte string). - - When :mod:`multiprocessing` is initialized the main process is assigned a - random string using :func:`os.urandom`. - - When a :class:`Process` object is created, it will inherit the - authentication key of its parent process, although this may be changed by - setting :attr:`authkey` to another byte string. - - See :ref:`multiprocessing-auth-keys`. - - .. attribute:: sentinel - - A numeric handle of a system object which will become "ready" when - the process ends. - - You can use this value if you want to wait on several events at - once using :func:`multiprocessing.connection.wait`. Otherwise - calling :meth:`join()` is simpler. - - On Windows, this is an OS handle usable with the ``WaitForSingleObject`` - and ``WaitForMultipleObjects`` family of API calls. On Unix, this is - a file descriptor usable with primitives from the :mod:`select` module. - - .. versionadded:: 3.3 - - .. method:: terminate() - - Terminate the process. On Unix this is done using the ``SIGTERM`` signal; - on Windows :c:func:`TerminateProcess` is used. Note that exit handlers and - finally clauses, etc., will not be executed. - - Note that descendant processes of the process will *not* be terminated -- - they will simply become orphaned. - - .. warning:: - - If this method is used when the associated process is using a pipe or - queue then the pipe or queue is liable to become corrupted and may - become unusable by other process. Similarly, if the process has - acquired a lock or semaphore etc. then terminating it is liable to - cause other processes to deadlock. - - .. method:: kill() - - Same as :meth:`terminate()` but using the ``SIGKILL`` signal on Unix. - - .. versionadded:: 3.7 - - .. method:: close() - - Close the :class:`Process` object, releasing all resources associated - with it. :exc:`ValueError` is raised if the underlying process - is still running. Once :meth:`close` returns successfully, most - other methods and attributes of the :class:`Process` object will - raise :exc:`ValueError`. - - .. versionadded:: 3.7 - - Note that the :meth:`start`, :meth:`join`, :meth:`is_alive`, - :meth:`terminate` and :attr:`exitcode` methods should only be called by - the process that created the process object. - - Example usage of some of the methods of :class:`Process`: - - .. doctest:: - - >>> import multiprocessing, time, signal - >>> p = multiprocessing.Process(target=time.sleep, args=(1000,)) - >>> print(p, p.is_alive()) - False - >>> p.start() - >>> print(p, p.is_alive()) - True - >>> p.terminate() - >>> time.sleep(0.1) - >>> print(p, p.is_alive()) - False - >>> p.exitcode == -signal.SIGTERM - True - -.. exception:: ProcessError - - The base class of all :mod:`multiprocessing` exceptions. - -.. exception:: BufferTooShort - - Exception raised by :meth:`Connection.recv_bytes_into()` when the supplied - buffer object is too small for the message read. - - If ``e`` is an instance of :exc:`BufferTooShort` then ``e.args[0]`` will give - the message as a byte string. - -.. exception:: AuthenticationError - - Raised when there is an authentication error. - -.. exception:: TimeoutError - - Raised by methods with a timeout when the timeout expires. - -Pipes and Queues -~~~~~~~~~~~~~~~~ - -When using multiple processes, one generally uses message passing for -communication between processes and avoids having to use any synchronization -primitives like locks. - -For passing messages one can use :func:`Pipe` (for a connection between two -processes) or a queue (which allows multiple producers and consumers). - -The :class:`Queue`, :class:`SimpleQueue` and :class:`JoinableQueue` types -are multi-producer, multi-consumer :abbr:`FIFO (first-in, first-out)` -queues modelled on the :class:`queue.Queue` class in the -standard library. They differ in that :class:`Queue` lacks the -:meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join` methods introduced -into Python 2.5's :class:`queue.Queue` class. - -If you use :class:`JoinableQueue` then you **must** call -:meth:`JoinableQueue.task_done` for each task removed from the queue or else the -semaphore used to count the number of unfinished tasks may eventually overflow, -raising an exception. - -Note that one can also create a shared queue by using a manager object -- see -:ref:`multiprocessing-managers`. - -.. note:: - - :mod:`multiprocessing` uses the usual :exc:`queue.Empty` and - :exc:`queue.Full` exceptions to signal a timeout. They are not available in - the :mod:`multiprocessing` namespace so you need to import them from - :mod:`queue`. - -.. note:: - - When an object is put on a queue, the object is pickled and a - background thread later flushes the pickled data to an underlying - pipe. This has some consequences which are a little surprising, - but should not cause any practical difficulties -- if they really - bother you then you can instead use a queue created with a - :ref:`manager `. - - (1) After putting an object on an empty queue there may be an - infinitesimal delay before the queue's :meth:`~Queue.empty` - method returns :const:`False` and :meth:`~Queue.get_nowait` can - return without raising :exc:`queue.Empty`. - - (2) If multiple processes are enqueuing objects, it is possible for - the objects to be received at the other end out-of-order. - However, objects enqueued by the same process will always be in - the expected order with respect to each other. - -.. warning:: - - If a process is killed using :meth:`Process.terminate` or :func:`os.kill` - while it is trying to use a :class:`Queue`, then the data in the queue is - likely to become corrupted. This may cause any other process to get an - exception when it tries to use the queue later on. - -.. warning:: - - As mentioned above, if a child process has put items on a queue (and it has - not used :meth:`JoinableQueue.cancel_join_thread - `), then that process will - not terminate until all buffered items have been flushed to the pipe. - - This means that if you try joining that process you may get a deadlock unless - you are sure that all items which have been put on the queue have been - consumed. Similarly, if the child process is non-daemonic then the parent - process may hang on exit when it tries to join all its non-daemonic children. - - Note that a queue created using a manager does not have this issue. See - :ref:`multiprocessing-programming`. - -For an example of the usage of queues for interprocess communication see -:ref:`multiprocessing-examples`. - - -.. function:: Pipe([duplex]) - - Returns a pair ``(conn1, conn2)`` of - :class:`~multiprocessing.connection.Connection` objects representing the - ends of a pipe. - - If *duplex* is ``True`` (the default) then the pipe is bidirectional. If - *duplex* is ``False`` then the pipe is unidirectional: ``conn1`` can only be - used for receiving messages and ``conn2`` can only be used for sending - messages. - - -.. class:: Queue([maxsize]) - - Returns a process shared queue implemented using a pipe and a few - locks/semaphores. When a process first puts an item on the queue a feeder - thread is started which transfers objects from a buffer into the pipe. - - The usual :exc:`queue.Empty` and :exc:`queue.Full` exceptions from the - standard library's :mod:`queue` module are raised to signal timeouts. - - :class:`Queue` implements all the methods of :class:`queue.Queue` except for - :meth:`~queue.Queue.task_done` and :meth:`~queue.Queue.join`. - - .. method:: qsize() - - Return the approximate size of the queue. Because of - multithreading/multiprocessing semantics, this number is not reliable. - - Note that this may raise :exc:`NotImplementedError` on Unix platforms like - macOS where ``sem_getvalue()`` is not implemented. - - .. method:: empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. Because of - multithreading/multiprocessing semantics, this is not reliable. - - .. method:: full() - - Return ``True`` if the queue is full, ``False`` otherwise. Because of - multithreading/multiprocessing semantics, this is not reliable. - - .. method:: put(obj[, block[, timeout]]) - - Put obj into the queue. If the optional argument *block* is ``True`` - (the default) and *timeout* is ``None`` (the default), block if necessary until - a free slot is available. If *timeout* is a positive number, it blocks at - most *timeout* seconds and raises the :exc:`queue.Full` exception if no - free slot was available within that time. Otherwise (*block* is - ``False``), put an item on the queue if a free slot is immediately - available, else raise the :exc:`queue.Full` exception (*timeout* is - ignored in that case). - - .. versionchanged:: 3.8 - If the queue is closed, :exc:`ValueError` is raised instead of - :exc:`AssertionError`. - - .. method:: put_nowait(obj) - - Equivalent to ``put(obj, False)``. - - .. method:: get([block[, timeout]]) - - Remove and return an item from the queue. If optional args *block* is - ``True`` (the default) and *timeout* is ``None`` (the default), block if - necessary until an item is available. If *timeout* is a positive number, - it blocks at most *timeout* seconds and raises the :exc:`queue.Empty` - exception if no item was available within that time. Otherwise (block is - ``False``), return an item if one is immediately available, else raise the - :exc:`queue.Empty` exception (*timeout* is ignored in that case). - - .. versionchanged:: 3.8 - If the queue is closed, :exc:`ValueError` is raised instead of - :exc:`OSError`. - - .. method:: get_nowait() - - Equivalent to ``get(False)``. - - :class:`multiprocessing.Queue` has a few additional methods not found in - :class:`queue.Queue`. These methods are usually unnecessary for most - code: - - .. method:: close() - - Indicate that no more data will be put on this queue by the current - process. The background thread will quit once it has flushed all buffered - data to the pipe. This is called automatically when the queue is garbage - collected. - - .. method:: join_thread() - - Join the background thread. This can only be used after :meth:`close` has - been called. It blocks until the background thread exits, ensuring that - all data in the buffer has been flushed to the pipe. - - By default if a process is not the creator of the queue then on exit it - will attempt to join the queue's background thread. The process can call - :meth:`cancel_join_thread` to make :meth:`join_thread` do nothing. - - .. method:: cancel_join_thread() - - Prevent :meth:`join_thread` from blocking. In particular, this prevents - the background thread from being joined automatically when the process - exits -- see :meth:`join_thread`. - - A better name for this method might be - ``allow_exit_without_flush()``. It is likely to cause enqueued - data to be lost, and you almost certainly will not need to use it. - It is really only there if you need the current process to exit - immediately without waiting to flush enqueued data to the - underlying pipe, and you don't care about lost data. - - .. note:: - - This class's functionality requires a functioning shared semaphore - implementation on the host operating system. Without one, the - functionality in this class will be disabled, and attempts to - instantiate a :class:`Queue` will result in an :exc:`ImportError`. See - :issue:`3770` for additional information. The same holds true for any - of the specialized queue types listed below. - -.. class:: SimpleQueue() - - It is a simplified :class:`Queue` type, very close to a locked :class:`Pipe`. - - .. method:: close() - - Close the queue: release internal resources. - - A queue must not be used anymore after it is closed. For example, - :meth:`get`, :meth:`put` and :meth:`empty` methods must no longer be - called. - - .. versionadded:: 3.9 - - .. method:: empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. - - .. method:: get() - - Remove and return an item from the queue. - - .. method:: put(item) - - Put *item* into the queue. - - -.. class:: JoinableQueue([maxsize]) - - :class:`JoinableQueue`, a :class:`Queue` subclass, is a queue which - additionally has :meth:`task_done` and :meth:`join` methods. - - .. method:: task_done() - - Indicate that a formerly enqueued task is complete. Used by queue - consumers. For each :meth:`~Queue.get` used to fetch a task, a subsequent - call to :meth:`task_done` tells the queue that the processing on the task - is complete. - - If a :meth:`~queue.Queue.join` is currently blocking, it will resume when all - items have been processed (meaning that a :meth:`task_done` call was - received for every item that had been :meth:`~Queue.put` into the queue). - - Raises a :exc:`ValueError` if called more times than there were items - placed in the queue. - - - .. method:: join() - - Block until all items in the queue have been gotten and processed. - - The count of unfinished tasks goes up whenever an item is added to the - queue. The count goes down whenever a consumer calls - :meth:`task_done` to indicate that the item was retrieved and all work on - it is complete. When the count of unfinished tasks drops to zero, - :meth:`~queue.Queue.join` unblocks. - - -Miscellaneous -~~~~~~~~~~~~~ - -.. function:: active_children() - - Return list of all live children of the current process. - - Calling this has the side effect of "joining" any processes which have - already finished. - -.. function:: cpu_count() - - Return the number of CPUs in the system. - - This number is not equivalent to the number of CPUs the current process can - use. The number of usable CPUs can be obtained with - ``len(os.sched_getaffinity(0))`` - - When the number of CPUs cannot be determined a :exc:`NotImplementedError` - is raised. - - .. seealso:: - :func:`os.cpu_count` - -.. function:: current_process() - - Return the :class:`Process` object corresponding to the current process. - - An analogue of :func:`threading.current_thread`. - -.. function:: parent_process() - - Return the :class:`Process` object corresponding to the parent process of - the :func:`current_process`. For the main process, ``parent_process`` will - be ``None``. - - .. versionadded:: 3.8 - -.. function:: freeze_support() - - Add support for when a program which uses :mod:`multiprocessing` has been - frozen to produce a Windows executable. (Has been tested with **py2exe**, - **PyInstaller** and **cx_Freeze**.) - - One needs to call this function straight after the ``if __name__ == - '__main__'`` line of the main module. For example:: - - from multiprocessing import Process, freeze_support - - def f(): - print('hello world!') - - if __name__ == '__main__': - freeze_support() - Process(target=f).start() - - If the ``freeze_support()`` line is omitted then trying to run the frozen - executable will raise :exc:`RuntimeError`. - - Calling ``freeze_support()`` has no effect when invoked on any operating - system other than Windows. In addition, if the module is being run - normally by the Python interpreter on Windows (the program has not been - frozen), then ``freeze_support()`` has no effect. - -.. function:: get_all_start_methods() - - Returns a list of the supported start methods, the first of which - is the default. The possible start methods are ``'fork'``, - ``'spawn'`` and ``'forkserver'``. On Windows only ``'spawn'`` is - available. On Unix ``'fork'`` and ``'spawn'`` are always - supported, with ``'fork'`` being the default. - - .. versionadded:: 3.4 - -.. function:: get_context(method=None) - - Return a context object which has the same attributes as the - :mod:`multiprocessing` module. - - If *method* is ``None`` then the default context is returned. - Otherwise *method* should be ``'fork'``, ``'spawn'``, - ``'forkserver'``. :exc:`ValueError` is raised if the specified - start method is not available. - - .. versionadded:: 3.4 - -.. function:: get_start_method(allow_none=False) - - Return the name of start method used for starting processes. - - If the start method has not been fixed and *allow_none* is false, - then the start method is fixed to the default and the name is - returned. If the start method has not been fixed and *allow_none* - is true then ``None`` is returned. - - The return value can be ``'fork'``, ``'spawn'``, ``'forkserver'`` - or ``None``. ``'fork'`` is the default on Unix, while ``'spawn'`` is - the default on Windows and macOS. - -.. versionchanged:: 3.8 - - On macOS, the *spawn* start method is now the default. The *fork* start - method should be considered unsafe as it can lead to crashes of the - subprocess. See :issue:`33725`. - - .. versionadded:: 3.4 - -.. function:: set_executable(executable) - - Set the path of the Python interpreter to use when starting a child process. - (By default :data:`sys.executable` is used). Embedders will probably need to - do some thing like :: - - set_executable(os.path.join(sys.exec_prefix, 'pythonw.exe')) - - before they can create child processes. - - .. versionchanged:: 3.4 - Now supported on Unix when the ``'spawn'`` start method is used. - - .. versionchanged:: 3.11 - Accepts a :term:`path-like object`. - -.. function:: set_start_method(method, force=False) - - Set the method which should be used to start child processes. - The *method* argument can be ``'fork'``, ``'spawn'`` or ``'forkserver'``. - Raises :exc:`RuntimeError` if the start method has already been set and *force* - is not ``True``. If *method* is ``None`` and *force* is ``True`` then the start - method is set to ``None``. If *method* is ``None`` and *force* is ``False`` - then the context is set to the default context. - - Note that this should be called at most once, and it should be - protected inside the ``if __name__ == '__main__'`` clause of the - main module. - - .. versionadded:: 3.4 - -.. note:: - - :mod:`multiprocessing` contains no analogues of - :func:`threading.active_count`, :func:`threading.enumerate`, - :func:`threading.settrace`, :func:`threading.setprofile`, - :class:`threading.Timer`, or :class:`threading.local`. - - -Connection Objects -~~~~~~~~~~~~~~~~~~ - -.. currentmodule:: multiprocessing.connection - -Connection objects allow the sending and receiving of picklable objects or -strings. They can be thought of as message oriented connected sockets. - -Connection objects are usually created using -:func:`Pipe ` -- see also -:ref:`multiprocessing-listeners-clients`. - -.. class:: Connection - - .. method:: send(obj) - - Send an object to the other end of the connection which should be read - using :meth:`recv`. - - The object must be picklable. Very large pickles (approximately 32 MiB+, - though it depends on the OS) may raise a :exc:`ValueError` exception. - - .. method:: recv() - - Return an object sent from the other end of the connection using - :meth:`send`. Blocks until there is something to receive. Raises - :exc:`EOFError` if there is nothing left to receive - and the other end was closed. - - .. method:: fileno() - - Return the file descriptor or handle used by the connection. - - .. method:: close() - - Close the connection. - - This is called automatically when the connection is garbage collected. - - .. method:: poll([timeout]) - - Return whether there is any data available to be read. - - If *timeout* is not specified then it will return immediately. If - *timeout* is a number then this specifies the maximum time in seconds to - block. If *timeout* is ``None`` then an infinite timeout is used. - - Note that multiple connection objects may be polled at once by - using :func:`multiprocessing.connection.wait`. - - .. method:: send_bytes(buffer[, offset[, size]]) - - Send byte data from a :term:`bytes-like object` as a complete message. - - If *offset* is given then data is read from that position in *buffer*. If - *size* is given then that many bytes will be read from buffer. Very large - buffers (approximately 32 MiB+, though it depends on the OS) may raise a - :exc:`ValueError` exception - - .. method:: recv_bytes([maxlength]) - - Return a complete message of byte data sent from the other end of the - connection as a string. Blocks until there is something to receive. - Raises :exc:`EOFError` if there is nothing left - to receive and the other end has closed. - - If *maxlength* is specified and the message is longer than *maxlength* - then :exc:`OSError` is raised and the connection will no longer be - readable. - - .. versionchanged:: 3.3 - This function used to raise :exc:`IOError`, which is now an - alias of :exc:`OSError`. - - - .. method:: recv_bytes_into(buffer[, offset]) - - Read into *buffer* a complete message of byte data sent from the other end - of the connection and return the number of bytes in the message. Blocks - until there is something to receive. Raises - :exc:`EOFError` if there is nothing left to receive and the other end was - closed. - - *buffer* must be a writable :term:`bytes-like object`. If - *offset* is given then the message will be written into the buffer from - that position. Offset must be a non-negative integer less than the - length of *buffer* (in bytes). - - If the buffer is too short then a :exc:`BufferTooShort` exception is - raised and the complete message is available as ``e.args[0]`` where ``e`` - is the exception instance. - - .. versionchanged:: 3.3 - Connection objects themselves can now be transferred between processes - using :meth:`Connection.send` and :meth:`Connection.recv`. - - .. versionadded:: 3.3 - Connection objects now support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the - connection object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. - -For example: - -.. doctest:: - - >>> from multiprocessing import Pipe - >>> a, b = Pipe() - >>> a.send([1, 'hello', None]) - >>> b.recv() - [1, 'hello', None] - >>> b.send_bytes(b'thank you') - >>> a.recv_bytes() - b'thank you' - >>> import array - >>> arr1 = array.array('i', range(5)) - >>> arr2 = array.array('i', [0] * 10) - >>> a.send_bytes(arr1) - >>> count = b.recv_bytes_into(arr2) - >>> assert count == len(arr1) * arr1.itemsize - >>> arr2 - array('i', [0, 1, 2, 3, 4, 0, 0, 0, 0, 0]) - -.. _multiprocessing-recv-pickle-security: - -.. warning:: - - The :meth:`Connection.recv` method automatically unpickles the data it - receives, which can be a security risk unless you can trust the process - which sent the message. - - Therefore, unless the connection object was produced using :func:`Pipe` you - should only use the :meth:`~Connection.recv` and :meth:`~Connection.send` - methods after performing some sort of authentication. See - :ref:`multiprocessing-auth-keys`. - -.. warning:: - - If a process is killed while it is trying to read or write to a pipe then - the data in the pipe is likely to become corrupted, because it may become - impossible to be sure where the message boundaries lie. - - -Synchronization primitives -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. currentmodule:: multiprocessing - -Generally synchronization primitives are not as necessary in a multiprocess -program as they are in a multithreaded program. See the documentation for -:mod:`threading` module. - -Note that one can also create synchronization primitives by using a manager -object -- see :ref:`multiprocessing-managers`. - -.. class:: Barrier(parties[, action[, timeout]]) - - A barrier object: a clone of :class:`threading.Barrier`. - - .. versionadded:: 3.3 - -.. class:: BoundedSemaphore([value]) - - A bounded semaphore object: a close analog of - :class:`threading.BoundedSemaphore`. - - A solitary difference from its close analog exists: its ``acquire`` method's - first argument is named *block*, as is consistent with :meth:`Lock.acquire`. - - .. note:: - On macOS, this is indistinguishable from :class:`Semaphore` because - ``sem_getvalue()`` is not implemented on that platform. - -.. class:: Condition([lock]) - - A condition variable: an alias for :class:`threading.Condition`. - - If *lock* is specified then it should be a :class:`Lock` or :class:`RLock` - object from :mod:`multiprocessing`. - - .. versionchanged:: 3.3 - The :meth:`~threading.Condition.wait_for` method was added. - -.. class:: Event() - - A clone of :class:`threading.Event`. - - -.. class:: Lock() - - A non-recursive lock object: a close analog of :class:`threading.Lock`. - Once a process or thread has acquired a lock, subsequent attempts to - acquire it from any process or thread will block until it is released; - any process or thread may release it. The concepts and behaviors of - :class:`threading.Lock` as it applies to threads are replicated here in - :class:`multiprocessing.Lock` as it applies to either processes or threads, - except as noted. - - Note that :class:`Lock` is actually a factory function which returns an - instance of ``multiprocessing.synchronize.Lock`` initialized with a - default context. - - :class:`Lock` supports the :term:`context manager` protocol and thus may be - used in :keyword:`with` statements. - - .. method:: acquire(block=True, timeout=None) - - Acquire a lock, blocking or non-blocking. - - With the *block* argument set to ``True`` (the default), the method call - will block until the lock is in an unlocked state, then set it to locked - and return ``True``. Note that the name of this first argument differs - from that in :meth:`threading.Lock.acquire`. - - With the *block* argument set to ``False``, the method call does not - block. If the lock is currently in a locked state, return ``False``; - otherwise set the lock to a locked state and return ``True``. - - When invoked with a positive, floating-point value for *timeout*, block - for at most the number of seconds specified by *timeout* as long as - the lock can not be acquired. Invocations with a negative value for - *timeout* are equivalent to a *timeout* of zero. Invocations with a - *timeout* value of ``None`` (the default) set the timeout period to - infinite. Note that the treatment of negative or ``None`` values for - *timeout* differs from the implemented behavior in - :meth:`threading.Lock.acquire`. The *timeout* argument has no practical - implications if the *block* argument is set to ``False`` and is thus - ignored. Returns ``True`` if the lock has been acquired or ``False`` if - the timeout period has elapsed. - - - .. method:: release() - - Release a lock. This can be called from any process or thread, not only - the process or thread which originally acquired the lock. - - Behavior is the same as in :meth:`threading.Lock.release` except that - when invoked on an unlocked lock, a :exc:`ValueError` is raised. - - -.. class:: RLock() - - A recursive lock object: a close analog of :class:`threading.RLock`. A - recursive lock must be released by the process or thread that acquired it. - Once a process or thread has acquired a recursive lock, the same process - or thread may acquire it again without blocking; that process or thread - must release it once for each time it has been acquired. - - Note that :class:`RLock` is actually a factory function which returns an - instance of ``multiprocessing.synchronize.RLock`` initialized with a - default context. - - :class:`RLock` supports the :term:`context manager` protocol and thus may be - used in :keyword:`with` statements. - - - .. method:: acquire(block=True, timeout=None) - - Acquire a lock, blocking or non-blocking. - - When invoked with the *block* argument set to ``True``, block until the - lock is in an unlocked state (not owned by any process or thread) unless - the lock is already owned by the current process or thread. The current - process or thread then takes ownership of the lock (if it does not - already have ownership) and the recursion level inside the lock increments - by one, resulting in a return value of ``True``. Note that there are - several differences in this first argument's behavior compared to the - implementation of :meth:`threading.RLock.acquire`, starting with the name - of the argument itself. - - When invoked with the *block* argument set to ``False``, do not block. - If the lock has already been acquired (and thus is owned) by another - process or thread, the current process or thread does not take ownership - and the recursion level within the lock is not changed, resulting in - a return value of ``False``. If the lock is in an unlocked state, the - current process or thread takes ownership and the recursion level is - incremented, resulting in a return value of ``True``. - - Use and behaviors of the *timeout* argument are the same as in - :meth:`Lock.acquire`. Note that some of these behaviors of *timeout* - differ from the implemented behaviors in :meth:`threading.RLock.acquire`. - - - .. method:: release() - - Release a lock, decrementing the recursion level. If after the - decrement the recursion level is zero, reset the lock to unlocked (not - owned by any process or thread) and if any other processes or threads - are blocked waiting for the lock to become unlocked, allow exactly one - of them to proceed. If after the decrement the recursion level is still - nonzero, the lock remains locked and owned by the calling process or - thread. - - Only call this method when the calling process or thread owns the lock. - An :exc:`AssertionError` is raised if this method is called by a process - or thread other than the owner or if the lock is in an unlocked (unowned) - state. Note that the type of exception raised in this situation - differs from the implemented behavior in :meth:`threading.RLock.release`. - - -.. class:: Semaphore([value]) - - A semaphore object: a close analog of :class:`threading.Semaphore`. - - A solitary difference from its close analog exists: its ``acquire`` method's - first argument is named *block*, as is consistent with :meth:`Lock.acquire`. - -.. note:: - - On macOS, ``sem_timedwait`` is unsupported, so calling ``acquire()`` with - a timeout will emulate that function's behavior using a sleeping loop. - -.. note:: - - If the SIGINT signal generated by :kbd:`Ctrl-C` arrives while the main thread is - blocked by a call to :meth:`BoundedSemaphore.acquire`, :meth:`Lock.acquire`, - :meth:`RLock.acquire`, :meth:`Semaphore.acquire`, :meth:`Condition.acquire` - or :meth:`Condition.wait` then the call will be immediately interrupted and - :exc:`KeyboardInterrupt` will be raised. - - This differs from the behaviour of :mod:`threading` where SIGINT will be - ignored while the equivalent blocking calls are in progress. - -.. note:: - - Some of this package's functionality requires a functioning shared semaphore - implementation on the host operating system. Without one, the - :mod:`multiprocessing.synchronize` module will be disabled, and attempts to - import it will result in an :exc:`ImportError`. See - :issue:`3770` for additional information. - - -Shared :mod:`ctypes` Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -It is possible to create shared objects using shared memory which can be -inherited by child processes. - -.. function:: Value(typecode_or_type, *args, lock=True) - - Return a :mod:`ctypes` object allocated from shared memory. By default the - return value is actually a synchronized wrapper for the object. The object - itself can be accessed via the *value* attribute of a :class:`Value`. - - *typecode_or_type* determines the type of the returned object: it is either a - ctypes type or a one character typecode of the kind used by the :mod:`array` - module. *\*args* is passed on to the constructor for the type. - - If *lock* is ``True`` (the default) then a new recursive lock - object is created to synchronize access to the value. If *lock* is - a :class:`Lock` or :class:`RLock` object then that will be used to - synchronize access to the value. If *lock* is ``False`` then - access to the returned object will not be automatically protected - by a lock, so it will not necessarily be "process-safe". - - Operations like ``+=`` which involve a read and write are not - atomic. So if, for instance, you want to atomically increment a - shared value it is insufficient to just do :: - - counter.value += 1 - - Assuming the associated lock is recursive (which it is by default) - you can instead do :: - - with counter.get_lock(): - counter.value += 1 - - Note that *lock* is a keyword-only argument. - -.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) - - Return a ctypes array allocated from shared memory. By default the return - value is actually a synchronized wrapper for the array. - - *typecode_or_type* determines the type of the elements of the returned array: - it is either a ctypes type or a one character typecode of the kind used by - the :mod:`array` module. If *size_or_initializer* is an integer, then it - determines the length of the array, and the array will be initially zeroed. - Otherwise, *size_or_initializer* is a sequence which is used to initialize - the array and whose length determines the length of the array. - - If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a :class:`Lock` or - :class:`RLock` object then that will be used to synchronize access to the - value. If *lock* is ``False`` then access to the returned object will not be - automatically protected by a lock, so it will not necessarily be - "process-safe". - - Note that *lock* is a keyword only argument. - - Note that an array of :data:`ctypes.c_char` has *value* and *raw* - attributes which allow one to use it to store and retrieve strings. - - -The :mod:`multiprocessing.sharedctypes` module ->>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> - -.. module:: multiprocessing.sharedctypes - :synopsis: Allocate ctypes objects from shared memory. - -The :mod:`multiprocessing.sharedctypes` module provides functions for allocating -:mod:`ctypes` objects from shared memory which can be inherited by child -processes. - -.. note:: - - Although it is possible to store a pointer in shared memory remember that - this will refer to a location in the address space of a specific process. - However, the pointer is quite likely to be invalid in the context of a second - process and trying to dereference the pointer from the second process may - cause a crash. - -.. function:: RawArray(typecode_or_type, size_or_initializer) - - Return a ctypes array allocated from shared memory. - - *typecode_or_type* determines the type of the elements of the returned array: - it is either a ctypes type or a one character typecode of the kind used by - the :mod:`array` module. If *size_or_initializer* is an integer then it - determines the length of the array, and the array will be initially zeroed. - Otherwise *size_or_initializer* is a sequence which is used to initialize the - array and whose length determines the length of the array. - - Note that setting and getting an element is potentially non-atomic -- use - :func:`Array` instead to make sure that access is automatically synchronized - using a lock. - -.. function:: RawValue(typecode_or_type, *args) - - Return a ctypes object allocated from shared memory. - - *typecode_or_type* determines the type of the returned object: it is either a - ctypes type or a one character typecode of the kind used by the :mod:`array` - module. *\*args* is passed on to the constructor for the type. - - Note that setting and getting the value is potentially non-atomic -- use - :func:`Value` instead to make sure that access is automatically synchronized - using a lock. - - Note that an array of :data:`ctypes.c_char` has ``value`` and ``raw`` - attributes which allow one to use it to store and retrieve strings -- see - documentation for :mod:`ctypes`. - -.. function:: Array(typecode_or_type, size_or_initializer, *, lock=True) - - The same as :func:`RawArray` except that depending on the value of *lock* a - process-safe synchronization wrapper may be returned instead of a raw ctypes - array. - - If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a - :class:`~multiprocessing.Lock` or :class:`~multiprocessing.RLock` object - then that will be used to synchronize access to the - value. If *lock* is ``False`` then access to the returned object will not be - automatically protected by a lock, so it will not necessarily be - "process-safe". - - Note that *lock* is a keyword-only argument. - -.. function:: Value(typecode_or_type, *args, lock=True) - - The same as :func:`RawValue` except that depending on the value of *lock* a - process-safe synchronization wrapper may be returned instead of a raw ctypes - object. - - If *lock* is ``True`` (the default) then a new lock object is created to - synchronize access to the value. If *lock* is a :class:`~multiprocessing.Lock` or - :class:`~multiprocessing.RLock` object then that will be used to synchronize access to the - value. If *lock* is ``False`` then access to the returned object will not be - automatically protected by a lock, so it will not necessarily be - "process-safe". - - Note that *lock* is a keyword-only argument. - -.. function:: copy(obj) - - Return a ctypes object allocated from shared memory which is a copy of the - ctypes object *obj*. - -.. function:: synchronized(obj[, lock]) - - Return a process-safe wrapper object for a ctypes object which uses *lock* to - synchronize access. If *lock* is ``None`` (the default) then a - :class:`multiprocessing.RLock` object is created automatically. - - A synchronized wrapper will have two methods in addition to those of the - object it wraps: :meth:`get_obj` returns the wrapped object and - :meth:`get_lock` returns the lock object used for synchronization. - - Note that accessing the ctypes object through the wrapper can be a lot slower - than accessing the raw ctypes object. - - .. versionchanged:: 3.5 - Synchronized objects support the :term:`context manager` protocol. - - -The table below compares the syntax for creating shared ctypes objects from -shared memory with the normal ctypes syntax. (In the table ``MyStruct`` is some -subclass of :class:`ctypes.Structure`.) - -==================== ========================== =========================== -ctypes sharedctypes using type sharedctypes using typecode -==================== ========================== =========================== -c_double(2.4) RawValue(c_double, 2.4) RawValue('d', 2.4) -MyStruct(4, 6) RawValue(MyStruct, 4, 6) -(c_short * 7)() RawArray(c_short, 7) RawArray('h', 7) -(c_int * 3)(9, 2, 8) RawArray(c_int, (9, 2, 8)) RawArray('i', (9, 2, 8)) -==================== ========================== =========================== - - -Below is an example where a number of ctypes objects are modified by a child -process:: - - from multiprocessing import Process, Lock - from multiprocessing.sharedctypes import Value, Array - from ctypes import Structure, c_double - - class Point(Structure): - _fields_ = [('x', c_double), ('y', c_double)] - - def modify(n, x, s, A): - n.value **= 2 - x.value **= 2 - s.value = s.value.upper() - for a in A: - a.x **= 2 - a.y **= 2 - - if __name__ == '__main__': - lock = Lock() - - n = Value('i', 7) - x = Value(c_double, 1.0/3.0, lock=False) - s = Array('c', b'hello world', lock=lock) - A = Array(Point, [(1.875,-6.25), (-5.75,2.0), (2.375,9.5)], lock=lock) - - p = Process(target=modify, args=(n, x, s, A)) - p.start() - p.join() - - print(n.value) - print(x.value) - print(s.value) - print([(a.x, a.y) for a in A]) - - -.. highlight:: none - -The results printed are :: - - 49 - 0.1111111111111111 - HELLO WORLD - [(3.515625, 39.0625), (33.0625, 4.0), (5.640625, 90.25)] - -.. highlight:: python3 - - -.. _multiprocessing-managers: - -Managers -~~~~~~~~ - -Managers provide a way to create data which can be shared between different -processes, including sharing over a network between processes running on -different machines. A manager object controls a server process which manages -*shared objects*. Other processes can access the shared objects by using -proxies. - -.. function:: multiprocessing.Manager() - :module: - - Returns a started :class:`~multiprocessing.managers.SyncManager` object which - can be used for sharing objects between processes. The returned manager - object corresponds to a spawned child process and has methods which will - create shared objects and return corresponding proxies. - -.. module:: multiprocessing.managers - :synopsis: Share data between process with shared objects. - -Manager processes will be shutdown as soon as they are garbage collected or -their parent process exits. The manager classes are defined in the -:mod:`multiprocessing.managers` module: - -.. class:: BaseManager(address=None, authkey=None, serializer='pickle', ctx=None, *, shutdown_timeout=1.0) - - Create a BaseManager object. - - Once created one should call :meth:`start` or ``get_server().serve_forever()`` to ensure - that the manager object refers to a started manager process. - - *address* is the address on which the manager process listens for new - connections. If *address* is ``None`` then an arbitrary one is chosen. - - *authkey* is the authentication key which will be used to check the - validity of incoming connections to the server process. If - *authkey* is ``None`` then ``current_process().authkey`` is used. - Otherwise *authkey* is used and it must be a byte string. - - *serializer* must be ``'pickle'`` (use :mod:`pickle` serialization) or - ``'xmlrpclib'`` (use :mod:`xmlrpc.client` serialization). - - *ctx* is a context object, or ``None`` (use the current context). See the - :func:`get_context` function. - - *shutdown_timeout* is a timeout in seconds used to wait until the process - used by the manager completes in the :meth:`shutdown` method. If the - shutdown times out, the process is terminated. If terminating the process - also times out, the process is killed. - - .. versionchanged:: 3.11 - Added the *shutdown_timeout* parameter. - - .. method:: start([initializer[, initargs]]) - - Start a subprocess to start the manager. If *initializer* is not ``None`` - then the subprocess will call ``initializer(*initargs)`` when it starts. - - .. method:: get_server() - - Returns a :class:`Server` object which represents the actual server under - the control of the Manager. The :class:`Server` object supports the - :meth:`serve_forever` method:: - - >>> from multiprocessing.managers import BaseManager - >>> manager = BaseManager(address=('', 50000), authkey=b'abc') - >>> server = manager.get_server() - >>> server.serve_forever() - - :class:`Server` additionally has an :attr:`address` attribute. - - .. method:: connect() - - Connect a local manager object to a remote manager process:: - - >>> from multiprocessing.managers import BaseManager - >>> m = BaseManager(address=('127.0.0.1', 50000), authkey=b'abc') - >>> m.connect() - - .. method:: shutdown() - - Stop the process used by the manager. This is only available if - :meth:`start` has been used to start the server process. - - This can be called multiple times. - - .. method:: register(typeid[, callable[, proxytype[, exposed[, method_to_typeid[, create_method]]]]]) - - A classmethod which can be used for registering a type or callable with - the manager class. - - *typeid* is a "type identifier" which is used to identify a particular - type of shared object. This must be a string. - - *callable* is a callable used for creating objects for this type - identifier. If a manager instance will be connected to the - server using the :meth:`connect` method, or if the - *create_method* argument is ``False`` then this can be left as - ``None``. - - *proxytype* is a subclass of :class:`BaseProxy` which is used to create - proxies for shared objects with this *typeid*. If ``None`` then a proxy - class is created automatically. - - *exposed* is used to specify a sequence of method names which proxies for - this typeid should be allowed to access using - :meth:`BaseProxy._callmethod`. (If *exposed* is ``None`` then - :attr:`proxytype._exposed_` is used instead if it exists.) In the case - where no exposed list is specified, all "public methods" of the shared - object will be accessible. (Here a "public method" means any attribute - which has a :meth:`~object.__call__` method and whose name does not begin - with ``'_'``.) - - *method_to_typeid* is a mapping used to specify the return type of those - exposed methods which should return a proxy. It maps method names to - typeid strings. (If *method_to_typeid* is ``None`` then - :attr:`proxytype._method_to_typeid_` is used instead if it exists.) If a - method's name is not a key of this mapping or if the mapping is ``None`` - then the object returned by the method will be copied by value. - - *create_method* determines whether a method should be created with name - *typeid* which can be used to tell the server process to create a new - shared object and return a proxy for it. By default it is ``True``. - - :class:`BaseManager` instances also have one read-only property: - - .. attribute:: address - - The address used by the manager. - - .. versionchanged:: 3.3 - Manager objects support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` starts the - server process (if it has not already started) and then returns the - manager object. :meth:`~contextmanager.__exit__` calls :meth:`shutdown`. - - In previous versions :meth:`~contextmanager.__enter__` did not start the - manager's server process if it was not already started. - -.. class:: SyncManager - - A subclass of :class:`BaseManager` which can be used for the synchronization - of processes. Objects of this type are returned by - :func:`multiprocessing.Manager`. - - Its methods create and return :ref:`multiprocessing-proxy_objects` for a - number of commonly used data types to be synchronized across processes. - This notably includes shared lists and dictionaries. - - .. method:: Barrier(parties[, action[, timeout]]) - - Create a shared :class:`threading.Barrier` object and return a - proxy for it. - - .. versionadded:: 3.3 - - .. method:: BoundedSemaphore([value]) - - Create a shared :class:`threading.BoundedSemaphore` object and return a - proxy for it. - - .. method:: Condition([lock]) - - Create a shared :class:`threading.Condition` object and return a proxy for - it. - - If *lock* is supplied then it should be a proxy for a - :class:`threading.Lock` or :class:`threading.RLock` object. - - .. versionchanged:: 3.3 - The :meth:`~threading.Condition.wait_for` method was added. - - .. method:: Event() - - Create a shared :class:`threading.Event` object and return a proxy for it. - - .. method:: Lock() - - Create a shared :class:`threading.Lock` object and return a proxy for it. - - .. method:: Namespace() - - Create a shared :class:`Namespace` object and return a proxy for it. - - .. method:: Queue([maxsize]) - - Create a shared :class:`queue.Queue` object and return a proxy for it. - - .. method:: RLock() - - Create a shared :class:`threading.RLock` object and return a proxy for it. - - .. method:: Semaphore([value]) - - Create a shared :class:`threading.Semaphore` object and return a proxy for - it. - - .. method:: Array(typecode, sequence) - - Create an array and return a proxy for it. - - .. method:: Value(typecode, value) - - Create an object with a writable ``value`` attribute and return a proxy - for it. - - .. method:: dict() - dict(mapping) - dict(sequence) - - Create a shared :class:`dict` object and return a proxy for it. - - .. method:: list() - list(sequence) - - Create a shared :class:`list` object and return a proxy for it. - - .. versionchanged:: 3.6 - Shared objects are capable of being nested. For example, a shared - container object such as a shared list can contain other shared objects - which will all be managed and synchronized by the :class:`SyncManager`. - -.. class:: Namespace - - A type that can register with :class:`SyncManager`. - - A namespace object has no public methods, but does have writable attributes. - Its representation shows the values of its attributes. - - However, when using a proxy for a namespace object, an attribute beginning - with ``'_'`` will be an attribute of the proxy and not an attribute of the - referent: - - .. doctest:: - - >>> manager = multiprocessing.Manager() - >>> Global = manager.Namespace() - >>> Global.x = 10 - >>> Global.y = 'hello' - >>> Global._z = 12.3 # this is an attribute of the proxy - >>> print(Global) - Namespace(x=10, y='hello') - - -Customized managers ->>>>>>>>>>>>>>>>>>> - -To create one's own manager, one creates a subclass of :class:`BaseManager` and -uses the :meth:`~BaseManager.register` classmethod to register new types or -callables with the manager class. For example:: - - from multiprocessing.managers import BaseManager - - class MathsClass: - def add(self, x, y): - return x + y - def mul(self, x, y): - return x * y - - class MyManager(BaseManager): - pass - - MyManager.register('Maths', MathsClass) - - if __name__ == '__main__': - with MyManager() as manager: - maths = manager.Maths() - print(maths.add(4, 3)) # prints 7 - print(maths.mul(7, 8)) # prints 56 - - -Using a remote manager ->>>>>>>>>>>>>>>>>>>>>> - -It is possible to run a manager server on one machine and have clients use it -from other machines (assuming that the firewalls involved allow it). - -Running the following commands creates a server for a single shared queue which -remote clients can access:: - - >>> from multiprocessing.managers import BaseManager - >>> from queue import Queue - >>> queue = Queue() - >>> class QueueManager(BaseManager): pass - >>> QueueManager.register('get_queue', callable=lambda:queue) - >>> m = QueueManager(address=('', 50000), authkey=b'abracadabra') - >>> s = m.get_server() - >>> s.serve_forever() - -One client can access the server as follows:: - - >>> from multiprocessing.managers import BaseManager - >>> class QueueManager(BaseManager): pass - >>> QueueManager.register('get_queue') - >>> m = QueueManager(address=('foo.bar.org', 50000), authkey=b'abracadabra') - >>> m.connect() - >>> queue = m.get_queue() - >>> queue.put('hello') - -Another client can also use it:: - - >>> from multiprocessing.managers import BaseManager - >>> class QueueManager(BaseManager): pass - >>> QueueManager.register('get_queue') - >>> m = QueueManager(address=('foo.bar.org', 50000), authkey=b'abracadabra') - >>> m.connect() - >>> queue = m.get_queue() - >>> queue.get() - 'hello' - -Local processes can also access that queue, using the code from above on the -client to access it remotely:: - - >>> from multiprocessing import Process, Queue - >>> from multiprocessing.managers import BaseManager - >>> class Worker(Process): - ... def __init__(self, q): - ... self.q = q - ... super().__init__() - ... def run(self): - ... self.q.put('local hello') - ... - >>> queue = Queue() - >>> w = Worker(queue) - >>> w.start() - >>> class QueueManager(BaseManager): pass - ... - >>> QueueManager.register('get_queue', callable=lambda: queue) - >>> m = QueueManager(address=('', 50000), authkey=b'abracadabra') - >>> s = m.get_server() - >>> s.serve_forever() - -.. _multiprocessing-proxy_objects: - -Proxy Objects -~~~~~~~~~~~~~ - -A proxy is an object which *refers* to a shared object which lives (presumably) -in a different process. The shared object is said to be the *referent* of the -proxy. Multiple proxy objects may have the same referent. - -A proxy object has methods which invoke corresponding methods of its referent -(although not every method of the referent will necessarily be available through -the proxy). In this way, a proxy can be used just like its referent can: - -.. doctest:: - - >>> from multiprocessing import Manager - >>> manager = Manager() - >>> l = manager.list([i*i for i in range(10)]) - >>> print(l) - [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] - >>> print(repr(l)) - - >>> l[4] - 16 - >>> l[2:5] - [4, 9, 16] - -Notice that applying :func:`str` to a proxy will return the representation of -the referent, whereas applying :func:`repr` will return the representation of -the proxy. - -An important feature of proxy objects is that they are picklable so they can be -passed between processes. As such, a referent can contain -:ref:`multiprocessing-proxy_objects`. This permits nesting of these managed -lists, dicts, and other :ref:`multiprocessing-proxy_objects`: - -.. doctest:: - - >>> a = manager.list() - >>> b = manager.list() - >>> a.append(b) # referent of a now contains referent of b - >>> print(a, b) - [] [] - >>> b.append('hello') - >>> print(a[0], b) - ['hello'] ['hello'] - -Similarly, dict and list proxies may be nested inside one another:: - - >>> l_outer = manager.list([ manager.dict() for i in range(2) ]) - >>> d_first_inner = l_outer[0] - >>> d_first_inner['a'] = 1 - >>> d_first_inner['b'] = 2 - >>> l_outer[1]['c'] = 3 - >>> l_outer[1]['z'] = 26 - >>> print(l_outer[0]) - {'a': 1, 'b': 2} - >>> print(l_outer[1]) - {'c': 3, 'z': 26} - -If standard (non-proxy) :class:`list` or :class:`dict` objects are contained -in a referent, modifications to those mutable values will not be propagated -through the manager because the proxy has no way of knowing when the values -contained within are modified. However, storing a value in a container proxy -(which triggers a ``__setitem__`` on the proxy object) does propagate through -the manager and so to effectively modify such an item, one could re-assign the -modified value to the container proxy:: - - # create a list proxy and append a mutable object (a dictionary) - lproxy = manager.list() - lproxy.append({}) - # now mutate the dictionary - d = lproxy[0] - d['a'] = 1 - d['b'] = 2 - # at this point, the changes to d are not yet synced, but by - # updating the dictionary, the proxy is notified of the change - lproxy[0] = d - -This approach is perhaps less convenient than employing nested -:ref:`multiprocessing-proxy_objects` for most use cases but also -demonstrates a level of control over the synchronization. - -.. note:: - - The proxy types in :mod:`multiprocessing` do nothing to support comparisons - by value. So, for instance, we have: - - .. doctest:: - - >>> manager.list([1,2,3]) == [1,2,3] - False - - One should just use a copy of the referent instead when making comparisons. - -.. class:: BaseProxy - - Proxy objects are instances of subclasses of :class:`BaseProxy`. - - .. method:: _callmethod(methodname[, args[, kwds]]) - - Call and return the result of a method of the proxy's referent. - - If ``proxy`` is a proxy whose referent is ``obj`` then the expression :: - - proxy._callmethod(methodname, args, kwds) - - will evaluate the expression :: - - getattr(obj, methodname)(*args, **kwds) - - in the manager's process. - - The returned value will be a copy of the result of the call or a proxy to - a new shared object -- see documentation for the *method_to_typeid* - argument of :meth:`BaseManager.register`. - - If an exception is raised by the call, then is re-raised by - :meth:`_callmethod`. If some other exception is raised in the manager's - process then this is converted into a :exc:`RemoteError` exception and is - raised by :meth:`_callmethod`. - - Note in particular that an exception will be raised if *methodname* has - not been *exposed*. - - An example of the usage of :meth:`_callmethod`: - - .. doctest:: - - >>> l = manager.list(range(10)) - >>> l._callmethod('__len__') - 10 - >>> l._callmethod('__getitem__', (slice(2, 7),)) # equivalent to l[2:7] - [2, 3, 4, 5, 6] - >>> l._callmethod('__getitem__', (20,)) # equivalent to l[20] - Traceback (most recent call last): - ... - IndexError: list index out of range - - .. method:: _getvalue() - - Return a copy of the referent. - - If the referent is unpicklable then this will raise an exception. - - .. method:: __repr__ - - Return a representation of the proxy object. - - .. method:: __str__ - - Return the representation of the referent. - - -Cleanup ->>>>>>> - -A proxy object uses a weakref callback so that when it gets garbage collected it -deregisters itself from the manager which owns its referent. - -A shared object gets deleted from the manager process when there are no longer -any proxies referring to it. - - -Process Pools -~~~~~~~~~~~~~ - -.. module:: multiprocessing.pool - :synopsis: Create pools of processes. - -One can create a pool of processes which will carry out tasks submitted to it -with the :class:`Pool` class. - -.. class:: Pool([processes[, initializer[, initargs[, maxtasksperchild [, context]]]]]) - - A process pool object which controls a pool of worker processes to which jobs - can be submitted. It supports asynchronous results with timeouts and - callbacks and has a parallel map implementation. - - *processes* is the number of worker processes to use. If *processes* is - ``None`` then the number returned by :func:`os.cpu_count` is used. - - If *initializer* is not ``None`` then each worker process will call - ``initializer(*initargs)`` when it starts. - - *maxtasksperchild* is the number of tasks a worker process can complete - before it will exit and be replaced with a fresh worker process, to enable - unused resources to be freed. The default *maxtasksperchild* is ``None``, which - means worker processes will live as long as the pool. - - *context* can be used to specify the context used for starting - the worker processes. Usually a pool is created using the - function :func:`multiprocessing.Pool` or the :meth:`Pool` method - of a context object. In both cases *context* is set - appropriately. - - Note that the methods of the pool object should only be called by - the process which created the pool. - - .. warning:: - :class:`multiprocessing.pool` objects have internal resources that need to be - properly managed (like any other resource) by using the pool as a context manager - or by calling :meth:`close` and :meth:`terminate` manually. Failure to do this - can lead to the process hanging on finalization. - - Note that it is **not correct** to rely on the garbage collector to destroy the pool - as CPython does not assure that the finalizer of the pool will be called - (see :meth:`object.__del__` for more information). - - .. versionadded:: 3.2 - *maxtasksperchild* - - .. versionadded:: 3.4 - *context* - - .. note:: - - Worker processes within a :class:`Pool` typically live for the complete - duration of the Pool's work queue. A frequent pattern found in other - systems (such as Apache, mod_wsgi, etc) to free resources held by - workers is to allow a worker within a pool to complete only a set - amount of work before being exiting, being cleaned up and a new - process spawned to replace the old one. The *maxtasksperchild* - argument to the :class:`Pool` exposes this ability to the end user. - - .. method:: apply(func[, args[, kwds]]) - - Call *func* with arguments *args* and keyword arguments *kwds*. It blocks - until the result is ready. Given this blocks, :meth:`apply_async` is - better suited for performing work in parallel. Additionally, *func* - is only executed in one of the workers of the pool. - - .. method:: apply_async(func[, args[, kwds[, callback[, error_callback]]]]) - - A variant of the :meth:`apply` method which returns a - :class:`~multiprocessing.pool.AsyncResult` object. - - If *callback* is specified then it should be a callable which accepts a - single argument. When the result becomes ready *callback* is applied to - it, that is unless the call failed, in which case the *error_callback* - is applied instead. - - If *error_callback* is specified then it should be a callable which - accepts a single argument. If the target function fails, then - the *error_callback* is called with the exception instance. - - Callbacks should complete immediately since otherwise the thread which - handles the results will get blocked. - - .. method:: map(func, iterable[, chunksize]) - - A parallel equivalent of the :func:`map` built-in function (it supports only - one *iterable* argument though, for multiple iterables see :meth:`starmap`). - It blocks until the result is ready. - - This method chops the iterable into a number of chunks which it submits to - the process pool as separate tasks. The (approximate) size of these - chunks can be specified by setting *chunksize* to a positive integer. - - Note that it may cause high memory usage for very long iterables. Consider - using :meth:`imap` or :meth:`imap_unordered` with explicit *chunksize* - option for better efficiency. - - .. method:: map_async(func, iterable[, chunksize[, callback[, error_callback]]]) - - A variant of the :meth:`.map` method which returns a - :class:`~multiprocessing.pool.AsyncResult` object. - - If *callback* is specified then it should be a callable which accepts a - single argument. When the result becomes ready *callback* is applied to - it, that is unless the call failed, in which case the *error_callback* - is applied instead. - - If *error_callback* is specified then it should be a callable which - accepts a single argument. If the target function fails, then - the *error_callback* is called with the exception instance. - - Callbacks should complete immediately since otherwise the thread which - handles the results will get blocked. - - .. method:: imap(func, iterable[, chunksize]) - - A lazier version of :meth:`.map`. - - The *chunksize* argument is the same as the one used by the :meth:`.map` - method. For very long iterables using a large value for *chunksize* can - make the job complete **much** faster than using the default value of - ``1``. - - Also if *chunksize* is ``1`` then the :meth:`!next` method of the iterator - returned by the :meth:`imap` method has an optional *timeout* parameter: - ``next(timeout)`` will raise :exc:`multiprocessing.TimeoutError` if the - result cannot be returned within *timeout* seconds. - - .. method:: imap_unordered(func, iterable[, chunksize]) - - The same as :meth:`imap` except that the ordering of the results from the - returned iterator should be considered arbitrary. (Only when there is - only one worker process is the order guaranteed to be "correct".) - - .. method:: starmap(func, iterable[, chunksize]) - - Like :meth:`~multiprocessing.pool.Pool.map` except that the - elements of the *iterable* are expected to be iterables that are - unpacked as arguments. - - Hence an *iterable* of ``[(1,2), (3, 4)]`` results in ``[func(1,2), - func(3,4)]``. - - .. versionadded:: 3.3 - - .. method:: starmap_async(func, iterable[, chunksize[, callback[, error_callback]]]) - - A combination of :meth:`starmap` and :meth:`map_async` that iterates over - *iterable* of iterables and calls *func* with the iterables unpacked. - Returns a result object. - - .. versionadded:: 3.3 - - .. method:: close() - - Prevents any more tasks from being submitted to the pool. Once all the - tasks have been completed the worker processes will exit. - - .. method:: terminate() - - Stops the worker processes immediately without completing outstanding - work. When the pool object is garbage collected :meth:`terminate` will be - called immediately. - - .. method:: join() - - Wait for the worker processes to exit. One must call :meth:`close` or - :meth:`terminate` before using :meth:`join`. - - .. versionadded:: 3.3 - Pool objects now support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the - pool object, and :meth:`~contextmanager.__exit__` calls :meth:`terminate`. - - -.. class:: AsyncResult - - The class of the result returned by :meth:`Pool.apply_async` and - :meth:`Pool.map_async`. - - .. method:: get([timeout]) - - Return the result when it arrives. If *timeout* is not ``None`` and the - result does not arrive within *timeout* seconds then - :exc:`multiprocessing.TimeoutError` is raised. If the remote call raised - an exception then that exception will be reraised by :meth:`get`. - - .. method:: wait([timeout]) - - Wait until the result is available or until *timeout* seconds pass. - - .. method:: ready() - - Return whether the call has completed. - - .. method:: successful() - - Return whether the call completed without raising an exception. Will - raise :exc:`ValueError` if the result is not ready. - - .. versionchanged:: 3.7 - If the result is not ready, :exc:`ValueError` is raised instead of - :exc:`AssertionError`. - -The following example demonstrates the use of a pool:: - - from multiprocessing import Pool - import time - - def f(x): - return x*x - - if __name__ == '__main__': - with Pool(processes=4) as pool: # start 4 worker processes - result = pool.apply_async(f, (10,)) # evaluate "f(10)" asynchronously in a single process - print(result.get(timeout=1)) # prints "100" unless your computer is *very* slow - - print(pool.map(f, range(10))) # prints "[0, 1, 4,..., 81]" - - it = pool.imap(f, range(10)) - print(next(it)) # prints "0" - print(next(it)) # prints "1" - print(it.next(timeout=1)) # prints "4" unless your computer is *very* slow - - result = pool.apply_async(time.sleep, (10,)) - print(result.get(timeout=1)) # raises multiprocessing.TimeoutError - - -.. _multiprocessing-listeners-clients: - -Listeners and Clients -~~~~~~~~~~~~~~~~~~~~~ - -.. module:: multiprocessing.connection - :synopsis: API for dealing with sockets. - -Usually message passing between processes is done using queues or by using -:class:`~Connection` objects returned by -:func:`~multiprocessing.Pipe`. - -However, the :mod:`multiprocessing.connection` module allows some extra -flexibility. It basically gives a high level message oriented API for dealing -with sockets or Windows named pipes. It also has support for *digest -authentication* using the :mod:`hmac` module, and for polling -multiple connections at the same time. - - -.. function:: deliver_challenge(connection, authkey) - - Send a randomly generated message to the other end of the connection and wait - for a reply. - - If the reply matches the digest of the message using *authkey* as the key - then a welcome message is sent to the other end of the connection. Otherwise - :exc:`~multiprocessing.AuthenticationError` is raised. - -.. function:: answer_challenge(connection, authkey) - - Receive a message, calculate the digest of the message using *authkey* as the - key, and then send the digest back. - - If a welcome message is not received, then - :exc:`~multiprocessing.AuthenticationError` is raised. - -.. function:: Client(address[, family[, authkey]]) - - Attempt to set up a connection to the listener which is using address - *address*, returning a :class:`~Connection`. - - The type of the connection is determined by *family* argument, but this can - generally be omitted since it can usually be inferred from the format of - *address*. (See :ref:`multiprocessing-address-formats`) - - If *authkey* is given and not None, it should be a byte string and will be - used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. - :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. - See :ref:`multiprocessing-auth-keys`. - -.. class:: Listener([address[, family[, backlog[, authkey]]]]) - - A wrapper for a bound socket or Windows named pipe which is 'listening' for - connections. - - *address* is the address to be used by the bound socket or named pipe of the - listener object. - - .. note:: - - If an address of '0.0.0.0' is used, the address will not be a connectable - end point on Windows. If you require a connectable end-point, - you should use '127.0.0.1'. - - *family* is the type of socket (or named pipe) to use. This can be one of - the strings ``'AF_INET'`` (for a TCP socket), ``'AF_UNIX'`` (for a Unix - domain socket) or ``'AF_PIPE'`` (for a Windows named pipe). Of these only - the first is guaranteed to be available. If *family* is ``None`` then the - family is inferred from the format of *address*. If *address* is also - ``None`` then a default is chosen. This default is the family which is - assumed to be the fastest available. See - :ref:`multiprocessing-address-formats`. Note that if *family* is - ``'AF_UNIX'`` and address is ``None`` then the socket will be created in a - private temporary directory created using :func:`tempfile.mkstemp`. - - If the listener object uses a socket then *backlog* (1 by default) is passed - to the :meth:`~socket.socket.listen` method of the socket once it has been - bound. - - If *authkey* is given and not None, it should be a byte string and will be - used as the secret key for an HMAC-based authentication challenge. No - authentication is done if *authkey* is None. - :exc:`~multiprocessing.AuthenticationError` is raised if authentication fails. - See :ref:`multiprocessing-auth-keys`. - - .. method:: accept() - - Accept a connection on the bound socket or named pipe of the listener - object and return a :class:`~Connection` object. - If authentication is attempted and fails, then - :exc:`~multiprocessing.AuthenticationError` is raised. - - .. method:: close() - - Close the bound socket or named pipe of the listener object. This is - called automatically when the listener is garbage collected. However it - is advisable to call it explicitly. - - Listener objects have the following read-only properties: - - .. attribute:: address - - The address which is being used by the Listener object. - - .. attribute:: last_accepted - - The address from which the last accepted connection came. If this is - unavailable then it is ``None``. - - .. versionadded:: 3.3 - Listener objects now support the context management protocol -- see - :ref:`typecontextmanager`. :meth:`~contextmanager.__enter__` returns the - listener object, and :meth:`~contextmanager.__exit__` calls :meth:`close`. - -.. function:: wait(object_list, timeout=None) - - Wait till an object in *object_list* is ready. Returns the list of - those objects in *object_list* which are ready. If *timeout* is a - float then the call blocks for at most that many seconds. If - *timeout* is ``None`` then it will block for an unlimited period. - A negative timeout is equivalent to a zero timeout. - - For both Unix and Windows, an object can appear in *object_list* if - it is - - * a readable :class:`~multiprocessing.connection.Connection` object; - * a connected and readable :class:`socket.socket` object; or - * the :attr:`~multiprocessing.Process.sentinel` attribute of a - :class:`~multiprocessing.Process` object. - - A connection or socket object is ready when there is data available - to be read from it, or the other end has been closed. - - **Unix**: ``wait(object_list, timeout)`` almost equivalent - ``select.select(object_list, [], [], timeout)``. The difference is - that, if :func:`select.select` is interrupted by a signal, it can - raise :exc:`OSError` with an error number of ``EINTR``, whereas - :func:`wait` will not. - - **Windows**: An item in *object_list* must either be an integer - handle which is waitable (according to the definition used by the - documentation of the Win32 function ``WaitForMultipleObjects()``) - or it can be an object with a :meth:`~io.IOBase.fileno` method which returns a - socket handle or pipe handle. (Note that pipe handles and socket - handles are **not** waitable handles.) - - .. versionadded:: 3.3 - - -**Examples** - -The following server code creates a listener which uses ``'secret password'`` as -an authentication key. It then waits for a connection and sends some data to -the client:: - - from multiprocessing.connection import Listener - from array import array - - address = ('localhost', 6000) # family is deduced to be 'AF_INET' - - with Listener(address, authkey=b'secret password') as listener: - with listener.accept() as conn: - print('connection accepted from', listener.last_accepted) - - conn.send([2.25, None, 'junk', float]) - - conn.send_bytes(b'hello') - - conn.send_bytes(array('i', [42, 1729])) - -The following code connects to the server and receives some data from the -server:: - - from multiprocessing.connection import Client - from array import array - - address = ('localhost', 6000) - - with Client(address, authkey=b'secret password') as conn: - print(conn.recv()) # => [2.25, None, 'junk', float] - - print(conn.recv_bytes()) # => 'hello' - - arr = array('i', [0, 0, 0, 0, 0]) - print(conn.recv_bytes_into(arr)) # => 8 - print(arr) # => array('i', [42, 1729, 0, 0, 0]) - -The following code uses :func:`~multiprocessing.connection.wait` to -wait for messages from multiple processes at once:: - - import time, random - from multiprocessing import Process, Pipe, current_process - from multiprocessing.connection import wait - - def foo(w): - for i in range(10): - w.send((i, current_process().name)) - w.close() - - if __name__ == '__main__': - readers = [] - - for i in range(4): - r, w = Pipe(duplex=False) - readers.append(r) - p = Process(target=foo, args=(w,)) - p.start() - # We close the writable end of the pipe now to be sure that - # p is the only process which owns a handle for it. This - # ensures that when p closes its handle for the writable end, - # wait() will promptly report the readable end as being ready. - w.close() - - while readers: - for r in wait(readers): - try: - msg = r.recv() - except EOFError: - readers.remove(r) - else: - print(msg) - - -.. _multiprocessing-address-formats: - -Address Formats ->>>>>>>>>>>>>>> - -* An ``'AF_INET'`` address is a tuple of the form ``(hostname, port)`` where - *hostname* is a string and *port* is an integer. - -* An ``'AF_UNIX'`` address is a string representing a filename on the - filesystem. - -* An ``'AF_PIPE'`` address is a string of the form - :samp:`r'\\\\\\.\\pipe\\\\{PipeName}'`. To use :func:`Client` to connect to a named - pipe on a remote computer called *ServerName* one should use an address of the - form :samp:`r'\\\\\\\\{ServerName}\\pipe\\\\{PipeName}'` instead. - -Note that any string beginning with two backslashes is assumed by default to be -an ``'AF_PIPE'`` address rather than an ``'AF_UNIX'`` address. - - -.. _multiprocessing-auth-keys: - -Authentication keys -~~~~~~~~~~~~~~~~~~~ - -When one uses :meth:`Connection.recv `, the -data received is automatically -unpickled. Unfortunately unpickling data from an untrusted source is a security -risk. Therefore :class:`Listener` and :func:`Client` use the :mod:`hmac` module -to provide digest authentication. - -An authentication key is a byte string which can be thought of as a -password: once a connection is established both ends will demand proof -that the other knows the authentication key. (Demonstrating that both -ends are using the same key does **not** involve sending the key over -the connection.) - -If authentication is requested but no authentication key is specified then the -return value of ``current_process().authkey`` is used (see -:class:`~multiprocessing.Process`). This value will be automatically inherited by -any :class:`~multiprocessing.Process` object that the current process creates. -This means that (by default) all processes of a multi-process program will share -a single authentication key which can be used when setting up connections -between themselves. - -Suitable authentication keys can also be generated by using :func:`os.urandom`. - - -Logging -~~~~~~~ - -Some support for logging is available. Note, however, that the :mod:`logging` -package does not use process shared locks so it is possible (depending on the -handler type) for messages from different processes to get mixed up. - -.. currentmodule:: multiprocessing -.. function:: get_logger() - - Returns the logger used by :mod:`multiprocessing`. If necessary, a new one - will be created. - - When first created the logger has level :const:`logging.NOTSET` and no - default handler. Messages sent to this logger will not by default propagate - to the root logger. - - Note that on Windows child processes will only inherit the level of the - parent process's logger -- any other customization of the logger will not be - inherited. - -.. currentmodule:: multiprocessing -.. function:: log_to_stderr(level=None) - - This function performs a call to :func:`get_logger` but in addition to - returning the logger created by get_logger, it adds a handler which sends - output to :data:`sys.stderr` using format - ``'[%(levelname)s/%(processName)s] %(message)s'``. - You can modify ``levelname`` of the logger by passing a ``level`` argument. - -Below is an example session with logging turned on:: - - >>> import multiprocessing, logging - >>> logger = multiprocessing.log_to_stderr() - >>> logger.setLevel(logging.INFO) - >>> logger.warning('doomed') - [WARNING/MainProcess] doomed - >>> m = multiprocessing.Manager() - [INFO/SyncManager-...] child process calling self.run() - [INFO/SyncManager-...] created temp directory /.../pymp-... - [INFO/SyncManager-...] manager serving at '/.../listener-...' - >>> del m - [INFO/MainProcess] sending shutdown message to manager - [INFO/SyncManager-...] manager exiting with exitcode 0 - -For a full table of logging levels, see the :mod:`logging` module. - - -The :mod:`multiprocessing.dummy` module -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. module:: multiprocessing.dummy - :synopsis: Dumb wrapper around threading. - -:mod:`multiprocessing.dummy` replicates the API of :mod:`multiprocessing` but is -no more than a wrapper around the :mod:`threading` module. - -.. currentmodule:: multiprocessing.pool - -In particular, the ``Pool`` function provided by :mod:`multiprocessing.dummy` -returns an instance of :class:`ThreadPool`, which is a subclass of -:class:`Pool` that supports all the same method calls but uses a pool of -worker threads rather than worker processes. - - -.. class:: ThreadPool([processes[, initializer[, initargs]]]) - - A thread pool object which controls a pool of worker threads to which jobs - can be submitted. :class:`ThreadPool` instances are fully interface - compatible with :class:`Pool` instances, and their resources must also be - properly managed, either by using the pool as a context manager or by - calling :meth:`~multiprocessing.pool.Pool.close` and - :meth:`~multiprocessing.pool.Pool.terminate` manually. - - *processes* is the number of worker threads to use. If *processes* is - ``None`` then the number returned by :func:`os.cpu_count` is used. - - If *initializer* is not ``None`` then each worker process will call - ``initializer(*initargs)`` when it starts. - - Unlike :class:`Pool`, *maxtasksperchild* and *context* cannot be provided. - - .. note:: - - A :class:`ThreadPool` shares the same interface as :class:`Pool`, which - is designed around a pool of processes and predates the introduction of - the :class:`concurrent.futures` module. As such, it inherits some - operations that don't make sense for a pool backed by threads, and it - has its own type for representing the status of asynchronous jobs, - :class:`AsyncResult`, that is not understood by any other libraries. - - Users should generally prefer to use - :class:`concurrent.futures.ThreadPoolExecutor`, which has a simpler - interface that was designed around threads from the start, and which - returns :class:`concurrent.futures.Future` instances that are - compatible with many other libraries, including :mod:`asyncio`. - - -.. _multiprocessing-programming: - -Programming guidelines ----------------------- - -There are certain guidelines and idioms which should be adhered to when using -:mod:`multiprocessing`. - - -All start methods -~~~~~~~~~~~~~~~~~ - -The following applies to all start methods. - -Avoid shared state - - As far as possible one should try to avoid shifting large amounts of data - between processes. - - It is probably best to stick to using queues or pipes for communication - between processes rather than using the lower level synchronization - primitives. - -Picklability - - Ensure that the arguments to the methods of proxies are picklable. - -Thread safety of proxies - - Do not use a proxy object from more than one thread unless you protect it - with a lock. - - (There is never a problem with different processes using the *same* proxy.) - -Joining zombie processes - - On Unix when a process finishes but has not been joined it becomes a zombie. - There should never be very many because each time a new process starts (or - :func:`~multiprocessing.active_children` is called) all completed processes - which have not yet been joined will be joined. Also calling a finished - process's :meth:`Process.is_alive ` will - join the process. Even so it is probably good - practice to explicitly join all the processes that you start. - -Better to inherit than pickle/unpickle - - When using the *spawn* or *forkserver* start methods many types - from :mod:`multiprocessing` need to be picklable so that child - processes can use them. However, one should generally avoid - sending shared objects to other processes using pipes or queues. - Instead you should arrange the program so that a process which - needs access to a shared resource created elsewhere can inherit it - from an ancestor process. - -Avoid terminating processes - - Using the :meth:`Process.terminate ` - method to stop a process is liable to - cause any shared resources (such as locks, semaphores, pipes and queues) - currently being used by the process to become broken or unavailable to other - processes. - - Therefore it is probably best to only consider using - :meth:`Process.terminate ` on processes - which never use any shared resources. - -Joining processes that use queues - - Bear in mind that a process that has put items in a queue will wait before - terminating until all the buffered items are fed by the "feeder" thread to - the underlying pipe. (The child process can call the - :meth:`Queue.cancel_join_thread ` - method of the queue to avoid this behaviour.) - - This means that whenever you use a queue you need to make sure that all - items which have been put on the queue will eventually be removed before the - process is joined. Otherwise you cannot be sure that processes which have - put items on the queue will terminate. Remember also that non-daemonic - processes will be joined automatically. - - An example which will deadlock is the following:: - - from multiprocessing import Process, Queue - - def f(q): - q.put('X' * 1000000) - - if __name__ == '__main__': - queue = Queue() - p = Process(target=f, args=(queue,)) - p.start() - p.join() # this deadlocks - obj = queue.get() - - A fix here would be to swap the last two lines (or simply remove the - ``p.join()`` line). - -Explicitly pass resources to child processes - - On Unix using the *fork* start method, a child process can make - use of a shared resource created in a parent process using a - global resource. However, it is better to pass the object as an - argument to the constructor for the child process. - - Apart from making the code (potentially) compatible with Windows - and the other start methods this also ensures that as long as the - child process is still alive the object will not be garbage - collected in the parent process. This might be important if some - resource is freed when the object is garbage collected in the - parent process. - - So for instance :: - - from multiprocessing import Process, Lock - - def f(): - ... do something using "lock" ... - - if __name__ == '__main__': - lock = Lock() - for i in range(10): - Process(target=f).start() - - should be rewritten as :: - - from multiprocessing import Process, Lock - - def f(l): - ... do something using "l" ... - - if __name__ == '__main__': - lock = Lock() - for i in range(10): - Process(target=f, args=(lock,)).start() - -Beware of replacing :data:`sys.stdin` with a "file like object" - - :mod:`multiprocessing` originally unconditionally called:: - - os.close(sys.stdin.fileno()) - - in the :meth:`multiprocessing.Process._bootstrap` method --- this resulted - in issues with processes-in-processes. This has been changed to:: - - sys.stdin.close() - sys.stdin = open(os.open(os.devnull, os.O_RDONLY), closefd=False) - - Which solves the fundamental issue of processes colliding with each other - resulting in a bad file descriptor error, but introduces a potential danger - to applications which replace :func:`sys.stdin` with a "file-like object" - with output buffering. This danger is that if multiple processes call - :meth:`~io.IOBase.close()` on this file-like object, it could result in the same - data being flushed to the object multiple times, resulting in corruption. - - If you write a file-like object and implement your own caching, you can - make it fork-safe by storing the pid whenever you append to the cache, - and discarding the cache when the pid changes. For example:: - - @property - def cache(self): - pid = os.getpid() - if pid != self._pid: - self._pid = pid - self._cache = [] - return self._cache - - For more information, see :issue:`5155`, :issue:`5313` and :issue:`5331` - -The *spawn* and *forkserver* start methods -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are a few extra restriction which don't apply to the *fork* -start method. - -More picklability - - Ensure that all arguments to :meth:`Process.__init__` are picklable. - Also, if you subclass :class:`~multiprocessing.Process` then make sure that - instances will be picklable when the :meth:`Process.start - ` method is called. - -Global variables - - Bear in mind that if code run in a child process tries to access a global - variable, then the value it sees (if any) may not be the same as the value - in the parent process at the time that :meth:`Process.start - ` was called. - - However, global variables which are just module level constants cause no - problems. - -.. _multiprocessing-safe-main-import: - -Safe importing of main module - - Make sure that the main module can be safely imported by a new Python - interpreter without causing unintended side effects (such as starting a new - process). - - For example, using the *spawn* or *forkserver* start method - running the following module would fail with a - :exc:`RuntimeError`:: - - from multiprocessing import Process - - def foo(): - print('hello') - - p = Process(target=foo) - p.start() - - Instead one should protect the "entry point" of the program by using ``if - __name__ == '__main__':`` as follows:: - - from multiprocessing import Process, freeze_support, set_start_method - - def foo(): - print('hello') - - if __name__ == '__main__': - freeze_support() - set_start_method('spawn') - p = Process(target=foo) - p.start() - - (The ``freeze_support()`` line can be omitted if the program will be run - normally instead of frozen.) - - This allows the newly spawned Python interpreter to safely import the module - and then run the module's ``foo()`` function. - - Similar restrictions apply if a pool or manager is created in the main - module. - - -.. _multiprocessing-examples: - -Examples --------- - -Demonstration of how to create and use customized managers and proxies: - -.. literalinclude:: ../includes/mp_newtype.py - :language: python3 - - -Using :class:`~multiprocessing.pool.Pool`: - -.. literalinclude:: ../includes/mp_pool.py - :language: python3 - - -An example showing how to use queues to feed tasks to a collection of worker -processes and collect the results: - -.. literalinclude:: ../includes/mp_workers.py diff --git a/Doc/library/multiprocessing.shared_memory.rst b/Doc/library/multiprocessing.shared_memory.rst deleted file mode 100644 index f453e6403d932d..00000000000000 --- a/Doc/library/multiprocessing.shared_memory.rst +++ /dev/null @@ -1,401 +0,0 @@ -:mod:`multiprocessing.shared_memory` --- Shared memory for direct access across processes -========================================================================================= - -.. module:: multiprocessing.shared_memory - :synopsis: Provides shared memory for direct access across processes. - -**Source code:** :source:`Lib/multiprocessing/shared_memory.py` - -.. versionadded:: 3.8 - -.. index:: - single: Shared Memory - single: POSIX Shared Memory - single: Named Shared Memory - --------------- - -This module provides a class, :class:`SharedMemory`, for the allocation -and management of shared memory to be accessed by one or more processes -on a multicore or symmetric multiprocessor (SMP) machine. To assist with -the life-cycle management of shared memory especially across distinct -processes, a :class:`~multiprocessing.managers.BaseManager` subclass, -:class:`SharedMemoryManager`, is also provided in the -``multiprocessing.managers`` module. - -In this module, shared memory refers to "System V style" shared memory blocks -(though is not necessarily implemented explicitly as such) and does not refer -to "distributed shared memory". This style of shared memory permits distinct -processes to potentially read and write to a common (or shared) region of -volatile memory. Processes are conventionally limited to only have access to -their own process memory space but shared memory permits the sharing -of data between processes, avoiding the need to instead send messages between -processes containing that data. Sharing data directly via memory can provide -significant performance benefits compared to sharing data via disk or socket -or other communications requiring the serialization/deserialization and -copying of data. - - -.. class:: SharedMemory(name=None, create=False, size=0) - - Creates a new shared memory block or attaches to an existing shared - memory block. Each shared memory block is assigned a unique name. - In this way, one process can create a shared memory block with a - particular name and a different process can attach to that same shared - memory block using that same name. - - As a resource for sharing data across processes, shared memory blocks - may outlive the original process that created them. When one process - no longer needs access to a shared memory block that might still be - needed by other processes, the :meth:`close()` method should be called. - When a shared memory block is no longer needed by any process, the - :meth:`unlink()` method should be called to ensure proper cleanup. - - *name* is the unique name for the requested shared memory, specified as - a string. When creating a new shared memory block, if ``None`` (the - default) is supplied for the name, a novel name will be generated. - - *create* controls whether a new shared memory block is created (``True``) - or an existing shared memory block is attached (``False``). - - *size* specifies the requested number of bytes when creating a new shared - memory block. Because some platforms choose to allocate chunks of memory - based upon that platform's memory page size, the exact size of the shared - memory block may be larger or equal to the size requested. When attaching - to an existing shared memory block, the ``size`` parameter is ignored. - - .. method:: close() - - Closes access to the shared memory from this instance. In order to - ensure proper cleanup of resources, all instances should call - ``close()`` once the instance is no longer needed. Note that calling - ``close()`` does not cause the shared memory block itself to be - destroyed. - - .. method:: unlink() - - Requests that the underlying shared memory block be destroyed. In - order to ensure proper cleanup of resources, ``unlink()`` should be - called once (and only once) across all processes which have need - for the shared memory block. After requesting its destruction, a - shared memory block may or may not be immediately destroyed and - this behavior may differ across platforms. Attempts to access data - inside the shared memory block after ``unlink()`` has been called may - result in memory access errors. Note: the last process relinquishing - its hold on a shared memory block may call ``unlink()`` and - :meth:`close()` in either order. - - .. attribute:: buf - - A memoryview of contents of the shared memory block. - - .. attribute:: name - - Read-only access to the unique name of the shared memory block. - - .. attribute:: size - - Read-only access to size in bytes of the shared memory block. - - -The following example demonstrates low-level use of :class:`SharedMemory` -instances:: - - >>> from multiprocessing import shared_memory - >>> shm_a = shared_memory.SharedMemory(create=True, size=10) - >>> type(shm_a.buf) - - >>> buffer = shm_a.buf - >>> len(buffer) - 10 - >>> buffer[:4] = bytearray([22, 33, 44, 55]) # Modify multiple at once - >>> buffer[4] = 100 # Modify single byte at a time - >>> # Attach to an existing shared memory block - >>> shm_b = shared_memory.SharedMemory(shm_a.name) - >>> import array - >>> array.array('b', shm_b.buf[:5]) # Copy the data into a new array.array - array('b', [22, 33, 44, 55, 100]) - >>> shm_b.buf[:5] = b'howdy' # Modify via shm_b using bytes - >>> bytes(shm_a.buf[:5]) # Access via shm_a - b'howdy' - >>> shm_b.close() # Close each SharedMemory instance - >>> shm_a.close() - >>> shm_a.unlink() # Call unlink only once to release the shared memory - - - -The following example demonstrates a practical use of the :class:`SharedMemory` -class with `NumPy arrays `_, accessing the -same ``numpy.ndarray`` from two distinct Python shells: - -.. doctest:: - :options: +SKIP - - >>> # In the first Python interactive shell - >>> import numpy as np - >>> a = np.array([1, 1, 2, 3, 5, 8]) # Start with an existing NumPy array - >>> from multiprocessing import shared_memory - >>> shm = shared_memory.SharedMemory(create=True, size=a.nbytes) - >>> # Now create a NumPy array backed by shared memory - >>> b = np.ndarray(a.shape, dtype=a.dtype, buffer=shm.buf) - >>> b[:] = a[:] # Copy the original data into shared memory - >>> b - array([1, 1, 2, 3, 5, 8]) - >>> type(b) - - >>> type(a) - - >>> shm.name # We did not specify a name so one was chosen for us - 'psm_21467_46075' - - >>> # In either the same shell or a new Python shell on the same machine - >>> import numpy as np - >>> from multiprocessing import shared_memory - >>> # Attach to the existing shared memory block - >>> existing_shm = shared_memory.SharedMemory(name='psm_21467_46075') - >>> # Note that a.shape is (6,) and a.dtype is np.int64 in this example - >>> c = np.ndarray((6,), dtype=np.int64, buffer=existing_shm.buf) - >>> c - array([1, 1, 2, 3, 5, 8]) - >>> c[-1] = 888 - >>> c - array([ 1, 1, 2, 3, 5, 888]) - - >>> # Back in the first Python interactive shell, b reflects this change - >>> b - array([ 1, 1, 2, 3, 5, 888]) - - >>> # Clean up from within the second Python shell - >>> del c # Unnecessary; merely emphasizing the array is no longer used - >>> existing_shm.close() - - >>> # Clean up from within the first Python shell - >>> del b # Unnecessary; merely emphasizing the array is no longer used - >>> shm.close() - >>> shm.unlink() # Free and release the shared memory block at the very end - - -.. class:: SharedMemoryManager([address[, authkey]]) - :module: multiprocessing.managers - - A subclass of :class:`~multiprocessing.managers.BaseManager` which can be - used for the management of shared memory blocks across processes. - - A call to :meth:`~multiprocessing.managers.BaseManager.start` on a - :class:`SharedMemoryManager` instance causes a new process to be started. - This new process's sole purpose is to manage the life cycle - of all shared memory blocks created through it. To trigger the release - of all shared memory blocks managed by that process, call - :meth:`~multiprocessing.managers.BaseManager.shutdown()` on the instance. - This triggers a :meth:`SharedMemory.unlink()` call on all of the - :class:`SharedMemory` objects managed by that process and then - stops the process itself. By creating ``SharedMemory`` instances - through a ``SharedMemoryManager``, we avoid the need to manually track - and trigger the freeing of shared memory resources. - - This class provides methods for creating and returning :class:`SharedMemory` - instances and for creating a list-like object (:class:`ShareableList`) - backed by shared memory. - - Refer to :class:`multiprocessing.managers.BaseManager` for a description - of the inherited *address* and *authkey* optional input arguments and how - they may be used to connect to an existing ``SharedMemoryManager`` service - from other processes. - - .. method:: SharedMemory(size) - - Create and return a new :class:`SharedMemory` object with the - specified ``size`` in bytes. - - .. method:: ShareableList(sequence) - - Create and return a new :class:`ShareableList` object, initialized - by the values from the input ``sequence``. - - -The following example demonstrates the basic mechanisms of a -:class:`SharedMemoryManager`: - -.. doctest:: - :options: +SKIP - - >>> from multiprocessing.managers import SharedMemoryManager - >>> smm = SharedMemoryManager() - >>> smm.start() # Start the process that manages the shared memory blocks - >>> sl = smm.ShareableList(range(4)) - >>> sl - ShareableList([0, 1, 2, 3], name='psm_6572_7512') - >>> raw_shm = smm.SharedMemory(size=128) - >>> another_sl = smm.ShareableList('alpha') - >>> another_sl - ShareableList(['a', 'l', 'p', 'h', 'a'], name='psm_6572_12221') - >>> smm.shutdown() # Calls unlink() on sl, raw_shm, and another_sl - -The following example depicts a potentially more convenient pattern for using -:class:`SharedMemoryManager` objects via the :keyword:`with` statement to -ensure that all shared memory blocks are released after they are no longer -needed: - -.. doctest:: - :options: +SKIP - - >>> with SharedMemoryManager() as smm: - ... sl = smm.ShareableList(range(2000)) - ... # Divide the work among two processes, storing partial results in sl - ... p1 = Process(target=do_work, args=(sl, 0, 1000)) - ... p2 = Process(target=do_work, args=(sl, 1000, 2000)) - ... p1.start() - ... p2.start() # A multiprocessing.Pool might be more efficient - ... p1.join() - ... p2.join() # Wait for all work to complete in both processes - ... total_result = sum(sl) # Consolidate the partial results now in sl - -When using a :class:`SharedMemoryManager` in a :keyword:`with` statement, the -shared memory blocks created using that manager are all released when the -:keyword:`with` statement's code block finishes execution. - - -.. class:: ShareableList(sequence=None, \*, name=None) - - Provides a mutable list-like object where all values stored within are - stored in a shared memory block. This constrains storable values to - only the ``int`` (signed 64-bit), ``float``, ``bool``, ``str`` (less - than 10M bytes each when encoded as utf-8), ``bytes`` (less than 10M - bytes each), and ``None`` built-in data types. It also notably - differs from the built-in ``list`` type in that these lists can not - change their overall length (i.e. no append, insert, etc.) and do not - support the dynamic creation of new :class:`ShareableList` instances - via slicing. - - *sequence* is used in populating a new ``ShareableList`` full of values. - Set to ``None`` to instead attach to an already existing - ``ShareableList`` by its unique shared memory name. - - *name* is the unique name for the requested shared memory, as described - in the definition for :class:`SharedMemory`. When attaching to an - existing ``ShareableList``, specify its shared memory block's unique - name while leaving ``sequence`` set to ``None``. - - .. note:: - - A known issue exists for :class:`bytes` and :class:`str` values. - If they end with ``\x00`` nul bytes or characters, those may be - *silently stripped* when fetching them by index from the - :class:`ShareableList`. This ``.rstrip(b'\x00')`` behavior is - considered a bug and may go away in the future. See :gh:`106939`. - - For applications where rstripping of trailing nulls is a problem, - work around it by always unconditionally appending an extra non-0 - byte to the end of such values when storing and unconditionally - removing it when fetching: - - .. doctest:: - - >>> from multiprocessing import shared_memory - >>> nul_bug_demo = shared_memory.ShareableList(['?\x00', b'\x03\x02\x01\x00\x00\x00']) - >>> nul_bug_demo[0] - '?' - >>> nul_bug_demo[1] - b'\x03\x02\x01' - >>> nul_bug_demo.shm.unlink() - >>> padded = shared_memory.ShareableList(['?\x00\x07', b'\x03\x02\x01\x00\x00\x00\x07']) - >>> padded[0][:-1] - '?\x00' - >>> padded[1][:-1] - b'\x03\x02\x01\x00\x00\x00' - >>> padded.shm.unlink() - - .. method:: count(value) - - Returns the number of occurrences of ``value``. - - .. method:: index(value) - - Returns first index position of ``value``. Raises :exc:`ValueError` if - ``value`` is not present. - - .. attribute:: format - - Read-only attribute containing the :mod:`struct` packing format used by - all currently stored values. - - .. attribute:: shm - - The :class:`SharedMemory` instance where the values are stored. - - -The following example demonstrates basic use of a :class:`ShareableList` -instance: - - >>> from multiprocessing import shared_memory - >>> a = shared_memory.ShareableList(['howdy', b'HoWdY', -273.154, 100, None, True, 42]) - >>> [ type(entry) for entry in a ] - [, , , , , , ] - >>> a[2] - -273.154 - >>> a[2] = -78.5 - >>> a[2] - -78.5 - >>> a[2] = 'dry ice' # Changing data types is supported as well - >>> a[2] - 'dry ice' - >>> a[2] = 'larger than previously allocated storage space' - Traceback (most recent call last): - ... - ValueError: exceeds available storage for existing str - >>> a[2] - 'dry ice' - >>> len(a) - 7 - >>> a.index(42) - 6 - >>> a.count(b'howdy') - 0 - >>> a.count(b'HoWdY') - 1 - >>> a.shm.close() - >>> a.shm.unlink() - >>> del a # Use of a ShareableList after call to unlink() is unsupported - -The following example depicts how one, two, or many processes may access the -same :class:`ShareableList` by supplying the name of the shared memory block -behind it: - - >>> b = shared_memory.ShareableList(range(5)) # In a first process - >>> c = shared_memory.ShareableList(name=b.shm.name) # In a second process - >>> c - ShareableList([0, 1, 2, 3, 4], name='...') - >>> c[-1] = -999 - >>> b[-1] - -999 - >>> b.shm.close() - >>> c.shm.close() - >>> c.shm.unlink() - -The following examples demonstrates that ``ShareableList`` -(and underlying ``SharedMemory``) objects -can be pickled and unpickled if needed. -Note, that it will still be the same shared object. -This happens, because the deserialized object has -the same unique name and is just attached to an existing -object with the same name (if the object is still alive): - - >>> import pickle - >>> from multiprocessing import shared_memory - >>> sl = shared_memory.ShareableList(range(10)) - >>> list(sl) - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - - >>> deserialized_sl = pickle.loads(pickle.dumps(sl)) - >>> list(deserialized_sl) - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - - >>> sl[0] = -1 - >>> deserialized_sl[1] = -2 - >>> list(sl) - [-1, -2, 2, 3, 4, 5, 6, 7, 8, 9] - >>> list(deserialized_sl) - [-1, -2, 2, 3, 4, 5, 6, 7, 8, 9] - - >>> sl.shm.close() - >>> sl.shm.unlink() diff --git a/Doc/library/netdata.rst b/Doc/library/netdata.rst deleted file mode 100644 index 1541e2a5444590..00000000000000 --- a/Doc/library/netdata.rst +++ /dev/null @@ -1,20 +0,0 @@ - -.. _netdata: - -********************** -Internet Data Handling -********************** - -This chapter describes modules which support handling data formats commonly used -on the internet. - - -.. toctree:: - - email.rst - json.rst - mailbox.rst - mimetypes.rst - base64.rst - binascii.rst - quopri.rst diff --git a/Doc/library/netrc.rst b/Doc/library/netrc.rst deleted file mode 100644 index c36e5cfecfc6a8..00000000000000 --- a/Doc/library/netrc.rst +++ /dev/null @@ -1,101 +0,0 @@ - -:mod:`netrc` --- netrc file processing -====================================== - -.. module:: netrc - :synopsis: Loading of .netrc files. - -.. moduleauthor:: Eric S. Raymond -.. sectionauthor:: Eric S. Raymond - -**Source code:** :source:`Lib/netrc.py` - --------------- - -The :class:`~netrc.netrc` class parses and encapsulates the netrc file format used by -the Unix :program:`ftp` program and other FTP clients. - - -.. class:: netrc([file]) - - A :class:`~netrc.netrc` instance or subclass instance encapsulates data from a netrc - file. The initialization argument, if present, specifies the file to parse. If - no argument is given, the file :file:`.netrc` in the user's home directory -- - as determined by :func:`os.path.expanduser` -- will be read. Otherwise, - a :exc:`FileNotFoundError` exception will be raised. - Parse errors will raise :exc:`NetrcParseError` with diagnostic - information including the file name, line number, and terminating token. - If no argument is specified on a POSIX system, the presence of passwords in - the :file:`.netrc` file will raise a :exc:`NetrcParseError` if the file - ownership or permissions are insecure (owned by a user other than the user - running the process, or accessible for read or write by any other user). - This implements security behavior equivalent to that of ftp and other - programs that use :file:`.netrc`. - - .. versionchanged:: 3.4 Added the POSIX permission check. - - .. versionchanged:: 3.7 - :func:`os.path.expanduser` is used to find the location of the - :file:`.netrc` file when *file* is not passed as argument. - - .. versionchanged:: 3.10 - :class:`netrc` try UTF-8 encoding before using locale specific - encoding. - The entry in the netrc file no longer needs to contain all tokens. The missing - tokens' value default to an empty string. All the tokens and their values now - can contain arbitrary characters, like whitespace and non-ASCII characters. - If the login name is anonymous, it won't trigger the security check. - - -.. exception:: NetrcParseError - - Exception raised by the :class:`~netrc.netrc` class when syntactical errors are - encountered in source text. Instances of this exception provide three - interesting attributes: - - .. attribute:: msg - - Textual explanation of the error. - - .. attribute:: filename - - The name of the source file. - - .. attribute:: lineno - - The line number on which the error was found. - - -.. _netrc-objects: - -netrc Objects -------------- - -A :class:`~netrc.netrc` instance has the following methods: - - -.. method:: netrc.authenticators(host) - - Return a 3-tuple ``(login, account, password)`` of authenticators for *host*. - If the netrc file did not contain an entry for the given host, return the tuple - associated with the 'default' entry. If neither matching host nor default entry - is available, return ``None``. - - -.. method:: netrc.__repr__() - - Dump the class data as a string in the format of a netrc file. (This discards - comments and may reorder the entries.) - -Instances of :class:`~netrc.netrc` have public instance variables: - - -.. attribute:: netrc.hosts - - Dictionary mapping host names to ``(login, account, password)`` tuples. The - 'default' entry, if any, is represented as a pseudo-host by that name. - - -.. attribute:: netrc.macros - - Dictionary mapping macro names to string lists. diff --git a/Doc/library/nis.rst b/Doc/library/nis.rst deleted file mode 100644 index 3fa7916c37b6a5..00000000000000 --- a/Doc/library/nis.rst +++ /dev/null @@ -1,72 +0,0 @@ - -:mod:`nis` --- Interface to Sun's NIS (Yellow Pages) -==================================================== - -.. module:: nis - :platform: Unix - :synopsis: Interface to Sun's NIS (Yellow Pages) library. - :deprecated: - -.. moduleauthor:: Fred Gansevles -.. sectionauthor:: Moshe Zadka - -.. deprecated-removed:: 3.11 3.13 - The :mod:`nis` module is deprecated - (see :pep:`PEP 594 <594#nis>` for details). - --------------- - -The :mod:`nis` module gives a thin wrapper around the NIS library, useful for -central administration of several hosts. - -Because NIS exists only on Unix systems, this module is only available for Unix. - -.. include:: ../includes/wasm-notavail.rst - -The :mod:`nis` module defines the following functions: - - -.. function:: match(key, mapname, domain=default_domain) - - Return the match for *key* in map *mapname*, or raise an error - (:exc:`nis.error`) if there is none. Both should be strings, *key* is 8-bit - clean. Return value is an arbitrary array of bytes (may contain ``NULL`` and - other joys). - - Note that *mapname* is first checked if it is an alias to another name. - - The *domain* argument allows overriding the NIS domain used for the lookup. If - unspecified, lookup is in the default NIS domain. - - -.. function:: cat(mapname, domain=default_domain) - - Return a dictionary mapping *key* to *value* such that ``match(key, - mapname)==value``. Note that both keys and values of the dictionary are - arbitrary arrays of bytes. - - Note that *mapname* is first checked if it is an alias to another name. - - The *domain* argument allows overriding the NIS domain used for the lookup. If - unspecified, lookup is in the default NIS domain. - - -.. function:: maps(domain=default_domain) - - Return a list of all valid maps. - - The *domain* argument allows overriding the NIS domain used for the lookup. If - unspecified, lookup is in the default NIS domain. - - -.. function:: get_default_domain() - - Return the system default NIS domain. - - -The :mod:`nis` module defines the following exception: - -.. exception:: error - - An error raised when a NIS function returns an error code. - diff --git a/Doc/library/nntplib.rst b/Doc/library/nntplib.rst deleted file mode 100644 index a36c8a5dccec5d..00000000000000 --- a/Doc/library/nntplib.rst +++ /dev/null @@ -1,569 +0,0 @@ -:mod:`nntplib` --- NNTP protocol client -======================================= - -.. module:: nntplib - :synopsis: NNTP protocol client (requires sockets). - :deprecated: - -**Source code:** :source:`Lib/nntplib.py` - -.. index:: - pair: NNTP; protocol - single: Network News Transfer Protocol - -.. deprecated:: 3.11 - The :mod:`nntplib` module is deprecated (see :pep:`594` for details). - --------------- - -This module defines the class :class:`NNTP` which implements the client side of -the Network News Transfer Protocol. It can be used to implement a news reader -or poster, or automated news processors. It is compatible with :rfc:`3977` -as well as the older :rfc:`977` and :rfc:`2980`. - -.. include:: ../includes/wasm-notavail.rst - -Here are two small examples of how it can be used. To list some statistics -about a newsgroup and print the subjects of the last 10 articles:: - - >>> s = nntplib.NNTP('news.gmane.io') - >>> resp, count, first, last, name = s.group('gmane.comp.python.committers') - >>> print('Group', name, 'has', count, 'articles, range', first, 'to', last) - Group gmane.comp.python.committers has 1096 articles, range 1 to 1096 - >>> resp, overviews = s.over((last - 9, last)) - >>> for id, over in overviews: - ... print(id, nntplib.decode_header(over['subject'])) - ... - 1087 Re: Commit privileges for Łukasz Langa - 1088 Re: 3.2 alpha 2 freeze - 1089 Re: 3.2 alpha 2 freeze - 1090 Re: Commit privileges for Łukasz Langa - 1091 Re: Commit privileges for Łukasz Langa - 1092 Updated ssh key - 1093 Re: Updated ssh key - 1094 Re: Updated ssh key - 1095 Hello fellow committers! - 1096 Re: Hello fellow committers! - >>> s.quit() - '205 Bye!' - -To post an article from a binary file (this assumes that the article has valid -headers, and that you have right to post on the particular newsgroup):: - - >>> s = nntplib.NNTP('news.gmane.io') - >>> f = open('article.txt', 'rb') - >>> s.post(f) - '240 Article posted successfully.' - >>> s.quit() - '205 Bye!' - -The module itself defines the following classes: - - -.. class:: NNTP(host, port=119, user=None, password=None, readermode=None, usenetrc=False, [timeout]) - - Return a new :class:`NNTP` object, representing a connection - to the NNTP server running on host *host*, listening at port *port*. - An optional *timeout* can be specified for the socket connection. - If the optional *user* and *password* are provided, or if suitable - credentials are present in :file:`/.netrc` and the optional flag *usenetrc* - is true, the ``AUTHINFO USER`` and ``AUTHINFO PASS`` commands are used - to identify and authenticate the user to the server. If the optional - flag *readermode* is true, then a ``mode reader`` command is sent before - authentication is performed. Reader mode is sometimes necessary if you are - connecting to an NNTP server on the local machine and intend to call - reader-specific commands, such as ``group``. If you get unexpected - :exc:`NNTPPermanentError`\ s, you might need to set *readermode*. - The :class:`NNTP` class supports the :keyword:`with` statement to - unconditionally consume :exc:`OSError` exceptions and to close the NNTP - connection when done, e.g.: - - >>> from nntplib import NNTP - >>> with NNTP('news.gmane.io') as n: - ... n.group('gmane.comp.python.committers') - ... # doctest: +SKIP - ('211 1755 1 1755 gmane.comp.python.committers', 1755, 1, 1755, 'gmane.comp.python.committers') - >>> - - .. audit-event:: nntplib.connect self,host,port nntplib.NNTP - - .. audit-event:: nntplib.putline self,line nntplib.NNTP - - All commands will raise an :ref:`auditing event ` - ``nntplib.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. - - .. versionchanged:: 3.2 - *usenetrc* is now ``False`` by default. - - .. versionchanged:: 3.3 - Support for the :keyword:`with` statement was added. - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - -.. class:: NNTP_SSL(host, port=563, user=None, password=None, ssl_context=None, readermode=None, usenetrc=False, [timeout]) - - Return a new :class:`NNTP_SSL` object, representing an encrypted - connection to the NNTP server running on host *host*, listening at - port *port*. :class:`NNTP_SSL` objects have the same methods as - :class:`NNTP` objects. If *port* is omitted, port 563 (NNTPS) is used. - *ssl_context* is also optional, and is a :class:`~ssl.SSLContext` object. - Please read :ref:`ssl-security` for best practices. - All other parameters behave the same as for :class:`NNTP`. - - Note that SSL-on-563 is discouraged per :rfc:`4642`, in favor of - STARTTLS as described below. However, some servers only support the - former. - - .. audit-event:: nntplib.connect self,host,port nntplib.NNTP_SSL - - .. audit-event:: nntplib.putline self,line nntplib.NNTP_SSL - - All commands will raise an :ref:`auditing event ` - ``nntplib.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - -.. exception:: NNTPError - - Derived from the standard exception :exc:`Exception`, this is the base - class for all exceptions raised by the :mod:`nntplib` module. Instances - of this class have the following attribute: - - .. attribute:: response - - The response of the server if available, as a :class:`str` object. - - -.. exception:: NNTPReplyError - - Exception raised when an unexpected reply is received from the server. - - -.. exception:: NNTPTemporaryError - - Exception raised when a response code in the range 400--499 is received. - - -.. exception:: NNTPPermanentError - - Exception raised when a response code in the range 500--599 is received. - - -.. exception:: NNTPProtocolError - - Exception raised when a reply is received from the server that does not begin - with a digit in the range 1--5. - - -.. exception:: NNTPDataError - - Exception raised when there is some error in the response data. - - -.. _nntp-objects: - -NNTP Objects ------------- - -When connected, :class:`NNTP` and :class:`NNTP_SSL` objects support the -following methods and attributes. - -Attributes -^^^^^^^^^^ - -.. attribute:: NNTP.nntp_version - - An integer representing the version of the NNTP protocol supported by the - server. In practice, this should be ``2`` for servers advertising - :rfc:`3977` compliance and ``1`` for others. - - .. versionadded:: 3.2 - -.. attribute:: NNTP.nntp_implementation - - A string describing the software name and version of the NNTP server, - or :const:`None` if not advertised by the server. - - .. versionadded:: 3.2 - -Methods -^^^^^^^ - -The *response* that is returned as the first item in the return tuple of almost -all methods is the server's response: a string beginning with a three-digit -code. If the server's response indicates an error, the method raises one of -the above exceptions. - -Many of the following methods take an optional keyword-only argument *file*. -When the *file* argument is supplied, it must be either a :term:`file object` -opened for binary writing, or the name of an on-disk file to be written to. -The method will then write any data returned by the server (except for the -response line and the terminating dot) to the file; any list of lines, -tuples or objects that the method normally returns will be empty. - -.. versionchanged:: 3.2 - Many of the following methods have been reworked and fixed, which makes - them incompatible with their 3.1 counterparts. - - -.. method:: NNTP.quit() - - Send a ``QUIT`` command and close the connection. Once this method has been - called, no other methods of the NNTP object should be called. - - -.. method:: NNTP.getwelcome() - - Return the welcome message sent by the server in reply to the initial - connection. (This message sometimes contains disclaimers or help information - that may be relevant to the user.) - - -.. method:: NNTP.getcapabilities() - - Return the :rfc:`3977` capabilities advertised by the server, as a - :class:`dict` instance mapping capability names to (possibly empty) lists - of values. On legacy servers which don't understand the ``CAPABILITIES`` - command, an empty dictionary is returned instead. - - >>> s = NNTP('news.gmane.io') - >>> 'POST' in s.getcapabilities() - True - - .. versionadded:: 3.2 - - -.. method:: NNTP.login(user=None, password=None, usenetrc=True) - - Send ``AUTHINFO`` commands with the user name and password. If *user* - and *password* are ``None`` and *usenetrc* is true, credentials from - ``~/.netrc`` will be used if possible. - - Unless intentionally delayed, login is normally performed during the - :class:`NNTP` object initialization and separately calling this function - is unnecessary. To force authentication to be delayed, you must not set - *user* or *password* when creating the object, and must set *usenetrc* to - False. - - .. versionadded:: 3.2 - - -.. method:: NNTP.starttls(context=None) - - Send a ``STARTTLS`` command. This will enable encryption on the NNTP - connection. The *context* argument is optional and should be a - :class:`ssl.SSLContext` object. Please read :ref:`ssl-security` for best - practices. - - Note that this may not be done after authentication information has - been transmitted, and authentication occurs by default if possible during a - :class:`NNTP` object initialization. See :meth:`NNTP.login` for information - on suppressing this behavior. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :data:`ssl.HAS_SNI`). - -.. method:: NNTP.newgroups(date, *, file=None) - - Send a ``NEWGROUPS`` command. The *date* argument should be a - :class:`datetime.date` or :class:`datetime.datetime` object. - Return a pair ``(response, groups)`` where *groups* is a list representing - the groups that are new since the given *date*. If *file* is supplied, - though, then *groups* will be empty. - - >>> from datetime import date, timedelta - >>> resp, groups = s.newgroups(date.today() - timedelta(days=3)) - >>> len(groups) # doctest: +SKIP - 85 - >>> groups[0] # doctest: +SKIP - GroupInfo(group='gmane.network.tor.devel', last='4', first='1', flag='m') - - -.. method:: NNTP.newnews(group, date, *, file=None) - - Send a ``NEWNEWS`` command. Here, *group* is a group name or ``'*'``, and - *date* has the same meaning as for :meth:`newgroups`. Return a pair - ``(response, articles)`` where *articles* is a list of message ids. - - This command is frequently disabled by NNTP server administrators. - - -.. method:: NNTP.list(group_pattern=None, *, file=None) - - Send a ``LIST`` or ``LIST ACTIVE`` command. Return a pair - ``(response, list)`` where *list* is a list of tuples representing all - the groups available from this NNTP server, optionally matching the - pattern string *group_pattern*. Each tuple has the form - ``(group, last, first, flag)``, where *group* is a group name, *last* - and *first* are the last and first article numbers, and *flag* usually - takes one of these values: - - * ``y``: Local postings and articles from peers are allowed. - * ``m``: The group is moderated and all postings must be approved. - * ``n``: No local postings are allowed, only articles from peers. - * ``j``: Articles from peers are filed in the junk group instead. - * ``x``: No local postings, and articles from peers are ignored. - * ``=foo.bar``: Articles are filed in the ``foo.bar`` group instead. - - If *flag* has another value, then the status of the newsgroup should be - considered unknown. - - This command can return very large results, especially if *group_pattern* - is not specified. It is best to cache the results offline unless you - really need to refresh them. - - .. versionchanged:: 3.2 - *group_pattern* was added. - - -.. method:: NNTP.descriptions(grouppattern) - - Send a ``LIST NEWSGROUPS`` command, where *grouppattern* is a wildmat string as - specified in :rfc:`3977` (it's essentially the same as DOS or UNIX shell wildcard - strings). Return a pair ``(response, descriptions)``, where *descriptions* - is a dictionary mapping group names to textual descriptions. - - >>> resp, descs = s.descriptions('gmane.comp.python.*') - >>> len(descs) # doctest: +SKIP - 295 - >>> descs.popitem() # doctest: +SKIP - ('gmane.comp.python.bio.general', 'BioPython discussion list (Moderated)') - - -.. method:: NNTP.description(group) - - Get a description for a single group *group*. If more than one group matches - (if 'group' is a real wildmat string), return the first match. If no group - matches, return an empty string. - - This elides the response code from the server. If the response code is needed, - use :meth:`descriptions`. - - -.. method:: NNTP.group(name) - - Send a ``GROUP`` command, where *name* is the group name. The group is - selected as the current group, if it exists. Return a tuple - ``(response, count, first, last, name)`` where *count* is the (estimated) - number of articles in the group, *first* is the first article number in - the group, *last* is the last article number in the group, and *name* - is the group name. - - -.. method:: NNTP.over(message_spec, *, file=None) - - Send an ``OVER`` command, or an ``XOVER`` command on legacy servers. - *message_spec* can be either a string representing a message id, or - a ``(first, last)`` tuple of numbers indicating a range of articles in - the current group, or a ``(first, None)`` tuple indicating a range of - articles starting from *first* to the last article in the current group, - or :const:`None` to select the current article in the current group. - - Return a pair ``(response, overviews)``. *overviews* is a list of - ``(article_number, overview)`` tuples, one for each article selected - by *message_spec*. Each *overview* is a dictionary with the same number - of items, but this number depends on the server. These items are either - message headers (the key is then the lower-cased header name) or metadata - items (the key is then the metadata name prepended with ``":"``). The - following items are guaranteed to be present by the NNTP specification: - - * the ``subject``, ``from``, ``date``, ``message-id`` and ``references`` - headers - * the ``:bytes`` metadata: the number of bytes in the entire raw article - (including headers and body) - * the ``:lines`` metadata: the number of lines in the article body - - The value of each item is either a string, or :const:`None` if not present. - - It is advisable to use the :func:`decode_header` function on header - values when they may contain non-ASCII characters:: - - >>> _, _, first, last, _ = s.group('gmane.comp.python.devel') - >>> resp, overviews = s.over((last, last)) - >>> art_num, over = overviews[0] - >>> art_num - 117216 - >>> list(over.keys()) - ['xref', 'from', ':lines', ':bytes', 'references', 'date', 'message-id', 'subject'] - >>> over['from'] - '=?UTF-8?B?Ik1hcnRpbiB2LiBMw7Z3aXMi?= ' - >>> nntplib.decode_header(over['from']) - '"Martin v. Löwis" ' - - .. versionadded:: 3.2 - - -.. method:: NNTP.help(*, file=None) - - Send a ``HELP`` command. Return a pair ``(response, list)`` where *list* is a - list of help strings. - - -.. method:: NNTP.stat(message_spec=None) - - Send a ``STAT`` command, where *message_spec* is either a message id - (enclosed in ``'<'`` and ``'>'``) or an article number in the current group. - If *message_spec* is omitted or :const:`None`, the current article in the - current group is considered. Return a triple ``(response, number, id)`` - where *number* is the article number and *id* is the message id. - - >>> _, _, first, last, _ = s.group('gmane.comp.python.devel') - >>> resp, number, message_id = s.stat(first) - >>> number, message_id - (9099, '<20030112190404.GE29873@epoch.metaslash.com>') - - -.. method:: NNTP.next() - - Send a ``NEXT`` command. Return as for :meth:`.stat`. - - -.. method:: NNTP.last() - - Send a ``LAST`` command. Return as for :meth:`.stat`. - - -.. method:: NNTP.article(message_spec=None, *, file=None) - - Send an ``ARTICLE`` command, where *message_spec* has the same meaning as - for :meth:`.stat`. Return a tuple ``(response, info)`` where *info* - is a :class:`~collections.namedtuple` with three attributes *number*, - *message_id* and *lines* (in that order). *number* is the article number - in the group (or 0 if the information is not available), *message_id* the - message id as a string, and *lines* a list of lines (without terminating - newlines) comprising the raw message including headers and body. - - >>> resp, info = s.article('<20030112190404.GE29873@epoch.metaslash.com>') - >>> info.number - 0 - >>> info.message_id - '<20030112190404.GE29873@epoch.metaslash.com>' - >>> len(info.lines) - 65 - >>> info.lines[0] - b'Path: main.gmane.org!not-for-mail' - >>> info.lines[1] - b'From: Neal Norwitz ' - >>> info.lines[-3:] - [b'There is a patch for 2.3 as well as 2.2.', b'', b'Neal'] - - -.. method:: NNTP.head(message_spec=None, *, file=None) - - Same as :meth:`article()`, but sends a ``HEAD`` command. The *lines* - returned (or written to *file*) will only contain the message headers, not - the body. - - -.. method:: NNTP.body(message_spec=None, *, file=None) - - Same as :meth:`article()`, but sends a ``BODY`` command. The *lines* - returned (or written to *file*) will only contain the message body, not the - headers. - - -.. method:: NNTP.post(data) - - Post an article using the ``POST`` command. The *data* argument is either - a :term:`file object` opened for binary reading, or any iterable of bytes - objects (representing raw lines of the article to be posted). It should - represent a well-formed news article, including the required headers. The - :meth:`post` method automatically escapes lines beginning with ``.`` and - appends the termination line. - - If the method succeeds, the server's response is returned. If the server - refuses posting, a :class:`NNTPReplyError` is raised. - - -.. method:: NNTP.ihave(message_id, data) - - Send an ``IHAVE`` command. *message_id* is the id of the message to send - to the server (enclosed in ``'<'`` and ``'>'``). The *data* parameter - and the return value are the same as for :meth:`post()`. - - -.. method:: NNTP.date() - - Return a pair ``(response, date)``. *date* is a :class:`~datetime.datetime` - object containing the current date and time of the server. - - -.. method:: NNTP.slave() - - Send a ``SLAVE`` command. Return the server's *response*. - - -.. method:: NNTP.set_debuglevel(level) - - Set the instance's debugging level. This controls the amount of debugging - output printed. The default, ``0``, produces no debugging output. A value of - ``1`` produces a moderate amount of debugging output, generally a single line - per request or response. A value of ``2`` or higher produces the maximum amount - of debugging output, logging each line sent and received on the connection - (including message text). - - -The following are optional NNTP extensions defined in :rfc:`2980`. Some of -them have been superseded by newer commands in :rfc:`3977`. - - -.. method:: NNTP.xhdr(hdr, str, *, file=None) - - Send an ``XHDR`` command. The *hdr* argument is a header keyword, e.g. - ``'subject'``. The *str* argument should have the form ``'first-last'`` - where *first* and *last* are the first and last article numbers to search. - Return a pair ``(response, list)``, where *list* is a list of pairs ``(id, - text)``, where *id* is an article number (as a string) and *text* is the text of - the requested header for that article. If the *file* parameter is supplied, then - the output of the ``XHDR`` command is stored in a file. If *file* is a string, - then the method will open a file with that name, write to it then close it. - If *file* is a :term:`file object`, then it will start calling :meth:`write` on - it to store the lines of the command output. If *file* is supplied, then the - returned *list* is an empty list. - - -.. method:: NNTP.xover(start, end, *, file=None) - - Send an ``XOVER`` command. *start* and *end* are article numbers - delimiting the range of articles to select. The return value is the - same of for :meth:`over()`. It is recommended to use :meth:`over()` - instead, since it will automatically use the newer ``OVER`` command - if available. - - -Utility functions ------------------ - -The module also defines the following utility function: - - -.. function:: decode_header(header_str) - - Decode a header value, un-escaping any escaped non-ASCII characters. - *header_str* must be a :class:`str` object. The unescaped value is - returned. Using this function is recommended to display some headers - in a human readable form:: - - >>> decode_header("Some subject") - 'Some subject' - >>> decode_header("=?ISO-8859-15?Q?D=E9buter_en_Python?=") - 'Débuter en Python' - >>> decode_header("Re: =?UTF-8?B?cHJvYmzDqG1lIGRlIG1hdHJpY2U=?=") - 'Re: problème de matrice' diff --git a/Doc/library/numbers.rst b/Doc/library/numbers.rst deleted file mode 100644 index 2a05b56db051f9..00000000000000 --- a/Doc/library/numbers.rst +++ /dev/null @@ -1,227 +0,0 @@ -:mod:`numbers` --- Numeric abstract base classes -================================================ - -.. module:: numbers - :synopsis: Numeric abstract base classes (Complex, Real, Integral, etc.). - -**Source code:** :source:`Lib/numbers.py` - --------------- - -The :mod:`numbers` module (:pep:`3141`) defines a hierarchy of numeric -:term:`abstract base classes ` which progressively define -more operations. None of the types defined in this module are intended to be instantiated. - - -.. class:: Number - - The root of the numeric hierarchy. If you just want to check if an argument - *x* is a number, without caring what kind, use ``isinstance(x, Number)``. - - -The numeric tower ------------------ - -.. class:: Complex - - Subclasses of this type describe complex numbers and include the operations - that work on the built-in :class:`complex` type. These are: conversions to - :class:`complex` and :class:`bool`, :attr:`.real`, :attr:`.imag`, ``+``, - ``-``, ``*``, ``/``, ``**``, :func:`abs`, :meth:`conjugate`, ``==``, and - ``!=``. All except ``-`` and ``!=`` are abstract. - - .. attribute:: real - - Abstract. Retrieves the real component of this number. - - .. attribute:: imag - - Abstract. Retrieves the imaginary component of this number. - - .. abstractmethod:: conjugate() - - Abstract. Returns the complex conjugate. For example, ``(1+3j).conjugate() - == (1-3j)``. - -.. class:: Real - - To :class:`Complex`, :class:`Real` adds the operations that work on real - numbers. - - In short, those are: a conversion to :class:`float`, :func:`math.trunc`, - :func:`round`, :func:`math.floor`, :func:`math.ceil`, :func:`divmod`, ``//``, - ``%``, ``<``, ``<=``, ``>``, and ``>=``. - - Real also provides defaults for :func:`complex`, :attr:`~Complex.real`, - :attr:`~Complex.imag`, and :meth:`~Complex.conjugate`. - - -.. class:: Rational - - Subtypes :class:`Real` and adds :attr:`~Rational.numerator` and - :attr:`~Rational.denominator` properties. It also provides a default for - :func:`float`. - - The :attr:`~Rational.numerator` and :attr:`~Rational.denominator` values - should be instances of :class:`Integral` and should be in lowest terms with - :attr:`~Rational.denominator` positive. - - .. attribute:: numerator - - Abstract. - - .. attribute:: denominator - - Abstract. - - -.. class:: Integral - - Subtypes :class:`Rational` and adds a conversion to :class:`int`. Provides - defaults for :func:`float`, :attr:`~Rational.numerator`, and - :attr:`~Rational.denominator`. Adds abstract methods for :func:`pow` with - modulus and bit-string operations: ``<<``, ``>>``, ``&``, ``^``, ``|``, - ``~``. - - -Notes for type implementors ---------------------------- - -Implementors should be careful to make equal numbers equal and hash -them to the same values. This may be subtle if there are two different -extensions of the real numbers. For example, :class:`fractions.Fraction` -implements :func:`hash` as follows:: - - def __hash__(self): - if self.denominator == 1: - # Get integers right. - return hash(self.numerator) - # Expensive check, but definitely correct. - if self == float(self): - return hash(float(self)) - else: - # Use tuple's hash to avoid a high collision rate on - # simple fractions. - return hash((self.numerator, self.denominator)) - - -Adding More Numeric ABCs -~~~~~~~~~~~~~~~~~~~~~~~~ - -There are, of course, more possible ABCs for numbers, and this would -be a poor hierarchy if it precluded the possibility of adding -those. You can add ``MyFoo`` between :class:`Complex` and -:class:`Real` with:: - - class MyFoo(Complex): ... - MyFoo.register(Real) - - -.. _implementing-the-arithmetic-operations: - -Implementing the arithmetic operations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -We want to implement the arithmetic operations so that mixed-mode -operations either call an implementation whose author knew about the -types of both arguments, or convert both to the nearest built in type -and do the operation there. For subtypes of :class:`Integral`, this -means that :meth:`__add__` and :meth:`__radd__` should be defined as:: - - class MyIntegral(Integral): - - def __add__(self, other): - if isinstance(other, MyIntegral): - return do_my_adding_stuff(self, other) - elif isinstance(other, OtherTypeIKnowAbout): - return do_my_other_adding_stuff(self, other) - else: - return NotImplemented - - def __radd__(self, other): - if isinstance(other, MyIntegral): - return do_my_adding_stuff(other, self) - elif isinstance(other, OtherTypeIKnowAbout): - return do_my_other_adding_stuff(other, self) - elif isinstance(other, Integral): - return int(other) + int(self) - elif isinstance(other, Real): - return float(other) + float(self) - elif isinstance(other, Complex): - return complex(other) + complex(self) - else: - return NotImplemented - - -There are 5 different cases for a mixed-type operation on subclasses -of :class:`Complex`. I'll refer to all of the above code that doesn't -refer to ``MyIntegral`` and ``OtherTypeIKnowAbout`` as -"boilerplate". ``a`` will be an instance of ``A``, which is a subtype -of :class:`Complex` (``a : A <: Complex``), and ``b : B <: -Complex``. I'll consider ``a + b``: - -1. If ``A`` defines an :meth:`__add__` which accepts ``b``, all is - well. -2. If ``A`` falls back to the boilerplate code, and it were to - return a value from :meth:`__add__`, we'd miss the possibility - that ``B`` defines a more intelligent :meth:`__radd__`, so the - boilerplate should return :const:`NotImplemented` from - :meth:`__add__`. (Or ``A`` may not implement :meth:`__add__` at - all.) -3. Then ``B``'s :meth:`__radd__` gets a chance. If it accepts - ``a``, all is well. -4. If it falls back to the boilerplate, there are no more possible - methods to try, so this is where the default implementation - should live. -5. If ``B <: A``, Python tries ``B.__radd__`` before - ``A.__add__``. This is ok, because it was implemented with - knowledge of ``A``, so it can handle those instances before - delegating to :class:`Complex`. - -If ``A <: Complex`` and ``B <: Real`` without sharing any other knowledge, -then the appropriate shared operation is the one involving the built -in :class:`complex`, and both :meth:`__radd__` s land there, so ``a+b -== b+a``. - -Because most of the operations on any given type will be very similar, -it can be useful to define a helper function which generates the -forward and reverse instances of any given operator. For example, -:class:`fractions.Fraction` uses:: - - def _operator_fallbacks(monomorphic_operator, fallback_operator): - def forward(a, b): - if isinstance(b, (int, Fraction)): - return monomorphic_operator(a, b) - elif isinstance(b, float): - return fallback_operator(float(a), b) - elif isinstance(b, complex): - return fallback_operator(complex(a), b) - else: - return NotImplemented - forward.__name__ = '__' + fallback_operator.__name__ + '__' - forward.__doc__ = monomorphic_operator.__doc__ - - def reverse(b, a): - if isinstance(a, Rational): - # Includes ints. - return monomorphic_operator(a, b) - elif isinstance(a, Real): - return fallback_operator(float(a), float(b)) - elif isinstance(a, Complex): - return fallback_operator(complex(a), complex(b)) - else: - return NotImplemented - reverse.__name__ = '__r' + fallback_operator.__name__ + '__' - reverse.__doc__ = monomorphic_operator.__doc__ - - return forward, reverse - - def _add(a, b): - """a + b""" - return Fraction(a.numerator * b.denominator + - b.numerator * a.denominator, - a.denominator * b.denominator) - - __add__, __radd__ = _operator_fallbacks(_add, operator.add) - - # ... diff --git a/Doc/library/numeric.rst b/Doc/library/numeric.rst deleted file mode 100644 index 7c76a479d73b26..00000000000000 --- a/Doc/library/numeric.rst +++ /dev/null @@ -1,26 +0,0 @@ - -.. _numeric: - -******************************** -Numeric and Mathematical Modules -******************************** - -The modules described in this chapter provide numeric and math-related functions -and data types. The :mod:`numbers` module defines an abstract hierarchy of -numeric types. The :mod:`math` and :mod:`cmath` modules contain various -mathematical functions for floating-point and complex numbers. The :mod:`decimal` -module supports exact representations of decimal numbers, using arbitrary precision -arithmetic. - -The following modules are documented in this chapter: - - -.. toctree:: - - numbers.rst - math.rst - cmath.rst - decimal.rst - fractions.rst - random.rst - statistics.rst diff --git a/Doc/library/operator.rst b/Doc/library/operator.rst deleted file mode 100644 index 96f2c287875d41..00000000000000 --- a/Doc/library/operator.rst +++ /dev/null @@ -1,571 +0,0 @@ -:mod:`operator` --- Standard operators as functions -=================================================== - -.. module:: operator - :synopsis: Functions corresponding to the standard operators. - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/operator.py` - -.. testsetup:: - - import operator - from operator import itemgetter, iadd - --------------- - -The :mod:`operator` module exports a set of efficient functions corresponding to -the intrinsic operators of Python. For example, ``operator.add(x, y)`` is -equivalent to the expression ``x+y``. Many function names are those used for -special methods, without the double underscores. For backward compatibility, -many of these have a variant with the double underscores kept. The variants -without the double underscores are preferred for clarity. - -The functions fall into categories that perform object comparisons, logical -operations, mathematical operations and sequence operations. - -The object comparison functions are useful for all objects, and are named after -the rich comparison operators they support: - - -.. function:: lt(a, b) - le(a, b) - eq(a, b) - ne(a, b) - ge(a, b) - gt(a, b) - __lt__(a, b) - __le__(a, b) - __eq__(a, b) - __ne__(a, b) - __ge__(a, b) - __gt__(a, b) - - Perform "rich comparisons" between *a* and *b*. Specifically, ``lt(a, b)`` is - equivalent to ``a < b``, ``le(a, b)`` is equivalent to ``a <= b``, ``eq(a, - b)`` is equivalent to ``a == b``, ``ne(a, b)`` is equivalent to ``a != b``, - ``gt(a, b)`` is equivalent to ``a > b`` and ``ge(a, b)`` is equivalent to ``a - >= b``. Note that these functions can return any value, which may - or may not be interpretable as a Boolean value. See - :ref:`comparisons` for more information about rich comparisons. - - -The logical operations are also generally applicable to all objects, and support -truth tests, identity tests, and boolean operations: - - -.. function:: not_(obj) - __not__(obj) - - Return the outcome of :keyword:`not` *obj*. (Note that there is no - :meth:`!__not__` method for object instances; only the interpreter core defines - this operation. The result is affected by the :meth:`~object.__bool__` and - :meth:`~object.__len__` methods.) - - -.. function:: truth(obj) - - Return :const:`True` if *obj* is true, and :const:`False` otherwise. This is - equivalent to using the :class:`bool` constructor. - - -.. function:: is_(a, b) - - Return ``a is b``. Tests object identity. - - -.. function:: is_not(a, b) - - Return ``a is not b``. Tests object identity. - - -The mathematical and bitwise operations are the most numerous: - - -.. function:: abs(obj) - __abs__(obj) - - Return the absolute value of *obj*. - - -.. function:: add(a, b) - __add__(a, b) - - Return ``a + b``, for *a* and *b* numbers. - - -.. function:: and_(a, b) - __and__(a, b) - - Return the bitwise and of *a* and *b*. - - -.. function:: floordiv(a, b) - __floordiv__(a, b) - - Return ``a // b``. - - -.. function:: index(a) - __index__(a) - - Return *a* converted to an integer. Equivalent to ``a.__index__()``. - - .. versionchanged:: 3.10 - The result always has exact type :class:`int`. Previously, the result - could have been an instance of a subclass of ``int``. - - -.. function:: inv(obj) - invert(obj) - __inv__(obj) - __invert__(obj) - - Return the bitwise inverse of the number *obj*. This is equivalent to ``~obj``. - - -.. function:: lshift(a, b) - __lshift__(a, b) - - Return *a* shifted left by *b*. - - -.. function:: mod(a, b) - __mod__(a, b) - - Return ``a % b``. - - -.. function:: mul(a, b) - __mul__(a, b) - - Return ``a * b``, for *a* and *b* numbers. - - -.. function:: matmul(a, b) - __matmul__(a, b) - - Return ``a @ b``. - - .. versionadded:: 3.5 - - -.. function:: neg(obj) - __neg__(obj) - - Return *obj* negated (``-obj``). - - -.. function:: or_(a, b) - __or__(a, b) - - Return the bitwise or of *a* and *b*. - - -.. function:: pos(obj) - __pos__(obj) - - Return *obj* positive (``+obj``). - - -.. function:: pow(a, b) - __pow__(a, b) - - Return ``a ** b``, for *a* and *b* numbers. - - -.. function:: rshift(a, b) - __rshift__(a, b) - - Return *a* shifted right by *b*. - - -.. function:: sub(a, b) - __sub__(a, b) - - Return ``a - b``. - - -.. function:: truediv(a, b) - __truediv__(a, b) - - Return ``a / b`` where 2/3 is .66 rather than 0. This is also known as - "true" division. - - -.. function:: xor(a, b) - __xor__(a, b) - - Return the bitwise exclusive or of *a* and *b*. - - -Operations which work with sequences (some of them with mappings too) include: - -.. function:: concat(a, b) - __concat__(a, b) - - Return ``a + b`` for *a* and *b* sequences. - - -.. function:: contains(a, b) - __contains__(a, b) - - Return the outcome of the test ``b in a``. Note the reversed operands. - - -.. function:: countOf(a, b) - - Return the number of occurrences of *b* in *a*. - - -.. function:: delitem(a, b) - __delitem__(a, b) - - Remove the value of *a* at index *b*. - - -.. function:: getitem(a, b) - __getitem__(a, b) - - Return the value of *a* at index *b*. - - -.. function:: indexOf(a, b) - - Return the index of the first of occurrence of *b* in *a*. - - -.. function:: setitem(a, b, c) - __setitem__(a, b, c) - - Set the value of *a* at index *b* to *c*. - - -.. function:: length_hint(obj, default=0) - - Return an estimated length for the object *obj*. First try to return its - actual length, then an estimate using :meth:`object.__length_hint__`, and - finally return the default value. - - .. versionadded:: 3.4 - - -The following operation works with callables: - -.. function:: call(obj, /, *args, **kwargs) - __call__(obj, /, *args, **kwargs) - - Return ``obj(*args, **kwargs)``. - - .. versionadded:: 3.11 - - -The :mod:`operator` module also defines tools for generalized attribute and item -lookups. These are useful for making fast field extractors as arguments for -:func:`map`, :func:`sorted`, :meth:`itertools.groupby`, or other functions that -expect a function argument. - - -.. function:: attrgetter(attr) - attrgetter(*attrs) - - Return a callable object that fetches *attr* from its operand. - If more than one attribute is requested, returns a tuple of attributes. - The attribute names can also contain dots. For example: - - * After ``f = attrgetter('name')``, the call ``f(b)`` returns ``b.name``. - - * After ``f = attrgetter('name', 'date')``, the call ``f(b)`` returns - ``(b.name, b.date)``. - - * After ``f = attrgetter('name.first', 'name.last')``, the call ``f(b)`` - returns ``(b.name.first, b.name.last)``. - - Equivalent to:: - - def attrgetter(*items): - if any(not isinstance(item, str) for item in items): - raise TypeError('attribute name must be a string') - if len(items) == 1: - attr = items[0] - def g(obj): - return resolve_attr(obj, attr) - else: - def g(obj): - return tuple(resolve_attr(obj, attr) for attr in items) - return g - - def resolve_attr(obj, attr): - for name in attr.split("."): - obj = getattr(obj, name) - return obj - - -.. function:: itemgetter(item) - itemgetter(*items) - - Return a callable object that fetches *item* from its operand using the - operand's :meth:`~object.__getitem__` method. If multiple items are specified, - returns a tuple of lookup values. For example: - - * After ``f = itemgetter(2)``, the call ``f(r)`` returns ``r[2]``. - - * After ``g = itemgetter(2, 5, 3)``, the call ``g(r)`` returns - ``(r[2], r[5], r[3])``. - - Equivalent to:: - - def itemgetter(*items): - if len(items) == 1: - item = items[0] - def g(obj): - return obj[item] - else: - def g(obj): - return tuple(obj[item] for item in items) - return g - - The items can be any type accepted by the operand's :meth:`~object.__getitem__` - method. Dictionaries accept any :term:`hashable` value. Lists, tuples, and - strings accept an index or a slice: - - >>> itemgetter(1)('ABCDEFG') - 'B' - >>> itemgetter(1, 3, 5)('ABCDEFG') - ('B', 'D', 'F') - >>> itemgetter(slice(2, None))('ABCDEFG') - 'CDEFG' - >>> soldier = dict(rank='captain', name='dotterbart') - >>> itemgetter('rank')(soldier) - 'captain' - - Example of using :func:`itemgetter` to retrieve specific fields from a - tuple record: - - >>> inventory = [('apple', 3), ('banana', 2), ('pear', 5), ('orange', 1)] - >>> getcount = itemgetter(1) - >>> list(map(getcount, inventory)) - [3, 2, 5, 1] - >>> sorted(inventory, key=getcount) - [('orange', 1), ('banana', 2), ('apple', 3), ('pear', 5)] - - -.. function:: methodcaller(name, /, *args, **kwargs) - - Return a callable object that calls the method *name* on its operand. If - additional arguments and/or keyword arguments are given, they will be given - to the method as well. For example: - - * After ``f = methodcaller('name')``, the call ``f(b)`` returns ``b.name()``. - - * After ``f = methodcaller('name', 'foo', bar=1)``, the call ``f(b)`` - returns ``b.name('foo', bar=1)``. - - Equivalent to:: - - def methodcaller(name, /, *args, **kwargs): - def caller(obj): - return getattr(obj, name)(*args, **kwargs) - return caller - - -.. _operator-map: - -Mapping Operators to Functions ------------------------------- - -This table shows how abstract operations correspond to operator symbols in the -Python syntax and the functions in the :mod:`operator` module. - -+-----------------------+-------------------------+---------------------------------------+ -| Operation | Syntax | Function | -+=======================+=========================+=======================================+ -| Addition | ``a + b`` | ``add(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Concatenation | ``seq1 + seq2`` | ``concat(seq1, seq2)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Containment Test | ``obj in seq`` | ``contains(seq, obj)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Division | ``a / b`` | ``truediv(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Division | ``a // b`` | ``floordiv(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Bitwise And | ``a & b`` | ``and_(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Bitwise Exclusive Or | ``a ^ b`` | ``xor(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Bitwise Inversion | ``~ a`` | ``invert(a)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Bitwise Or | ``a | b`` | ``or_(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Exponentiation | ``a ** b`` | ``pow(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Identity | ``a is b`` | ``is_(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Identity | ``a is not b`` | ``is_not(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Indexed Assignment | ``obj[k] = v`` | ``setitem(obj, k, v)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Indexed Deletion | ``del obj[k]`` | ``delitem(obj, k)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Indexing | ``obj[k]`` | ``getitem(obj, k)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Left Shift | ``a << b`` | ``lshift(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Modulo | ``a % b`` | ``mod(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Multiplication | ``a * b`` | ``mul(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Matrix Multiplication | ``a @ b`` | ``matmul(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Negation (Arithmetic) | ``- a`` | ``neg(a)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Negation (Logical) | ``not a`` | ``not_(a)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Positive | ``+ a`` | ``pos(a)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Right Shift | ``a >> b`` | ``rshift(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Slice Assignment | ``seq[i:j] = values`` | ``setitem(seq, slice(i, j), values)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Slice Deletion | ``del seq[i:j]`` | ``delitem(seq, slice(i, j))`` | -+-----------------------+-------------------------+---------------------------------------+ -| Slicing | ``seq[i:j]`` | ``getitem(seq, slice(i, j))`` | -+-----------------------+-------------------------+---------------------------------------+ -| String Formatting | ``s % obj`` | ``mod(s, obj)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Subtraction | ``a - b`` | ``sub(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Truth Test | ``obj`` | ``truth(obj)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Ordering | ``a < b`` | ``lt(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Ordering | ``a <= b`` | ``le(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Equality | ``a == b`` | ``eq(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Difference | ``a != b`` | ``ne(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Ordering | ``a >= b`` | ``ge(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ -| Ordering | ``a > b`` | ``gt(a, b)`` | -+-----------------------+-------------------------+---------------------------------------+ - -In-place Operators ------------------- - -Many operations have an "in-place" version. Listed below are functions -providing a more primitive access to in-place operators than the usual syntax -does; for example, the :term:`statement` ``x += y`` is equivalent to -``x = operator.iadd(x, y)``. Another way to put it is to say that -``z = operator.iadd(x, y)`` is equivalent to the compound statement -``z = x; z += y``. - -In those examples, note that when an in-place method is called, the computation -and assignment are performed in two separate steps. The in-place functions -listed below only do the first step, calling the in-place method. The second -step, assignment, is not handled. - -For immutable targets such as strings, numbers, and tuples, the updated -value is computed, but not assigned back to the input variable: - ->>> a = 'hello' ->>> iadd(a, ' world') -'hello world' ->>> a -'hello' - -For mutable targets such as lists and dictionaries, the in-place method -will perform the update, so no subsequent assignment is necessary: - ->>> s = ['h', 'e', 'l', 'l', 'o'] ->>> iadd(s, [' ', 'w', 'o', 'r', 'l', 'd']) -['h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd'] ->>> s -['h', 'e', 'l', 'l', 'o', ' ', 'w', 'o', 'r', 'l', 'd'] - -.. function:: iadd(a, b) - __iadd__(a, b) - - ``a = iadd(a, b)`` is equivalent to ``a += b``. - - -.. function:: iand(a, b) - __iand__(a, b) - - ``a = iand(a, b)`` is equivalent to ``a &= b``. - - -.. function:: iconcat(a, b) - __iconcat__(a, b) - - ``a = iconcat(a, b)`` is equivalent to ``a += b`` for *a* and *b* sequences. - - -.. function:: ifloordiv(a, b) - __ifloordiv__(a, b) - - ``a = ifloordiv(a, b)`` is equivalent to ``a //= b``. - - -.. function:: ilshift(a, b) - __ilshift__(a, b) - - ``a = ilshift(a, b)`` is equivalent to ``a <<= b``. - - -.. function:: imod(a, b) - __imod__(a, b) - - ``a = imod(a, b)`` is equivalent to ``a %= b``. - - -.. function:: imul(a, b) - __imul__(a, b) - - ``a = imul(a, b)`` is equivalent to ``a *= b``. - - -.. function:: imatmul(a, b) - __imatmul__(a, b) - - ``a = imatmul(a, b)`` is equivalent to ``a @= b``. - - .. versionadded:: 3.5 - - -.. function:: ior(a, b) - __ior__(a, b) - - ``a = ior(a, b)`` is equivalent to ``a |= b``. - - -.. function:: ipow(a, b) - __ipow__(a, b) - - ``a = ipow(a, b)`` is equivalent to ``a **= b``. - - -.. function:: irshift(a, b) - __irshift__(a, b) - - ``a = irshift(a, b)`` is equivalent to ``a >>= b``. - - -.. function:: isub(a, b) - __isub__(a, b) - - ``a = isub(a, b)`` is equivalent to ``a -= b``. - - -.. function:: itruediv(a, b) - __itruediv__(a, b) - - ``a = itruediv(a, b)`` is equivalent to ``a /= b``. - - -.. function:: ixor(a, b) - __ixor__(a, b) - - ``a = ixor(a, b)`` is equivalent to ``a ^= b``. diff --git a/Doc/library/optparse.rst b/Doc/library/optparse.rst deleted file mode 100644 index 6ab61ba1d13e02..00000000000000 --- a/Doc/library/optparse.rst +++ /dev/null @@ -1,2081 +0,0 @@ -:mod:`optparse` --- Parser for command line options -=================================================== - -.. module:: optparse - :synopsis: Command-line option parsing library. - :deprecated: - -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - -**Source code:** :source:`Lib/optparse.py` - -.. deprecated:: 3.2 - The :mod:`optparse` module is deprecated and will not be developed further; - development will continue with the :mod:`argparse` module. - --------------- - -:mod:`optparse` is a more convenient, flexible, and powerful library for parsing -command-line options than the old :mod:`getopt` module. :mod:`optparse` uses a -more declarative style of command-line parsing: you create an instance of -:class:`OptionParser`, populate it with options, and parse the command -line. :mod:`optparse` allows users to specify options in the conventional -GNU/POSIX syntax, and additionally generates usage and help messages for you. - -Here's an example of using :mod:`optparse` in a simple script:: - - from optparse import OptionParser - ... - parser = OptionParser() - parser.add_option("-f", "--file", dest="filename", - help="write report to FILE", metavar="FILE") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose", default=True, - help="don't print status messages to stdout") - - (options, args) = parser.parse_args() - -With these few lines of code, users of your script can now do the "usual thing" -on the command-line, for example:: - - --file=outfile -q - -As it parses the command line, :mod:`optparse` sets attributes of the -``options`` object returned by :meth:`~OptionParser.parse_args` based on user-supplied -command-line values. When :meth:`~OptionParser.parse_args` returns from parsing this command -line, ``options.filename`` will be ``"outfile"`` and ``options.verbose`` will be -``False``. :mod:`optparse` supports both long and short options, allows short -options to be merged together, and allows options to be associated with their -arguments in a variety of ways. Thus, the following command lines are all -equivalent to the above example:: - - -f outfile --quiet - --quiet --file outfile - -q -foutfile - -qfoutfile - -Additionally, users can run one of the following :: - - -h - --help - -and :mod:`optparse` will print out a brief summary of your script's options: - -.. code-block:: text - - Usage: [options] - - Options: - -h, --help show this help message and exit - -f FILE, --file=FILE write report to FILE - -q, --quiet don't print status messages to stdout - -where the value of *yourscript* is determined at runtime (normally from -``sys.argv[0]``). - - -.. _optparse-background: - -Background ----------- - -:mod:`optparse` was explicitly designed to encourage the creation of programs -with straightforward, conventional command-line interfaces. To that end, it -supports only the most common command-line syntax and semantics conventionally -used under Unix. If you are unfamiliar with these conventions, read this -section to acquaint yourself with them. - - -.. _optparse-terminology: - -Terminology -^^^^^^^^^^^ - -argument - a string entered on the command-line, and passed by the shell to ``execl()`` - or ``execv()``. In Python, arguments are elements of ``sys.argv[1:]`` - (``sys.argv[0]`` is the name of the program being executed). Unix shells - also use the term "word". - - It is occasionally desirable to substitute an argument list other than - ``sys.argv[1:]``, so you should read "argument" as "an element of - ``sys.argv[1:]``, or of some other list provided as a substitute for - ``sys.argv[1:]``". - -option - an argument used to supply extra information to guide or customize the - execution of a program. There are many different syntaxes for options; the - traditional Unix syntax is a hyphen ("-") followed by a single letter, - e.g. ``-x`` or ``-F``. Also, traditional Unix syntax allows multiple - options to be merged into a single argument, e.g. ``-x -F`` is equivalent - to ``-xF``. The GNU project introduced ``--`` followed by a series of - hyphen-separated words, e.g. ``--file`` or ``--dry-run``. These are the - only two option syntaxes provided by :mod:`optparse`. - - Some other option syntaxes that the world has seen include: - - * a hyphen followed by a few letters, e.g. ``-pf`` (this is *not* the same - as multiple options merged into a single argument) - - * a hyphen followed by a whole word, e.g. ``-file`` (this is technically - equivalent to the previous syntax, but they aren't usually seen in the same - program) - - * a plus sign followed by a single letter, or a few letters, or a word, e.g. - ``+f``, ``+rgb`` - - * a slash followed by a letter, or a few letters, or a word, e.g. ``/f``, - ``/file`` - - These option syntaxes are not supported by :mod:`optparse`, and they never - will be. This is deliberate: the first three are non-standard on any - environment, and the last only makes sense if you're exclusively targeting - Windows or certain legacy platforms (e.g. VMS, MS-DOS). - -option argument - an argument that follows an option, is closely associated with that option, - and is consumed from the argument list when that option is. With - :mod:`optparse`, option arguments may either be in a separate argument from - their option: - - .. code-block:: text - - -f foo - --file foo - - or included in the same argument: - - .. code-block:: text - - -ffoo - --file=foo - - Typically, a given option either takes an argument or it doesn't. Lots of - people want an "optional option arguments" feature, meaning that some options - will take an argument if they see it, and won't if they don't. This is - somewhat controversial, because it makes parsing ambiguous: if ``-a`` takes - an optional argument and ``-b`` is another option entirely, how do we - interpret ``-ab``? Because of this ambiguity, :mod:`optparse` does not - support this feature. - -positional argument - something leftover in the argument list after options have been parsed, i.e. - after options and their arguments have been parsed and removed from the - argument list. - -required option - an option that must be supplied on the command-line; note that the phrase - "required option" is self-contradictory in English. :mod:`optparse` doesn't - prevent you from implementing required options, but doesn't give you much - help at it either. - -For example, consider this hypothetical command-line:: - - prog -v --report report.txt foo bar - -``-v`` and ``--report`` are both options. Assuming that ``--report`` -takes one argument, ``report.txt`` is an option argument. ``foo`` and -``bar`` are positional arguments. - - -.. _optparse-what-options-for: - -What are options for? -^^^^^^^^^^^^^^^^^^^^^ - -Options are used to provide extra information to tune or customize the execution -of a program. In case it wasn't clear, options are usually *optional*. A -program should be able to run just fine with no options whatsoever. (Pick a -random program from the Unix or GNU toolsets. Can it run without any options at -all and still make sense? The main exceptions are ``find``, ``tar``, and -``dd``\ ---all of which are mutant oddballs that have been rightly criticized -for their non-standard syntax and confusing interfaces.) - -Lots of people want their programs to have "required options". Think about it. -If it's required, then it's *not optional*! If there is a piece of information -that your program absolutely requires in order to run successfully, that's what -positional arguments are for. - -As an example of good command-line interface design, consider the humble ``cp`` -utility, for copying files. It doesn't make much sense to try to copy files -without supplying a destination and at least one source. Hence, ``cp`` fails if -you run it with no arguments. However, it has a flexible, useful syntax that -does not require any options at all:: - - cp SOURCE DEST - cp SOURCE ... DEST-DIR - -You can get pretty far with just that. Most ``cp`` implementations provide a -bunch of options to tweak exactly how the files are copied: you can preserve -mode and modification time, avoid following symlinks, ask before clobbering -existing files, etc. But none of this distracts from the core mission of -``cp``, which is to copy either one file to another, or several files to another -directory. - - -.. _optparse-what-positional-arguments-for: - -What are positional arguments for? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Positional arguments are for those pieces of information that your program -absolutely, positively requires to run. - -A good user interface should have as few absolute requirements as possible. If -your program requires 17 distinct pieces of information in order to run -successfully, it doesn't much matter *how* you get that information from the -user---most people will give up and walk away before they successfully run the -program. This applies whether the user interface is a command-line, a -configuration file, or a GUI: if you make that many demands on your users, most -of them will simply give up. - -In short, try to minimize the amount of information that users are absolutely -required to supply---use sensible defaults whenever possible. Of course, you -also want to make your programs reasonably flexible. That's what options are -for. Again, it doesn't matter if they are entries in a config file, widgets in -the "Preferences" dialog of a GUI, or command-line options---the more options -you implement, the more flexible your program is, and the more complicated its -implementation becomes. Too much flexibility has drawbacks as well, of course; -too many options can overwhelm users and make your code much harder to maintain. - - -.. _optparse-tutorial: - -Tutorial --------- - -While :mod:`optparse` is quite flexible and powerful, it's also straightforward -to use in most cases. This section covers the code patterns that are common to -any :mod:`optparse`\ -based program. - -First, you need to import the OptionParser class; then, early in the main -program, create an OptionParser instance:: - - from optparse import OptionParser - ... - parser = OptionParser() - -Then you can start defining options. The basic syntax is:: - - parser.add_option(opt_str, ..., - attr=value, ...) - -Each option has one or more option strings, such as ``-f`` or ``--file``, -and several option attributes that tell :mod:`optparse` what to expect and what -to do when it encounters that option on the command line. - -Typically, each option will have one short option string and one long option -string, e.g.:: - - parser.add_option("-f", "--file", ...) - -You're free to define as many short option strings and as many long option -strings as you like (including zero), as long as there is at least one option -string overall. - -The option strings passed to :meth:`OptionParser.add_option` are effectively -labels for the -option defined by that call. For brevity, we will frequently refer to -*encountering an option* on the command line; in reality, :mod:`optparse` -encounters *option strings* and looks up options from them. - -Once all of your options are defined, instruct :mod:`optparse` to parse your -program's command line:: - - (options, args) = parser.parse_args() - -(If you like, you can pass a custom argument list to :meth:`~OptionParser.parse_args`, but -that's rarely necessary: by default it uses ``sys.argv[1:]``.) - -:meth:`~OptionParser.parse_args` returns two values: - -* ``options``, an object containing values for all of your options---e.g. if - ``--file`` takes a single string argument, then ``options.file`` will be the - filename supplied by the user, or ``None`` if the user did not supply that - option - -* ``args``, the list of positional arguments leftover after parsing options - -This tutorial section only covers the four most important option attributes: -:attr:`~Option.action`, :attr:`~Option.type`, :attr:`~Option.dest` -(destination), and :attr:`~Option.help`. Of these, :attr:`~Option.action` is the -most fundamental. - - -.. _optparse-understanding-option-actions: - -Understanding option actions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Actions tell :mod:`optparse` what to do when it encounters an option on the -command line. There is a fixed set of actions hard-coded into :mod:`optparse`; -adding new actions is an advanced topic covered in section -:ref:`optparse-extending-optparse`. Most actions tell :mod:`optparse` to store -a value in some variable---for example, take a string from the command line and -store it in an attribute of ``options``. - -If you don't specify an option action, :mod:`optparse` defaults to ``store``. - - -.. _optparse-store-action: - -The store action -^^^^^^^^^^^^^^^^ - -The most common option action is ``store``, which tells :mod:`optparse` to take -the next argument (or the remainder of the current argument), ensure that it is -of the correct type, and store it to your chosen destination. - -For example:: - - parser.add_option("-f", "--file", - action="store", type="string", dest="filename") - -Now let's make up a fake command line and ask :mod:`optparse` to parse it:: - - args = ["-f", "foo.txt"] - (options, args) = parser.parse_args(args) - -When :mod:`optparse` sees the option string ``-f``, it consumes the next -argument, ``foo.txt``, and stores it in ``options.filename``. So, after this -call to :meth:`~OptionParser.parse_args`, ``options.filename`` is ``"foo.txt"``. - -Some other option types supported by :mod:`optparse` are ``int`` and ``float``. -Here's an option that expects an integer argument:: - - parser.add_option("-n", type="int", dest="num") - -Note that this option has no long option string, which is perfectly acceptable. -Also, there's no explicit action, since the default is ``store``. - -Let's parse another fake command-line. This time, we'll jam the option argument -right up against the option: since ``-n42`` (one argument) is equivalent to -``-n 42`` (two arguments), the code :: - - (options, args) = parser.parse_args(["-n42"]) - print(options.num) - -will print ``42``. - -If you don't specify a type, :mod:`optparse` assumes ``string``. Combined with -the fact that the default action is ``store``, that means our first example can -be a lot shorter:: - - parser.add_option("-f", "--file", dest="filename") - -If you don't supply a destination, :mod:`optparse` figures out a sensible -default from the option strings: if the first long option string is -``--foo-bar``, then the default destination is ``foo_bar``. If there are no -long option strings, :mod:`optparse` looks at the first short option string: the -default destination for ``-f`` is ``f``. - -:mod:`optparse` also includes the built-in ``complex`` type. Adding -types is covered in section :ref:`optparse-extending-optparse`. - - -.. _optparse-handling-boolean-options: - -Handling boolean (flag) options -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Flag options---set a variable to true or false when a particular option is -seen---are quite common. :mod:`optparse` supports them with two separate actions, -``store_true`` and ``store_false``. For example, you might have a ``verbose`` -flag that is turned on with ``-v`` and off with ``-q``:: - - parser.add_option("-v", action="store_true", dest="verbose") - parser.add_option("-q", action="store_false", dest="verbose") - -Here we have two different options with the same destination, which is perfectly -OK. (It just means you have to be a bit careful when setting default -values---see below.) - -When :mod:`optparse` encounters ``-v`` on the command line, it sets -``options.verbose`` to ``True``; when it encounters ``-q``, -``options.verbose`` is set to ``False``. - - -.. _optparse-other-actions: - -Other actions -^^^^^^^^^^^^^ - -Some other actions supported by :mod:`optparse` are: - -``"store_const"`` - store a constant value, pre-set via :attr:`Option.const` - -``"append"`` - append this option's argument to a list - -``"count"`` - increment a counter by one - -``"callback"`` - call a specified function - -These are covered in section :ref:`optparse-reference-guide`, -and section :ref:`optparse-option-callbacks`. - - -.. _optparse-default-values: - -Default values -^^^^^^^^^^^^^^ - -All of the above examples involve setting some variable (the "destination") when -certain command-line options are seen. What happens if those options are never -seen? Since we didn't supply any defaults, they are all set to ``None``. This -is usually fine, but sometimes you want more control. :mod:`optparse` lets you -supply a default value for each destination, which is assigned before the -command line is parsed. - -First, consider the verbose/quiet example. If we want :mod:`optparse` to set -``verbose`` to ``True`` unless ``-q`` is seen, then we can do this:: - - parser.add_option("-v", action="store_true", dest="verbose", default=True) - parser.add_option("-q", action="store_false", dest="verbose") - -Since default values apply to the *destination* rather than to any particular -option, and these two options happen to have the same destination, this is -exactly equivalent:: - - parser.add_option("-v", action="store_true", dest="verbose") - parser.add_option("-q", action="store_false", dest="verbose", default=True) - -Consider this:: - - parser.add_option("-v", action="store_true", dest="verbose", default=False) - parser.add_option("-q", action="store_false", dest="verbose", default=True) - -Again, the default value for ``verbose`` will be ``True``: the last default -value supplied for any particular destination is the one that counts. - -A clearer way to specify default values is the :meth:`set_defaults` method of -OptionParser, which you can call at any time before calling -:meth:`~OptionParser.parse_args`:: - - parser.set_defaults(verbose=True) - parser.add_option(...) - (options, args) = parser.parse_args() - -As before, the last value specified for a given option destination is the one -that counts. For clarity, try to use one method or the other of setting default -values, not both. - - -.. _optparse-generating-help: - -Generating help -^^^^^^^^^^^^^^^ - -:mod:`optparse`'s ability to generate help and usage text automatically is -useful for creating user-friendly command-line interfaces. All you have to do -is supply a :attr:`~Option.help` value for each option, and optionally a short -usage message for your whole program. Here's an OptionParser populated with -user-friendly (documented) options:: - - usage = "usage: %prog [options] arg1 arg2" - parser = OptionParser(usage=usage) - parser.add_option("-v", "--verbose", - action="store_true", dest="verbose", default=True, - help="make lots of noise [default]") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose", - help="be vewwy quiet (I'm hunting wabbits)") - parser.add_option("-f", "--filename", - metavar="FILE", help="write output to FILE") - parser.add_option("-m", "--mode", - default="intermediate", - help="interaction mode: novice, intermediate, " - "or expert [default: %default]") - -If :mod:`optparse` encounters either ``-h`` or ``--help`` on the -command-line, or if you just call :meth:`parser.print_help`, it prints the -following to standard output: - -.. code-block:: text - - Usage: [options] arg1 arg2 - - Options: - -h, --help show this help message and exit - -v, --verbose make lots of noise [default] - -q, --quiet be vewwy quiet (I'm hunting wabbits) - -f FILE, --filename=FILE - write output to FILE - -m MODE, --mode=MODE interaction mode: novice, intermediate, or - expert [default: intermediate] - -(If the help output is triggered by a help option, :mod:`optparse` exits after -printing the help text.) - -There's a lot going on here to help :mod:`optparse` generate the best possible -help message: - -* the script defines its own usage message:: - - usage = "usage: %prog [options] arg1 arg2" - - :mod:`optparse` expands ``%prog`` in the usage string to the name of the - current program, i.e. ``os.path.basename(sys.argv[0])``. The expanded string - is then printed before the detailed option help. - - If you don't supply a usage string, :mod:`optparse` uses a bland but sensible - default: ``"Usage: %prog [options]"``, which is fine if your script doesn't - take any positional arguments. - -* every option defines a help string, and doesn't worry about - line-wrapping---\ :mod:`optparse` takes care of wrapping lines and making - the help output look good. - -* options that take a value indicate this fact in their automatically generated - help message, e.g. for the "mode" option:: - - -m MODE, --mode=MODE - - Here, "MODE" is called the meta-variable: it stands for the argument that the - user is expected to supply to ``-m``/``--mode``. By default, - :mod:`optparse` converts the destination variable name to uppercase and uses - that for the meta-variable. Sometimes, that's not what you want---for - example, the ``--filename`` option explicitly sets ``metavar="FILE"``, - resulting in this automatically generated option description:: - - -f FILE, --filename=FILE - - This is important for more than just saving space, though: the manually - written help text uses the meta-variable ``FILE`` to clue the user in that - there's a connection between the semi-formal syntax ``-f FILE`` and the informal - semantic description "write output to FILE". This is a simple but effective - way to make your help text a lot clearer and more useful for end users. - -* options that have a default value can include ``%default`` in the help - string---\ :mod:`optparse` will replace it with :func:`str` of the option's - default value. If an option has no default value (or the default value is - ``None``), ``%default`` expands to ``none``. - -Grouping Options -++++++++++++++++ - -When dealing with many options, it is convenient to group these options for -better help output. An :class:`OptionParser` can contain several option groups, -each of which can contain several options. - -An option group is obtained using the class :class:`OptionGroup`: - -.. class:: OptionGroup(parser, title, description=None) - - where - - * parser is the :class:`OptionParser` instance the group will be inserted in - to - * title is the group title - * description, optional, is a long description of the group - -:class:`OptionGroup` inherits from :class:`OptionContainer` (like -:class:`OptionParser`) and so the :meth:`add_option` method can be used to add -an option to the group. - -Once all the options are declared, using the :class:`OptionParser` method -:meth:`add_option_group` the group is added to the previously defined parser. - -Continuing with the parser defined in the previous section, adding an -:class:`OptionGroup` to a parser is easy:: - - group = OptionGroup(parser, "Dangerous Options", - "Caution: use these options at your own risk. " - "It is believed that some of them bite.") - group.add_option("-g", action="store_true", help="Group option.") - parser.add_option_group(group) - -This would result in the following help output: - -.. code-block:: text - - Usage: [options] arg1 arg2 - - Options: - -h, --help show this help message and exit - -v, --verbose make lots of noise [default] - -q, --quiet be vewwy quiet (I'm hunting wabbits) - -f FILE, --filename=FILE - write output to FILE - -m MODE, --mode=MODE interaction mode: novice, intermediate, or - expert [default: intermediate] - - Dangerous Options: - Caution: use these options at your own risk. It is believed that some - of them bite. - - -g Group option. - -A bit more complete example might involve using more than one group: still -extending the previous example:: - - group = OptionGroup(parser, "Dangerous Options", - "Caution: use these options at your own risk. " - "It is believed that some of them bite.") - group.add_option("-g", action="store_true", help="Group option.") - parser.add_option_group(group) - - group = OptionGroup(parser, "Debug Options") - group.add_option("-d", "--debug", action="store_true", - help="Print debug information") - group.add_option("-s", "--sql", action="store_true", - help="Print all SQL statements executed") - group.add_option("-e", action="store_true", help="Print every action done") - parser.add_option_group(group) - -that results in the following output: - -.. code-block:: text - - Usage: [options] arg1 arg2 - - Options: - -h, --help show this help message and exit - -v, --verbose make lots of noise [default] - -q, --quiet be vewwy quiet (I'm hunting wabbits) - -f FILE, --filename=FILE - write output to FILE - -m MODE, --mode=MODE interaction mode: novice, intermediate, or expert - [default: intermediate] - - Dangerous Options: - Caution: use these options at your own risk. It is believed that some - of them bite. - - -g Group option. - - Debug Options: - -d, --debug Print debug information - -s, --sql Print all SQL statements executed - -e Print every action done - -Another interesting method, in particular when working programmatically with -option groups is: - -.. method:: OptionParser.get_option_group(opt_str) - - Return the :class:`OptionGroup` to which the short or long option - string *opt_str* (e.g. ``'-o'`` or ``'--option'``) belongs. If - there's no such :class:`OptionGroup`, return ``None``. - -.. _optparse-printing-version-string: - -Printing a version string -^^^^^^^^^^^^^^^^^^^^^^^^^ - -Similar to the brief usage string, :mod:`optparse` can also print a version -string for your program. You have to supply the string as the ``version`` -argument to OptionParser:: - - parser = OptionParser(usage="%prog [-f] [-q]", version="%prog 1.0") - -``%prog`` is expanded just like it is in ``usage``. Apart from that, -``version`` can contain anything you like. When you supply it, :mod:`optparse` -automatically adds a ``--version`` option to your parser. If it encounters -this option on the command line, it expands your ``version`` string (by -replacing ``%prog``), prints it to stdout, and exits. - -For example, if your script is called ``/usr/bin/foo``: - -.. code-block:: shell-session - - $ /usr/bin/foo --version - foo 1.0 - -The following two methods can be used to print and get the ``version`` string: - -.. method:: OptionParser.print_version(file=None) - - Print the version message for the current program (``self.version``) to - *file* (default stdout). As with :meth:`print_usage`, any occurrence - of ``%prog`` in ``self.version`` is replaced with the name of the current - program. Does nothing if ``self.version`` is empty or undefined. - -.. method:: OptionParser.get_version() - - Same as :meth:`print_version` but returns the version string instead of - printing it. - - -.. _optparse-how-optparse-handles-errors: - -How :mod:`optparse` handles errors -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are two broad classes of errors that :mod:`optparse` has to worry about: -programmer errors and user errors. Programmer errors are usually erroneous -calls to :func:`OptionParser.add_option`, e.g. invalid option strings, unknown -option attributes, missing option attributes, etc. These are dealt with in the -usual way: raise an exception (either :exc:`optparse.OptionError` or -:exc:`TypeError`) and let the program crash. - -Handling user errors is much more important, since they are guaranteed to happen -no matter how stable your code is. :mod:`optparse` can automatically detect -some user errors, such as bad option arguments (passing ``-n 4x`` where -``-n`` takes an integer argument), missing arguments (``-n`` at the end -of the command line, where ``-n`` takes an argument of any type). Also, -you can call :func:`OptionParser.error` to signal an application-defined error -condition:: - - (options, args) = parser.parse_args() - ... - if options.a and options.b: - parser.error("options -a and -b are mutually exclusive") - -In either case, :mod:`optparse` handles the error the same way: it prints the -program's usage message and an error message to standard error and exits with -error status 2. - -Consider the first example above, where the user passes ``4x`` to an option -that takes an integer: - -.. code-block:: shell-session - - $ /usr/bin/foo -n 4x - Usage: foo [options] - - foo: error: option -n: invalid integer value: '4x' - -Or, where the user fails to pass a value at all: - -.. code-block:: shell-session - - $ /usr/bin/foo -n - Usage: foo [options] - - foo: error: -n option requires an argument - -:mod:`optparse`\ -generated error messages take care always to mention the -option involved in the error; be sure to do the same when calling -:func:`OptionParser.error` from your application code. - -If :mod:`optparse`'s default error-handling behaviour does not suit your needs, -you'll need to subclass OptionParser and override its :meth:`~OptionParser.exit` -and/or :meth:`~OptionParser.error` methods. - - -.. _optparse-putting-it-all-together: - -Putting it all together -^^^^^^^^^^^^^^^^^^^^^^^ - -Here's what :mod:`optparse`\ -based scripts usually look like:: - - from optparse import OptionParser - ... - def main(): - usage = "usage: %prog [options] arg" - parser = OptionParser(usage) - parser.add_option("-f", "--file", dest="filename", - help="read data from FILENAME") - parser.add_option("-v", "--verbose", - action="store_true", dest="verbose") - parser.add_option("-q", "--quiet", - action="store_false", dest="verbose") - ... - (options, args) = parser.parse_args() - if len(args) != 1: - parser.error("incorrect number of arguments") - if options.verbose: - print("reading %s..." % options.filename) - ... - - if __name__ == "__main__": - main() - - -.. _optparse-reference-guide: - -Reference Guide ---------------- - - -.. _optparse-creating-parser: - -Creating the parser -^^^^^^^^^^^^^^^^^^^ - -The first step in using :mod:`optparse` is to create an OptionParser instance. - -.. class:: OptionParser(...) - - The OptionParser constructor has no required arguments, but a number of - optional keyword arguments. You should always pass them as keyword - arguments, i.e. do not rely on the order in which the arguments are declared. - - ``usage`` (default: ``"%prog [options]"``) - The usage summary to print when your program is run incorrectly or with a - help option. When :mod:`optparse` prints the usage string, it expands - ``%prog`` to ``os.path.basename(sys.argv[0])`` (or to ``prog`` if you - passed that keyword argument). To suppress a usage message, pass the - special value :const:`optparse.SUPPRESS_USAGE`. - - ``option_list`` (default: ``[]``) - A list of Option objects to populate the parser with. The options in - ``option_list`` are added after any options in ``standard_option_list`` (a - class attribute that may be set by OptionParser subclasses), but before - any version or help options. Deprecated; use :meth:`add_option` after - creating the parser instead. - - ``option_class`` (default: optparse.Option) - Class to use when adding options to the parser in :meth:`add_option`. - - ``version`` (default: ``None``) - A version string to print when the user supplies a version option. If you - supply a true value for ``version``, :mod:`optparse` automatically adds a - version option with the single option string ``--version``. The - substring ``%prog`` is expanded the same as for ``usage``. - - ``conflict_handler`` (default: ``"error"``) - Specifies what to do when options with conflicting option strings are - added to the parser; see section - :ref:`optparse-conflicts-between-options`. - - ``description`` (default: ``None``) - A paragraph of text giving a brief overview of your program. - :mod:`optparse` reformats this paragraph to fit the current terminal width - and prints it when the user requests help (after ``usage``, but before the - list of options). - - ``formatter`` (default: a new :class:`IndentedHelpFormatter`) - An instance of optparse.HelpFormatter that will be used for printing help - text. :mod:`optparse` provides two concrete classes for this purpose: - IndentedHelpFormatter and TitledHelpFormatter. - - ``add_help_option`` (default: ``True``) - If true, :mod:`optparse` will add a help option (with option strings ``-h`` - and ``--help``) to the parser. - - ``prog`` - The string to use when expanding ``%prog`` in ``usage`` and ``version`` - instead of ``os.path.basename(sys.argv[0])``. - - ``epilog`` (default: ``None``) - A paragraph of help text to print after the option help. - -.. _optparse-populating-parser: - -Populating the parser -^^^^^^^^^^^^^^^^^^^^^ - -There are several ways to populate the parser with options. The preferred way -is by using :meth:`OptionParser.add_option`, as shown in section -:ref:`optparse-tutorial`. :meth:`add_option` can be called in one of two ways: - -* pass it an Option instance (as returned by :func:`make_option`) - -* pass it any combination of positional and keyword arguments that are - acceptable to :func:`make_option` (i.e., to the Option constructor), and it - will create the Option instance for you - -The other alternative is to pass a list of pre-constructed Option instances to -the OptionParser constructor, as in:: - - option_list = [ - make_option("-f", "--filename", - action="store", type="string", dest="filename"), - make_option("-q", "--quiet", - action="store_false", dest="verbose"), - ] - parser = OptionParser(option_list=option_list) - -(:func:`make_option` is a factory function for creating Option instances; -currently it is an alias for the Option constructor. A future version of -:mod:`optparse` may split Option into several classes, and :func:`make_option` -will pick the right class to instantiate. Do not instantiate Option directly.) - - -.. _optparse-defining-options: - -Defining options -^^^^^^^^^^^^^^^^ - -Each Option instance represents a set of synonymous command-line option strings, -e.g. ``-f`` and ``--file``. You can specify any number of short or -long option strings, but you must specify at least one overall option string. - -The canonical way to create an :class:`Option` instance is with the -:meth:`add_option` method of :class:`OptionParser`. - -.. method:: OptionParser.add_option(option) - OptionParser.add_option(*opt_str, attr=value, ...) - - To define an option with only a short option string:: - - parser.add_option("-f", attr=value, ...) - - And to define an option with only a long option string:: - - parser.add_option("--foo", attr=value, ...) - - The keyword arguments define attributes of the new Option object. The most - important option attribute is :attr:`~Option.action`, and it largely - determines which other attributes are relevant or required. If you pass - irrelevant option attributes, or fail to pass required ones, :mod:`optparse` - raises an :exc:`OptionError` exception explaining your mistake. - - An option's *action* determines what :mod:`optparse` does when it encounters - this option on the command-line. The standard option actions hard-coded into - :mod:`optparse` are: - - ``"store"`` - store this option's argument (default) - - ``"store_const"`` - store a constant value, pre-set via :attr:`Option.const` - - ``"store_true"`` - store ``True`` - - ``"store_false"`` - store ``False`` - - ``"append"`` - append this option's argument to a list - - ``"append_const"`` - append a constant value to a list, pre-set via :attr:`Option.const` - - ``"count"`` - increment a counter by one - - ``"callback"`` - call a specified function - - ``"help"`` - print a usage message including all options and the documentation for them - - (If you don't supply an action, the default is ``"store"``. For this action, - you may also supply :attr:`~Option.type` and :attr:`~Option.dest` option - attributes; see :ref:`optparse-standard-option-actions`.) - -As you can see, most actions involve storing or updating a value somewhere. -:mod:`optparse` always creates a special object for this, conventionally called -``options``, which is an instance of :class:`optparse.Values`. - -.. class:: Values - - An object holding parsed argument names and values as attributes. - Normally created by calling when calling :meth:`OptionParser.parse_args`, - and can be overridden by a custom subclass passed to the *values* argument of - :meth:`OptionParser.parse_args` (as described in :ref:`optparse-parsing-arguments`). - -Option -arguments (and various other values) are stored as attributes of this object, -according to the :attr:`~Option.dest` (destination) option attribute. - -For example, when you call :: - - parser.parse_args() - -one of the first things :mod:`optparse` does is create the ``options`` object:: - - options = Values() - -If one of the options in this parser is defined with :: - - parser.add_option("-f", "--file", action="store", type="string", dest="filename") - -and the command-line being parsed includes any of the following:: - - -ffoo - -f foo - --file=foo - --file foo - -then :mod:`optparse`, on seeing this option, will do the equivalent of :: - - options.filename = "foo" - -The :attr:`~Option.type` and :attr:`~Option.dest` option attributes are almost -as important as :attr:`~Option.action`, but :attr:`~Option.action` is the only -one that makes sense for *all* options. - - -.. _optparse-option-attributes: - -Option attributes -^^^^^^^^^^^^^^^^^ - -.. class:: Option - - A single command line argument, - with various attributes passed by keyword to the constructor. - Normally created with :meth:`OptionParser.add_option` rather than directly, - and can be overridden by a custom class via the *option_class* argument - to :class:`OptionParser`. - -The following option attributes may be passed as keyword arguments to -:meth:`OptionParser.add_option`. If you pass an option attribute that is not -relevant to a particular option, or fail to pass a required option attribute, -:mod:`optparse` raises :exc:`OptionError`. - -.. attribute:: Option.action - - (default: ``"store"``) - - Determines :mod:`optparse`'s behaviour when this option is seen on the - command line; the available options are documented :ref:`here - `. - -.. attribute:: Option.type - - (default: ``"string"``) - - The argument type expected by this option (e.g., ``"string"`` or ``"int"``); - the available option types are documented :ref:`here - `. - -.. attribute:: Option.dest - - (default: derived from option strings) - - If the option's action implies writing or modifying a value somewhere, this - tells :mod:`optparse` where to write it: :attr:`~Option.dest` names an - attribute of the ``options`` object that :mod:`optparse` builds as it parses - the command line. - -.. attribute:: Option.default - - The value to use for this option's destination if the option is not seen on - the command line. See also :meth:`OptionParser.set_defaults`. - -.. attribute:: Option.nargs - - (default: 1) - - How many arguments of type :attr:`~Option.type` should be consumed when this - option is seen. If > 1, :mod:`optparse` will store a tuple of values to - :attr:`~Option.dest`. - -.. attribute:: Option.const - - For actions that store a constant value, the constant value to store. - -.. attribute:: Option.choices - - For options of type ``"choice"``, the list of strings the user may choose - from. - -.. attribute:: Option.callback - - For options with action ``"callback"``, the callable to call when this option - is seen. See section :ref:`optparse-option-callbacks` for detail on the - arguments passed to the callable. - -.. attribute:: Option.callback_args - Option.callback_kwargs - - Additional positional and keyword arguments to pass to ``callback`` after the - four standard callback arguments. - -.. attribute:: Option.help - - Help text to print for this option when listing all available options after - the user supplies a :attr:`~Option.help` option (such as ``--help``). If - no help text is supplied, the option will be listed without help text. To - hide this option, use the special value :const:`optparse.SUPPRESS_HELP`. - -.. attribute:: Option.metavar - - (default: derived from option strings) - - Stand-in for the option argument(s) to use when printing help text. See - section :ref:`optparse-tutorial` for an example. - - -.. _optparse-standard-option-actions: - -Standard option actions -^^^^^^^^^^^^^^^^^^^^^^^ - -The various option actions all have slightly different requirements and effects. -Most actions have several relevant option attributes which you may specify to -guide :mod:`optparse`'s behaviour; a few have required attributes, which you -must specify for any option using that action. - -* ``"store"`` [relevant: :attr:`~Option.type`, :attr:`~Option.dest`, - :attr:`~Option.nargs`, :attr:`~Option.choices`] - - The option must be followed by an argument, which is converted to a value - according to :attr:`~Option.type` and stored in :attr:`~Option.dest`. If - :attr:`~Option.nargs` > 1, multiple arguments will be consumed from the - command line; all will be converted according to :attr:`~Option.type` and - stored to :attr:`~Option.dest` as a tuple. See the - :ref:`optparse-standard-option-types` section. - - If :attr:`~Option.choices` is supplied (a list or tuple of strings), the type - defaults to ``"choice"``. - - If :attr:`~Option.type` is not supplied, it defaults to ``"string"``. - - If :attr:`~Option.dest` is not supplied, :mod:`optparse` derives a destination - from the first long option string (e.g., ``--foo-bar`` implies - ``foo_bar``). If there are no long option strings, :mod:`optparse` derives a - destination from the first short option string (e.g., ``-f`` implies ``f``). - - Example:: - - parser.add_option("-f") - parser.add_option("-p", type="float", nargs=3, dest="point") - - As it parses the command line :: - - -f foo.txt -p 1 -3.5 4 -fbar.txt - - :mod:`optparse` will set :: - - options.f = "foo.txt" - options.point = (1.0, -3.5, 4.0) - options.f = "bar.txt" - -* ``"store_const"`` [required: :attr:`~Option.const`; relevant: - :attr:`~Option.dest`] - - The value :attr:`~Option.const` is stored in :attr:`~Option.dest`. - - Example:: - - parser.add_option("-q", "--quiet", - action="store_const", const=0, dest="verbose") - parser.add_option("-v", "--verbose", - action="store_const", const=1, dest="verbose") - parser.add_option("--noisy", - action="store_const", const=2, dest="verbose") - - If ``--noisy`` is seen, :mod:`optparse` will set :: - - options.verbose = 2 - -* ``"store_true"`` [relevant: :attr:`~Option.dest`] - - A special case of ``"store_const"`` that stores ``True`` to - :attr:`~Option.dest`. - -* ``"store_false"`` [relevant: :attr:`~Option.dest`] - - Like ``"store_true"``, but stores ``False``. - - Example:: - - parser.add_option("--clobber", action="store_true", dest="clobber") - parser.add_option("--no-clobber", action="store_false", dest="clobber") - -* ``"append"`` [relevant: :attr:`~Option.type`, :attr:`~Option.dest`, - :attr:`~Option.nargs`, :attr:`~Option.choices`] - - The option must be followed by an argument, which is appended to the list in - :attr:`~Option.dest`. If no default value for :attr:`~Option.dest` is - supplied, an empty list is automatically created when :mod:`optparse` first - encounters this option on the command-line. If :attr:`~Option.nargs` > 1, - multiple arguments are consumed, and a tuple of length :attr:`~Option.nargs` - is appended to :attr:`~Option.dest`. - - The defaults for :attr:`~Option.type` and :attr:`~Option.dest` are the same as - for the ``"store"`` action. - - Example:: - - parser.add_option("-t", "--tracks", action="append", type="int") - - If ``-t3`` is seen on the command-line, :mod:`optparse` does the equivalent - of:: - - options.tracks = [] - options.tracks.append(int("3")) - - If, a little later on, ``--tracks=4`` is seen, it does:: - - options.tracks.append(int("4")) - - The ``append`` action calls the ``append`` method on the current value of the - option. This means that any default value specified must have an ``append`` - method. It also means that if the default value is non-empty, the default - elements will be present in the parsed value for the option, with any values - from the command line appended after those default values:: - - >>> parser.add_option("--files", action="append", default=['~/.mypkg/defaults']) - >>> opts, args = parser.parse_args(['--files', 'overrides.mypkg']) - >>> opts.files - ['~/.mypkg/defaults', 'overrides.mypkg'] - -* ``"append_const"`` [required: :attr:`~Option.const`; relevant: - :attr:`~Option.dest`] - - Like ``"store_const"``, but the value :attr:`~Option.const` is appended to - :attr:`~Option.dest`; as with ``"append"``, :attr:`~Option.dest` defaults to - ``None``, and an empty list is automatically created the first time the option - is encountered. - -* ``"count"`` [relevant: :attr:`~Option.dest`] - - Increment the integer stored at :attr:`~Option.dest`. If no default value is - supplied, :attr:`~Option.dest` is set to zero before being incremented the - first time. - - Example:: - - parser.add_option("-v", action="count", dest="verbosity") - - The first time ``-v`` is seen on the command line, :mod:`optparse` does the - equivalent of:: - - options.verbosity = 0 - options.verbosity += 1 - - Every subsequent occurrence of ``-v`` results in :: - - options.verbosity += 1 - -* ``"callback"`` [required: :attr:`~Option.callback`; relevant: - :attr:`~Option.type`, :attr:`~Option.nargs`, :attr:`~Option.callback_args`, - :attr:`~Option.callback_kwargs`] - - Call the function specified by :attr:`~Option.callback`, which is called as :: - - func(option, opt_str, value, parser, *args, **kwargs) - - See section :ref:`optparse-option-callbacks` for more detail. - -* ``"help"`` - - Prints a complete help message for all the options in the current option - parser. The help message is constructed from the ``usage`` string passed to - OptionParser's constructor and the :attr:`~Option.help` string passed to every - option. - - If no :attr:`~Option.help` string is supplied for an option, it will still be - listed in the help message. To omit an option entirely, use the special value - :const:`optparse.SUPPRESS_HELP`. - - :mod:`optparse` automatically adds a :attr:`~Option.help` option to all - OptionParsers, so you do not normally need to create one. - - Example:: - - from optparse import OptionParser, SUPPRESS_HELP - - # usually, a help option is added automatically, but that can - # be suppressed using the add_help_option argument - parser = OptionParser(add_help_option=False) - - parser.add_option("-h", "--help", action="help") - parser.add_option("-v", action="store_true", dest="verbose", - help="Be moderately verbose") - parser.add_option("--file", dest="filename", - help="Input file to read data from") - parser.add_option("--secret", help=SUPPRESS_HELP) - - If :mod:`optparse` sees either ``-h`` or ``--help`` on the command line, - it will print something like the following help message to stdout (assuming - ``sys.argv[0]`` is ``"foo.py"``): - - .. code-block:: text - - Usage: foo.py [options] - - Options: - -h, --help Show this help message and exit - -v Be moderately verbose - --file=FILENAME Input file to read data from - - After printing the help message, :mod:`optparse` terminates your process with - ``sys.exit(0)``. - -* ``"version"`` - - Prints the version number supplied to the OptionParser to stdout and exits. - The version number is actually formatted and printed by the - ``print_version()`` method of OptionParser. Generally only relevant if the - ``version`` argument is supplied to the OptionParser constructor. As with - :attr:`~Option.help` options, you will rarely create ``version`` options, - since :mod:`optparse` automatically adds them when needed. - - -.. _optparse-standard-option-types: - -Standard option types -^^^^^^^^^^^^^^^^^^^^^ - -:mod:`optparse` has five built-in option types: ``"string"``, ``"int"``, -``"choice"``, ``"float"`` and ``"complex"``. If you need to add new -option types, see section :ref:`optparse-extending-optparse`. - -Arguments to string options are not checked or converted in any way: the text on -the command line is stored in the destination (or passed to the callback) as-is. - -Integer arguments (type ``"int"``) are parsed as follows: - -* if the number starts with ``0x``, it is parsed as a hexadecimal number - -* if the number starts with ``0``, it is parsed as an octal number - -* if the number starts with ``0b``, it is parsed as a binary number - -* otherwise, the number is parsed as a decimal number - - -The conversion is done by calling :func:`int` with the appropriate base (2, 8, -10, or 16). If this fails, so will :mod:`optparse`, although with a more useful -error message. - -``"float"`` and ``"complex"`` option arguments are converted directly with -:func:`float` and :func:`complex`, with similar error-handling. - -``"choice"`` options are a subtype of ``"string"`` options. The -:attr:`~Option.choices` option attribute (a sequence of strings) defines the -set of allowed option arguments. :func:`optparse.check_choice` compares -user-supplied option arguments against this master list and raises -:exc:`OptionValueError` if an invalid string is given. - - -.. _optparse-parsing-arguments: - -Parsing arguments -^^^^^^^^^^^^^^^^^ - -The whole point of creating and populating an OptionParser is to call its -:meth:`~OptionParser.parse_args` method. - -.. method:: OptionParser.parse_args(args=None, values=None) - - Parse the command-line options found in *args*. - - The input parameters are - - ``args`` - the list of arguments to process (default: ``sys.argv[1:]``) - - ``values`` - an :class:`Values` object to store option arguments in (default: a - new instance of :class:`Values`) -- if you give an existing object, the - option defaults will not be initialized on it - - and the return value is a pair ``(options, args)`` where - - ``options`` - the same object that was passed in as *values*, or the ``optparse.Values`` - instance created by :mod:`optparse` - - ``args`` - the leftover positional arguments after all options have been processed - -The most common usage is to supply neither keyword argument. If you supply -``values``, it will be modified with repeated :func:`setattr` calls (roughly one -for every option argument stored to an option destination) and returned by -:meth:`~OptionParser.parse_args`. - -If :meth:`~OptionParser.parse_args` encounters any errors in the argument list, it calls the -OptionParser's :meth:`error` method with an appropriate end-user error message. -This ultimately terminates your process with an exit status of 2 (the -traditional Unix exit status for command-line errors). - - -.. _optparse-querying-manipulating-option-parser: - -Querying and manipulating your option parser -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The default behavior of the option parser can be customized slightly, and you -can also poke around your option parser and see what's there. OptionParser -provides several methods to help you out: - -.. method:: OptionParser.disable_interspersed_args() - - Set parsing to stop on the first non-option. For example, if ``-a`` and - ``-b`` are both simple options that take no arguments, :mod:`optparse` - normally accepts this syntax:: - - prog -a arg1 -b arg2 - - and treats it as equivalent to :: - - prog -a -b arg1 arg2 - - To disable this feature, call :meth:`disable_interspersed_args`. This - restores traditional Unix syntax, where option parsing stops with the first - non-option argument. - - Use this if you have a command processor which runs another command which has - options of its own and you want to make sure these options don't get - confused. For example, each command might have a different set of options. - -.. method:: OptionParser.enable_interspersed_args() - - Set parsing to not stop on the first non-option, allowing interspersing - switches with command arguments. This is the default behavior. - -.. method:: OptionParser.get_option(opt_str) - - Returns the Option instance with the option string *opt_str*, or ``None`` if - no options have that option string. - -.. method:: OptionParser.has_option(opt_str) - - Return ``True`` if the OptionParser has an option with option string *opt_str* - (e.g., ``-q`` or ``--verbose``). - -.. method:: OptionParser.remove_option(opt_str) - - If the :class:`OptionParser` has an option corresponding to *opt_str*, that - option is removed. If that option provided any other option strings, all of - those option strings become invalid. If *opt_str* does not occur in any - option belonging to this :class:`OptionParser`, raises :exc:`ValueError`. - - -.. _optparse-conflicts-between-options: - -Conflicts between options -^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you're not careful, it's easy to define options with conflicting option -strings:: - - parser.add_option("-n", "--dry-run", ...) - ... - parser.add_option("-n", "--noisy", ...) - -(This is particularly true if you've defined your own OptionParser subclass with -some standard options.) - -Every time you add an option, :mod:`optparse` checks for conflicts with existing -options. If it finds any, it invokes the current conflict-handling mechanism. -You can set the conflict-handling mechanism either in the constructor:: - - parser = OptionParser(..., conflict_handler=handler) - -or with a separate call:: - - parser.set_conflict_handler(handler) - -The available conflict handlers are: - - ``"error"`` (default) - assume option conflicts are a programming error and raise - :exc:`OptionConflictError` - - ``"resolve"`` - resolve option conflicts intelligently (see below) - - -As an example, let's define an :class:`OptionParser` that resolves conflicts -intelligently and add conflicting options to it:: - - parser = OptionParser(conflict_handler="resolve") - parser.add_option("-n", "--dry-run", ..., help="do no harm") - parser.add_option("-n", "--noisy", ..., help="be noisy") - -At this point, :mod:`optparse` detects that a previously added option is already -using the ``-n`` option string. Since ``conflict_handler`` is ``"resolve"``, -it resolves the situation by removing ``-n`` from the earlier option's list of -option strings. Now ``--dry-run`` is the only way for the user to activate -that option. If the user asks for help, the help message will reflect that:: - - Options: - --dry-run do no harm - ... - -n, --noisy be noisy - -It's possible to whittle away the option strings for a previously added option -until there are none left, and the user has no way of invoking that option from -the command-line. In that case, :mod:`optparse` removes that option completely, -so it doesn't show up in help text or anywhere else. Carrying on with our -existing OptionParser:: - - parser.add_option("--dry-run", ..., help="new dry-run option") - -At this point, the original ``-n``/``--dry-run`` option is no longer -accessible, so :mod:`optparse` removes it, leaving this help text:: - - Options: - ... - -n, --noisy be noisy - --dry-run new dry-run option - - -.. _optparse-cleanup: - -Cleanup -^^^^^^^ - -OptionParser instances have several cyclic references. This should not be a -problem for Python's garbage collector, but you may wish to break the cyclic -references explicitly by calling :meth:`~OptionParser.destroy` on your -OptionParser once you are done with it. This is particularly useful in -long-running applications where large object graphs are reachable from your -OptionParser. - - -.. _optparse-other-methods: - -Other methods -^^^^^^^^^^^^^ - -OptionParser supports several other public methods: - -.. method:: OptionParser.set_usage(usage) - - Set the usage string according to the rules described above for the ``usage`` - constructor keyword argument. Passing ``None`` sets the default usage - string; use :const:`optparse.SUPPRESS_USAGE` to suppress a usage message. - -.. method:: OptionParser.print_usage(file=None) - - Print the usage message for the current program (``self.usage``) to *file* - (default stdout). Any occurrence of the string ``%prog`` in ``self.usage`` - is replaced with the name of the current program. Does nothing if - ``self.usage`` is empty or not defined. - -.. method:: OptionParser.get_usage() - - Same as :meth:`print_usage` but returns the usage string instead of - printing it. - -.. method:: OptionParser.set_defaults(dest=value, ...) - - Set default values for several option destinations at once. Using - :meth:`set_defaults` is the preferred way to set default values for options, - since multiple options can share the same destination. For example, if - several "mode" options all set the same destination, any one of them can set - the default, and the last one wins:: - - parser.add_option("--advanced", action="store_const", - dest="mode", const="advanced", - default="novice") # overridden below - parser.add_option("--novice", action="store_const", - dest="mode", const="novice", - default="advanced") # overrides above setting - - To avoid this confusion, use :meth:`set_defaults`:: - - parser.set_defaults(mode="advanced") - parser.add_option("--advanced", action="store_const", - dest="mode", const="advanced") - parser.add_option("--novice", action="store_const", - dest="mode", const="novice") - - -.. _optparse-option-callbacks: - -Option Callbacks ----------------- - -When :mod:`optparse`'s built-in actions and types aren't quite enough for your -needs, you have two choices: extend :mod:`optparse` or define a callback option. -Extending :mod:`optparse` is more general, but overkill for a lot of simple -cases. Quite often a simple callback is all you need. - -There are two steps to defining a callback option: - -* define the option itself using the ``"callback"`` action - -* write the callback; this is a function (or method) that takes at least four - arguments, as described below - - -.. _optparse-defining-callback-option: - -Defining a callback option -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As always, the easiest way to define a callback option is by using the -:meth:`OptionParser.add_option` method. Apart from :attr:`~Option.action`, the -only option attribute you must specify is ``callback``, the function to call:: - - parser.add_option("-c", action="callback", callback=my_callback) - -``callback`` is a function (or other callable object), so you must have already -defined ``my_callback()`` when you create this callback option. In this simple -case, :mod:`optparse` doesn't even know if ``-c`` takes any arguments, -which usually means that the option takes no arguments---the mere presence of -``-c`` on the command-line is all it needs to know. In some -circumstances, though, you might want your callback to consume an arbitrary -number of command-line arguments. This is where writing callbacks gets tricky; -it's covered later in this section. - -:mod:`optparse` always passes four particular arguments to your callback, and it -will only pass additional arguments if you specify them via -:attr:`~Option.callback_args` and :attr:`~Option.callback_kwargs`. Thus, the -minimal callback function signature is:: - - def my_callback(option, opt, value, parser): - -The four arguments to a callback are described below. - -There are several other option attributes that you can supply when you define a -callback option: - -:attr:`~Option.type` - has its usual meaning: as with the ``"store"`` or ``"append"`` actions, it - instructs :mod:`optparse` to consume one argument and convert it to - :attr:`~Option.type`. Rather than storing the converted value(s) anywhere, - though, :mod:`optparse` passes it to your callback function. - -:attr:`~Option.nargs` - also has its usual meaning: if it is supplied and > 1, :mod:`optparse` will - consume :attr:`~Option.nargs` arguments, each of which must be convertible to - :attr:`~Option.type`. It then passes a tuple of converted values to your - callback. - -:attr:`~Option.callback_args` - a tuple of extra positional arguments to pass to the callback - -:attr:`~Option.callback_kwargs` - a dictionary of extra keyword arguments to pass to the callback - - -.. _optparse-how-callbacks-called: - -How callbacks are called -^^^^^^^^^^^^^^^^^^^^^^^^ - -All callbacks are called as follows:: - - func(option, opt_str, value, parser, *args, **kwargs) - -where - -``option`` - is the Option instance that's calling the callback - -``opt_str`` - is the option string seen on the command-line that's triggering the callback. - (If an abbreviated long option was used, ``opt_str`` will be the full, - canonical option string---e.g. if the user puts ``--foo`` on the - command-line as an abbreviation for ``--foobar``, then ``opt_str`` will be - ``"--foobar"``.) - -``value`` - is the argument to this option seen on the command-line. :mod:`optparse` will - only expect an argument if :attr:`~Option.type` is set; the type of ``value`` will be - the type implied by the option's type. If :attr:`~Option.type` for this option is - ``None`` (no argument expected), then ``value`` will be ``None``. If :attr:`~Option.nargs` - > 1, ``value`` will be a tuple of values of the appropriate type. - -``parser`` - is the OptionParser instance driving the whole thing, mainly useful because - you can access some other interesting data through its instance attributes: - - ``parser.largs`` - the current list of leftover arguments, ie. arguments that have been - consumed but are neither options nor option arguments. Feel free to modify - ``parser.largs``, e.g. by adding more arguments to it. (This list will - become ``args``, the second return value of :meth:`~OptionParser.parse_args`.) - - ``parser.rargs`` - the current list of remaining arguments, ie. with ``opt_str`` and - ``value`` (if applicable) removed, and only the arguments following them - still there. Feel free to modify ``parser.rargs``, e.g. by consuming more - arguments. - - ``parser.values`` - the object where option values are by default stored (an instance of - optparse.OptionValues). This lets callbacks use the same mechanism as the - rest of :mod:`optparse` for storing option values; you don't need to mess - around with globals or closures. You can also access or modify the - value(s) of any options already encountered on the command-line. - -``args`` - is a tuple of arbitrary positional arguments supplied via the - :attr:`~Option.callback_args` option attribute. - -``kwargs`` - is a dictionary of arbitrary keyword arguments supplied via - :attr:`~Option.callback_kwargs`. - - -.. _optparse-raising-errors-in-callback: - -Raising errors in a callback -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The callback function should raise :exc:`OptionValueError` if there are any -problems with the option or its argument(s). :mod:`optparse` catches this and -terminates the program, printing the error message you supply to stderr. Your -message should be clear, concise, accurate, and mention the option at fault. -Otherwise, the user will have a hard time figuring out what they did wrong. - - -.. _optparse-callback-example-1: - -Callback example 1: trivial callback -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Here's an example of a callback option that takes no arguments, and simply -records that the option was seen:: - - def record_foo_seen(option, opt_str, value, parser): - parser.values.saw_foo = True - - parser.add_option("--foo", action="callback", callback=record_foo_seen) - -Of course, you could do that with the ``"store_true"`` action. - - -.. _optparse-callback-example-2: - -Callback example 2: check option order -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Here's a slightly more interesting example: record the fact that ``-a`` is -seen, but blow up if it comes after ``-b`` in the command-line. :: - - def check_order(option, opt_str, value, parser): - if parser.values.b: - raise OptionValueError("can't use -a after -b") - parser.values.a = 1 - ... - parser.add_option("-a", action="callback", callback=check_order) - parser.add_option("-b", action="store_true", dest="b") - - -.. _optparse-callback-example-3: - -Callback example 3: check option order (generalized) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you want to re-use this callback for several similar options (set a flag, but -blow up if ``-b`` has already been seen), it needs a bit of work: the error -message and the flag that it sets must be generalized. :: - - def check_order(option, opt_str, value, parser): - if parser.values.b: - raise OptionValueError("can't use %s after -b" % opt_str) - setattr(parser.values, option.dest, 1) - ... - parser.add_option("-a", action="callback", callback=check_order, dest='a') - parser.add_option("-b", action="store_true", dest="b") - parser.add_option("-c", action="callback", callback=check_order, dest='c') - - -.. _optparse-callback-example-4: - -Callback example 4: check arbitrary condition -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Of course, you could put any condition in there---you're not limited to checking -the values of already-defined options. For example, if you have options that -should not be called when the moon is full, all you have to do is this:: - - def check_moon(option, opt_str, value, parser): - if is_moon_full(): - raise OptionValueError("%s option invalid when moon is full" - % opt_str) - setattr(parser.values, option.dest, 1) - ... - parser.add_option("--foo", - action="callback", callback=check_moon, dest="foo") - -(The definition of ``is_moon_full()`` is left as an exercise for the reader.) - - -.. _optparse-callback-example-5: - -Callback example 5: fixed arguments -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Things get slightly more interesting when you define callback options that take -a fixed number of arguments. Specifying that a callback option takes arguments -is similar to defining a ``"store"`` or ``"append"`` option: if you define -:attr:`~Option.type`, then the option takes one argument that must be -convertible to that type; if you further define :attr:`~Option.nargs`, then the -option takes :attr:`~Option.nargs` arguments. - -Here's an example that just emulates the standard ``"store"`` action:: - - def store_value(option, opt_str, value, parser): - setattr(parser.values, option.dest, value) - ... - parser.add_option("--foo", - action="callback", callback=store_value, - type="int", nargs=3, dest="foo") - -Note that :mod:`optparse` takes care of consuming 3 arguments and converting -them to integers for you; all you have to do is store them. (Or whatever; -obviously you don't need a callback for this example.) - - -.. _optparse-callback-example-6: - -Callback example 6: variable arguments -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Things get hairy when you want an option to take a variable number of arguments. -For this case, you must write a callback, as :mod:`optparse` doesn't provide any -built-in capabilities for it. And you have to deal with certain intricacies of -conventional Unix command-line parsing that :mod:`optparse` normally handles for -you. In particular, callbacks should implement the conventional rules for bare -``--`` and ``-`` arguments: - -* either ``--`` or ``-`` can be option arguments - -* bare ``--`` (if not the argument to some option): halt command-line - processing and discard the ``--`` - -* bare ``-`` (if not the argument to some option): halt command-line - processing but keep the ``-`` (append it to ``parser.largs``) - -If you want an option that takes a variable number of arguments, there are -several subtle, tricky issues to worry about. The exact implementation you -choose will be based on which trade-offs you're willing to make for your -application (which is why :mod:`optparse` doesn't support this sort of thing -directly). - -Nevertheless, here's a stab at a callback for an option with variable -arguments:: - - def vararg_callback(option, opt_str, value, parser): - assert value is None - value = [] - - def floatable(str): - try: - float(str) - return True - except ValueError: - return False - - for arg in parser.rargs: - # stop on --foo like options - if arg[:2] == "--" and len(arg) > 2: - break - # stop on -a, but not on -3 or -3.0 - if arg[:1] == "-" and len(arg) > 1 and not floatable(arg): - break - value.append(arg) - - del parser.rargs[:len(value)] - setattr(parser.values, option.dest, value) - - ... - parser.add_option("-c", "--callback", dest="vararg_attr", - action="callback", callback=vararg_callback) - - -.. _optparse-extending-optparse: - -Extending :mod:`optparse` -------------------------- - -Since the two major controlling factors in how :mod:`optparse` interprets -command-line options are the action and type of each option, the most likely -direction of extension is to add new actions and new types. - - -.. _optparse-adding-new-types: - -Adding new types -^^^^^^^^^^^^^^^^ - -To add new types, you need to define your own subclass of :mod:`optparse`'s -:class:`Option` class. This class has a couple of attributes that define -:mod:`optparse`'s types: :attr:`~Option.TYPES` and :attr:`~Option.TYPE_CHECKER`. - -.. attribute:: Option.TYPES - - A tuple of type names; in your subclass, simply define a new tuple - :attr:`TYPES` that builds on the standard one. - -.. attribute:: Option.TYPE_CHECKER - - A dictionary mapping type names to type-checking functions. A type-checking - function has the following signature:: - - def check_mytype(option, opt, value) - - where ``option`` is an :class:`Option` instance, ``opt`` is an option string - (e.g., ``-f``), and ``value`` is the string from the command line that must - be checked and converted to your desired type. ``check_mytype()`` should - return an object of the hypothetical type ``mytype``. The value returned by - a type-checking function will wind up in the OptionValues instance returned - by :meth:`OptionParser.parse_args`, or be passed to a callback as the - ``value`` parameter. - - Your type-checking function should raise :exc:`OptionValueError` if it - encounters any problems. :exc:`OptionValueError` takes a single string - argument, which is passed as-is to :class:`OptionParser`'s :meth:`error` - method, which in turn prepends the program name and the string ``"error:"`` - and prints everything to stderr before terminating the process. - -Here's a silly example that demonstrates adding a ``"complex"`` option type to -parse Python-style complex numbers on the command line. (This is even sillier -than it used to be, because :mod:`optparse` 1.3 added built-in support for -complex numbers, but never mind.) - -First, the necessary imports:: - - from copy import copy - from optparse import Option, OptionValueError - -You need to define your type-checker first, since it's referred to later (in the -:attr:`~Option.TYPE_CHECKER` class attribute of your Option subclass):: - - def check_complex(option, opt, value): - try: - return complex(value) - except ValueError: - raise OptionValueError( - "option %s: invalid complex value: %r" % (opt, value)) - -Finally, the Option subclass:: - - class MyOption (Option): - TYPES = Option.TYPES + ("complex",) - TYPE_CHECKER = copy(Option.TYPE_CHECKER) - TYPE_CHECKER["complex"] = check_complex - -(If we didn't make a :func:`copy` of :attr:`Option.TYPE_CHECKER`, we would end -up modifying the :attr:`~Option.TYPE_CHECKER` attribute of :mod:`optparse`'s -Option class. This being Python, nothing stops you from doing that except good -manners and common sense.) - -That's it! Now you can write a script that uses the new option type just like -any other :mod:`optparse`\ -based script, except you have to instruct your -OptionParser to use MyOption instead of Option:: - - parser = OptionParser(option_class=MyOption) - parser.add_option("-c", type="complex") - -Alternately, you can build your own option list and pass it to OptionParser; if -you don't use :meth:`add_option` in the above way, you don't need to tell -OptionParser which option class to use:: - - option_list = [MyOption("-c", action="store", type="complex", dest="c")] - parser = OptionParser(option_list=option_list) - - -.. _optparse-adding-new-actions: - -Adding new actions -^^^^^^^^^^^^^^^^^^ - -Adding new actions is a bit trickier, because you have to understand that -:mod:`optparse` has a couple of classifications for actions: - -"store" actions - actions that result in :mod:`optparse` storing a value to an attribute of the - current OptionValues instance; these options require a :attr:`~Option.dest` - attribute to be supplied to the Option constructor. - -"typed" actions - actions that take a value from the command line and expect it to be of a - certain type; or rather, a string that can be converted to a certain type. - These options require a :attr:`~Option.type` attribute to the Option - constructor. - -These are overlapping sets: some default "store" actions are ``"store"``, -``"store_const"``, ``"append"``, and ``"count"``, while the default "typed" -actions are ``"store"``, ``"append"``, and ``"callback"``. - -When you add an action, you need to categorize it by listing it in at least one -of the following class attributes of Option (all are lists of strings): - -.. attribute:: Option.ACTIONS - - All actions must be listed in ACTIONS. - -.. attribute:: Option.STORE_ACTIONS - - "store" actions are additionally listed here. - -.. attribute:: Option.TYPED_ACTIONS - - "typed" actions are additionally listed here. - -.. attribute:: Option.ALWAYS_TYPED_ACTIONS - - Actions that always take a type (i.e. whose options always take a value) are - additionally listed here. The only effect of this is that :mod:`optparse` - assigns the default type, ``"string"``, to options with no explicit type - whose action is listed in :attr:`ALWAYS_TYPED_ACTIONS`. - -In order to actually implement your new action, you must override Option's -:meth:`take_action` method and add a case that recognizes your action. - -For example, let's add an ``"extend"`` action. This is similar to the standard -``"append"`` action, but instead of taking a single value from the command-line -and appending it to an existing list, ``"extend"`` will take multiple values in -a single comma-delimited string, and extend an existing list with them. That -is, if ``--names`` is an ``"extend"`` option of type ``"string"``, the command -line :: - - --names=foo,bar --names blah --names ding,dong - -would result in a list :: - - ["foo", "bar", "blah", "ding", "dong"] - -Again we define a subclass of Option:: - - class MyOption(Option): - - ACTIONS = Option.ACTIONS + ("extend",) - STORE_ACTIONS = Option.STORE_ACTIONS + ("extend",) - TYPED_ACTIONS = Option.TYPED_ACTIONS + ("extend",) - ALWAYS_TYPED_ACTIONS = Option.ALWAYS_TYPED_ACTIONS + ("extend",) - - def take_action(self, action, dest, opt, value, values, parser): - if action == "extend": - lvalue = value.split(",") - values.ensure_value(dest, []).extend(lvalue) - else: - Option.take_action( - self, action, dest, opt, value, values, parser) - -Features of note: - -* ``"extend"`` both expects a value on the command-line and stores that value - somewhere, so it goes in both :attr:`~Option.STORE_ACTIONS` and - :attr:`~Option.TYPED_ACTIONS`. - -* to ensure that :mod:`optparse` assigns the default type of ``"string"`` to - ``"extend"`` actions, we put the ``"extend"`` action in - :attr:`~Option.ALWAYS_TYPED_ACTIONS` as well. - -* :meth:`MyOption.take_action` implements just this one new action, and passes - control back to :meth:`Option.take_action` for the standard :mod:`optparse` - actions. - -* ``values`` is an instance of the optparse_parser.Values class, which provides - the very useful :meth:`ensure_value` method. :meth:`ensure_value` is - essentially :func:`getattr` with a safety valve; it is called as :: - - values.ensure_value(attr, value) - - If the ``attr`` attribute of ``values`` doesn't exist or is ``None``, then - ensure_value() first sets it to ``value``, and then returns 'value. This is - very handy for actions like ``"extend"``, ``"append"``, and ``"count"``, all - of which accumulate data in a variable and expect that variable to be of a - certain type (a list for the first two, an integer for the latter). Using - :meth:`ensure_value` means that scripts using your action don't have to worry - about setting a default value for the option destinations in question; they - can just leave the default as ``None`` and :meth:`ensure_value` will take care of - getting it right when it's needed. - -Exceptions ----------- - -.. exception:: OptionError - - Raised if an :class:`Option` instance is created with invalid or - inconsistent arguments. - -.. exception:: OptionConflictError - - Raised if conflicting options are added to an :class:`OptionParser`. - -.. exception:: OptionValueError - - Raised if an invalid option value is encountered on the command line. - -.. exception:: BadOptionError - - Raised if an invalid option is passed on the command line. - -.. exception:: AmbiguousOptionError - - Raised if an ambiguous option is passed on the command line. diff --git a/Doc/library/os.path.rst b/Doc/library/os.path.rst deleted file mode 100644 index 8f6dffd04fa54a..00000000000000 --- a/Doc/library/os.path.rst +++ /dev/null @@ -1,516 +0,0 @@ -:mod:`os.path` --- Common pathname manipulations -================================================ - -.. module:: os.path - :synopsis: Operations on pathnames. - -**Source code:** :source:`Lib/posixpath.py` (for POSIX) and -:source:`Lib/ntpath.py` (for Windows). - -.. index:: single: path; operations - --------------- - -This module implements some useful functions on pathnames. To read or write -files see :func:`open`, and for accessing the filesystem see the :mod:`os` -module. The path parameters can be passed as strings, or bytes, or any object -implementing the :class:`os.PathLike` protocol. - -Unlike a Unix shell, Python does not do any *automatic* path expansions. -Functions such as :func:`expanduser` and :func:`expandvars` can be invoked -explicitly when an application desires shell-like path expansion. (See also -the :mod:`glob` module.) - - -.. seealso:: - The :mod:`pathlib` module offers high-level path objects. - - -.. note:: - - All of these functions accept either only bytes or only string objects as - their parameters. The result is an object of the same type, if a path or - file name is returned. - -.. note:: - - Since different operating systems have different path name conventions, there - are several versions of this module in the standard library. The - :mod:`os.path` module is always the path module suitable for the operating - system Python is running on, and therefore usable for local paths. However, - you can also import and use the individual modules if you want to manipulate - a path that is *always* in one of the different formats. They all have the - same interface: - - * :mod:`posixpath` for UNIX-style paths - * :mod:`ntpath` for Windows paths - - -.. versionchanged:: 3.8 - - :func:`exists`, :func:`lexists`, :func:`isdir`, :func:`isfile`, - :func:`islink`, and :func:`ismount` now return ``False`` instead of - raising an exception for paths that contain characters or bytes - unrepresentable at the OS level. - - -.. function:: abspath(path) - - Return a normalized absolutized version of the pathname *path*. On most - platforms, this is equivalent to calling the function :func:`normpath` as - follows: ``normpath(join(os.getcwd(), path))``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: basename(path) - - Return the base name of pathname *path*. This is the second element of the - pair returned by passing *path* to the function :func:`split`. Note that - the result of this function is different - from the Unix :program:`basename` program; where :program:`basename` for - ``'/foo/bar/'`` returns ``'bar'``, the :func:`basename` function returns an - empty string (``''``). - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: commonpath(paths) - - Return the longest common sub-path of each pathname in the sequence - *paths*. Raise :exc:`ValueError` if *paths* contain both absolute - and relative pathnames, the *paths* are on the different drives or - if *paths* is empty. Unlike :func:`commonprefix`, this returns a - valid path. - - .. availability:: Unix, Windows. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.6 - Accepts a sequence of :term:`path-like objects `. - - -.. function:: commonprefix(list) - - Return the longest path prefix (taken character-by-character) that is a - prefix of all paths in *list*. If *list* is empty, return the empty string - (``''``). - - .. note:: - - This function may return invalid paths because it works a - character at a time. To obtain a valid path, see - :func:`commonpath`. - - :: - - >>> os.path.commonprefix(['/usr/lib', '/usr/local/lib']) - '/usr/l' - - >>> os.path.commonpath(['/usr/lib', '/usr/local/lib']) - '/usr' - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: dirname(path) - - Return the directory name of pathname *path*. This is the first element of - the pair returned by passing *path* to the function :func:`split`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: exists(path) - - Return ``True`` if *path* refers to an existing path or an open - file descriptor. Returns ``False`` for broken symbolic links. On - some platforms, this function may return ``False`` if permission is - not granted to execute :func:`os.stat` on the requested file, even - if the *path* physically exists. - - .. versionchanged:: 3.3 - *path* can now be an integer: ``True`` is returned if it is an - open file descriptor, ``False`` otherwise. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: lexists(path) - - Return ``True`` if *path* refers to an existing path. Returns ``True`` for - broken symbolic links. Equivalent to :func:`exists` on platforms lacking - :func:`os.lstat`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. index:: single: ~ (tilde); home directory expansion - -.. function:: expanduser(path) - - On Unix and Windows, return the argument with an initial component of ``~`` or - ``~user`` replaced by that *user*'s home directory. - - .. index:: pair: module; pwd - - On Unix, an initial ``~`` is replaced by the environment variable :envvar:`HOME` - if it is set; otherwise the current user's home directory is looked up in the - password directory through the built-in module :mod:`pwd`. An initial ``~user`` - is looked up directly in the password directory. - - On Windows, :envvar:`USERPROFILE` will be used if set, otherwise a combination - of :envvar:`HOMEPATH` and :envvar:`HOMEDRIVE` will be used. An initial - ``~user`` is handled by checking that the last directory component of the current - user's home directory matches :envvar:`USERNAME`, and replacing it if so. - - If the expansion fails or if the path does not begin with a tilde, the path is - returned unchanged. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. versionchanged:: 3.8 - No longer uses :envvar:`HOME` on Windows. - -.. index:: - single: $ (dollar); environment variables expansion - single: % (percent); environment variables expansion (Windows) - -.. function:: expandvars(path) - - Return the argument with environment variables expanded. Substrings of the form - ``$name`` or ``${name}`` are replaced by the value of environment variable - *name*. Malformed variable names and references to non-existing variables are - left unchanged. - - On Windows, ``%name%`` expansions are supported in addition to ``$name`` and - ``${name}``. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: getatime(path) - - Return the time of last access of *path*. The return value is a floating point number giving - the number of seconds since the epoch (see the :mod:`time` module). Raise - :exc:`OSError` if the file does not exist or is inaccessible. - - -.. function:: getmtime(path) - - Return the time of last modification of *path*. The return value is a floating point number - giving the number of seconds since the epoch (see the :mod:`time` module). - Raise :exc:`OSError` if the file does not exist or is inaccessible. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: getctime(path) - - Return the system's ctime which, on some systems (like Unix) is the time of the - last metadata change, and, on others (like Windows), is the creation time for *path*. - The return value is a number giving the number of seconds since the epoch (see - the :mod:`time` module). Raise :exc:`OSError` if the file does not exist or - is inaccessible. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: getsize(path) - - Return the size, in bytes, of *path*. Raise :exc:`OSError` if the file does - not exist or is inaccessible. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: isabs(path) - - Return ``True`` if *path* is an absolute pathname. On Unix, that means it - begins with a slash, on Windows that it begins with a (back)slash after chopping - off a potential drive letter. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: isfile(path) - - Return ``True`` if *path* is an :func:`existing ` regular file. - This follows symbolic links, so both :func:`islink` and :func:`isfile` can - be true for the same path. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: isdir(path) - - Return ``True`` if *path* is an :func:`existing ` directory. This - follows symbolic links, so both :func:`islink` and :func:`isdir` can be true - for the same path. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: islink(path) - - Return ``True`` if *path* refers to an :func:`existing ` directory - entry that is a symbolic link. Always ``False`` if symbolic links are not - supported by the Python runtime. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: ismount(path) - - Return ``True`` if pathname *path* is a :dfn:`mount point`: a point in a - file system where a different file system has been mounted. On POSIX, the - function checks whether *path*'s parent, :file:`{path}/..`, is on a different - device than *path*, or whether :file:`{path}/..` and *path* point to the same - i-node on the same device --- this should detect mount points for all Unix - and POSIX variants. It is not able to reliably detect bind mounts on the - same filesystem. On Windows, a drive letter root and a share UNC are - always mount points, and for any other path ``GetVolumePathName`` is called - to see if it is different from the input path. - - .. versionadded:: 3.4 - Support for detecting non-root mount points on Windows. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: join(path, *paths) - - Join one or more path segments intelligently. The return value is the - concatenation of *path* and all members of *\*paths*, with exactly one - directory separator following each non-empty part, except the last. That is, - the result will only end in a separator if the last part is either empty or - ends in a separator. If a segment is an absolute path (which on Windows - requires both a drive and a root), then all previous segments are ignored and - joining continues from the absolute path segment. - - On Windows, the drive is not reset when a rooted path segment (e.g., - ``r'\foo'``) is encountered. If a segment is on a different drive or is an - absolute path, all previous segments are ignored and the drive is reset. Note - that since there is a current directory for each drive, - ``os.path.join("c:", "foo")`` represents a path relative to the current - directory on drive :file:`C:` (:file:`c:foo`), not :file:`c:\\foo`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *path* and *paths*. - - -.. function:: normcase(path) - - Normalize the case of a pathname. On Windows, convert all characters in the - pathname to lowercase, and also convert forward slashes to backward slashes. - On other operating systems, return the path unchanged. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: normpath(path) - - Normalize a pathname by collapsing redundant separators and up-level - references so that ``A//B``, ``A/B/``, ``A/./B`` and ``A/foo/../B`` all - become ``A/B``. This string manipulation may change the meaning of a path - that contains symbolic links. On Windows, it converts forward slashes to - backward slashes. To normalize case, use :func:`normcase`. - - .. note:: - On POSIX systems, in accordance with `IEEE Std 1003.1 2013 Edition; 4.13 - Pathname Resolution `_, - if a pathname begins with exactly two slashes, the first component - following the leading characters may be interpreted in an implementation-defined - manner, although more than two leading characters shall be treated as a - single character. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: realpath(path, *, strict=False) - - Return the canonical path of the specified filename, eliminating any symbolic - links encountered in the path (if they are supported by the operating - system). - - If a path doesn't exist or a symlink loop is encountered, and *strict* is - ``True``, :exc:`OSError` is raised. If *strict* is ``False``, the path is - resolved as far as possible and any remainder is appended without checking - whether it exists. - - .. note:: - This function emulates the operating system's procedure for making a path - canonical, which differs slightly between Windows and UNIX with respect - to how links and subsequent path components interact. - - Operating system APIs make paths canonical as needed, so it's not - normally necessary to call this function. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. versionchanged:: 3.8 - Symbolic links and junctions are now resolved on Windows. - - .. versionchanged:: 3.10 - The *strict* parameter was added. - - -.. function:: relpath(path, start=os.curdir) - - Return a relative filepath to *path* either from the current directory or - from an optional *start* directory. This is a path computation: the - filesystem is not accessed to confirm the existence or nature of *path* or - *start*. On Windows, :exc:`ValueError` is raised when *path* and *start* - are on different drives. - - *start* defaults to :data:`os.curdir`. - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: samefile(path1, path2) - - Return ``True`` if both pathname arguments refer to the same file or directory. - This is determined by the device number and i-node number and raises an - exception if an :func:`os.stat` call on either pathname fails. - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.2 - Added Windows support. - - .. versionchanged:: 3.4 - Windows now uses the same implementation as all other platforms. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: sameopenfile(fp1, fp2) - - Return ``True`` if the file descriptors *fp1* and *fp2* refer to the same file. - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.2 - Added Windows support. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: samestat(stat1, stat2) - - Return ``True`` if the stat tuples *stat1* and *stat2* refer to the same file. - These structures may have been returned by :func:`os.fstat`, - :func:`os.lstat`, or :func:`os.stat`. This function implements the - underlying comparison used by :func:`samefile` and :func:`sameopenfile`. - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.4 - Added Windows support. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: split(path) - - Split the pathname *path* into a pair, ``(head, tail)`` where *tail* is the - last pathname component and *head* is everything leading up to that. The - *tail* part will never contain a slash; if *path* ends in a slash, *tail* - will be empty. If there is no slash in *path*, *head* will be empty. If - *path* is empty, both *head* and *tail* are empty. Trailing slashes are - stripped from *head* unless it is the root (one or more slashes only). In - all cases, ``join(head, tail)`` returns a path to the same location as *path* - (but the strings may differ). Also see the functions :func:`dirname` and - :func:`basename`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: splitdrive(path) - - Split the pathname *path* into a pair ``(drive, tail)`` where *drive* is either - a mount point or the empty string. On systems which do not use drive - specifications, *drive* will always be the empty string. In all cases, ``drive - + tail`` will be the same as *path*. - - On Windows, splits a pathname into drive/UNC sharepoint and relative path. - - If the path contains a drive letter, drive will contain everything - up to and including the colon:: - - >>> splitdrive("c:/dir") - ("c:", "/dir") - - If the path contains a UNC path, drive will contain the host name - and share, up to but not including the fourth separator:: - - >>> splitdrive("//host/computer/dir") - ("//host/computer", "/dir") - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: splitext(path) - - Split the pathname *path* into a pair ``(root, ext)`` such that ``root + ext == - path``, and the extension, *ext*, is empty or begins with a period and contains at - most one period. - - If the path contains no extension, *ext* will be ``''``:: - - >>> splitext('bar') - ('bar', '') - - If the path contains an extension, then *ext* will be set to this extension, - including the leading period. Note that previous periods will be ignored:: - - >>> splitext('foo.bar.exe') - ('foo.bar', '.exe') - >>> splitext('/foo/bar.exe') - ('/foo/bar', '.exe') - - Leading periods of the last component of the path are considered to - be part of the root:: - - >>> splitext('.cshrc') - ('.cshrc', '') - >>> splitext('/foo/....jpg') - ('/foo/....jpg', '') - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. data:: supports_unicode_filenames - - ``True`` if arbitrary Unicode strings can be used as file names (within limitations - imposed by the file system). diff --git a/Doc/library/os.rst b/Doc/library/os.rst deleted file mode 100644 index 087a01d2d9efc1..00000000000000 --- a/Doc/library/os.rst +++ /dev/null @@ -1,5141 +0,0 @@ -:mod:`os` --- Miscellaneous operating system interfaces -======================================================= - -.. module:: os - :synopsis: Miscellaneous operating system interfaces. - -**Source code:** :source:`Lib/os.py` - --------------- - -This module provides a portable way of using operating system dependent -functionality. If you just want to read or write a file see :func:`open`, if -you want to manipulate paths, see the :mod:`os.path` module, and if you want to -read all the lines in all the files on the command line see the :mod:`fileinput` -module. For creating temporary files and directories see the :mod:`tempfile` -module, and for high-level file and directory handling see the :mod:`shutil` -module. - -Notes on the availability of these functions: - -* The design of all built-in operating system dependent modules of Python is - such that as long as the same functionality is available, it uses the same - interface; for example, the function ``os.stat(path)`` returns stat - information about *path* in the same format (which happens to have originated - with the POSIX interface). - -* Extensions peculiar to a particular operating system are also available - through the :mod:`os` module, but using them is of course a threat to - portability. - -* All functions accepting path or file names accept both bytes and string - objects, and result in an object of the same type, if a path or file name is - returned. - -* On VxWorks, os.popen, os.fork, os.execv and os.spawn*p* are not supported. - -* On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, large - parts of the :mod:`os` module are not available or behave differently. API - related to processes (e.g. :func:`~os.fork`, :func:`~os.execve`), signals - (e.g. :func:`~os.kill`, :func:`~os.wait`), and resources - (e.g. :func:`~os.nice`) are not available. Others like :func:`~os.getuid` - and :func:`~os.getpid` are emulated or stubs. - - -.. note:: - - All functions in this module raise :exc:`OSError` (or subclasses thereof) in - the case of invalid or inaccessible file names and paths, or other arguments - that have the correct type, but are not accepted by the operating system. - -.. exception:: error - - An alias for the built-in :exc:`OSError` exception. - - -.. data:: name - - The name of the operating system dependent module imported. The following - names have currently been registered: ``'posix'``, ``'nt'``, - ``'java'``. - - .. seealso:: - :data:`sys.platform` has a finer granularity. :func:`os.uname` gives - system-dependent version information. - - The :mod:`platform` module provides detailed checks for the - system's identity. - - -.. _os-filenames: -.. _filesystem-encoding: - -File Names, Command Line Arguments, and Environment Variables -------------------------------------------------------------- - -In Python, file names, command line arguments, and environment variables are -represented using the string type. On some systems, decoding these strings to -and from bytes is necessary before passing them to the operating system. Python -uses the :term:`filesystem encoding and error handler` to perform this -conversion (see :func:`sys.getfilesystemencoding`). - -The :term:`filesystem encoding and error handler` are configured at Python -startup by the :c:func:`PyConfig_Read` function: see -:c:member:`~PyConfig.filesystem_encoding` and -:c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. - -.. versionchanged:: 3.1 - On some systems, conversion using the file system encoding may fail. In this - case, Python uses the :ref:`surrogateescape encoding error handler - `, which means that undecodable bytes are replaced by a - Unicode character U+DC\ *xx* on decoding, and these are again - translated to the original byte on encoding. - - -The :term:`file system encoding ` must -guarantee to successfully decode all bytes below 128. If the file system -encoding fails to provide this guarantee, API functions can raise -:exc:`UnicodeError`. - -See also the :term:`locale encoding`. - - -.. _utf8-mode: - -Python UTF-8 Mode ------------------ - -.. versionadded:: 3.7 - See :pep:`540` for more details. - -The Python UTF-8 Mode ignores the :term:`locale encoding` and forces the usage -of the UTF-8 encoding: - -* Use UTF-8 as the :term:`filesystem encoding `. -* :func:`sys.getfilesystemencoding()` returns ``'utf-8'``. -* :func:`locale.getpreferredencoding()` returns ``'utf-8'`` (the *do_setlocale* - argument has no effect). -* :data:`sys.stdin`, :data:`sys.stdout`, and :data:`sys.stderr` all use - UTF-8 as their text encoding, with the ``surrogateescape`` - :ref:`error handler ` being enabled for :data:`sys.stdin` - and :data:`sys.stdout` (:data:`sys.stderr` continues to use - ``backslashreplace`` as it does in the default locale-aware mode) -* On Unix, :func:`os.device_encoding` returns ``'utf-8'`` rather than the - device encoding. - -Note that the standard stream settings in UTF-8 mode can be overridden by -:envvar:`PYTHONIOENCODING` (just as they can be in the default locale-aware -mode). - -As a consequence of the changes in those lower level APIs, other higher -level APIs also exhibit different default behaviours: - -* Command line arguments, environment variables and filenames are decoded - to text using the UTF-8 encoding. -* :func:`os.fsdecode()` and :func:`os.fsencode()` use the UTF-8 encoding. -* :func:`open()`, :func:`io.open()`, and :func:`codecs.open()` use the UTF-8 - encoding by default. However, they still use the strict error handler by - default so that attempting to open a binary file in text mode is likely - to raise an exception rather than producing nonsense data. - -The :ref:`Python UTF-8 Mode ` is enabled if the LC_CTYPE locale is -``C`` or ``POSIX`` at Python startup (see the :c:func:`PyConfig_Read` -function). - -It can be enabled or disabled using the :option:`-X utf8 <-X>` command line -option and the :envvar:`PYTHONUTF8` environment variable. - -If the :envvar:`PYTHONUTF8` environment variable is not set at all, then the -interpreter defaults to using the current locale settings, *unless* the current -locale is identified as a legacy ASCII-based locale (as described for -:envvar:`PYTHONCOERCECLOCALE`), and locale coercion is either disabled or -fails. In such legacy locales, the interpreter will default to enabling UTF-8 -mode unless explicitly instructed not to do so. - -The Python UTF-8 Mode can only be enabled at the Python startup. Its value -can be read from :data:`sys.flags.utf8_mode `. - -See also the :ref:`UTF-8 mode on Windows ` -and the :term:`filesystem encoding and error handler`. - -.. seealso:: - - :pep:`686` - Python 3.15 will make :ref:`utf8-mode` default. - - -.. _os-procinfo: - -Process Parameters ------------------- - -These functions and data items provide information and operate on the current -process and user. - - -.. function:: ctermid() - - Return the filename corresponding to the controlling terminal of the process. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: environ - - A :term:`mapping` object where keys and values are strings that represent - the process environment. For example, ``environ['HOME']`` is the pathname - of your home directory (on some platforms), and is equivalent to - ``getenv("HOME")`` in C. - - This mapping is captured the first time the :mod:`os` module is imported, - typically during Python startup as part of processing :file:`site.py`. Changes - to the environment made after this time are not reflected in :data:`os.environ`, - except for changes made by modifying :data:`os.environ` directly. - - This mapping may be used to modify the environment as well as query the - environment. :func:`putenv` will be called automatically when the mapping - is modified. - - On Unix, keys and values use :func:`sys.getfilesystemencoding` and - ``'surrogateescape'`` error handler. Use :data:`environb` if you would like - to use a different encoding. - - On Windows, the keys are converted to uppercase. This also applies when - getting, setting, or deleting an item. For example, - ``environ['monty'] = 'python'`` maps the key ``'MONTY'`` to the value - ``'python'``. - - .. note:: - - Calling :func:`putenv` directly does not change :data:`os.environ`, so it's better - to modify :data:`os.environ`. - - .. note:: - - On some platforms, including FreeBSD and macOS, setting ``environ`` may - cause memory leaks. Refer to the system documentation for - :c:func:`!putenv`. - - You can delete items in this mapping to unset environment variables. - :func:`unsetenv` will be called automatically when an item is deleted from - :data:`os.environ`, and when one of the :meth:`pop` or :meth:`clear` methods is - called. - - .. versionchanged:: 3.9 - Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. - - -.. data:: environb - - Bytes version of :data:`environ`: a :term:`mapping` object where both keys - and values are :class:`bytes` objects representing the process environment. - :data:`environ` and :data:`environb` are synchronized (modifying - :data:`environb` updates :data:`environ`, and vice versa). - - :data:`environb` is only available if :const:`supports_bytes_environ` is - ``True``. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.9 - Updated to support :pep:`584`'s merge (``|``) and update (``|=``) operators. - - -.. function:: chdir(path) - fchdir(fd) - getcwd() - :noindex: - - These functions are described in :ref:`os-file-dir`. - - -.. function:: fsencode(filename) - - Encode :term:`path-like ` *filename* to the - :term:`filesystem encoding and error handler`; return :class:`bytes` - unchanged. - - :func:`fsdecode` is the reverse function. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.6 - Support added to accept objects implementing the :class:`os.PathLike` - interface. - - -.. function:: fsdecode(filename) - - Decode the :term:`path-like ` *filename* from the - :term:`filesystem encoding and error handler`; return :class:`str` - unchanged. - - :func:`fsencode` is the reverse function. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.6 - Support added to accept objects implementing the :class:`os.PathLike` - interface. - - -.. function:: fspath(path) - - Return the file system representation of the path. - - If :class:`str` or :class:`bytes` is passed in, it is returned unchanged. - Otherwise :meth:`~os.PathLike.__fspath__` is called and its value is - returned as long as it is a :class:`str` or :class:`bytes` object. - In all other cases, :exc:`TypeError` is raised. - - .. versionadded:: 3.6 - - -.. class:: PathLike - - An :term:`abstract base class` for objects representing a file system path, - e.g. :class:`pathlib.PurePath`. - - .. versionadded:: 3.6 - - .. abstractmethod:: __fspath__() - - Return the file system path representation of the object. - - The method should only return a :class:`str` or :class:`bytes` object, - with the preference being for :class:`str`. - - -.. function:: getenv(key, default=None) - - Return the value of the environment variable *key* as a string if it exists, or - *default* if it doesn't. *key* is a string. Note that - since :func:`getenv` uses :data:`os.environ`, the mapping of :func:`getenv` is - similarly also captured on import, and the function may not reflect - future environment changes. - - On Unix, keys and values are decoded with :func:`sys.getfilesystemencoding` - and ``'surrogateescape'`` error handler. Use :func:`os.getenvb` if you - would like to use a different encoding. - - .. availability:: Unix, Windows. - - -.. function:: getenvb(key, default=None) - - Return the value of the environment variable *key* as bytes if it exists, or - *default* if it doesn't. *key* must be bytes. Note that - since :func:`getenvb` uses :data:`os.environb`, the mapping of :func:`getenvb` is - similarly also captured on import, and the function may not reflect - future environment changes. - - - :func:`getenvb` is only available if :const:`supports_bytes_environ` - is ``True``. - - .. availability:: Unix. - - .. versionadded:: 3.2 - - -.. function:: get_exec_path(env=None) - - Returns the list of directories that will be searched for a named - executable, similar to a shell, when launching a process. - *env*, when specified, should be an environment variable dictionary - to lookup the PATH in. - By default, when *env* is ``None``, :data:`environ` is used. - - .. versionadded:: 3.2 - - -.. function:: getegid() - - Return the effective group id of the current process. This corresponds to the - "set id" bit on the file being executed in the current process. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: geteuid() - - .. index:: single: user; effective id - - Return the current process's effective user id. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: getgid() - - .. index:: single: process; group - - Return the real group id of the current process. - - .. availability:: Unix. - - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - -.. function:: getgrouplist(user, group, /) - - Return list of group ids that *user* belongs to. If *group* is not in the - list, it is included; typically, *group* is specified as the group ID - field from the password record for *user*, because that group ID will - otherwise be potentially omitted. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - -.. function:: getgroups() - - Return list of supplemental group ids associated with the current process. - - .. availability:: Unix, not Emscripten, not WASI. - - .. note:: - - On macOS, :func:`getgroups` behavior differs somewhat from - other Unix platforms. If the Python interpreter was built with a - deployment target of ``10.5`` or earlier, :func:`getgroups` returns - the list of effective group ids associated with the current user process; - this list is limited to a system-defined number of entries, typically 16, - and may be modified by calls to :func:`setgroups` if suitably privileged. - If built with a deployment target greater than ``10.5``, - :func:`getgroups` returns the current group access list for the user - associated with the effective user id of the process; the group access - list may change over the lifetime of the process, it is not affected by - calls to :func:`setgroups`, and its length is not limited to 16. The - deployment target value, :const:`MACOSX_DEPLOYMENT_TARGET`, can be - obtained with :func:`sysconfig.get_config_var`. - - -.. function:: getlogin() - - Return the name of the user logged in on the controlling terminal of the - process. For most purposes, it is more useful to use - :func:`getpass.getuser` since the latter checks the environment variables - :envvar:`LOGNAME` or :envvar:`USERNAME` to find out who the user is, and - falls back to ``pwd.getpwuid(os.getuid())[0]`` to get the login name of the - current real user id. - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - -.. function:: getpgid(pid) - - Return the process group id of the process with process id *pid*. If *pid* is 0, - the process group id of the current process is returned. - - .. availability:: Unix, not Emscripten, not WASI. - -.. function:: getpgrp() - - .. index:: single: process; group - - Return the id of the current process group. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: getpid() - - .. index:: single: process; id - - Return the current process id. - - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - -.. function:: getppid() - - .. index:: single: process; id of parent - - Return the parent's process id. When the parent process has exited, on Unix - the id returned is the one of the init process (1), on Windows it is still - the same id, which may be already reused by another process. - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - .. versionchanged:: 3.2 - Added support for Windows. - - -.. function:: getpriority(which, who) - - .. index:: single: process; scheduling priority - - Get program scheduling priority. The value *which* is one of - :const:`PRIO_PROCESS`, :const:`PRIO_PGRP`, or :const:`PRIO_USER`, and *who* - is interpreted relative to *which* (a process identifier for - :const:`PRIO_PROCESS`, process group identifier for :const:`PRIO_PGRP`, and a - user ID for :const:`PRIO_USER`). A zero value for *who* denotes - (respectively) the calling process, the process group of the calling process, - or the real user ID of the calling process. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - -.. data:: PRIO_PROCESS - PRIO_PGRP - PRIO_USER - - Parameters for the :func:`getpriority` and :func:`setpriority` functions. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - -.. function:: getresuid() - - Return a tuple (ruid, euid, suid) denoting the current process's - real, effective, and saved user ids. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.2 - - -.. function:: getresgid() - - Return a tuple (rgid, egid, sgid) denoting the current process's - real, effective, and saved group ids. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.2 - - -.. function:: getuid() - - .. index:: single: user; id - - Return the current process's real user id. - - .. availability:: Unix. - - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - -.. function:: initgroups(username, gid, /) - - Call the system initgroups() to initialize the group access list with all of - the groups of which the specified username is a member, plus the specified - group id. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.2 - - -.. function:: putenv(key, value, /) - - .. index:: single: environment variables; setting - - Set the environment variable named *key* to the string *value*. Such - changes to the environment affect subprocesses started with :func:`os.system`, - :func:`popen` or :func:`fork` and :func:`execv`. - - Assignments to items in :data:`os.environ` are automatically translated into - corresponding calls to :func:`putenv`; however, calls to :func:`putenv` - don't update :data:`os.environ`, so it is actually preferable to assign to items - of :data:`os.environ`. This also applies to :func:`getenv` and :func:`getenvb`, which - respectively use :data:`os.environ` and :data:`os.environb` in their implementations. - - .. note:: - - On some platforms, including FreeBSD and macOS, setting ``environ`` may - cause memory leaks. Refer to the system documentation for :c:func:`!putenv`. - - .. audit-event:: os.putenv key,value os.putenv - - .. versionchanged:: 3.9 - The function is now always available. - - -.. function:: setegid(egid, /) - - Set the current process's effective group id. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: seteuid(euid, /) - - Set the current process's effective user id. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: setgid(gid, /) - - Set the current process' group id. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: setgroups(groups, /) - - Set the list of supplemental group ids associated with the current process to - *groups*. *groups* must be a sequence, and each element must be an integer - identifying a group. This operation is typically available only to the superuser. - - .. availability:: Unix, not Emscripten, not WASI. - - .. note:: On macOS, the length of *groups* may not exceed the - system-defined maximum number of effective group ids, typically 16. - See the documentation for :func:`getgroups` for cases where it may not - return the same group list set by calling setgroups(). - -.. function:: setpgrp() - - Call the system call :c:func:`!setpgrp` or ``setpgrp(0, 0)`` depending on - which version is implemented (if any). See the Unix manual for the semantics. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: setpgid(pid, pgrp, /) - - Call the system call :c:func:`!setpgid` to set the process group id of the - process with id *pid* to the process group with id *pgrp*. See the Unix manual - for the semantics. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: setpriority(which, who, priority) - - .. index:: single: process; scheduling priority - - Set program scheduling priority. The value *which* is one of - :const:`PRIO_PROCESS`, :const:`PRIO_PGRP`, or :const:`PRIO_USER`, and *who* - is interpreted relative to *which* (a process identifier for - :const:`PRIO_PROCESS`, process group identifier for :const:`PRIO_PGRP`, and a - user ID for :const:`PRIO_USER`). A zero value for *who* denotes - (respectively) the calling process, the process group of the calling process, - or the real user ID of the calling process. - *priority* is a value in the range -20 to 19. The default priority is 0; - lower priorities cause more favorable scheduling. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - -.. function:: setregid(rgid, egid, /) - - Set the current process's real and effective group ids. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: setresgid(rgid, egid, sgid, /) - - Set the current process's real, effective, and saved group ids. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.2 - - -.. function:: setresuid(ruid, euid, suid, /) - - Set the current process's real, effective, and saved user ids. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.2 - - -.. function:: setreuid(ruid, euid, /) - - Set the current process's real and effective user ids. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: getsid(pid, /) - - Call the system call :c:func:`!getsid`. See the Unix manual for the semantics. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: setsid() - - Call the system call :c:func:`!setsid`. See the Unix manual for the semantics. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: setuid(uid, /) - - .. index:: single: user; id, setting - - Set the current process's user id. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. placed in this section since it relates to errno.... a little weak -.. function:: strerror(code, /) - - Return the error message corresponding to the error code in *code*. - On platforms where :c:func:`!strerror` returns ``NULL`` when given an unknown - error number, :exc:`ValueError` is raised. - - -.. data:: supports_bytes_environ - - ``True`` if the native OS type of the environment is bytes (eg. ``False`` on - Windows). - - .. versionadded:: 3.2 - - -.. function:: umask(mask, /) - - Set the current numeric umask and return the previous umask. - - The function is a stub on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - -.. function:: uname() - - .. index:: - single: gethostname() (in module socket) - single: gethostbyaddr() (in module socket) - - Returns information identifying the current operating system. - The return value is an object with five attributes: - - * :attr:`sysname` - operating system name - * :attr:`nodename` - name of machine on network (implementation-defined) - * :attr:`release` - operating system release - * :attr:`version` - operating system version - * :attr:`machine` - hardware identifier - - For backwards compatibility, this object is also iterable, behaving - like a five-tuple containing :attr:`sysname`, :attr:`nodename`, - :attr:`release`, :attr:`version`, and :attr:`machine` - in that order. - - Some systems truncate :attr:`nodename` to 8 characters or to the - leading component; a better way to get the hostname is - :func:`socket.gethostname` or even - ``socket.gethostbyaddr(socket.gethostname())``. - - .. availability:: Unix. - - .. versionchanged:: 3.3 - Return type changed from a tuple to a tuple-like object - with named attributes. - - -.. function:: unsetenv(key, /) - - .. index:: single: environment variables; deleting - - Unset (delete) the environment variable named *key*. Such changes to the - environment affect subprocesses started with :func:`os.system`, :func:`popen` or - :func:`fork` and :func:`execv`. - - Deletion of items in :data:`os.environ` is automatically translated into a - corresponding call to :func:`unsetenv`; however, calls to :func:`unsetenv` - don't update :data:`os.environ`, so it is actually preferable to delete items of - :data:`os.environ`. - - .. audit-event:: os.unsetenv key os.unsetenv - - .. versionchanged:: 3.9 - The function is now always available and is also available on Windows. - - -.. _os-newstreams: - -File Object Creation --------------------- - -These functions create new :term:`file objects `. (See also -:func:`~os.open` for opening file descriptors.) - - -.. function:: fdopen(fd, *args, **kwargs) - - Return an open file object connected to the file descriptor *fd*. This is an - alias of the :func:`open` built-in function and accepts the same arguments. - The only difference is that the first argument of :func:`fdopen` must always - be an integer. - - -.. _os-fd-ops: - -File Descriptor Operations --------------------------- - -These functions operate on I/O streams referenced using file descriptors. - -File descriptors are small integers corresponding to a file that has been opened -by the current process. For example, standard input is usually file descriptor -0, standard output is 1, and standard error is 2. Further files opened by a -process will then be assigned 3, 4, 5, and so forth. The name "file descriptor" -is slightly deceptive; on Unix platforms, sockets and pipes are also referenced -by file descriptors. - -The :meth:`~io.IOBase.fileno` method can be used to obtain the file descriptor -associated with a :term:`file object` when required. Note that using the file -descriptor directly will bypass the file object methods, ignoring aspects such -as internal buffering of data. - - -.. function:: close(fd) - - Close file descriptor *fd*. - - .. note:: - - This function is intended for low-level I/O and must be applied to a file - descriptor as returned by :func:`os.open` or :func:`pipe`. To close a "file - object" returned by the built-in function :func:`open` or by :func:`popen` or - :func:`fdopen`, use its :meth:`~io.IOBase.close` method. - - -.. function:: closerange(fd_low, fd_high, /) - - Close all file descriptors from *fd_low* (inclusive) to *fd_high* (exclusive), - ignoring errors. Equivalent to (but much faster than):: - - for fd in range(fd_low, fd_high): - try: - os.close(fd) - except OSError: - pass - - -.. function:: copy_file_range(src, dst, count, offset_src=None, offset_dst=None) - - Copy *count* bytes from file descriptor *src*, starting from offset - *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. - If *offset_src* is None, then *src* is read from the current position; - respectively for *offset_dst*. The files pointed by *src* and *dst* - must reside in the same filesystem, otherwise an :exc:`OSError` is - raised with :attr:`~OSError.errno` set to :const:`errno.EXDEV`. - - This copy is done without the additional cost of transferring data - from the kernel to user space and then back into the kernel. Additionally, - some filesystems could implement extra optimizations. The copy is done as if - both files are opened as binary. - - The return value is the amount of bytes copied. This could be less than the - amount requested. - - .. availability:: Linux >= 4.5 with glibc >= 2.27. - - .. versionadded:: 3.8 - - -.. function:: device_encoding(fd) - - Return a string describing the encoding of the device associated with *fd* - if it is connected to a terminal; else return :const:`None`. - - On Unix, if the :ref:`Python UTF-8 Mode ` is enabled, return - ``'UTF-8'`` rather than the device encoding. - - .. versionchanged:: 3.10 - On Unix, the function now implements the Python UTF-8 Mode. - - -.. function:: dup(fd, /) - - Return a duplicate of file descriptor *fd*. The new file descriptor is - :ref:`non-inheritable `. - - On Windows, when duplicating a standard stream (0: stdin, 1: stdout, - 2: stderr), the new file descriptor is :ref:`inheritable - `. - - .. availability:: not WASI. - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - - -.. function:: dup2(fd, fd2, inheritable=True) - - Duplicate file descriptor *fd* to *fd2*, closing the latter first if - necessary. Return *fd2*. The new file descriptor is :ref:`inheritable - ` by default or non-inheritable if *inheritable* - is ``False``. - - .. availability:: not WASI. - - .. versionchanged:: 3.4 - Add the optional *inheritable* parameter. - - .. versionchanged:: 3.7 - Return *fd2* on success. Previously, ``None`` was always returned. - - -.. function:: fchmod(fd, mode) - - Change the mode of the file given by *fd* to the numeric *mode*. See the - docs for :func:`chmod` for possible values of *mode*. As of Python 3.3, this - is equivalent to ``os.chmod(fd, mode)``. - - .. audit-event:: os.chmod path,mode,dir_fd os.fchmod - - .. availability:: Unix. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - -.. function:: fchown(fd, uid, gid) - - Change the owner and group id of the file given by *fd* to the numeric *uid* - and *gid*. To leave one of the ids unchanged, set it to -1. See - :func:`chown`. As of Python 3.3, this is equivalent to ``os.chown(fd, uid, - gid)``. - - .. audit-event:: os.chown path,uid,gid,dir_fd os.fchown - - .. availability:: Unix. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - -.. function:: fdatasync(fd) - - Force write of file with filedescriptor *fd* to disk. Does not force update of - metadata. - - .. availability:: Unix. - - .. note:: - This function is not available on MacOS. - - -.. function:: fpathconf(fd, name, /) - - Return system configuration information relevant to an open file. *name* - specifies the configuration value to retrieve; it may be a string which is the - name of a defined system value; these names are specified in a number of - standards (POSIX.1, Unix 95, Unix 98, and others). Some platforms define - additional names as well. The names known to the host operating system are - given in the ``pathconf_names`` dictionary. For configuration variables not - included in that mapping, passing an integer for *name* is also accepted. - - If *name* is a string and is not known, :exc:`ValueError` is raised. If a - specific value for *name* is not supported by the host system, even if it is - included in ``pathconf_names``, an :exc:`OSError` is raised with - :const:`errno.EINVAL` for the error number. - - As of Python 3.3, this is equivalent to ``os.pathconf(fd, name)``. - - .. availability:: Unix. - - -.. function:: fstat(fd) - - Get the status of the file descriptor *fd*. Return a :class:`stat_result` - object. - - As of Python 3.3, this is equivalent to ``os.stat(fd)``. - - .. seealso:: - - The :func:`.stat` function. - - -.. function:: fstatvfs(fd, /) - - Return information about the filesystem containing the file associated with - file descriptor *fd*, like :func:`statvfs`. As of Python 3.3, this is - equivalent to ``os.statvfs(fd)``. - - .. availability:: Unix. - - -.. function:: fsync(fd) - - Force write of file with filedescriptor *fd* to disk. On Unix, this calls the - native :c:func:`!fsync` function; on Windows, the MS :c:func:`!_commit` function. - - If you're starting with a buffered Python :term:`file object` *f*, first do - ``f.flush()``, and then do ``os.fsync(f.fileno())``, to ensure that all internal - buffers associated with *f* are written to disk. - - .. availability:: Unix, Windows. - - -.. function:: ftruncate(fd, length, /) - - Truncate the file corresponding to file descriptor *fd*, so that it is at - most *length* bytes in size. As of Python 3.3, this is equivalent to - ``os.truncate(fd, length)``. - - .. audit-event:: os.truncate fd,length os.ftruncate - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.5 - Added support for Windows - - -.. function:: get_blocking(fd, /) - - Get the blocking mode of the file descriptor: ``False`` if the - :data:`O_NONBLOCK` flag is set, ``True`` if the flag is cleared. - - See also :func:`set_blocking` and :meth:`socket.socket.setblocking`. - - .. availability:: Unix. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - .. versionadded:: 3.5 - - -.. function:: isatty(fd, /) - - Return ``True`` if the file descriptor *fd* is open and connected to a - tty(-like) device, else ``False``. - - -.. function:: lockf(fd, cmd, len, /) - - Apply, test or remove a POSIX lock on an open file descriptor. - *fd* is an open file descriptor. - *cmd* specifies the command to use - one of :data:`F_LOCK`, :data:`F_TLOCK`, - :data:`F_ULOCK` or :data:`F_TEST`. - *len* specifies the section of the file to lock. - - .. audit-event:: os.lockf fd,cmd,len os.lockf - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. data:: F_LOCK - F_TLOCK - F_ULOCK - F_TEST - - Flags that specify what action :func:`lockf` will take. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: login_tty(fd, /) - - Prepare the tty of which fd is a file descriptor for a new login session. - Make the calling process a session leader; make the tty the controlling tty, - the stdin, the stdout, and the stderr of the calling process; close fd. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.11 - - -.. function:: lseek(fd, pos, whence, /) - - Set the current position of file descriptor *fd* to position *pos*, modified - by *whence*, and return the new position in bytes relative to - the start of the file. - Valid values for *whence* are: - - * :const:`SEEK_SET` or ``0`` -- set *pos* relative to the beginning of the file - * :const:`SEEK_CUR` or ``1`` -- set *pos* relative to the current file position - * :const:`SEEK_END` or ``2`` -- set *pos* relative to the end of the file - * :const:`SEEK_HOLE` -- set *pos* to the next data location, relative to *pos* - * :const:`SEEK_DATA` -- set *pos* to the next data hole, relative to *pos* - - .. versionchanged:: 3.3 - - Add support for :const:`!SEEK_HOLE` and :const:`!SEEK_DATA`. - - -.. data:: SEEK_SET - SEEK_CUR - SEEK_END - - Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` - method on :term:`file-like objects `, - for whence to adjust the file position indicator. - - :const:`SEEK_SET` - Adjust the file position relative to the beginning of the file. - :const:`SEEK_CUR` - Adjust the file position relative to the current file position. - :const:`SEEK_END` - Adjust the file position relative to the end of the file. - - Their values are 0, 1, and 2, respectively. - - -.. data:: SEEK_HOLE - SEEK_DATA - - Parameters to the :func:`lseek` function and the :meth:`~io.IOBase.seek` - method on :term:`file-like objects `, - for seeking file data and holes on sparsely allocated files. - - :data:`!SEEK_DATA` - Adjust the file offset to the next location containing data, - relative to the seek position. - - :data:`!SEEK_HOLE` - Adjust the file offset to the next location containing a hole, - relative to the seek position. - A hole is defined as a sequence of zeros. - - .. note:: - - These operations only make sense for filesystems that support them. - - .. availability:: Linux >= 3.1, macOS, Unix - - .. versionadded:: 3.3 - - -.. function:: open(path, flags, mode=0o777, *, dir_fd=None) - - Open the file *path* and set various flags according to *flags* and possibly - its mode according to *mode*. When computing *mode*, the current umask value - is first masked out. Return the file descriptor for the newly opened file. - The new file descriptor is :ref:`non-inheritable `. - - For a description of the flag and mode values, see the C run-time documentation; - flag constants (like :const:`O_RDONLY` and :const:`O_WRONLY`) are defined in - the :mod:`os` module. In particular, on Windows adding - :const:`O_BINARY` is needed to open files in binary mode. - - This function can support :ref:`paths relative to directory descriptors - ` with the *dir_fd* parameter. - - .. audit-event:: open path,mode,flags os.open - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - - .. note:: - - This function is intended for low-level I/O. For normal usage, use the - built-in function :func:`open`, which returns a :term:`file object` with - :meth:`~file.read` and :meth:`~file.write` methods (and many more). To - wrap a file descriptor in a file object, use :func:`fdopen`. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -The following constants are options for the *flags* parameter to the -:func:`~os.open` function. They can be combined using the bitwise OR operator -``|``. Some of them are not available on all platforms. For descriptions of -their availability and use, consult the :manpage:`open(2)` manual page on Unix -or `the MSDN `_ on Windows. - - -.. data:: O_RDONLY - O_WRONLY - O_RDWR - O_APPEND - O_CREAT - O_EXCL - O_TRUNC - - The above constants are available on Unix and Windows. - - -.. data:: O_DSYNC - O_RSYNC - O_SYNC - O_NDELAY - O_NONBLOCK - O_NOCTTY - O_CLOEXEC - - The above constants are only available on Unix. - - .. versionchanged:: 3.3 - Add :data:`O_CLOEXEC` constant. - -.. data:: O_BINARY - O_NOINHERIT - O_SHORT_LIVED - O_TEMPORARY - O_RANDOM - O_SEQUENTIAL - O_TEXT - - The above constants are only available on Windows. - -.. data:: O_EVTONLY - O_FSYNC - O_SYMLINK - O_NOFOLLOW_ANY - - The above constants are only available on macOS. - - .. versionchanged:: 3.10 - Add :data:`O_EVTONLY`, :data:`O_FSYNC`, :data:`O_SYMLINK` - and :data:`O_NOFOLLOW_ANY` constants. - -.. data:: O_ASYNC - O_DIRECT - O_DIRECTORY - O_NOFOLLOW - O_NOATIME - O_PATH - O_TMPFILE - O_SHLOCK - O_EXLOCK - - The above constants are extensions and not present if they are not defined by - the C library. - - .. versionchanged:: 3.4 - Add :data:`O_PATH` on systems that support it. - Add :data:`O_TMPFILE`, only available on Linux Kernel 3.11 - or newer. - - -.. function:: openpty() - - .. index:: pair: module; pty - - Open a new pseudo-terminal pair. Return a pair of file descriptors - ``(master, slave)`` for the pty and the tty, respectively. The new file - descriptors are :ref:`non-inheritable `. For a (slightly) more - portable approach, use the :mod:`pty` module. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionchanged:: 3.4 - The new file descriptors are now non-inheritable. - - -.. function:: pipe() - - Create a pipe. Return a pair of file descriptors ``(r, w)`` usable for - reading and writing, respectively. The new file descriptor is - :ref:`non-inheritable `. - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.4 - The new file descriptors are now non-inheritable. - - -.. function:: pipe2(flags, /) - - Create a pipe with *flags* set atomically. - *flags* can be constructed by ORing together one or more of these values: - :data:`O_NONBLOCK`, :data:`O_CLOEXEC`. - Return a pair of file descriptors ``(r, w)`` usable for reading and writing, - respectively. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - -.. function:: posix_fallocate(fd, offset, len, /) - - Ensures that enough disk space is allocated for the file specified by *fd* - starting from *offset* and continuing for *len* bytes. - - .. availability:: Unix, not Emscripten. - - .. versionadded:: 3.3 - - -.. function:: posix_fadvise(fd, offset, len, advice, /) - - Announces an intention to access data in a specific pattern thus allowing - the kernel to make optimizations. - The advice applies to the region of the file specified by *fd* starting at - *offset* and continuing for *len* bytes. - *advice* is one of :data:`POSIX_FADV_NORMAL`, :data:`POSIX_FADV_SEQUENTIAL`, - :data:`POSIX_FADV_RANDOM`, :data:`POSIX_FADV_NOREUSE`, - :data:`POSIX_FADV_WILLNEED` or :data:`POSIX_FADV_DONTNEED`. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. data:: POSIX_FADV_NORMAL - POSIX_FADV_SEQUENTIAL - POSIX_FADV_RANDOM - POSIX_FADV_NOREUSE - POSIX_FADV_WILLNEED - POSIX_FADV_DONTNEED - - Flags that can be used in *advice* in :func:`posix_fadvise` that specify - the access pattern that is likely to be used. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: pread(fd, n, offset, /) - - Read at most *n* bytes from file descriptor *fd* at a position of *offset*, - leaving the file offset unchanged. - - Return a bytestring containing the bytes read. If the end of the file - referred to by *fd* has been reached, an empty bytes object is returned. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: preadv(fd, buffers, offset, flags=0, /) - - Read from a file descriptor *fd* at a position of *offset* into mutable - :term:`bytes-like objects ` *buffers*, leaving the file - offset unchanged. Transfer data into each buffer until it is full and then - move on to the next buffer in the sequence to hold the rest of the data. - - The flags argument contains a bitwise OR of zero or more of the following - flags: - - - :data:`RWF_HIPRI` - - :data:`RWF_NOWAIT` - - Return the total number of bytes actually read which can be less than the - total capacity of all the objects. - - The operating system may set a limit (:func:`sysconf` value - ``'SC_IOV_MAX'``) on the number of buffers that can be used. - - Combine the functionality of :func:`os.readv` and :func:`os.pread`. - - .. availability:: Linux >= 2.6.30, FreeBSD >= 6.0, OpenBSD >= 2.7, AIX >= 7.1. - - Using flags requires Linux >= 4.6. - - .. versionadded:: 3.7 - - -.. data:: RWF_NOWAIT - - Do not wait for data which is not immediately available. If this flag is - specified, the system call will return instantly if it would have to read - data from the backing storage or wait for a lock. - - If some data was successfully read, it will return the number of bytes read. - If no bytes were read, it will return ``-1`` and set errno to - :const:`errno.EAGAIN`. - - .. availability:: Linux >= 4.14. - - .. versionadded:: 3.7 - - -.. data:: RWF_HIPRI - - High priority read/write. Allows block-based filesystems to use polling - of the device, which provides lower latency, but may use additional - resources. - - Currently, on Linux, this feature is usable only on a file descriptor opened - using the :data:`O_DIRECT` flag. - - .. availability:: Linux >= 4.6. - - .. versionadded:: 3.7 - - -.. function:: pwrite(fd, str, offset, /) - - Write the bytestring in *str* to file descriptor *fd* at position of - *offset*, leaving the file offset unchanged. - - Return the number of bytes actually written. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: pwritev(fd, buffers, offset, flags=0, /) - - Write the *buffers* contents to file descriptor *fd* at a offset *offset*, - leaving the file offset unchanged. *buffers* must be a sequence of - :term:`bytes-like objects `. Buffers are processed in - array order. Entire contents of the first buffer is written before - proceeding to the second, and so on. - - The flags argument contains a bitwise OR of zero or more of the following - flags: - - - :data:`RWF_DSYNC` - - :data:`RWF_SYNC` - - :data:`RWF_APPEND` - - Return the total number of bytes actually written. - - The operating system may set a limit (:func:`sysconf` value - ``'SC_IOV_MAX'``) on the number of buffers that can be used. - - Combine the functionality of :func:`os.writev` and :func:`os.pwrite`. - - .. availability:: Linux >= 2.6.30, FreeBSD >= 6.0, OpenBSD >= 2.7, AIX >= 7.1. - - Using flags requires Linux >= 4.6. - - .. versionadded:: 3.7 - - -.. data:: RWF_DSYNC - - Provide a per-write equivalent of the :data:`O_DSYNC` :func:`os.open` flag. - This flag effect applies only to the data range written by the system call. - - .. availability:: Linux >= 4.7. - - .. versionadded:: 3.7 - - -.. data:: RWF_SYNC - - Provide a per-write equivalent of the :data:`O_SYNC` :func:`os.open` flag. - This flag effect applies only to the data range written by the system call. - - .. availability:: Linux >= 4.7. - - .. versionadded:: 3.7 - - -.. data:: RWF_APPEND - - Provide a per-write equivalent of the :data:`O_APPEND` :func:`os.open` - flag. This flag is meaningful only for :func:`os.pwritev`, and its - effect applies only to the data range written by the system call. The - *offset* argument does not affect the write operation; the data is always - appended to the end of the file. However, if the *offset* argument is - ``-1``, the current file *offset* is updated. - - .. availability:: Linux >= 4.16. - - .. versionadded:: 3.10 - - -.. function:: read(fd, n, /) - - Read at most *n* bytes from file descriptor *fd*. - - Return a bytestring containing the bytes read. If the end of the file - referred to by *fd* has been reached, an empty bytes object is returned. - - .. note:: - - This function is intended for low-level I/O and must be applied to a file - descriptor as returned by :func:`os.open` or :func:`pipe`. To read a - "file object" returned by the built-in function :func:`open` or by - :func:`popen` or :func:`fdopen`, or :data:`sys.stdin`, use its - :meth:`~file.read` or :meth:`~file.readline` methods. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. function:: sendfile(out_fd, in_fd, offset, count) - sendfile(out_fd, in_fd, offset, count, headers=(), trailers=(), flags=0) - - Copy *count* bytes from file descriptor *in_fd* to file descriptor *out_fd* - starting at *offset*. - Return the number of bytes sent. When EOF is reached return ``0``. - - The first function notation is supported by all platforms that define - :func:`sendfile`. - - On Linux, if *offset* is given as ``None``, the bytes are read from the - current position of *in_fd* and the position of *in_fd* is updated. - - The second case may be used on macOS and FreeBSD where *headers* and - *trailers* are arbitrary sequences of buffers that are written before and - after the data from *in_fd* is written. It returns the same as the first case. - - On macOS and FreeBSD, a value of ``0`` for *count* specifies to send until - the end of *in_fd* is reached. - - All platforms support sockets as *out_fd* file descriptor, and some platforms - allow other types (e.g. regular file, pipe) as well. - - Cross-platform applications should not use *headers*, *trailers* and *flags* - arguments. - - .. availability:: Unix, not Emscripten, not WASI. - - .. note:: - - For a higher-level wrapper of :func:`sendfile`, see - :meth:`socket.socket.sendfile`. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.9 - Parameters *out* and *in* was renamed to *out_fd* and *in_fd*. - - -.. data:: SF_NODISKIO - SF_MNOWAIT - SF_SYNC - - Parameters to the :func:`sendfile` function, if the implementation supports - them. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - -.. data:: SF_NOCACHE - - Parameter to the :func:`sendfile` function, if the implementation supports - it. The data won't be cached in the virtual memory and will be freed afterwards. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.11 - - -.. function:: set_blocking(fd, blocking, /) - - Set the blocking mode of the specified file descriptor. Set the - :data:`O_NONBLOCK` flag if blocking is ``False``, clear the flag otherwise. - - See also :func:`get_blocking` and :meth:`socket.socket.setblocking`. - - .. availability:: Unix. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - .. versionadded:: 3.5 - - -.. function:: splice(src, dst, count, offset_src=None, offset_dst=None) - - Transfer *count* bytes from file descriptor *src*, starting from offset - *offset_src*, to file descriptor *dst*, starting from offset *offset_dst*. - At least one of the file descriptors must refer to a pipe. If *offset_src* - is None, then *src* is read from the current position; respectively for - *offset_dst*. The offset associated to the file descriptor that refers to a - pipe must be ``None``. The files pointed by *src* and *dst* must reside in - the same filesystem, otherwise an :exc:`OSError` is raised with - :attr:`~OSError.errno` set to :const:`errno.EXDEV`. - - This copy is done without the additional cost of transferring data - from the kernel to user space and then back into the kernel. Additionally, - some filesystems could implement extra optimizations. The copy is done as if - both files are opened as binary. - - Upon successful completion, returns the number of bytes spliced to or from - the pipe. A return value of 0 means end of input. If *src* refers to a - pipe, then this means that there was no data to transfer, and it would not - make sense to block because there are no writers connected to the write end - of the pipe. - - .. availability:: Linux >= 2.6.17 with glibc >= 2.5 - - .. versionadded:: 3.10 - - -.. data:: SPLICE_F_MOVE - SPLICE_F_NONBLOCK - SPLICE_F_MORE - - .. versionadded:: 3.10 - -.. function:: readv(fd, buffers, /) - - Read from a file descriptor *fd* into a number of mutable :term:`bytes-like - objects ` *buffers*. Transfer data into each buffer until - it is full and then move on to the next buffer in the sequence to hold the - rest of the data. - - Return the total number of bytes actually read which can be less than the - total capacity of all the objects. - - The operating system may set a limit (:func:`sysconf` value - ``'SC_IOV_MAX'``) on the number of buffers that can be used. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: tcgetpgrp(fd, /) - - Return the process group associated with the terminal given by *fd* (an open - file descriptor as returned by :func:`os.open`). - - .. availability:: Unix, not WASI. - - -.. function:: tcsetpgrp(fd, pg, /) - - Set the process group associated with the terminal given by *fd* (an open file - descriptor as returned by :func:`os.open`) to *pg*. - - .. availability:: Unix, not WASI. - - -.. function:: ttyname(fd, /) - - Return a string which specifies the terminal device associated with - file descriptor *fd*. If *fd* is not associated with a terminal device, an - exception is raised. - - .. availability:: Unix. - - -.. function:: write(fd, str, /) - - Write the bytestring in *str* to file descriptor *fd*. - - Return the number of bytes actually written. - - .. note:: - - This function is intended for low-level I/O and must be applied to a file - descriptor as returned by :func:`os.open` or :func:`pipe`. To write a "file - object" returned by the built-in function :func:`open` or by :func:`popen` or - :func:`fdopen`, or :data:`sys.stdout` or :data:`sys.stderr`, use its - :meth:`~file.write` method. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. function:: writev(fd, buffers, /) - - Write the contents of *buffers* to file descriptor *fd*. *buffers* must be - a sequence of :term:`bytes-like objects `. Buffers are - processed in array order. Entire contents of the first buffer is written - before proceeding to the second, and so on. - - Returns the total number of bytes actually written. - - The operating system may set a limit (:func:`sysconf` value - ``'SC_IOV_MAX'``) on the number of buffers that can be used. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. _terminal-size: - -Querying the size of a terminal -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 3.3 - -.. function:: get_terminal_size(fd=STDOUT_FILENO, /) - - Return the size of the terminal window as ``(columns, lines)``, - tuple of type :class:`terminal_size`. - - The optional argument ``fd`` (default ``STDOUT_FILENO``, or standard - output) specifies which file descriptor should be queried. - - If the file descriptor is not connected to a terminal, an :exc:`OSError` - is raised. - - :func:`shutil.get_terminal_size` is the high-level function which - should normally be used, ``os.get_terminal_size`` is the low-level - implementation. - - .. availability:: Unix, Windows. - -.. class:: terminal_size - - A subclass of tuple, holding ``(columns, lines)`` of the terminal window size. - - .. attribute:: columns - - Width of the terminal window in characters. - - .. attribute:: lines - - Height of the terminal window in characters. - - -.. _fd_inheritance: - -Inheritance of File Descriptors -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 3.4 - -A file descriptor has an "inheritable" flag which indicates if the file descriptor -can be inherited by child processes. Since Python 3.4, file descriptors -created by Python are non-inheritable by default. - -On UNIX, non-inheritable file descriptors are closed in child processes at the -execution of a new program, other file descriptors are inherited. - -On Windows, non-inheritable handles and file descriptors are closed in child -processes, except for standard streams (file descriptors 0, 1 and 2: stdin, stdout -and stderr), which are always inherited. Using :func:`spawn\* ` functions, -all inheritable handles and all inheritable file descriptors are inherited. -Using the :mod:`subprocess` module, all file descriptors except standard -streams are closed, and inheritable handles are only inherited if the -*close_fds* parameter is ``False``. - -On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, the file -descriptor cannot be modified. - -.. function:: get_inheritable(fd, /) - - Get the "inheritable" flag of the specified file descriptor (a boolean). - -.. function:: set_inheritable(fd, inheritable, /) - - Set the "inheritable" flag of the specified file descriptor. - -.. function:: get_handle_inheritable(handle, /) - - Get the "inheritable" flag of the specified handle (a boolean). - - .. availability:: Windows. - -.. function:: set_handle_inheritable(handle, inheritable, /) - - Set the "inheritable" flag of the specified handle. - - .. availability:: Windows. - - -.. _os-file-dir: - -Files and Directories ---------------------- - -On some Unix platforms, many of these functions support one or more of these -features: - -.. _path_fd: - -* **specifying a file descriptor:** - Normally the *path* argument provided to functions in the :mod:`os` module - must be a string specifying a file path. However, some functions now - alternatively accept an open file descriptor for their *path* argument. - The function will then operate on the file referred to by the descriptor. - (For POSIX systems, Python will call the variant of the function prefixed - with ``f`` (e.g. call ``fchdir`` instead of ``chdir``).) - - You can check whether or not *path* can be specified as a file descriptor - for a particular function on your platform using :data:`os.supports_fd`. - If this functionality is unavailable, using it will raise a - :exc:`NotImplementedError`. - - If the function also supports *dir_fd* or *follow_symlinks* arguments, it's - an error to specify one of those when supplying *path* as a file descriptor. - -.. _dir_fd: - -* **paths relative to directory descriptors:** If *dir_fd* is not ``None``, it - should be a file descriptor referring to a directory, and the path to operate - on should be relative; path will then be relative to that directory. If the - path is absolute, *dir_fd* is ignored. (For POSIX systems, Python will call - the variant of the function with an ``at`` suffix and possibly prefixed with - ``f`` (e.g. call ``faccessat`` instead of ``access``). - - You can check whether or not *dir_fd* is supported for a particular function - on your platform using :data:`os.supports_dir_fd`. If it's unavailable, - using it will raise a :exc:`NotImplementedError`. - -.. _follow_symlinks: - -* **not following symlinks:** If *follow_symlinks* is - ``False``, and the last element of the path to operate on is a symbolic link, - the function will operate on the symbolic link itself rather than the file - pointed to by the link. (For POSIX systems, Python will call the ``l...`` - variant of the function.) - - You can check whether or not *follow_symlinks* is supported for a particular - function on your platform using :data:`os.supports_follow_symlinks`. - If it's unavailable, using it will raise a :exc:`NotImplementedError`. - - - -.. function:: access(path, mode, *, dir_fd=None, effective_ids=False, follow_symlinks=True) - - Use the real uid/gid to test for access to *path*. Note that most operations - will use the effective uid/gid, therefore this routine can be used in a - suid/sgid environment to test if the invoking user has the specified access to - *path*. *mode* should be :const:`F_OK` to test the existence of *path*, or it - can be the inclusive OR of one or more of :const:`R_OK`, :const:`W_OK`, and - :const:`X_OK` to test permissions. Return :const:`True` if access is allowed, - :const:`False` if not. See the Unix man page :manpage:`access(2)` for more - information. - - This function can support specifying :ref:`paths relative to directory - descriptors ` and :ref:`not following symlinks `. - - If *effective_ids* is ``True``, :func:`access` will perform its access - checks using the effective uid/gid instead of the real uid/gid. - *effective_ids* may not be supported on your platform; you can check whether - or not it is available using :data:`os.supports_effective_ids`. If it is - unavailable, using it will raise a :exc:`NotImplementedError`. - - .. note:: - - Using :func:`access` to check if a user is authorized to e.g. open a file - before actually doing so using :func:`open` creates a security hole, - because the user might exploit the short time interval between checking - and opening the file to manipulate it. It's preferable to use :term:`EAFP` - techniques. For example:: - - if os.access("myfile", os.R_OK): - with open("myfile") as fp: - return fp.read() - return "some default data" - - is better written as:: - - try: - fp = open("myfile") - except PermissionError: - return "some default data" - else: - with fp: - return fp.read() - - .. note:: - - I/O operations may fail even when :func:`access` indicates that they would - succeed, particularly for operations on network filesystems which may have - permissions semantics beyond the usual POSIX permission-bit model. - - .. versionchanged:: 3.3 - Added the *dir_fd*, *effective_ids*, and *follow_symlinks* parameters. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. data:: F_OK - R_OK - W_OK - X_OK - - Values to pass as the *mode* parameter of :func:`access` to test the - existence, readability, writability and executability of *path*, - respectively. - - -.. function:: chdir(path) - - .. index:: single: directory; changing - - Change the current working directory to *path*. - - This function can support :ref:`specifying a file descriptor `. The - descriptor must refer to an opened directory, not an open file. - - This function can raise :exc:`OSError` and subclasses such as - :exc:`FileNotFoundError`, :exc:`PermissionError`, and :exc:`NotADirectoryError`. - - .. audit-event:: os.chdir path os.chdir - - .. versionadded:: 3.3 - Added support for specifying *path* as a file descriptor - on some platforms. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: chflags(path, flags, *, follow_symlinks=True) - - Set the flags of *path* to the numeric *flags*. *flags* may take a combination - (bitwise OR) of the following values (as defined in the :mod:`stat` module): - - * :const:`stat.UF_NODUMP` - * :const:`stat.UF_IMMUTABLE` - * :const:`stat.UF_APPEND` - * :const:`stat.UF_OPAQUE` - * :const:`stat.UF_NOUNLINK` - * :const:`stat.UF_COMPRESSED` - * :const:`stat.UF_HIDDEN` - * :const:`stat.SF_ARCHIVED` - * :const:`stat.SF_IMMUTABLE` - * :const:`stat.SF_APPEND` - * :const:`stat.SF_NOUNLINK` - * :const:`stat.SF_SNAPSHOT` - - This function can support :ref:`not following symlinks `. - - .. audit-event:: os.chflags path,flags os.chflags - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - The *follow_symlinks* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: chmod(path, mode, *, dir_fd=None, follow_symlinks=True) - - Change the mode of *path* to the numeric *mode*. *mode* may take one of the - following values (as defined in the :mod:`stat` module) or bitwise ORed - combinations of them: - - * :const:`stat.S_ISUID` - * :const:`stat.S_ISGID` - * :const:`stat.S_ENFMT` - * :const:`stat.S_ISVTX` - * :const:`stat.S_IREAD` - * :const:`stat.S_IWRITE` - * :const:`stat.S_IEXEC` - * :const:`stat.S_IRWXU` - * :const:`stat.S_IRUSR` - * :const:`stat.S_IWUSR` - * :const:`stat.S_IXUSR` - * :const:`stat.S_IRWXG` - * :const:`stat.S_IRGRP` - * :const:`stat.S_IWGRP` - * :const:`stat.S_IXGRP` - * :const:`stat.S_IRWXO` - * :const:`stat.S_IROTH` - * :const:`stat.S_IWOTH` - * :const:`stat.S_IXOTH` - - This function can support :ref:`specifying a file descriptor `, - :ref:`paths relative to directory descriptors ` and :ref:`not - following symlinks `. - - .. note:: - - Although Windows supports :func:`chmod`, you can only set the file's - read-only flag with it (via the ``stat.S_IWRITE`` and ``stat.S_IREAD`` - constants or a corresponding integer value). All other bits are ignored. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - .. audit-event:: os.chmod path,mode,dir_fd os.chmod - - .. versionadded:: 3.3 - Added support for specifying *path* as an open file descriptor, - and the *dir_fd* and *follow_symlinks* arguments. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: chown(path, uid, gid, *, dir_fd=None, follow_symlinks=True) - - Change the owner and group id of *path* to the numeric *uid* and *gid*. To - leave one of the ids unchanged, set it to -1. - - This function can support :ref:`specifying a file descriptor `, - :ref:`paths relative to directory descriptors ` and :ref:`not - following symlinks `. - - See :func:`shutil.chown` for a higher-level function that accepts names in - addition to numeric ids. - - .. audit-event:: os.chown path,uid,gid,dir_fd os.chown - - .. availability:: Unix. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - .. versionadded:: 3.3 - Added support for specifying *path* as an open file descriptor, - and the *dir_fd* and *follow_symlinks* arguments. - - .. versionchanged:: 3.6 - Supports a :term:`path-like object`. - - -.. function:: chroot(path) - - Change the root directory of the current process to *path*. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: fchdir(fd) - - Change the current working directory to the directory represented by the file - descriptor *fd*. The descriptor must refer to an opened directory, not an - open file. As of Python 3.3, this is equivalent to ``os.chdir(fd)``. - - .. audit-event:: os.chdir path os.fchdir - - .. availability:: Unix. - - -.. function:: getcwd() - - Return a string representing the current working directory. - - -.. function:: getcwdb() - - Return a bytestring representing the current working directory. - - .. versionchanged:: 3.8 - The function now uses the UTF-8 encoding on Windows, rather than the ANSI - code page: see :pep:`529` for the rationale. The function is no longer - deprecated on Windows. - - -.. function:: lchflags(path, flags) - - Set the flags of *path* to the numeric *flags*, like :func:`chflags`, but do - not follow symbolic links. As of Python 3.3, this is equivalent to - ``os.chflags(path, flags, follow_symlinks=False)``. - - .. audit-event:: os.chflags path,flags os.lchflags - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: lchmod(path, mode) - - Change the mode of *path* to the numeric *mode*. If path is a symlink, this - affects the symlink rather than the target. See the docs for :func:`chmod` - for possible values of *mode*. As of Python 3.3, this is equivalent to - ``os.chmod(path, mode, follow_symlinks=False)``. - - .. audit-event:: os.chmod path,mode,dir_fd os.lchmod - - .. availability:: Unix. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. function:: lchown(path, uid, gid) - - Change the owner and group id of *path* to the numeric *uid* and *gid*. This - function will not follow symbolic links. As of Python 3.3, this is equivalent - to ``os.chown(path, uid, gid, follow_symlinks=False)``. - - .. audit-event:: os.chown path,uid,gid,dir_fd os.lchown - - .. availability:: Unix. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: link(src, dst, *, src_dir_fd=None, dst_dir_fd=None, follow_symlinks=True) - - Create a hard link pointing to *src* named *dst*. - - This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to - supply :ref:`paths relative to directory descriptors `, and :ref:`not - following symlinks `. - - .. audit-event:: os.link src,dst,src_dir_fd,dst_dir_fd os.link - - .. availability:: Unix, Windows, not Emscripten. - - .. versionchanged:: 3.2 - Added Windows support. - - .. versionadded:: 3.3 - Added the *src_dir_fd*, *dst_dir_fd*, and *follow_symlinks* arguments. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. - - -.. function:: listdir(path='.') - - Return a list containing the names of the entries in the directory given by - *path*. The list is in arbitrary order, and does not include the special - entries ``'.'`` and ``'..'`` even if they are present in the directory. - If a file is removed from or added to the directory during the call of - this function, whether a name for that file be included is unspecified. - - *path* may be a :term:`path-like object`. If *path* is of type ``bytes`` - (directly or indirectly through the :class:`PathLike` interface), - the filenames returned will also be of type ``bytes``; - in all other circumstances, they will be of type ``str``. - - This function can also support :ref:`specifying a file descriptor - `; the file descriptor must refer to a directory. - - .. audit-event:: os.listdir path os.listdir - - .. note:: - To encode ``str`` filenames to ``bytes``, use :func:`~os.fsencode`. - - .. seealso:: - - The :func:`scandir` function returns directory entries along with - file attribute information, giving better performance for many - common use cases. - - .. versionchanged:: 3.2 - The *path* parameter became optional. - - .. versionadded:: 3.3 - Added support for specifying *path* as an open file descriptor. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: lstat(path, *, dir_fd=None) - - Perform the equivalent of an :c:func:`!lstat` system call on the given path. - Similar to :func:`~os.stat`, but does not follow symbolic links. Return a - :class:`stat_result` object. - - On platforms that do not support symbolic links, this is an alias for - :func:`~os.stat`. - - As of Python 3.3, this is equivalent to ``os.stat(path, dir_fd=dir_fd, - follow_symlinks=False)``. - - This function can also support :ref:`paths relative to directory descriptors - `. - - .. seealso:: - - The :func:`.stat` function. - - .. versionchanged:: 3.2 - Added support for Windows 6.0 (Vista) symbolic links. - - .. versionchanged:: 3.3 - Added the *dir_fd* parameter. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. versionchanged:: 3.8 - On Windows, now opens reparse points that represent another path - (name surrogates), including symbolic links and directory junctions. - Other kinds of reparse points are resolved by the operating system as - for :func:`~os.stat`. - - -.. function:: mkdir(path, mode=0o777, *, dir_fd=None) - - Create a directory named *path* with numeric mode *mode*. - - If the directory already exists, :exc:`FileExistsError` is raised. If a parent - directory in the path does not exist, :exc:`FileNotFoundError` is raised. - - .. _mkdir_modebits: - - On some systems, *mode* is ignored. Where it is used, the current umask - value is first masked out. If bits other than the last 9 (i.e. the last 3 - digits of the octal representation of the *mode*) are set, their meaning is - platform-dependent. On some platforms, they are ignored and you should call - :func:`chmod` explicitly to set them. - - This function can also support :ref:`paths relative to directory descriptors - `. - - It is also possible to create temporary directories; see the - :mod:`tempfile` module's :func:`tempfile.mkdtemp` function. - - .. audit-event:: os.mkdir path,mode,dir_fd os.mkdir - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: makedirs(name, mode=0o777, exist_ok=False) - - .. index:: - single: directory; creating - single: UNC paths; and os.makedirs() - - Recursive directory creation function. Like :func:`mkdir`, but makes all - intermediate-level directories needed to contain the leaf directory. - - The *mode* parameter is passed to :func:`mkdir` for creating the leaf - directory; see :ref:`the mkdir() description ` for how it - is interpreted. To set the file permission bits of any newly created parent - directories you can set the umask before invoking :func:`makedirs`. The - file permission bits of existing parent directories are not changed. - - If *exist_ok* is ``False`` (the default), a :exc:`FileExistsError` is - raised if the target directory already exists. - - .. note:: - - :func:`makedirs` will become confused if the path elements to create - include :data:`pardir` (eg. ".." on UNIX systems). - - This function handles UNC paths correctly. - - .. audit-event:: os.mkdir path,mode,dir_fd os.makedirs - - .. versionadded:: 3.2 - The *exist_ok* parameter. - - .. versionchanged:: 3.4.1 - - Before Python 3.4.1, if *exist_ok* was ``True`` and the directory existed, - :func:`makedirs` would still raise an error if *mode* did not match the - mode of the existing directory. Since this behavior was impossible to - implement safely, it was removed in Python 3.4.1. See :issue:`21082`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. versionchanged:: 3.7 - The *mode* argument no longer affects the file permission bits of - newly created intermediate-level directories. - - -.. function:: mkfifo(path, mode=0o666, *, dir_fd=None) - - Create a FIFO (a named pipe) named *path* with numeric mode *mode*. - The current umask value is first masked out from the mode. - - This function can also support :ref:`paths relative to directory descriptors - `. - - FIFOs are pipes that can be accessed like regular files. FIFOs exist until they - are deleted (for example with :func:`os.unlink`). Generally, FIFOs are used as - rendezvous between "client" and "server" type processes: the server opens the - FIFO for reading, and the client opens it for writing. Note that :func:`mkfifo` - doesn't open the FIFO --- it just creates the rendezvous point. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: mknod(path, mode=0o600, device=0, *, dir_fd=None) - - Create a filesystem node (file, device special file or named pipe) named - *path*. *mode* specifies both the permissions to use and the type of node - to be created, being combined (bitwise OR) with one of ``stat.S_IFREG``, - ``stat.S_IFCHR``, ``stat.S_IFBLK``, and ``stat.S_IFIFO`` (those constants are - available in :mod:`stat`). For ``stat.S_IFCHR`` and ``stat.S_IFBLK``, - *device* defines the newly created device special file (probably using - :func:`os.makedev`), otherwise it is ignored. - - This function can also support :ref:`paths relative to directory descriptors - `. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: major(device, /) - - Extract the device major number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). - - -.. function:: minor(device, /) - - Extract the device minor number from a raw device number (usually the - :attr:`st_dev` or :attr:`st_rdev` field from :c:struct:`stat`). - - -.. function:: makedev(major, minor, /) - - Compose a raw device number from the major and minor device numbers. - - -.. function:: pathconf(path, name) - - Return system configuration information relevant to a named file. *name* - specifies the configuration value to retrieve; it may be a string which is the - name of a defined system value; these names are specified in a number of - standards (POSIX.1, Unix 95, Unix 98, and others). Some platforms define - additional names as well. The names known to the host operating system are - given in the ``pathconf_names`` dictionary. For configuration variables not - included in that mapping, passing an integer for *name* is also accepted. - - If *name* is a string and is not known, :exc:`ValueError` is raised. If a - specific value for *name* is not supported by the host system, even if it is - included in ``pathconf_names``, an :exc:`OSError` is raised with - :const:`errno.EINVAL` for the error number. - - This function can support :ref:`specifying a file descriptor - `. - - .. availability:: Unix. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. data:: pathconf_names - - Dictionary mapping names accepted by :func:`pathconf` and :func:`fpathconf` to - the integer values defined for those names by the host operating system. This - can be used to determine the set of names known to the system. - - .. availability:: Unix. - - -.. function:: readlink(path, *, dir_fd=None) - - Return a string representing the path to which the symbolic link points. The - result may be either an absolute or relative pathname; if it is relative, it - may be converted to an absolute pathname using - ``os.path.join(os.path.dirname(path), result)``. - - If the *path* is a string object (directly or indirectly through a - :class:`PathLike` interface), the result will also be a string object, - and the call may raise a UnicodeDecodeError. If the *path* is a bytes - object (direct or indirectly), the result will be a bytes object. - - This function can also support :ref:`paths relative to directory descriptors - `. - - When trying to resolve a path that may contain links, use - :func:`~os.path.realpath` to properly handle recursion and platform - differences. - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.2 - Added support for Windows 6.0 (Vista) symbolic links. - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` on Unix. - - .. versionchanged:: 3.8 - Accepts a :term:`path-like object` and a bytes object on Windows. - - .. versionchanged:: 3.8 - Added support for directory junctions, and changed to return the - substitution path (which typically includes ``\\?\`` prefix) rather - than the optional "print name" field that was previously returned. - -.. function:: remove(path, *, dir_fd=None) - - Remove (delete) the file *path*. If *path* is a directory, an - :exc:`OSError` is raised. Use :func:`rmdir` to remove directories. - If the file does not exist, a :exc:`FileNotFoundError` is raised. - - This function can support :ref:`paths relative to directory descriptors - `. - - On Windows, attempting to remove a file that is in use causes an exception to - be raised; on Unix, the directory entry is removed but the storage allocated - to the file is not made available until the original file is no longer in use. - - This function is semantically identical to :func:`unlink`. - - .. audit-event:: os.remove path,dir_fd os.remove - - .. versionadded:: 3.3 - The *dir_fd* argument. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: removedirs(name) - - .. index:: single: directory; deleting - - Remove directories recursively. Works like :func:`rmdir` except that, if the - leaf directory is successfully removed, :func:`removedirs` tries to - successively remove every parent directory mentioned in *path* until an error - is raised (which is ignored, because it generally means that a parent directory - is not empty). For example, ``os.removedirs('foo/bar/baz')`` will first remove - the directory ``'foo/bar/baz'``, and then remove ``'foo/bar'`` and ``'foo'`` if - they are empty. Raises :exc:`OSError` if the leaf directory could not be - successfully removed. - - .. audit-event:: os.remove path,dir_fd os.removedirs - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: rename(src, dst, *, src_dir_fd=None, dst_dir_fd=None) - - Rename the file or directory *src* to *dst*. If *dst* exists, the operation - will fail with an :exc:`OSError` subclass in a number of cases: - - On Windows, if *dst* exists a :exc:`FileExistsError` is always raised. - The operation may fail if *src* and *dst* are on different filesystems. Use - :func:`shutil.move` to support moves to a different filesystem. - - On Unix, if *src* is a file and *dst* is a directory or vice-versa, an - :exc:`IsADirectoryError` or a :exc:`NotADirectoryError` will be raised - respectively. If both are directories and *dst* is empty, *dst* will be - silently replaced. If *dst* is a non-empty directory, an :exc:`OSError` - is raised. If both are files, *dst* will be replaced silently if the user - has permission. The operation may fail on some Unix flavors if *src* and - *dst* are on different filesystems. If successful, the renaming will be an - atomic operation (this is a POSIX requirement). - - This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to - supply :ref:`paths relative to directory descriptors `. - - If you want cross-platform overwriting of the destination, use :func:`replace`. - - .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.rename - - .. versionadded:: 3.3 - The *src_dir_fd* and *dst_dir_fd* arguments. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. - - -.. function:: renames(old, new) - - Recursive directory or file renaming function. Works like :func:`rename`, except - creation of any intermediate directories needed to make the new pathname good is - attempted first. After the rename, directories corresponding to rightmost path - segments of the old name will be pruned away using :func:`removedirs`. - - .. note:: - - This function can fail with the new directory structure made if you lack - permissions needed to remove the leaf directory or file. - - .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.renames - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *old* and *new*. - - -.. function:: replace(src, dst, *, src_dir_fd=None, dst_dir_fd=None) - - Rename the file or directory *src* to *dst*. If *dst* is a non-empty directory, - :exc:`OSError` will be raised. If *dst* exists and is a file, it will - be replaced silently if the user has permission. The operation may fail - if *src* and *dst* are on different filesystems. If successful, - the renaming will be an atomic operation (this is a POSIX requirement). - - This function can support specifying *src_dir_fd* and/or *dst_dir_fd* to - supply :ref:`paths relative to directory descriptors `. - - .. audit-event:: os.rename src,dst,src_dir_fd,dst_dir_fd os.replace - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. - - -.. function:: rmdir(path, *, dir_fd=None) - - Remove (delete) the directory *path*. If the directory does not exist or is - not empty, a :exc:`FileNotFoundError` or an :exc:`OSError` is raised - respectively. In order to remove whole directory trees, - :func:`shutil.rmtree` can be used. - - This function can support :ref:`paths relative to directory descriptors - `. - - .. audit-event:: os.rmdir path,dir_fd os.rmdir - - .. versionadded:: 3.3 - The *dir_fd* parameter. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: scandir(path='.') - - Return an iterator of :class:`os.DirEntry` objects corresponding to the - entries in the directory given by *path*. The entries are yielded in - arbitrary order, and the special entries ``'.'`` and ``'..'`` are not - included. If a file is removed from or added to the directory after - creating the iterator, whether an entry for that file be included is - unspecified. - - Using :func:`scandir` instead of :func:`listdir` can significantly - increase the performance of code that also needs file type or file - attribute information, because :class:`os.DirEntry` objects expose this - information if the operating system provides it when scanning a directory. - All :class:`os.DirEntry` methods may perform a system call, but - :func:`~os.DirEntry.is_dir` and :func:`~os.DirEntry.is_file` usually only - require a system call for symbolic links; :func:`os.DirEntry.stat` - always requires a system call on Unix but only requires one for - symbolic links on Windows. - - *path* may be a :term:`path-like object`. If *path* is of type ``bytes`` - (directly or indirectly through the :class:`PathLike` interface), - the type of the :attr:`~os.DirEntry.name` and :attr:`~os.DirEntry.path` - attributes of each :class:`os.DirEntry` will be ``bytes``; in all other - circumstances, they will be of type ``str``. - - This function can also support :ref:`specifying a file descriptor - `; the file descriptor must refer to a directory. - - .. audit-event:: os.scandir path os.scandir - - The :func:`scandir` iterator supports the :term:`context manager` protocol - and has the following method: - - .. method:: scandir.close() - - Close the iterator and free acquired resources. - - This is called automatically when the iterator is exhausted or garbage - collected, or when an error happens during iterating. However it - is advisable to call it explicitly or use the :keyword:`with` - statement. - - .. versionadded:: 3.6 - - The following example shows a simple use of :func:`scandir` to display all - the files (excluding directories) in the given *path* that don't start with - ``'.'``. The ``entry.is_file()`` call will generally not make an additional - system call:: - - with os.scandir(path) as it: - for entry in it: - if not entry.name.startswith('.') and entry.is_file(): - print(entry.name) - - .. note:: - - On Unix-based systems, :func:`scandir` uses the system's - `opendir() `_ - and - `readdir() `_ - functions. On Windows, it uses the Win32 - `FindFirstFileW `_ - and - `FindNextFileW `_ - functions. - - .. versionadded:: 3.5 - - .. versionadded:: 3.6 - Added support for the :term:`context manager` protocol and the - :func:`~scandir.close()` method. If a :func:`scandir` iterator is neither - exhausted nor explicitly closed a :exc:`ResourceWarning` will be emitted - in its destructor. - - The function accepts a :term:`path-like object`. - - .. versionchanged:: 3.7 - Added support for :ref:`file descriptors ` on Unix. - - -.. class:: DirEntry - - Object yielded by :func:`scandir` to expose the file path and other file - attributes of a directory entry. - - :func:`scandir` will provide as much of this information as possible without - making additional system calls. When a ``stat()`` or ``lstat()`` system call - is made, the ``os.DirEntry`` object will cache the result. - - ``os.DirEntry`` instances are not intended to be stored in long-lived data - structures; if you know the file metadata has changed or if a long time has - elapsed since calling :func:`scandir`, call ``os.stat(entry.path)`` to fetch - up-to-date information. - - Because the ``os.DirEntry`` methods can make operating system calls, they may - also raise :exc:`OSError`. If you need very fine-grained - control over errors, you can catch :exc:`OSError` when calling one of the - ``os.DirEntry`` methods and handle as appropriate. - - To be directly usable as a :term:`path-like object`, ``os.DirEntry`` - implements the :class:`PathLike` interface. - - Attributes and methods on a ``os.DirEntry`` instance are as follows: - - .. attribute:: name - - The entry's base filename, relative to the :func:`scandir` *path* - argument. - - The :attr:`name` attribute will be ``bytes`` if the :func:`scandir` - *path* argument is of type ``bytes`` and ``str`` otherwise. Use - :func:`~os.fsdecode` to decode byte filenames. - - .. attribute:: path - - The entry's full path name: equivalent to ``os.path.join(scandir_path, - entry.name)`` where *scandir_path* is the :func:`scandir` *path* - argument. The path is only absolute if the :func:`scandir` *path* - argument was absolute. If the :func:`scandir` *path* - argument was a :ref:`file descriptor `, the :attr:`path` - attribute is the same as the :attr:`name` attribute. - - The :attr:`path` attribute will be ``bytes`` if the :func:`scandir` - *path* argument is of type ``bytes`` and ``str`` otherwise. Use - :func:`~os.fsdecode` to decode byte filenames. - - .. method:: inode() - - Return the inode number of the entry. - - The result is cached on the ``os.DirEntry`` object. Use - ``os.stat(entry.path, follow_symlinks=False).st_ino`` to fetch up-to-date - information. - - On the first, uncached call, a system call is required on Windows but - not on Unix. - - .. method:: is_dir(*, follow_symlinks=True) - - Return ``True`` if this entry is a directory or a symbolic link pointing - to a directory; return ``False`` if the entry is or points to any other - kind of file, or if it doesn't exist anymore. - - If *follow_symlinks* is ``False``, return ``True`` only if this entry - is a directory (without following symlinks); return ``False`` if the - entry is any other kind of file or if it doesn't exist anymore. - - The result is cached on the ``os.DirEntry`` object, with a separate cache - for *follow_symlinks* ``True`` and ``False``. Call :func:`os.stat` along - with :func:`stat.S_ISDIR` to fetch up-to-date information. - - On the first, uncached call, no system call is required in most cases. - Specifically, for non-symlinks, neither Windows or Unix require a system - call, except on certain Unix file systems, such as network file systems, - that return ``dirent.d_type == DT_UNKNOWN``. If the entry is a symlink, - a system call will be required to follow the symlink unless - *follow_symlinks* is ``False``. - - This method can raise :exc:`OSError`, such as :exc:`PermissionError`, - but :exc:`FileNotFoundError` is caught and not raised. - - .. method:: is_file(*, follow_symlinks=True) - - Return ``True`` if this entry is a file or a symbolic link pointing to a - file; return ``False`` if the entry is or points to a directory or other - non-file entry, or if it doesn't exist anymore. - - If *follow_symlinks* is ``False``, return ``True`` only if this entry - is a file (without following symlinks); return ``False`` if the entry is - a directory or other non-file entry, or if it doesn't exist anymore. - - The result is cached on the ``os.DirEntry`` object. Caching, system calls - made, and exceptions raised are as per :func:`~os.DirEntry.is_dir`. - - .. method:: is_symlink() - - Return ``True`` if this entry is a symbolic link (even if broken); - return ``False`` if the entry points to a directory or any kind of file, - or if it doesn't exist anymore. - - The result is cached on the ``os.DirEntry`` object. Call - :func:`os.path.islink` to fetch up-to-date information. - - On the first, uncached call, no system call is required in most cases. - Specifically, neither Windows or Unix require a system call, except on - certain Unix file systems, such as network file systems, that return - ``dirent.d_type == DT_UNKNOWN``. - - This method can raise :exc:`OSError`, such as :exc:`PermissionError`, - but :exc:`FileNotFoundError` is caught and not raised. - - .. method:: stat(*, follow_symlinks=True) - - Return a :class:`stat_result` object for this entry. This method - follows symbolic links by default; to stat a symbolic link add the - ``follow_symlinks=False`` argument. - - On Unix, this method always requires a system call. On Windows, it - only requires a system call if *follow_symlinks* is ``True`` and the - entry is a reparse point (for example, a symbolic link or directory - junction). - - On Windows, the ``st_ino``, ``st_dev`` and ``st_nlink`` attributes of the - :class:`stat_result` are always set to zero. Call :func:`os.stat` to - get these attributes. - - The result is cached on the ``os.DirEntry`` object, with a separate cache - for *follow_symlinks* ``True`` and ``False``. Call :func:`os.stat` to - fetch up-to-date information. - - Note that there is a nice correspondence between several attributes - and methods of ``os.DirEntry`` and of :class:`pathlib.Path`. In - particular, the ``name`` attribute has the same - meaning, as do the ``is_dir()``, ``is_file()``, ``is_symlink()`` - and ``stat()`` methods. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.6 - Added support for the :class:`~os.PathLike` interface. Added support - for :class:`bytes` paths on Windows. - - -.. function:: stat(path, *, dir_fd=None, follow_symlinks=True) - - Get the status of a file or a file descriptor. Perform the equivalent of a - :c:func:`stat` system call on the given path. *path* may be specified as - either a string or bytes -- directly or indirectly through the :class:`PathLike` - interface -- or as an open file descriptor. Return a :class:`stat_result` - object. - - This function normally follows symlinks; to stat a symlink add the argument - ``follow_symlinks=False``, or use :func:`lstat`. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - On Windows, passing ``follow_symlinks=False`` will disable following all - name-surrogate reparse points, which includes symlinks and directory - junctions. Other types of reparse points that do not resemble links or that - the operating system is unable to follow will be opened directly. When - following a chain of multiple links, this may result in the original link - being returned instead of the non-link that prevented full traversal. To - obtain stat results for the final path in this case, use the - :func:`os.path.realpath` function to resolve the path name as far as - possible and call :func:`lstat` on the result. This does not apply to - dangling symlinks or junction points, which will raise the usual exceptions. - - .. index:: pair: module; stat - - Example:: - - >>> import os - >>> statinfo = os.stat('somefile.txt') - >>> statinfo - os.stat_result(st_mode=33188, st_ino=7876932, st_dev=234881026, - st_nlink=1, st_uid=501, st_gid=501, st_size=264, st_atime=1297230295, - st_mtime=1297230027, st_ctime=1297230027) - >>> statinfo.st_size - 264 - - .. seealso:: - - :func:`fstat` and :func:`lstat` functions. - - .. versionadded:: 3.3 - Added the *dir_fd* and *follow_symlinks* arguments, specifying a file - descriptor instead of a path. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. versionchanged:: 3.8 - On Windows, all reparse points that can be resolved by the operating - system are now followed, and passing ``follow_symlinks=False`` - disables following all name surrogate reparse points. If the operating - system reaches a reparse point that it is not able to follow, *stat* now - returns the information for the original path as if - ``follow_symlinks=False`` had been specified instead of raising an error. - - -.. class:: stat_result - - Object whose attributes correspond roughly to the members of the - :c:struct:`stat` structure. It is used for the result of :func:`os.stat`, - :func:`os.fstat` and :func:`os.lstat`. - - Attributes: - - .. attribute:: st_mode - - File mode: file type and file mode bits (permissions). - - .. attribute:: st_ino - - Platform dependent, but if non-zero, uniquely identifies the - file for a given value of ``st_dev``. Typically: - - * the inode number on Unix, - * the `file index - `_ on - Windows - - .. attribute:: st_dev - - Identifier of the device on which this file resides. - - .. attribute:: st_nlink - - Number of hard links. - - .. attribute:: st_uid - - User identifier of the file owner. - - .. attribute:: st_gid - - Group identifier of the file owner. - - .. attribute:: st_size - - Size of the file in bytes, if it is a regular file or a symbolic link. - The size of a symbolic link is the length of the pathname it contains, - without a terminating null byte. - - Timestamps: - - .. attribute:: st_atime - - Time of most recent access expressed in seconds. - - .. attribute:: st_mtime - - Time of most recent content modification expressed in seconds. - - .. attribute:: st_ctime - - Platform dependent: - - * the time of most recent metadata change on Unix, - * the time of creation on Windows, expressed in seconds. - - .. attribute:: st_atime_ns - - Time of most recent access expressed in nanoseconds as an integer. - - .. attribute:: st_mtime_ns - - Time of most recent content modification expressed in nanoseconds as an - integer. - - .. attribute:: st_ctime_ns - - Platform dependent: - - * the time of most recent metadata change on Unix, - * the time of creation on Windows, expressed in nanoseconds as an - integer. - - .. note:: - - The exact meaning and resolution of the :attr:`st_atime`, - :attr:`st_mtime`, and :attr:`st_ctime` attributes depend on the operating - system and the file system. For example, on Windows systems using the FAT - or FAT32 file systems, :attr:`st_mtime` has 2-second resolution, and - :attr:`st_atime` has only 1-day resolution. See your operating system - documentation for details. - - Similarly, although :attr:`st_atime_ns`, :attr:`st_mtime_ns`, - and :attr:`st_ctime_ns` are always expressed in nanoseconds, many - systems do not provide nanosecond precision. On systems that do - provide nanosecond precision, the floating-point object used to - store :attr:`st_atime`, :attr:`st_mtime`, and :attr:`st_ctime` - cannot preserve all of it, and as such will be slightly inexact. - If you need the exact timestamps you should always use - :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and :attr:`st_ctime_ns`. - - On some Unix systems (such as Linux), the following attributes may also be - available: - - .. attribute:: st_blocks - - Number of 512-byte blocks allocated for file. - This may be smaller than :attr:`st_size`/512 when the file has holes. - - .. attribute:: st_blksize - - "Preferred" blocksize for efficient file system I/O. Writing to a file in - smaller chunks may cause an inefficient read-modify-rewrite. - - .. attribute:: st_rdev - - Type of device if an inode device. - - .. attribute:: st_flags - - User defined flags for file. - - On other Unix systems (such as FreeBSD), the following attributes may be - available (but may be only filled out if root tries to use them): - - .. attribute:: st_gen - - File generation number. - - .. attribute:: st_birthtime - - Time of file creation. - - On Solaris and derivatives, the following attributes may also be - available: - - .. attribute:: st_fstype - - String that uniquely identifies the type of the filesystem that - contains the file. - - On macOS systems, the following attributes may also be available: - - .. attribute:: st_rsize - - Real size of the file. - - .. attribute:: st_creator - - Creator of the file. - - .. attribute:: st_type - - File type. - - On Windows systems, the following attributes are also available: - - .. attribute:: st_file_attributes - - Windows file attributes: ``dwFileAttributes`` member of the - ``BY_HANDLE_FILE_INFORMATION`` structure returned by - :c:func:`!GetFileInformationByHandle`. - See the :const:`!FILE_ATTRIBUTE_* ` - constants in the :mod:`stat` module. - - .. attribute:: st_reparse_tag - - When :attr:`st_file_attributes` has the :const:`~stat.FILE_ATTRIBUTE_REPARSE_POINT` - set, this field contains the tag identifying the type of reparse point. - See the :const:`IO_REPARSE_TAG_* ` - constants in the :mod:`stat` module. - - The standard module :mod:`stat` defines functions and constants that are - useful for extracting information from a :c:struct:`stat` structure. (On - Windows, some items are filled with dummy values.) - - For backward compatibility, a :class:`stat_result` instance is also - accessible as a tuple of at least 10 integers giving the most important (and - portable) members of the :c:struct:`stat` structure, in the order - :attr:`st_mode`, :attr:`st_ino`, :attr:`st_dev`, :attr:`st_nlink`, - :attr:`st_uid`, :attr:`st_gid`, :attr:`st_size`, :attr:`st_atime`, - :attr:`st_mtime`, :attr:`st_ctime`. More items may be added at the end by - some implementations. For compatibility with older Python versions, - accessing :class:`stat_result` as a tuple always returns integers. - - .. versionadded:: 3.3 - Added the :attr:`st_atime_ns`, :attr:`st_mtime_ns`, and - :attr:`st_ctime_ns` members. - - .. versionadded:: 3.5 - Added the :attr:`st_file_attributes` member on Windows. - - .. versionchanged:: 3.5 - Windows now returns the file index as :attr:`st_ino` when - available. - - .. versionadded:: 3.7 - Added the :attr:`st_fstype` member to Solaris/derivatives. - - .. versionadded:: 3.8 - Added the :attr:`st_reparse_tag` member on Windows. - - .. versionchanged:: 3.8 - On Windows, the :attr:`st_mode` member now identifies special - files as :const:`S_IFCHR`, :const:`S_IFIFO` or :const:`S_IFBLK` - as appropriate. - -.. function:: statvfs(path) - - Perform a :c:func:`!statvfs` system call on the given path. The return value is - an object whose attributes describe the filesystem on the given path, and - correspond to the members of the :c:struct:`statvfs` structure, namely: - :attr:`f_bsize`, :attr:`f_frsize`, :attr:`f_blocks`, :attr:`f_bfree`, - :attr:`f_bavail`, :attr:`f_files`, :attr:`f_ffree`, :attr:`f_favail`, - :attr:`f_flag`, :attr:`f_namemax`, :attr:`f_fsid`. - - Two module-level constants are defined for the :attr:`f_flag` attribute's - bit-flags: if :const:`ST_RDONLY` is set, the filesystem is mounted - read-only, and if :const:`ST_NOSUID` is set, the semantics of - setuid/setgid bits are disabled or not supported. - - Additional module-level constants are defined for GNU/glibc based systems. - These are :const:`ST_NODEV` (disallow access to device special files), - :const:`ST_NOEXEC` (disallow program execution), :const:`ST_SYNCHRONOUS` - (writes are synced at once), :const:`ST_MANDLOCK` (allow mandatory locks on an FS), - :const:`ST_WRITE` (write on file/directory/symlink), :const:`ST_APPEND` - (append-only file), :const:`ST_IMMUTABLE` (immutable file), :const:`ST_NOATIME` - (do not update access times), :const:`ST_NODIRATIME` (do not update directory access - times), :const:`ST_RELATIME` (update atime relative to mtime/ctime). - - This function can support :ref:`specifying a file descriptor `. - - .. availability:: Unix. - - .. versionchanged:: 3.2 - The :const:`ST_RDONLY` and :const:`ST_NOSUID` constants were added. - - .. versionadded:: 3.3 - Added support for specifying *path* as an open file descriptor. - - .. versionchanged:: 3.4 - The :const:`ST_NODEV`, :const:`ST_NOEXEC`, :const:`ST_SYNCHRONOUS`, - :const:`ST_MANDLOCK`, :const:`ST_WRITE`, :const:`ST_APPEND`, - :const:`ST_IMMUTABLE`, :const:`ST_NOATIME`, :const:`ST_NODIRATIME`, - and :const:`ST_RELATIME` constants were added. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. versionadded:: 3.7 - Added :attr:`f_fsid`. - - -.. data:: supports_dir_fd - - A :class:`set` object indicating which functions in the :mod:`os` - module accept an open file descriptor for their *dir_fd* parameter. - Different platforms provide different features, and the underlying - functionality Python uses to implement the *dir_fd* parameter is not - available on all platforms Python supports. For consistency's sake, - functions that may support *dir_fd* always allow specifying the - parameter, but will throw an exception if the functionality is used - when it's not locally available. (Specifying ``None`` for *dir_fd* - is always supported on all platforms.) - - To check whether a particular function accepts an open file descriptor - for its *dir_fd* parameter, use the ``in`` operator on ``supports_dir_fd``. - As an example, this expression evaluates to ``True`` if :func:`os.stat` - accepts open file descriptors for *dir_fd* on the local platform:: - - os.stat in os.supports_dir_fd - - Currently *dir_fd* parameters only work on Unix platforms; - none of them work on Windows. - - .. versionadded:: 3.3 - - -.. data:: supports_effective_ids - - A :class:`set` object indicating whether :func:`os.access` permits - specifying ``True`` for its *effective_ids* parameter on the local platform. - (Specifying ``False`` for *effective_ids* is always supported on all - platforms.) If the local platform supports it, the collection will contain - :func:`os.access`; otherwise it will be empty. - - This expression evaluates to ``True`` if :func:`os.access` supports - ``effective_ids=True`` on the local platform:: - - os.access in os.supports_effective_ids - - Currently *effective_ids* is only supported on Unix platforms; - it does not work on Windows. - - .. versionadded:: 3.3 - - -.. data:: supports_fd - - A :class:`set` object indicating which functions in the - :mod:`os` module permit specifying their *path* parameter as an open file - descriptor on the local platform. Different platforms provide different - features, and the underlying functionality Python uses to accept open file - descriptors as *path* arguments is not available on all platforms Python - supports. - - To determine whether a particular function permits specifying an open file - descriptor for its *path* parameter, use the ``in`` operator on - ``supports_fd``. As an example, this expression evaluates to ``True`` if - :func:`os.chdir` accepts open file descriptors for *path* on your local - platform:: - - os.chdir in os.supports_fd - - .. versionadded:: 3.3 - - -.. data:: supports_follow_symlinks - - A :class:`set` object indicating which functions in the :mod:`os` module - accept ``False`` for their *follow_symlinks* parameter on the local platform. - Different platforms provide different features, and the underlying - functionality Python uses to implement *follow_symlinks* is not available - on all platforms Python supports. For consistency's sake, functions that - may support *follow_symlinks* always allow specifying the parameter, but - will throw an exception if the functionality is used when it's not locally - available. (Specifying ``True`` for *follow_symlinks* is always supported - on all platforms.) - - To check whether a particular function accepts ``False`` for its - *follow_symlinks* parameter, use the ``in`` operator on - ``supports_follow_symlinks``. As an example, this expression evaluates - to ``True`` if you may specify ``follow_symlinks=False`` when calling - :func:`os.stat` on the local platform:: - - os.stat in os.supports_follow_symlinks - - .. versionadded:: 3.3 - - -.. function:: symlink(src, dst, target_is_directory=False, *, dir_fd=None) - - Create a symbolic link pointing to *src* named *dst*. - - On Windows, a symlink represents either a file or a directory, and does not - morph to the target dynamically. If the target is present, the type of the - symlink will be created to match. Otherwise, the symlink will be created - as a directory if *target_is_directory* is ``True`` or a file symlink (the - default) otherwise. On non-Windows platforms, *target_is_directory* is ignored. - - This function can support :ref:`paths relative to directory descriptors - `. - - .. note:: - - On newer versions of Windows 10, unprivileged accounts can create symlinks - if Developer Mode is enabled. When Developer Mode is not available/enabled, - the *SeCreateSymbolicLinkPrivilege* privilege is required, or the process - must be run as an administrator. - - - :exc:`OSError` is raised when the function is called by an unprivileged - user. - - .. audit-event:: os.symlink src,dst,dir_fd os.symlink - - .. availability:: Unix, Windows. - - The function is limited on Emscripten and WASI, see - :ref:`wasm-availability` for more information. - - .. versionchanged:: 3.2 - Added support for Windows 6.0 (Vista) symbolic links. - - .. versionadded:: 3.3 - Added the *dir_fd* argument, and now allow *target_is_directory* - on non-Windows platforms. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *src* and *dst*. - - .. versionchanged:: 3.8 - Added support for unelevated symlinks on Windows with Developer Mode. - - -.. function:: sync() - - Force write of everything to disk. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: truncate(path, length) - - Truncate the file corresponding to *path*, so that it is at most - *length* bytes in size. - - This function can support :ref:`specifying a file descriptor `. - - .. audit-event:: os.truncate path,length os.truncate - - .. availability:: Unix, Windows. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - Added support for Windows - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: unlink(path, *, dir_fd=None) - - Remove (delete) the file *path*. This function is semantically - identical to :func:`remove`; the ``unlink`` name is its - traditional Unix name. Please see the documentation for - :func:`remove` for further information. - - .. audit-event:: os.remove path,dir_fd os.unlink - - .. versionadded:: 3.3 - The *dir_fd* parameter. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: utime(path, times=None, *[, ns], dir_fd=None, follow_symlinks=True) - - Set the access and modified times of the file specified by *path*. - - :func:`utime` takes two optional parameters, *times* and *ns*. - These specify the times set on *path* and are used as follows: - - - If *ns* is specified, - it must be a 2-tuple of the form ``(atime_ns, mtime_ns)`` - where each member is an int expressing nanoseconds. - - If *times* is not ``None``, - it must be a 2-tuple of the form ``(atime, mtime)`` - where each member is an int or float expressing seconds. - - If *times* is ``None`` and *ns* is unspecified, - this is equivalent to specifying ``ns=(atime_ns, mtime_ns)`` - where both times are the current time. - - It is an error to specify tuples for both *times* and *ns*. - - Note that the exact times you set here may not be returned by a subsequent - :func:`~os.stat` call, depending on the resolution with which your operating - system records access and modification times; see :func:`~os.stat`. The best - way to preserve exact times is to use the *st_atime_ns* and *st_mtime_ns* - fields from the :func:`os.stat` result object with the *ns* parameter to - :func:`utime`. - - This function can support :ref:`specifying a file descriptor `, - :ref:`paths relative to directory descriptors ` and :ref:`not - following symlinks `. - - .. audit-event:: os.utime path,times,ns,dir_fd os.utime - - .. versionadded:: 3.3 - Added support for specifying *path* as an open file descriptor, - and the *dir_fd*, *follow_symlinks*, and *ns* parameters. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: walk(top, topdown=True, onerror=None, followlinks=False) - - .. index:: - single: directory; walking - single: directory; traversal - - Generate the file names in a directory tree by walking the tree - either top-down or bottom-up. For each directory in the tree rooted at directory - *top* (including *top* itself), it yields a 3-tuple ``(dirpath, dirnames, - filenames)``. - - *dirpath* is a string, the path to the directory. *dirnames* is a list of the - names of the subdirectories in *dirpath* (including symlinks to directories, - and excluding ``'.'`` and ``'..'``). - *filenames* is a list of the names of the non-directory files in *dirpath*. - Note that the names in the lists contain no path components. To get a full path - (which begins with *top*) to a file or directory in *dirpath*, do - ``os.path.join(dirpath, name)``. Whether or not the lists are sorted - depends on the file system. If a file is removed from or added to the - *dirpath* directory during generating the lists, whether a name for that - file be included is unspecified. - - If optional argument *topdown* is ``True`` or not specified, the triple for a - directory is generated before the triples for any of its subdirectories - (directories are generated top-down). If *topdown* is ``False``, the triple - for a directory is generated after the triples for all of its subdirectories - (directories are generated bottom-up). No matter the value of *topdown*, the - list of subdirectories is retrieved before the tuples for the directory and - its subdirectories are generated. - - When *topdown* is ``True``, the caller can modify the *dirnames* list in-place - (perhaps using :keyword:`del` or slice assignment), and :func:`walk` will only - recurse into the subdirectories whose names remain in *dirnames*; this can be - used to prune the search, impose a specific order of visiting, or even to inform - :func:`walk` about directories the caller creates or renames before it resumes - :func:`walk` again. Modifying *dirnames* when *topdown* is ``False`` has - no effect on the behavior of the walk, because in bottom-up mode the directories - in *dirnames* are generated before *dirpath* itself is generated. - - By default, errors from the :func:`scandir` call are ignored. If optional - argument *onerror* is specified, it should be a function; it will be called with - one argument, an :exc:`OSError` instance. It can report the error to continue - with the walk, or raise the exception to abort the walk. Note that the filename - is available as the ``filename`` attribute of the exception object. - - By default, :func:`walk` will not walk down into symbolic links that resolve to - directories. Set *followlinks* to ``True`` to visit directories pointed to by - symlinks, on systems that support them. - - .. note:: - - Be aware that setting *followlinks* to ``True`` can lead to infinite - recursion if a link points to a parent directory of itself. :func:`walk` - does not keep track of the directories it visited already. - - .. note:: - - If you pass a relative pathname, don't change the current working directory - between resumptions of :func:`walk`. :func:`walk` never changes the current - directory, and assumes that its caller doesn't either. - - This example displays the number of bytes taken by non-directory files in each - directory under the starting directory, except that it doesn't look under any - CVS subdirectory:: - - import os - from os.path import join, getsize - for root, dirs, files in os.walk('python/Lib/email'): - print(root, "consumes", end=" ") - print(sum(getsize(join(root, name)) for name in files), end=" ") - print("bytes in", len(files), "non-directory files") - if 'CVS' in dirs: - dirs.remove('CVS') # don't visit CVS directories - - In the next example (simple implementation of :func:`shutil.rmtree`), - walking the tree bottom-up is essential, :func:`rmdir` doesn't allow - deleting a directory before the directory is empty:: - - # Delete everything reachable from the directory named in "top", - # assuming there are no symbolic links. - # CAUTION: This is dangerous! For example, if top == '/', it - # could delete all your disk files. - import os - for root, dirs, files in os.walk(top, topdown=False): - for name in files: - os.remove(os.path.join(root, name)) - for name in dirs: - os.rmdir(os.path.join(root, name)) - - .. audit-event:: os.walk top,topdown,onerror,followlinks os.walk - - .. versionchanged:: 3.5 - This function now calls :func:`os.scandir` instead of :func:`os.listdir`, - making it faster by reducing the number of calls to :func:`os.stat`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: fwalk(top='.', topdown=True, onerror=None, *, follow_symlinks=False, dir_fd=None) - - .. index:: - single: directory; walking - single: directory; traversal - - This behaves exactly like :func:`walk`, except that it yields a 4-tuple - ``(dirpath, dirnames, filenames, dirfd)``, and it supports ``dir_fd``. - - *dirpath*, *dirnames* and *filenames* are identical to :func:`walk` output, - and *dirfd* is a file descriptor referring to the directory *dirpath*. - - This function always supports :ref:`paths relative to directory descriptors - ` and :ref:`not following symlinks `. Note however - that, unlike other functions, the :func:`fwalk` default value for - *follow_symlinks* is ``False``. - - .. note:: - - Since :func:`fwalk` yields file descriptors, those are only valid until - the next iteration step, so you should duplicate them (e.g. with - :func:`dup`) if you want to keep them longer. - - This example displays the number of bytes taken by non-directory files in each - directory under the starting directory, except that it doesn't look under any - CVS subdirectory:: - - import os - for root, dirs, files, rootfd in os.fwalk('python/Lib/email'): - print(root, "consumes", end="") - print(sum([os.stat(name, dir_fd=rootfd).st_size for name in files]), - end="") - print("bytes in", len(files), "non-directory files") - if 'CVS' in dirs: - dirs.remove('CVS') # don't visit CVS directories - - In the next example, walking the tree bottom-up is essential: - :func:`rmdir` doesn't allow deleting a directory before the directory is - empty:: - - # Delete everything reachable from the directory named in "top", - # assuming there are no symbolic links. - # CAUTION: This is dangerous! For example, if top == '/', it - # could delete all your disk files. - import os - for root, dirs, files, rootfd in os.fwalk(top, topdown=False): - for name in files: - os.unlink(name, dir_fd=rootfd) - for name in dirs: - os.rmdir(name, dir_fd=rootfd) - - .. audit-event:: os.fwalk top,topdown,onerror,follow_symlinks,dir_fd os.fwalk - - .. availability:: Unix. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - .. versionchanged:: 3.7 - Added support for :class:`bytes` paths. - - -.. function:: memfd_create(name[, flags=os.MFD_CLOEXEC]) - - Create an anonymous file and return a file descriptor that refers to it. - *flags* must be one of the ``os.MFD_*`` constants available on the system - (or a bitwise ORed combination of them). By default, the new file - descriptor is :ref:`non-inheritable `. - - The name supplied in *name* is used as a filename and will be displayed as - the target of the corresponding symbolic link in the directory - ``/proc/self/fd/``. The displayed name is always prefixed with ``memfd:`` - and serves only for debugging purposes. Names do not affect the behavior of - the file descriptor, and as such multiple files can have the same name - without any side effects. - - .. availability:: Linux >= 3.17 with glibc >= 2.27. - - .. versionadded:: 3.8 - - -.. data:: MFD_CLOEXEC - MFD_ALLOW_SEALING - MFD_HUGETLB - MFD_HUGE_SHIFT - MFD_HUGE_MASK - MFD_HUGE_64KB - MFD_HUGE_512KB - MFD_HUGE_1MB - MFD_HUGE_2MB - MFD_HUGE_8MB - MFD_HUGE_16MB - MFD_HUGE_32MB - MFD_HUGE_256MB - MFD_HUGE_512MB - MFD_HUGE_1GB - MFD_HUGE_2GB - MFD_HUGE_16GB - - These flags can be passed to :func:`memfd_create`. - - .. availability:: Linux >= 3.17 with glibc >= 2.27 - - The ``MFD_HUGE*`` flags are only available since Linux 4.14. - - .. versionadded:: 3.8 - - -.. function:: eventfd(initval[, flags=os.EFD_CLOEXEC]) - - Create and return an event file descriptor. The file descriptors supports - raw :func:`read` and :func:`write` with a buffer size of 8, - :func:`~select.select`, :func:`~select.poll` and similar. See man page - :manpage:`eventfd(2)` for more information. By default, the - new file descriptor is :ref:`non-inheritable `. - - *initval* is the initial value of the event counter. The initial value - must be an 32 bit unsigned integer. Please note that the initial value is - limited to a 32 bit unsigned int although the event counter is an unsigned - 64 bit integer with a maximum value of 2\ :sup:`64`\ -\ 2. - - *flags* can be constructed from :const:`EFD_CLOEXEC`, - :const:`EFD_NONBLOCK`, and :const:`EFD_SEMAPHORE`. - - If :const:`EFD_SEMAPHORE` is specified and the event counter is non-zero, - :func:`eventfd_read` returns 1 and decrements the counter by one. - - If :const:`EFD_SEMAPHORE` is not specified and the event counter is - non-zero, :func:`eventfd_read` returns the current event counter value and - resets the counter to zero. - - If the event counter is zero and :const:`EFD_NONBLOCK` is not - specified, :func:`eventfd_read` blocks. - - :func:`eventfd_write` increments the event counter. Write blocks if the - write operation would increment the counter to a value larger than - 2\ :sup:`64`\ -\ 2. - - Example:: - - import os - - # semaphore with start value '1' - fd = os.eventfd(1, os.EFD_SEMAPHORE | os.EFC_CLOEXEC) - try: - # acquire semaphore - v = os.eventfd_read(fd) - try: - do_work() - finally: - # release semaphore - os.eventfd_write(fd, v) - finally: - os.close(fd) - - .. availability:: Linux >= 2.6.27 with glibc >= 2.8 - - .. versionadded:: 3.10 - -.. function:: eventfd_read(fd) - - Read value from an :func:`eventfd` file descriptor and return a 64 bit - unsigned int. The function does not verify that *fd* is an :func:`eventfd`. - - .. availability:: Linux >= 2.6.27 - - .. versionadded:: 3.10 - -.. function:: eventfd_write(fd, value) - - Add value to an :func:`eventfd` file descriptor. *value* must be a 64 bit - unsigned int. The function does not verify that *fd* is an :func:`eventfd`. - - .. availability:: Linux >= 2.6.27 - - .. versionadded:: 3.10 - -.. data:: EFD_CLOEXEC - - Set close-on-exec flag for new :func:`eventfd` file descriptor. - - .. availability:: Linux >= 2.6.27 - - .. versionadded:: 3.10 - -.. data:: EFD_NONBLOCK - - Set :const:`O_NONBLOCK` status flag for new :func:`eventfd` file - descriptor. - - .. availability:: Linux >= 2.6.27 - - .. versionadded:: 3.10 - -.. data:: EFD_SEMAPHORE - - Provide semaphore-like semantics for reads from a :func:`eventfd` file - descriptor. On read the internal counter is decremented by one. - - .. availability:: Linux >= 2.6.30 - - .. versionadded:: 3.10 - - -Linux extended attributes -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. versionadded:: 3.3 - -These functions are all available on Linux only. - -.. function:: getxattr(path, attribute, *, follow_symlinks=True) - - Return the value of the extended filesystem attribute *attribute* for - *path*. *attribute* can be bytes or str (directly or indirectly through the - :class:`PathLike` interface). If it is str, it is encoded with the filesystem - encoding. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. audit-event:: os.getxattr path,attribute os.getxattr - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *path* and *attribute*. - - -.. function:: listxattr(path=None, *, follow_symlinks=True) - - Return a list of the extended filesystem attributes on *path*. The - attributes in the list are represented as strings decoded with the filesystem - encoding. If *path* is ``None``, :func:`listxattr` will examine the current - directory. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. audit-event:: os.listxattr path os.listxattr - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. function:: removexattr(path, attribute, *, follow_symlinks=True) - - Removes the extended filesystem attribute *attribute* from *path*. - *attribute* should be bytes or str (directly or indirectly through the - :class:`PathLike` interface). If it is a string, it is encoded - with the :term:`filesystem encoding and error handler`. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. audit-event:: os.removexattr path,attribute os.removexattr - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *path* and *attribute*. - - -.. function:: setxattr(path, attribute, value, flags=0, *, follow_symlinks=True) - - Set the extended filesystem attribute *attribute* on *path* to *value*. - *attribute* must be a bytes or str with no embedded NULs (directly or - indirectly through the :class:`PathLike` interface). If it is a str, - it is encoded with the :term:`filesystem encoding and error handler`. *flags* may be - :data:`XATTR_REPLACE` or :data:`XATTR_CREATE`. If :data:`XATTR_REPLACE` is - given and the attribute does not exist, ``ENODATA`` will be raised. - If :data:`XATTR_CREATE` is given and the attribute already exists, the - attribute will not be created and ``EEXISTS`` will be raised. - - This function can support :ref:`specifying a file descriptor ` and - :ref:`not following symlinks `. - - .. note:: - - A bug in Linux kernel versions less than 2.6.39 caused the flags argument - to be ignored on some filesystems. - - .. audit-event:: os.setxattr path,attribute,value,flags os.setxattr - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object` for *path* and *attribute*. - - -.. data:: XATTR_SIZE_MAX - - The maximum size the value of an extended attribute can be. Currently, this - is 64 KiB on Linux. - - -.. data:: XATTR_CREATE - - This is a possible value for the flags argument in :func:`setxattr`. It - indicates the operation must create an attribute. - - -.. data:: XATTR_REPLACE - - This is a possible value for the flags argument in :func:`setxattr`. It - indicates the operation must replace an existing attribute. - - -.. _os-process: - -Process Management ------------------- - -These functions may be used to create and manage processes. - -The various :func:`exec\* ` functions take a list of arguments for the new -program loaded into the process. In each case, the first of these arguments is -passed to the new program as its own name rather than as an argument a user may -have typed on a command line. For the C programmer, this is the ``argv[0]`` -passed to a program's :c:func:`main`. For example, ``os.execv('/bin/echo', -['foo', 'bar'])`` will only print ``bar`` on standard output; ``foo`` will seem -to be ignored. - - -.. function:: abort() - - Generate a :const:`SIGABRT` signal to the current process. On Unix, the default - behavior is to produce a core dump; on Windows, the process immediately returns - an exit code of ``3``. Be aware that calling this function will not call the - Python signal handler registered for :const:`SIGABRT` with - :func:`signal.signal`. - - -.. function:: add_dll_directory(path) - - Add a path to the DLL search path. - - This search path is used when resolving dependencies for imported - extension modules (the module itself is resolved through - :data:`sys.path`), and also by :mod:`ctypes`. - - Remove the directory by calling **close()** on the returned object - or using it in a :keyword:`with` statement. - - See the `Microsoft documentation - `_ - for more information about how DLLs are loaded. - - .. audit-event:: os.add_dll_directory path os.add_dll_directory - - .. availability:: Windows. - - .. versionadded:: 3.8 - Previous versions of CPython would resolve DLLs using the default - behavior for the current process. This led to inconsistencies, - such as only sometimes searching :envvar:`PATH` or the current - working directory, and OS functions such as ``AddDllDirectory`` - having no effect. - - In 3.8, the two primary ways DLLs are loaded now explicitly - override the process-wide behavior to ensure consistency. See the - :ref:`porting notes ` for information on - updating libraries. - - -.. function:: execl(path, arg0, arg1, ...) - execle(path, arg0, arg1, ..., env) - execlp(file, arg0, arg1, ...) - execlpe(file, arg0, arg1, ..., env) - execv(path, args) - execve(path, args, env) - execvp(file, args) - execvpe(file, args, env) - - These functions all execute a new program, replacing the current process; they - do not return. On Unix, the new executable is loaded into the current process, - and will have the same process id as the caller. Errors will be reported as - :exc:`OSError` exceptions. - - The current process is replaced immediately. Open file objects and - descriptors are not flushed, so if there may be data buffered - on these open files, you should flush them using - :func:`sys.stdout.flush` or :func:`os.fsync` before calling an - :func:`exec\* ` function. - - The "l" and "v" variants of the :func:`exec\* ` functions differ in how - command-line arguments are passed. The "l" variants are perhaps the easiest - to work with if the number of parameters is fixed when the code is written; the - individual parameters simply become additional parameters to the :func:`execl\*` - functions. The "v" variants are good when the number of parameters is - variable, with the arguments being passed in a list or tuple as the *args* - parameter. In either case, the arguments to the child process should start with - the name of the command being run, but this is not enforced. - - The variants which include a "p" near the end (:func:`execlp`, - :func:`execlpe`, :func:`execvp`, and :func:`execvpe`) will use the - :envvar:`PATH` environment variable to locate the program *file*. When the - environment is being replaced (using one of the :func:`exec\*e ` variants, - discussed in the next paragraph), the new environment is used as the source of - the :envvar:`PATH` variable. The other variants, :func:`execl`, :func:`execle`, - :func:`execv`, and :func:`execve`, will not use the :envvar:`PATH` variable to - locate the executable; *path* must contain an appropriate absolute or relative - path. - - For :func:`execle`, :func:`execlpe`, :func:`execve`, and :func:`execvpe` (note - that these all end in "e"), the *env* parameter must be a mapping which is - used to define the environment variables for the new process (these are used - instead of the current process' environment); the functions :func:`execl`, - :func:`execlp`, :func:`execv`, and :func:`execvp` all cause the new process to - inherit the environment of the current process. - - For :func:`execve` on some platforms, *path* may also be specified as an open - file descriptor. This functionality may not be supported on your platform; - you can check whether or not it is available using :data:`os.supports_fd`. - If it is unavailable, using it will raise a :exc:`NotImplementedError`. - - .. audit-event:: os.exec path,args,env os.execl - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - .. versionadded:: 3.3 - Added support for specifying *path* as an open file descriptor - for :func:`execve`. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - -.. function:: _exit(n) - - Exit the process with status *n*, without calling cleanup handlers, flushing - stdio buffers, etc. - - .. note:: - - The standard way to exit is :func:`sys.exit(n) `. :func:`!_exit` should - normally only be used in the child process after a :func:`fork`. - -The following exit codes are defined and can be used with :func:`_exit`, -although they are not required. These are typically used for system programs -written in Python, such as a mail server's external command delivery program. - -.. note:: - - Some of these may not be available on all Unix platforms, since there is some - variation. These constants are defined where they are defined by the underlying - platform. - - -.. data:: EX_OK - - Exit code that means no error occurred. May be taken from the defined value of - ``EXIT_SUCCESS`` on some platforms. Generally has a value of zero. - - .. availability:: Unix, Windows. - - -.. data:: EX_USAGE - - Exit code that means the command was used incorrectly, such as when the wrong - number of arguments are given. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_DATAERR - - Exit code that means the input data was incorrect. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_NOINPUT - - Exit code that means an input file did not exist or was not readable. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_NOUSER - - Exit code that means a specified user did not exist. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_NOHOST - - Exit code that means a specified host did not exist. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_UNAVAILABLE - - Exit code that means that a required service is unavailable. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_SOFTWARE - - Exit code that means an internal software error was detected. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_OSERR - - Exit code that means an operating system error was detected, such as the - inability to fork or create a pipe. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_OSFILE - - Exit code that means some system file did not exist, could not be opened, or had - some other kind of error. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_CANTCREAT - - Exit code that means a user specified output file could not be created. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_IOERR - - Exit code that means that an error occurred while doing I/O on some file. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_TEMPFAIL - - Exit code that means a temporary failure occurred. This indicates something - that may not really be an error, such as a network connection that couldn't be - made during a retryable operation. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_PROTOCOL - - Exit code that means that a protocol exchange was illegal, invalid, or not - understood. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_NOPERM - - Exit code that means that there were insufficient permissions to perform the - operation (but not intended for file system problems). - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_CONFIG - - Exit code that means that some kind of configuration error occurred. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: EX_NOTFOUND - - Exit code that means something like "an entry was not found". - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: fork() - - Fork a child process. Return ``0`` in the child and the child's process id in the - parent. If an error occurs :exc:`OSError` is raised. - - Note that some platforms including FreeBSD <= 6.3 and Cygwin have - known issues when using ``fork()`` from a thread. - - .. audit-event:: os.fork "" os.fork - - .. versionchanged:: 3.8 - Calling ``fork()`` in a subinterpreter is no longer supported - (:exc:`RuntimeError` is raised). - - .. warning:: - - See :mod:`ssl` for applications that use the SSL module with fork(). - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: forkpty() - - Fork a child process, using a new pseudo-terminal as the child's controlling - terminal. Return a pair of ``(pid, fd)``, where *pid* is ``0`` in the child, the - new child's process id in the parent, and *fd* is the file descriptor of the - master end of the pseudo-terminal. For a more portable approach, use the - :mod:`pty` module. If an error occurs :exc:`OSError` is raised. - - .. audit-event:: os.forkpty "" os.forkpty - - .. versionchanged:: 3.8 - Calling ``forkpty()`` in a subinterpreter is no longer supported - (:exc:`RuntimeError` is raised). - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: kill(pid, sig, /) - - .. index:: - single: process; killing - single: process; signalling - - Send signal *sig* to the process *pid*. Constants for the specific signals - available on the host platform are defined in the :mod:`signal` module. - - Windows: The :const:`signal.CTRL_C_EVENT` and - :const:`signal.CTRL_BREAK_EVENT` signals are special signals which can - only be sent to console processes which share a common console window, - e.g., some subprocesses. Any other value for *sig* will cause the process - to be unconditionally killed by the TerminateProcess API, and the exit code - will be set to *sig*. The Windows version of :func:`kill` additionally takes - process handles to be killed. - - See also :func:`signal.pthread_kill`. - - .. audit-event:: os.kill pid,sig os.kill - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - .. versionadded:: 3.2 - Windows support. - - -.. function:: killpg(pgid, sig, /) - - .. index:: - single: process; killing - single: process; signalling - - Send the signal *sig* to the process group *pgid*. - - .. audit-event:: os.killpg pgid,sig os.killpg - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: nice(increment, /) - - Add *increment* to the process's "niceness". Return the new niceness. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: pidfd_open(pid, flags=0) - - Return a file descriptor referring to the process *pid*. This descriptor can - be used to perform process management without races and signals. The *flags* - argument is provided for future extensions; no flag values are currently - defined. - - See the :manpage:`pidfd_open(2)` man page for more details. - - .. availability:: Linux >= 5.3 - .. versionadded:: 3.9 - - -.. function:: plock(op, /) - - Lock program segments into memory. The value of *op* (defined in - ````) determines which segments are locked. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: popen(cmd, mode='r', buffering=-1) - - Open a pipe to or from command *cmd*. - The return value is an open file object - connected to the pipe, which can be read or written depending on whether *mode* - is ``'r'`` (default) or ``'w'``. - The *buffering* argument have the same meaning as - the corresponding argument to the built-in :func:`open` function. The - returned file object reads or writes text strings rather than bytes. - - The ``close`` method returns :const:`None` if the subprocess exited - successfully, or the subprocess's return code if there was an - error. On POSIX systems, if the return code is positive it - represents the return value of the process left-shifted by one - byte. If the return code is negative, the process was terminated - by the signal given by the negated value of the return code. (For - example, the return value might be ``- signal.SIGKILL`` if the - subprocess was killed.) On Windows systems, the return value - contains the signed integer return code from the child process. - - On Unix, :func:`waitstatus_to_exitcode` can be used to convert the ``close`` - method result (exit status) into an exit code if it is not ``None``. On - Windows, the ``close`` method result is directly the exit code - (or ``None``). - - This is implemented using :class:`subprocess.Popen`; see that class's - documentation for more powerful ways to manage and communicate with - subprocesses. - - .. availability:: not Emscripten, not WASI. - - .. note:: - The :ref:`Python UTF-8 Mode ` affects encodings used - for *cmd* and pipe contents. - - :func:`popen` is a simple wrapper around :class:`subprocess.Popen`. - Use :class:`subprocess.Popen` or :func:`subprocess.run` to - control options like encodings. - - -.. function:: posix_spawn(path, argv, env, *, file_actions=None, \ - setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ - setsigdef=(), scheduler=None) - - Wraps the :c:func:`!posix_spawn` C library API for use from Python. - - Most users should use :func:`subprocess.run` instead of :func:`posix_spawn`. - - The positional-only arguments *path*, *args*, and *env* are similar to - :func:`execve`. - - The *path* parameter is the path to the executable file. The *path* should - contain a directory. Use :func:`posix_spawnp` to pass an executable file - without directory. - - The *file_actions* argument may be a sequence of tuples describing actions - to take on specific file descriptors in the child process between the C - library implementation's :c:func:`fork` and :c:func:`exec` steps. - The first item in each tuple must be one of the three type indicator - listed below describing the remaining tuple elements: - - .. data:: POSIX_SPAWN_OPEN - - (``os.POSIX_SPAWN_OPEN``, *fd*, *path*, *flags*, *mode*) - - Performs ``os.dup2(os.open(path, flags, mode), fd)``. - - .. data:: POSIX_SPAWN_CLOSE - - (``os.POSIX_SPAWN_CLOSE``, *fd*) - - Performs ``os.close(fd)``. - - .. data:: POSIX_SPAWN_DUP2 - - (``os.POSIX_SPAWN_DUP2``, *fd*, *new_fd*) - - Performs ``os.dup2(fd, new_fd)``. - - These tuples correspond to the C library - :c:func:`!posix_spawn_file_actions_addopen`, - :c:func:`!posix_spawn_file_actions_addclose`, and - :c:func:`!posix_spawn_file_actions_adddup2` API calls used to prepare - for the :c:func:`!posix_spawn` call itself. - - The *setpgroup* argument will set the process group of the child to the value - specified. If the value specified is 0, the child's process group ID will be - made the same as its process ID. If the value of *setpgroup* is not set, the - child will inherit the parent's process group ID. This argument corresponds - to the C library :c:macro:`!POSIX_SPAWN_SETPGROUP` flag. - - If the *resetids* argument is ``True`` it will reset the effective UID and - GID of the child to the real UID and GID of the parent process. If the - argument is ``False``, then the child retains the effective UID and GID of - the parent. In either case, if the set-user-ID and set-group-ID permission - bits are enabled on the executable file, their effect will override the - setting of the effective UID and GID. This argument corresponds to the C - library :c:macro:`!POSIX_SPAWN_RESETIDS` flag. - - If the *setsid* argument is ``True``, it will create a new session ID - for ``posix_spawn``. *setsid* requires :c:macro:`!POSIX_SPAWN_SETSID` - or :c:macro:`!POSIX_SPAWN_SETSID_NP` flag. Otherwise, :exc:`NotImplementedError` - is raised. - - The *setsigmask* argument will set the signal mask to the signal set - specified. If the parameter is not used, then the child inherits the - parent's signal mask. This argument corresponds to the C library - :c:macro:`!POSIX_SPAWN_SETSIGMASK` flag. - - The *sigdef* argument will reset the disposition of all signals in the set - specified. This argument corresponds to the C library - :c:macro:`!POSIX_SPAWN_SETSIGDEF` flag. - - The *scheduler* argument must be a tuple containing the (optional) scheduler - policy and an instance of :class:`sched_param` with the scheduler parameters. - A value of ``None`` in the place of the scheduler policy indicates that is - not being provided. This argument is a combination of the C library - :c:macro:`!POSIX_SPAWN_SETSCHEDPARAM` and :c:macro:`!POSIX_SPAWN_SETSCHEDULER` - flags. - - .. audit-event:: os.posix_spawn path,argv,env os.posix_spawn - - .. versionadded:: 3.8 - - .. availability:: Unix, not Emscripten, not WASI. - -.. function:: posix_spawnp(path, argv, env, *, file_actions=None, \ - setpgroup=None, resetids=False, setsid=False, setsigmask=(), \ - setsigdef=(), scheduler=None) - - Wraps the :c:func:`!posix_spawnp` C library API for use from Python. - - Similar to :func:`posix_spawn` except that the system searches - for the *executable* file in the list of directories specified by the - :envvar:`PATH` environment variable (in the same way as for ``execvp(3)``). - - .. audit-event:: os.posix_spawn path,argv,env os.posix_spawnp - - .. versionadded:: 3.8 - - .. availability:: POSIX, not Emscripten, not WASI. - - See :func:`posix_spawn` documentation. - - -.. function:: register_at_fork(*, before=None, after_in_parent=None, \ - after_in_child=None) - - Register callables to be executed when a new child process is forked - using :func:`os.fork` or similar process cloning APIs. - The parameters are optional and keyword-only. - Each specifies a different call point. - - * *before* is a function called before forking a child process. - * *after_in_parent* is a function called from the parent process - after forking a child process. - * *after_in_child* is a function called from the child process. - - These calls are only made if control is expected to return to the - Python interpreter. A typical :mod:`subprocess` launch will not - trigger them as the child is not going to re-enter the interpreter. - - Functions registered for execution before forking are called in - reverse registration order. Functions registered for execution - after forking (either in the parent or in the child) are called - in registration order. - - Note that :c:func:`fork` calls made by third-party C code may not - call those functions, unless it explicitly calls :c:func:`PyOS_BeforeFork`, - :c:func:`PyOS_AfterFork_Parent` and :c:func:`PyOS_AfterFork_Child`. - - There is no way to unregister a function. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.7 - - -.. function:: spawnl(mode, path, ...) - spawnle(mode, path, ..., env) - spawnlp(mode, file, ...) - spawnlpe(mode, file, ..., env) - spawnv(mode, path, args) - spawnve(mode, path, args, env) - spawnvp(mode, file, args) - spawnvpe(mode, file, args, env) - - Execute the program *path* in a new process. - - (Note that the :mod:`subprocess` module provides more powerful facilities for - spawning new processes and retrieving their results; using that module is - preferable to using these functions. Check especially the - :ref:`subprocess-replacements` section.) - - If *mode* is :const:`P_NOWAIT`, this function returns the process id of the new - process; if *mode* is :const:`P_WAIT`, returns the process's exit code if it - exits normally, or ``-signal``, where *signal* is the signal that killed the - process. On Windows, the process id will actually be the process handle, so can - be used with the :func:`waitpid` function. - - Note on VxWorks, this function doesn't return ``-signal`` when the new process is - killed. Instead it raises OSError exception. - - The "l" and "v" variants of the :func:`spawn\* ` functions differ in how - command-line arguments are passed. The "l" variants are perhaps the easiest - to work with if the number of parameters is fixed when the code is written; the - individual parameters simply become additional parameters to the - :func:`spawnl\*` functions. The "v" variants are good when the number of - parameters is variable, with the arguments being passed in a list or tuple as - the *args* parameter. In either case, the arguments to the child process must - start with the name of the command being run. - - The variants which include a second "p" near the end (:func:`spawnlp`, - :func:`spawnlpe`, :func:`spawnvp`, and :func:`spawnvpe`) will use the - :envvar:`PATH` environment variable to locate the program *file*. When the - environment is being replaced (using one of the :func:`spawn\*e ` variants, - discussed in the next paragraph), the new environment is used as the source of - the :envvar:`PATH` variable. The other variants, :func:`spawnl`, - :func:`spawnle`, :func:`spawnv`, and :func:`spawnve`, will not use the - :envvar:`PATH` variable to locate the executable; *path* must contain an - appropriate absolute or relative path. - - For :func:`spawnle`, :func:`spawnlpe`, :func:`spawnve`, and :func:`spawnvpe` - (note that these all end in "e"), the *env* parameter must be a mapping - which is used to define the environment variables for the new process (they are - used instead of the current process' environment); the functions - :func:`spawnl`, :func:`spawnlp`, :func:`spawnv`, and :func:`spawnvp` all cause - the new process to inherit the environment of the current process. Note that - keys and values in the *env* dictionary must be strings; invalid keys or - values will cause the function to fail, with a return value of ``127``. - - As an example, the following calls to :func:`spawnlp` and :func:`spawnvpe` are - equivalent:: - - import os - os.spawnlp(os.P_WAIT, 'cp', 'cp', 'index.html', '/dev/null') - - L = ['cp', 'index.html', '/dev/null'] - os.spawnvpe(os.P_WAIT, 'cp', L, os.environ) - - .. audit-event:: os.spawn mode,path,args,env os.spawnl - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - :func:`spawnlp`, :func:`spawnlpe`, :func:`spawnvp` - and :func:`spawnvpe` are not available on Windows. :func:`spawnle` and - :func:`spawnve` are not thread-safe on Windows; we advise you to use the - :mod:`subprocess` module instead. - - .. versionchanged:: 3.6 - Accepts a :term:`path-like object`. - - -.. data:: P_NOWAIT - P_NOWAITO - - Possible values for the *mode* parameter to the :func:`spawn\* ` family of - functions. If either of these values is given, the :func:`spawn\*` functions - will return as soon as the new process has been created, with the process id as - the return value. - - .. availability:: Unix, Windows. - - -.. data:: P_WAIT - - Possible value for the *mode* parameter to the :func:`spawn\* ` family of - functions. If this is given as *mode*, the :func:`spawn\*` functions will not - return until the new process has run to completion and will return the exit code - of the process the run is successful, or ``-signal`` if a signal kills the - process. - - .. availability:: Unix, Windows. - - -.. data:: P_DETACH - P_OVERLAY - - Possible values for the *mode* parameter to the :func:`spawn\* ` family of - functions. These are less portable than those listed above. :const:`P_DETACH` - is similar to :const:`P_NOWAIT`, but the new process is detached from the - console of the calling process. If :const:`P_OVERLAY` is used, the current - process will be replaced; the :func:`spawn\* ` function will not return. - - .. availability:: Windows. - - -.. function:: startfile(path, [operation], [arguments], [cwd], [show_cmd]) - - Start a file with its associated application. - - When *operation* is not specified or ``'open'``, this acts like double-clicking - the file in Windows Explorer, or giving the file name as an argument to the - :program:`start` command from the interactive command shell: the file is opened - with whatever application (if any) its extension is associated. - - When another *operation* is given, it must be a "command verb" that specifies - what should be done with the file. Common verbs documented by Microsoft are - ``'print'`` and ``'edit'`` (to be used on files) as well as ``'explore'`` and - ``'find'`` (to be used on directories). - - When launching an application, specify *arguments* to be passed as a single - string. This argument may have no effect when using this function to launch a - document. - - The default working directory is inherited, but may be overridden by the *cwd* - argument. This should be an absolute path. A relative *path* will be resolved - against this argument. - - Use *show_cmd* to override the default window style. Whether this has any - effect will depend on the application being launched. Values are integers as - supported by the Win32 :c:func:`!ShellExecute` function. - - :func:`startfile` returns as soon as the associated application is launched. - There is no option to wait for the application to close, and no way to retrieve - the application's exit status. The *path* parameter is relative to the current - directory or *cwd*. If you want to use an absolute path, make sure the first - character is not a slash (``'/'``) Use :mod:`pathlib` or the - :func:`os.path.normpath` function to ensure that paths are properly encoded for - Win32. - - To reduce interpreter startup overhead, the Win32 :c:func:`!ShellExecute` - function is not resolved until this function is first called. If the function - cannot be resolved, :exc:`NotImplementedError` will be raised. - - .. audit-event:: os.startfile path,operation os.startfile - - .. audit-event:: os.startfile/2 path,operation,arguments,cwd,show_cmd os.startfile - - .. availability:: Windows. - - .. versionchanged:: 3.10 - Added the *arguments*, *cwd* and *show_cmd* arguments, and the - ``os.startfile/2`` audit event. - - -.. function:: system(command) - - Execute the command (a string) in a subshell. This is implemented by calling - the Standard C function :c:func:`system`, and has the same limitations. - Changes to :data:`sys.stdin`, etc. are not reflected in the environment of - the executed command. If *command* generates any output, it will be sent to - the interpreter standard output stream. The C standard does not - specify the meaning of the return value of the C function, so the return - value of the Python function is system-dependent. - - On Unix, the return value is the exit status of the process encoded in the - format specified for :func:`wait`. - - On Windows, the return value is that returned by the system shell after - running *command*. The shell is given by the Windows environment variable - :envvar:`COMSPEC`: it is usually :program:`cmd.exe`, which returns the exit - status of the command run; on systems using a non-native shell, consult your - shell documentation. - - The :mod:`subprocess` module provides more powerful facilities for spawning - new processes and retrieving their results; using that module is preferable - to using this function. See the :ref:`subprocess-replacements` section in - the :mod:`subprocess` documentation for some helpful recipes. - - On Unix, :func:`waitstatus_to_exitcode` can be used to convert the result - (exit status) into an exit code. On Windows, the result is directly the exit - code. - - .. audit-event:: os.system command os.system - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - -.. function:: times() - - Returns the current global process times. - The return value is an object with five attributes: - - * :attr:`!user` - user time - * :attr:`!system` - system time - * :attr:`!children_user` - user time of all child processes - * :attr:`!children_system` - system time of all child processes - * :attr:`!elapsed` - elapsed real time since a fixed point in the past - - For backwards compatibility, this object also behaves like a five-tuple - containing :attr:`!user`, :attr:`!system`, :attr:`!children_user`, - :attr:`!children_system`, and :attr:`!elapsed` in that order. - - See the Unix manual page - :manpage:`times(2)` and `times(3) `_ manual page on Unix or `the GetProcessTimes MSDN - `_ - on Windows. On Windows, only :attr:`!user` and :attr:`!system` are known; the other attributes are zero. - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.3 - Return type changed from a tuple to a tuple-like object - with named attributes. - - -.. function:: wait() - - Wait for completion of a child process, and return a tuple containing its pid - and exit status indication: a 16-bit number, whose low byte is the signal number - that killed the process, and whose high byte is the exit status (if the signal - number is zero); the high bit of the low byte is set if a core file was - produced. - - If there are no children that could be waited for, :exc:`ChildProcessError` - is raised. - - :func:`waitstatus_to_exitcode` can be used to convert the exit status into an - exit code. - - .. availability:: Unix, not Emscripten, not WASI. - - .. seealso:: - - The other :func:`!wait*` functions documented below can be used to wait for the - completion of a specific child process and have more options. - :func:`waitpid` is the only one also available on Windows. - - -.. function:: waitid(idtype, id, options, /) - - Wait for the completion of a child process. - - *idtype* can be :data:`P_PID`, :data:`P_PGID`, :data:`P_ALL`, or (on Linux) :data:`P_PIDFD`. - The interpretation of *id* depends on it; see their individual descriptions. - - *options* is an OR combination of flags. At least one of :data:`WEXITED`, - :data:`WSTOPPED` or :data:`WCONTINUED` is required; - :data:`WNOHANG` and :data:`WNOWAIT` are additional optional flags. - - The return value is an object representing the data contained in the - :c:type:`siginfo_t` structure with the following attributes: - - * :attr:`!si_pid` (process ID) - * :attr:`!si_uid` (real user ID of the child) - * :attr:`!si_signo` (always :const:`~signal.SIGCHLD`) - * :attr:`!si_status` (the exit status or signal number, depending on :attr:`!si_code`) - * :attr:`!si_code` (see :data:`CLD_EXITED` for possible values) - - If :data:`WNOHANG` is specified and there are no matching children in the - requested state, ``None`` is returned. - Otherwise, if there are no matching children - that could be waited for, :exc:`ChildProcessError` is raised. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - -.. function:: waitpid(pid, options, /) - - The details of this function differ on Unix and Windows. - - On Unix: Wait for completion of a child process given by process id *pid*, and - return a tuple containing its process id and exit status indication (encoded as - for :func:`wait`). The semantics of the call are affected by the value of the - integer *options*, which should be ``0`` for normal operation. - - If *pid* is greater than ``0``, :func:`waitpid` requests status information for - that specific process. If *pid* is ``0``, the request is for the status of any - child in the process group of the current process. If *pid* is ``-1``, the - request pertains to any child of the current process. If *pid* is less than - ``-1``, status is requested for any process in the process group ``-pid`` (the - absolute value of *pid*). - - *options* is an OR combination of flags. If it contains :data:`WNOHANG` and - there are no matching children in the requested state, ``(0, 0)`` is - returned. Otherwise, if there are no matching children that could be waited - for, :exc:`ChildProcessError` is raised. Other options that can be used are - :data:`WUNTRACED` and :data:`WCONTINUED`. - - On Windows: Wait for completion of a process given by process handle *pid*, and - return a tuple containing *pid*, and its exit status shifted left by 8 bits - (shifting makes cross-platform use of the function easier). A *pid* less than or - equal to ``0`` has no special meaning on Windows, and raises an exception. The - value of integer *options* has no effect. *pid* can refer to any process whose - id is known, not necessarily a child process. The :func:`spawn\* ` - functions called with :const:`P_NOWAIT` return suitable process handles. - - :func:`waitstatus_to_exitcode` can be used to convert the exit status into an - exit code. - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise an - exception, the function now retries the system call instead of raising an - :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. function:: wait3(options) - - Similar to :func:`waitpid`, except no process id argument is given and a - 3-element tuple containing the child's process id, exit status indication, - and resource usage information is returned. Refer to - :func:`resource.getrusage` for details on resource usage information. The - *options* argument is the same as that provided to :func:`waitpid` and - :func:`wait4`. - - :func:`waitstatus_to_exitcode` can be used to convert the exit status into an - exitcode. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: wait4(pid, options) - - Similar to :func:`waitpid`, except a 3-element tuple, containing the child's - process id, exit status indication, and resource usage information is - returned. Refer to :func:`resource.getrusage` for details on resource usage - information. The arguments to :func:`wait4` are the same as those provided - to :func:`waitpid`. - - :func:`waitstatus_to_exitcode` can be used to convert the exit status into an - exitcode. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: P_PID - P_PGID - P_ALL - P_PIDFD - - These are the possible values for *idtype* in :func:`waitid`. They affect - how *id* is interpreted: - - * :data:`!P_PID` - wait for the child whose PID is *id*. - * :data:`!P_PGID` - wait for any child whose progress group ID is *id*. - * :data:`!P_ALL` - wait for any child; *id* is ignored. - * :data:`!P_PIDFD` - wait for the child identified by the file descriptor - *id* (a process file descriptor created with :func:`pidfd_open`). - - .. availability:: Unix, not Emscripten, not WASI. - - .. note:: :data:`!P_PIDFD` is only available on Linux >= 5.4. - - .. versionadded:: 3.3 - .. versionadded:: 3.9 - The :data:`!P_PIDFD` constant. - - -.. data:: WCONTINUED - - This *options* flag for :func:`waitpid`, :func:`wait3`, :func:`wait4`, and - :func:`waitid` causes child processes to be reported if they have been - continued from a job control stop since they were last reported. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: WEXITED - - This *options* flag for :func:`waitid` causes child processes that have terminated to - be reported. - - The other ``wait*`` functions always report children that have terminated, - so this option is not available for them. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - -.. data:: WSTOPPED - - This *options* flag for :func:`waitid` causes child processes that have been stopped - by the delivery of a signal to be reported. - - This option is not available for the other ``wait*`` functions. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - -.. data:: WUNTRACED - - This *options* flag for :func:`waitpid`, :func:`wait3`, and :func:`wait4` causes - child processes to also be reported if they have been stopped but their - current state has not been reported since they were stopped. - - This option is not available for :func:`waitid`. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: WNOHANG - - This *options* flag causes :func:`waitpid`, :func:`wait3`, :func:`wait4`, and - :func:`waitid` to return right away if no child process status is available - immediately. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: WNOWAIT - - This *options* flag causes :func:`waitid` to leave the child in a waitable state, so that - a later :func:`!wait*` call can be used to retrieve the child status information again. - - This option is not available for the other ``wait*`` functions. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. data:: CLD_EXITED - CLD_KILLED - CLD_DUMPED - CLD_TRAPPED - CLD_STOPPED - CLD_CONTINUED - - These are the possible values for :attr:`!si_code` in the result returned by - :func:`waitid`. - - .. availability:: Unix, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.9 - Added :data:`CLD_KILLED` and :data:`CLD_STOPPED` values. - - -.. function:: waitstatus_to_exitcode(status) - - Convert a wait status to an exit code. - - On Unix: - - * If the process exited normally (if ``WIFEXITED(status)`` is true), - return the process exit status (return ``WEXITSTATUS(status)``): - result greater than or equal to 0. - * If the process was terminated by a signal (if ``WIFSIGNALED(status)`` is - true), return ``-signum`` where *signum* is the number of the signal that - caused the process to terminate (return ``-WTERMSIG(status)``): - result less than 0. - * Otherwise, raise a :exc:`ValueError`. - - On Windows, return *status* shifted right by 8 bits. - - On Unix, if the process is being traced or if :func:`waitpid` was called - with :data:`WUNTRACED` option, the caller must first check if - ``WIFSTOPPED(status)`` is true. This function must not be called if - ``WIFSTOPPED(status)`` is true. - - .. seealso:: - - :func:`WIFEXITED`, :func:`WEXITSTATUS`, :func:`WIFSIGNALED`, - :func:`WTERMSIG`, :func:`WIFSTOPPED`, :func:`WSTOPSIG` functions. - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - .. versionadded:: 3.9 - - -The following functions take a process status code as returned by -:func:`system`, :func:`wait`, or :func:`waitpid` as a parameter. They may be -used to determine the disposition of a process. - -.. function:: WCOREDUMP(status, /) - - Return ``True`` if a core dump was generated for the process, otherwise - return ``False``. - - This function should be employed only if :func:`WIFSIGNALED` is true. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: WIFCONTINUED(status) - - Return ``True`` if a stopped child has been resumed by delivery of - :const:`~signal.SIGCONT` (if the process has been continued from a job - control stop), otherwise return ``False``. - - See :data:`WCONTINUED` option. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: WIFSTOPPED(status) - - Return ``True`` if the process was stopped by delivery of a signal, - otherwise return ``False``. - - :func:`WIFSTOPPED` only returns ``True`` if the :func:`waitpid` call was - done using :data:`WUNTRACED` option or when the process is being traced (see - :manpage:`ptrace(2)`). - - .. availability:: Unix, not Emscripten, not WASI. - -.. function:: WIFSIGNALED(status) - - Return ``True`` if the process was terminated by a signal, otherwise return - ``False``. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: WIFEXITED(status) - - Return ``True`` if the process exited terminated normally, that is, - by calling ``exit()`` or ``_exit()``, or by returning from ``main()``; - otherwise return ``False``. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: WEXITSTATUS(status) - - Return the process exit status. - - This function should be employed only if :func:`WIFEXITED` is true. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: WSTOPSIG(status) - - Return the signal which caused the process to stop. - - This function should be employed only if :func:`WIFSTOPPED` is true. - - .. availability:: Unix, not Emscripten, not WASI. - - -.. function:: WTERMSIG(status) - - Return the number of the signal that caused the process to terminate. - - This function should be employed only if :func:`WIFSIGNALED` is true. - - .. availability:: Unix, not Emscripten, not WASI. - - -Interface to the scheduler --------------------------- - -These functions control how a process is allocated CPU time by the operating -system. They are only available on some Unix platforms. For more detailed -information, consult your Unix manpages. - -.. versionadded:: 3.3 - -The following scheduling policies are exposed if they are supported by the -operating system. - -.. data:: SCHED_OTHER - - The default scheduling policy. - -.. data:: SCHED_BATCH - - Scheduling policy for CPU-intensive processes that tries to preserve - interactivity on the rest of the computer. - -.. data:: SCHED_IDLE - - Scheduling policy for extremely low priority background tasks. - -.. data:: SCHED_SPORADIC - - Scheduling policy for sporadic server programs. - -.. data:: SCHED_FIFO - - A First In First Out scheduling policy. - -.. data:: SCHED_RR - - A round-robin scheduling policy. - -.. data:: SCHED_RESET_ON_FORK - - This flag can be OR'ed with any other scheduling policy. When a process with - this flag set forks, its child's scheduling policy and priority are reset to - the default. - - -.. class:: sched_param(sched_priority) - - This class represents tunable scheduling parameters used in - :func:`sched_setparam`, :func:`sched_setscheduler`, and - :func:`sched_getparam`. It is immutable. - - At the moment, there is only one possible parameter: - - .. attribute:: sched_priority - - The scheduling priority for a scheduling policy. - - -.. function:: sched_get_priority_min(policy) - - Get the minimum priority value for *policy*. *policy* is one of the - scheduling policy constants above. - - -.. function:: sched_get_priority_max(policy) - - Get the maximum priority value for *policy*. *policy* is one of the - scheduling policy constants above. - - -.. function:: sched_setscheduler(pid, policy, param, /) - - Set the scheduling policy for the process with PID *pid*. A *pid* of 0 means - the calling process. *policy* is one of the scheduling policy constants - above. *param* is a :class:`sched_param` instance. - - -.. function:: sched_getscheduler(pid, /) - - Return the scheduling policy for the process with PID *pid*. A *pid* of 0 - means the calling process. The result is one of the scheduling policy - constants above. - - -.. function:: sched_setparam(pid, param, /) - - Set the scheduling parameters for the process with PID *pid*. A *pid* of 0 means - the calling process. *param* is a :class:`sched_param` instance. - - -.. function:: sched_getparam(pid, /) - - Return the scheduling parameters as a :class:`sched_param` instance for the - process with PID *pid*. A *pid* of 0 means the calling process. - - -.. function:: sched_rr_get_interval(pid, /) - - Return the round-robin quantum in seconds for the process with PID *pid*. A - *pid* of 0 means the calling process. - - -.. function:: sched_yield() - - Voluntarily relinquish the CPU. - - -.. function:: sched_setaffinity(pid, mask, /) - - Restrict the process with PID *pid* (or the current process if zero) to a - set of CPUs. *mask* is an iterable of integers representing the set of - CPUs to which the process should be restricted. - - -.. function:: sched_getaffinity(pid, /) - - Return the set of CPUs the process with PID *pid* is restricted to. - - If *pid* is zero, return the set of CPUs the calling thread of the current - process is restricted to. - - -.. _os-path: - -Miscellaneous System Information --------------------------------- - - -.. function:: confstr(name, /) - - Return string-valued system configuration values. *name* specifies the - configuration value to retrieve; it may be a string which is the name of a - defined system value; these names are specified in a number of standards (POSIX, - Unix 95, Unix 98, and others). Some platforms define additional names as well. - The names known to the host operating system are given as the keys of the - ``confstr_names`` dictionary. For configuration variables not included in that - mapping, passing an integer for *name* is also accepted. - - If the configuration value specified by *name* isn't defined, ``None`` is - returned. - - If *name* is a string and is not known, :exc:`ValueError` is raised. If a - specific value for *name* is not supported by the host system, even if it is - included in ``confstr_names``, an :exc:`OSError` is raised with - :const:`errno.EINVAL` for the error number. - - .. availability:: Unix. - - -.. data:: confstr_names - - Dictionary mapping names accepted by :func:`confstr` to the integer values - defined for those names by the host operating system. This can be used to - determine the set of names known to the system. - - .. availability:: Unix. - - -.. function:: cpu_count() - - Return the number of logical CPUs in the system. Returns ``None`` if - undetermined. - - This number is not equivalent to the number of logical CPUs the current - process can use. ``len(os.sched_getaffinity(0))`` gets the number of logical - CPUs the calling thread of the current process is restricted to - - .. versionadded:: 3.4 - - -.. function:: getloadavg() - - Return the number of processes in the system run queue averaged over the last - 1, 5, and 15 minutes or raises :exc:`OSError` if the load average was - unobtainable. - - .. availability:: Unix. - - -.. function:: sysconf(name, /) - - Return integer-valued system configuration values. If the configuration value - specified by *name* isn't defined, ``-1`` is returned. The comments regarding - the *name* parameter for :func:`confstr` apply here as well; the dictionary that - provides information on the known names is given by ``sysconf_names``. - - .. availability:: Unix. - - -.. data:: sysconf_names - - Dictionary mapping names accepted by :func:`sysconf` to the integer values - defined for those names by the host operating system. This can be used to - determine the set of names known to the system. - - .. availability:: Unix. - - .. versionchanged:: 3.11 - Add ``'SC_MINSIGSTKSZ'`` name. - -The following data values are used to support path manipulation operations. These -are defined for all platforms. - -Higher-level operations on pathnames are defined in the :mod:`os.path` module. - - -.. index:: single: . (dot); in pathnames -.. data:: curdir - - The constant string used by the operating system to refer to the current - directory. This is ``'.'`` for Windows and POSIX. Also available via - :mod:`os.path`. - - -.. index:: single: ..; in pathnames -.. data:: pardir - - The constant string used by the operating system to refer to the parent - directory. This is ``'..'`` for Windows and POSIX. Also available via - :mod:`os.path`. - - -.. index:: single: / (slash); in pathnames -.. index:: single: \ (backslash); in pathnames (Windows) -.. data:: sep - - The character used by the operating system to separate pathname components. - This is ``'/'`` for POSIX and ``'\\'`` for Windows. Note that knowing this - is not sufficient to be able to parse or concatenate pathnames --- use - :func:`os.path.split` and :func:`os.path.join` --- but it is occasionally - useful. Also available via :mod:`os.path`. - - -.. index:: single: / (slash); in pathnames -.. data:: altsep - - An alternative character used by the operating system to separate pathname - components, or ``None`` if only one separator character exists. This is set to - ``'/'`` on Windows systems where ``sep`` is a backslash. Also available via - :mod:`os.path`. - - -.. index:: single: . (dot); in pathnames -.. data:: extsep - - The character which separates the base filename from the extension; for example, - the ``'.'`` in :file:`os.py`. Also available via :mod:`os.path`. - - -.. index:: single: : (colon); path separator (POSIX) - single: ; (semicolon) -.. data:: pathsep - - The character conventionally used by the operating system to separate search - path components (as in :envvar:`PATH`), such as ``':'`` for POSIX or ``';'`` for - Windows. Also available via :mod:`os.path`. - - -.. data:: defpath - - The default search path used by :func:`exec\*p\* ` and - :func:`spawn\*p\* ` if the environment doesn't have a ``'PATH'`` - key. Also available via :mod:`os.path`. - - -.. data:: linesep - - The string used to separate (or, rather, terminate) lines on the current - platform. This may be a single character, such as ``'\n'`` for POSIX, or - multiple characters, for example, ``'\r\n'`` for Windows. Do not use - *os.linesep* as a line terminator when writing files opened in text mode (the - default); use a single ``'\n'`` instead, on all platforms. - - -.. data:: devnull - - The file path of the null device. For example: ``'/dev/null'`` for - POSIX, ``'nul'`` for Windows. Also available via :mod:`os.path`. - -.. data:: RTLD_LAZY - RTLD_NOW - RTLD_GLOBAL - RTLD_LOCAL - RTLD_NODELETE - RTLD_NOLOAD - RTLD_DEEPBIND - - Flags for use with the :func:`~sys.setdlopenflags` and - :func:`~sys.getdlopenflags` functions. See the Unix manual page - :manpage:`dlopen(3)` for what the different flags mean. - - .. versionadded:: 3.3 - - -Random numbers --------------- - - -.. function:: getrandom(size, flags=0) - - Get up to *size* random bytes. The function can return less bytes than - requested. - - These bytes can be used to seed user-space random number generators or for - cryptographic purposes. - - ``getrandom()`` relies on entropy gathered from device drivers and other - sources of environmental noise. Unnecessarily reading large quantities of - data will have a negative impact on other users of the ``/dev/random`` and - ``/dev/urandom`` devices. - - The flags argument is a bit mask that can contain zero or more of the - following values ORed together: :py:const:`os.GRND_RANDOM` and - :py:data:`GRND_NONBLOCK`. - - See also the `Linux getrandom() manual page - `_. - - .. availability:: Linux >= 3.17. - - .. versionadded:: 3.6 - -.. function:: urandom(size, /) - - Return a bytestring of *size* random bytes suitable for cryptographic use. - - This function returns random bytes from an OS-specific randomness source. The - returned data should be unpredictable enough for cryptographic applications, - though its exact quality depends on the OS implementation. - - On Linux, if the ``getrandom()`` syscall is available, it is used in - blocking mode: block until the system urandom entropy pool is initialized - (128 bits of entropy are collected by the kernel). See the :pep:`524` for - the rationale. On Linux, the :func:`getrandom` function can be used to get - random bytes in non-blocking mode (using the :data:`GRND_NONBLOCK` flag) or - to poll until the system urandom entropy pool is initialized. - - On a Unix-like system, random bytes are read from the ``/dev/urandom`` - device. If the ``/dev/urandom`` device is not available or not readable, the - :exc:`NotImplementedError` exception is raised. - - On Windows, it will use ``BCryptGenRandom()``. - - .. seealso:: - The :mod:`secrets` module provides higher level functions. For an - easy-to-use interface to the random number generator provided by your - platform, please see :class:`random.SystemRandom`. - - .. versionchanged:: 3.6.0 - On Linux, ``getrandom()`` is now used in blocking mode to increase the - security. - - .. versionchanged:: 3.5.2 - On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool - is not initialized yet), fall back on reading ``/dev/urandom``. - - .. versionchanged:: 3.5 - On Linux 3.17 and newer, the ``getrandom()`` syscall is now used - when available. On OpenBSD 5.6 and newer, the C ``getentropy()`` - function is now used. These functions avoid the usage of an internal file - descriptor. - - .. versionchanged:: 3.11 - On Windows, ``BCryptGenRandom()`` is used instead of ``CryptGenRandom()`` - which is deprecated. - -.. data:: GRND_NONBLOCK - - By default, when reading from ``/dev/random``, :func:`getrandom` blocks if - no random bytes are available, and when reading from ``/dev/urandom``, it blocks - if the entropy pool has not yet been initialized. - - If the :py:data:`GRND_NONBLOCK` flag is set, then :func:`getrandom` does not - block in these cases, but instead immediately raises :exc:`BlockingIOError`. - - .. versionadded:: 3.6 - -.. data:: GRND_RANDOM - - If this bit is set, then random bytes are drawn from the - ``/dev/random`` pool instead of the ``/dev/urandom`` pool. - - .. versionadded:: 3.6 diff --git a/Doc/library/ossaudiodev.rst b/Doc/library/ossaudiodev.rst deleted file mode 100644 index e14c1bf8d5367e..00000000000000 --- a/Doc/library/ossaudiodev.rst +++ /dev/null @@ -1,453 +0,0 @@ -:mod:`ossaudiodev` --- Access to OSS-compatible audio devices -============================================================= - -.. module:: ossaudiodev - :platform: Linux, FreeBSD - :synopsis: Access to OSS-compatible audio devices. - :deprecated: - -.. deprecated-removed:: 3.11 3.13 - The :mod:`ossaudiodev` module is deprecated - (see :pep:`PEP 594 <594#ossaudiodev>` for details). - --------------- - -This module allows you to access the OSS (Open Sound System) audio interface. -OSS is available for a wide range of open-source and commercial Unices, and is -the standard audio interface for Linux and recent versions of FreeBSD. - -.. Things will get more complicated for future Linux versions, since - ALSA is in the standard kernel as of 2.5.x. Presumably if you - use ALSA, you'll have to make sure its OSS compatibility layer - is active to use ossaudiodev, but you're going to need it for the vast - majority of Linux audio apps anyway. - - Sounds like things are also complicated for other BSDs. In response - to my python-dev query, Thomas Wouters said: - - > Likewise, googling shows OpenBSD also uses OSS/Free -- the commercial - > OSS installation manual tells you to remove references to OSS/Free from the - > kernel :) - - but Aleksander Piotrowsk actually has an OpenBSD box, and he quotes - from its : - > * WARNING! WARNING! - > * This is an OSS (Linux) audio emulator. - > * Use the Native NetBSD API for developing new code, and this - > * only for compiling Linux programs. - - There's also an ossaudio manpage on OpenBSD that explains things - further. Presumably NetBSD and OpenBSD have a different standard - audio interface. That's the great thing about standards, there are so - many to choose from ... ;-) - - This probably all warrants a footnote or two, but I don't understand - things well enough right now to write it! --GPW - -.. versionchanged:: 3.3 - Operations in this module now raise :exc:`OSError` where :exc:`IOError` - was raised. - - -.. seealso:: - - `Open Sound System Programmer's Guide `_ - the official documentation for the OSS C API - - The module defines a large number of constants supplied by the OSS device - driver; see ```` on either Linux or FreeBSD for a listing. - -:mod:`ossaudiodev` defines the following variables and functions: - - -.. exception:: OSSAudioError - - This exception is raised on certain errors. The argument is a string describing - what went wrong. - - (If :mod:`ossaudiodev` receives an error from a system call such as - :c:func:`open`, :c:func:`write`, or :c:func:`ioctl`, it raises :exc:`OSError`. - Errors detected directly by :mod:`ossaudiodev` result in :exc:`OSSAudioError`.) - - (For backwards compatibility, the exception class is also available as - ``ossaudiodev.error``.) - - -.. function:: open(mode) - open(device, mode) - - Open an audio device and return an OSS audio device object. This object - supports many file-like methods, such as :meth:`read`, :meth:`write`, and - :meth:`fileno` (although there are subtle differences between conventional Unix - read/write semantics and those of OSS audio devices). It also supports a number - of audio-specific methods; see below for the complete list of methods. - - *device* is the audio device filename to use. If it is not specified, this - module first looks in the environment variable :envvar:`AUDIODEV` for a device - to use. If not found, it falls back to :file:`/dev/dsp`. - - *mode* is one of ``'r'`` for read-only (record) access, ``'w'`` for - write-only (playback) access and ``'rw'`` for both. Since many sound cards - only allow one process to have the recorder or player open at a time, it is a - good idea to open the device only for the activity needed. Further, some - sound cards are half-duplex: they can be opened for reading or writing, but - not both at once. - - Note the unusual calling syntax: the *first* argument is optional, and the - second is required. This is a historical artifact for compatibility with the - older :mod:`linuxaudiodev` module which :mod:`ossaudiodev` supersedes. - - .. XXX it might also be motivated - by my unfounded-but-still-possibly-true belief that the default - audio device varies unpredictably across operating systems. -GW - - -.. function:: openmixer([device]) - - Open a mixer device and return an OSS mixer device object. *device* is the - mixer device filename to use. If it is not specified, this module first looks - in the environment variable :envvar:`MIXERDEV` for a device to use. If not - found, it falls back to :file:`/dev/mixer`. - - -.. _ossaudio-device-objects: - -Audio Device Objects --------------------- - -Before you can write to or read from an audio device, you must call three -methods in the correct order: - -#. :meth:`setfmt` to set the output format - -#. :meth:`channels` to set the number of channels - -#. :meth:`speed` to set the sample rate - -Alternately, you can use the :meth:`setparameters` method to set all three audio -parameters at once. This is more convenient, but may not be as flexible in all -cases. - -The audio device objects returned by :func:`.open` define the following methods -and (read-only) attributes: - - -.. method:: oss_audio_device.close() - - Explicitly close the audio device. When you are done writing to or reading from - an audio device, you should explicitly close it. A closed device cannot be used - again. - - -.. method:: oss_audio_device.fileno() - - Return the file descriptor associated with the device. - - -.. method:: oss_audio_device.read(size) - - Read *size* bytes from the audio input and return them as a Python string. - Unlike most Unix device drivers, OSS audio devices in blocking mode (the - default) will block :func:`read` until the entire requested amount of data is - available. - - -.. method:: oss_audio_device.write(data) - - Write a :term:`bytes-like object` *data* to the audio device and return the - number of bytes written. If the audio device is in blocking mode (the - default), the entire data is always written (again, this is different from - usual Unix device semantics). If the device is in non-blocking mode, some - data may not be written---see :meth:`writeall`. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. method:: oss_audio_device.writeall(data) - - Write a :term:`bytes-like object` *data* to the audio device: waits until - the audio device is able to accept data, writes as much data as it will - accept, and repeats until *data* has been completely written. If the device - is in blocking mode (the default), this has the same effect as - :meth:`write`; :meth:`writeall` is only useful in non-blocking mode. Has - no return value, since the amount of data written is always equal to the - amount of data supplied. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. versionchanged:: 3.2 - Audio device objects also support the context management protocol, i.e. they can - be used in a :keyword:`with` statement. - - -The following methods each map to exactly one :c:func:`ioctl` system call. The -correspondence is obvious: for example, :meth:`setfmt` corresponds to the -``SNDCTL_DSP_SETFMT`` ioctl, and :meth:`sync` to ``SNDCTL_DSP_SYNC`` (this can -be useful when consulting the OSS documentation). If the underlying -:c:func:`ioctl` fails, they all raise :exc:`OSError`. - - -.. method:: oss_audio_device.nonblock() - - Put the device into non-blocking mode. Once in non-blocking mode, there is no - way to return it to blocking mode. - - -.. method:: oss_audio_device.getfmts() - - Return a bitmask of the audio output formats supported by the soundcard. Some - of the formats supported by OSS are: - - +-------------------------+---------------------------------------------+ - | Format | Description | - +=========================+=============================================+ - | :const:`AFMT_MU_LAW` | a logarithmic encoding (used by Sun ``.au`` | - | | files and :file:`/dev/audio`) | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_A_LAW` | a logarithmic encoding | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_IMA_ADPCM` | a 4:1 compressed format defined by the | - | | Interactive Multimedia Association | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_U8` | Unsigned, 8-bit audio | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_S16_LE` | Signed, 16-bit audio, little-endian byte | - | | order (as used by Intel processors) | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_S16_BE` | Signed, 16-bit audio, big-endian byte order | - | | (as used by 68k, PowerPC, Sparc) | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_S8` | Signed, 8 bit audio | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_U16_LE` | Unsigned, 16-bit little-endian audio | - +-------------------------+---------------------------------------------+ - | :const:`AFMT_U16_BE` | Unsigned, 16-bit big-endian audio | - +-------------------------+---------------------------------------------+ - - Consult the OSS documentation for a full list of audio formats, and note that - most devices support only a subset of these formats. Some older devices only - support :const:`AFMT_U8`; the most common format used today is - :const:`AFMT_S16_LE`. - - -.. method:: oss_audio_device.setfmt(format) - - Try to set the current audio format to *format*---see :meth:`getfmts` for a - list. Returns the audio format that the device was set to, which may not be the - requested format. May also be used to return the current audio format---do this - by passing an "audio format" of :const:`AFMT_QUERY`. - - -.. method:: oss_audio_device.channels(nchannels) - - Set the number of output channels to *nchannels*. A value of 1 indicates - monophonic sound, 2 stereophonic. Some devices may have more than 2 channels, - and some high-end devices may not support mono. Returns the number of channels - the device was set to. - - -.. method:: oss_audio_device.speed(samplerate) - - Try to set the audio sampling rate to *samplerate* samples per second. Returns - the rate actually set. Most sound devices don't support arbitrary sampling - rates. Common rates are: - - +-------+-------------------------------------------+ - | Rate | Description | - +=======+===========================================+ - | 8000 | default rate for :file:`/dev/audio` | - +-------+-------------------------------------------+ - | 11025 | speech recording | - +-------+-------------------------------------------+ - | 22050 | | - +-------+-------------------------------------------+ - | 44100 | CD quality audio (at 16 bits/sample and 2 | - | | channels) | - +-------+-------------------------------------------+ - | 96000 | DVD quality audio (at 24 bits/sample) | - +-------+-------------------------------------------+ - - -.. method:: oss_audio_device.sync() - - Wait until the sound device has played every byte in its buffer. (This happens - implicitly when the device is closed.) The OSS documentation recommends closing - and re-opening the device rather than using :meth:`sync`. - - -.. method:: oss_audio_device.reset() - - Immediately stop playing or recording and return the device to a state where it - can accept commands. The OSS documentation recommends closing and re-opening - the device after calling :meth:`reset`. - - -.. method:: oss_audio_device.post() - - Tell the driver that there is likely to be a pause in the output, making it - possible for the device to handle the pause more intelligently. You might use - this after playing a spot sound effect, before waiting for user input, or before - doing disk I/O. - -The following convenience methods combine several ioctls, or one ioctl and some -simple calculations. - - -.. method:: oss_audio_device.setparameters(format, nchannels, samplerate[, strict=False]) - - Set the key audio sampling parameters---sample format, number of channels, and - sampling rate---in one method call. *format*, *nchannels*, and *samplerate* - should be as specified in the :meth:`setfmt`, :meth:`channels`, and - :meth:`speed` methods. If *strict* is true, :meth:`setparameters` checks to - see if each parameter was actually set to the requested value, and raises - :exc:`OSSAudioError` if not. Returns a tuple (*format*, *nchannels*, - *samplerate*) indicating the parameter values that were actually set by the - device driver (i.e., the same as the return values of :meth:`setfmt`, - :meth:`channels`, and :meth:`speed`). - - For example, :: - - (fmt, channels, rate) = dsp.setparameters(fmt, channels, rate) - - is equivalent to :: - - fmt = dsp.setfmt(fmt) - channels = dsp.channels(channels) - rate = dsp.rate(rate) - - -.. method:: oss_audio_device.bufsize() - - Returns the size of the hardware buffer, in samples. - - -.. method:: oss_audio_device.obufcount() - - Returns the number of samples that are in the hardware buffer yet to be played. - - -.. method:: oss_audio_device.obuffree() - - Returns the number of samples that could be queued into the hardware buffer to - be played without blocking. - -Audio device objects also support several read-only attributes: - - -.. attribute:: oss_audio_device.closed - - Boolean indicating whether the device has been closed. - - -.. attribute:: oss_audio_device.name - - String containing the name of the device file. - - -.. attribute:: oss_audio_device.mode - - The I/O mode for the file, either ``"r"``, ``"rw"``, or ``"w"``. - - -.. _mixer-device-objects: - -Mixer Device Objects --------------------- - -The mixer object provides two file-like methods: - - -.. method:: oss_mixer_device.close() - - This method closes the open mixer device file. Any further attempts to use the - mixer after this file is closed will raise an :exc:`OSError`. - - -.. method:: oss_mixer_device.fileno() - - Returns the file handle number of the open mixer device file. - -.. versionchanged:: 3.2 - Mixer objects also support the context management protocol. - - -The remaining methods are specific to audio mixing: - - -.. method:: oss_mixer_device.controls() - - This method returns a bitmask specifying the available mixer controls ("Control" - being a specific mixable "channel", such as :const:`SOUND_MIXER_PCM` or - :const:`SOUND_MIXER_SYNTH`). This bitmask indicates a subset of all available - mixer controls---the :const:`SOUND_MIXER_\*` constants defined at module level. - To determine if, for example, the current mixer object supports a PCM mixer, use - the following Python code:: - - mixer=ossaudiodev.openmixer() - if mixer.controls() & (1 << ossaudiodev.SOUND_MIXER_PCM): - # PCM is supported - ... code ... - - For most purposes, the :const:`SOUND_MIXER_VOLUME` (master volume) and - :const:`SOUND_MIXER_PCM` controls should suffice---but code that uses the mixer - should be flexible when it comes to choosing mixer controls. On the Gravis - Ultrasound, for example, :const:`SOUND_MIXER_VOLUME` does not exist. - - -.. method:: oss_mixer_device.stereocontrols() - - Returns a bitmask indicating stereo mixer controls. If a bit is set, the - corresponding control is stereo; if it is unset, the control is either - monophonic or not supported by the mixer (use in combination with - :meth:`controls` to determine which). - - See the code example for the :meth:`controls` function for an example of getting - data from a bitmask. - - -.. method:: oss_mixer_device.reccontrols() - - Returns a bitmask specifying the mixer controls that may be used to record. See - the code example for :meth:`controls` for an example of reading from a bitmask. - - -.. method:: oss_mixer_device.get(control) - - Returns the volume of a given mixer control. The returned volume is a 2-tuple - ``(left_volume,right_volume)``. Volumes are specified as numbers from 0 - (silent) to 100 (full volume). If the control is monophonic, a 2-tuple is still - returned, but both volumes are the same. - - Raises :exc:`OSSAudioError` if an invalid control is specified, or - :exc:`OSError` if an unsupported control is specified. - - -.. method:: oss_mixer_device.set(control, (left, right)) - - Sets the volume for a given mixer control to ``(left,right)``. ``left`` and - ``right`` must be ints and between 0 (silent) and 100 (full volume). On - success, the new volume is returned as a 2-tuple. Note that this may not be - exactly the same as the volume specified, because of the limited resolution of - some soundcard's mixers. - - Raises :exc:`OSSAudioError` if an invalid mixer control was specified, or if the - specified volumes were out-of-range. - - -.. method:: oss_mixer_device.get_recsrc() - - This method returns a bitmask indicating which control(s) are currently being - used as a recording source. - - -.. method:: oss_mixer_device.set_recsrc(bitmask) - - Call this function to specify a recording source. Returns a bitmask indicating - the new recording source (or sources) if successful; raises :exc:`OSError` if an - invalid source was specified. To set the current recording source to the - microphone input:: - - mixer.setrecsrc (1 << ossaudiodev.SOUND_MIXER_MIC) diff --git a/Doc/library/pathlib-inheritance.png b/Doc/library/pathlib-inheritance.png deleted file mode 100644 index 628ae6e9adb6fa..00000000000000 Binary files a/Doc/library/pathlib-inheritance.png and /dev/null differ diff --git a/Doc/library/pathlib-inheritance.svg b/Doc/library/pathlib-inheritance.svg deleted file mode 100644 index 01f8684a2b201c..00000000000000 --- a/Doc/library/pathlib-inheritance.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/Doc/library/pathlib.rst b/Doc/library/pathlib.rst deleted file mode 100644 index 974ab424a57f54..00000000000000 --- a/Doc/library/pathlib.rst +++ /dev/null @@ -1,1339 +0,0 @@ - -:mod:`pathlib` --- Object-oriented filesystem paths -=================================================== - -.. module:: pathlib - :synopsis: Object-oriented filesystem paths - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/pathlib.py` - -.. index:: single: path; operations - --------------- - -This module offers classes representing filesystem paths with semantics -appropriate for different operating systems. Path classes are divided -between :ref:`pure paths `, which provide purely computational -operations without I/O, and :ref:`concrete paths `, which -inherit from pure paths but also provide I/O operations. - -.. image:: pathlib-inheritance.png - :align: center - :class: invert-in-dark-mode - -If you've never used this module before or just aren't sure which class is -right for your task, :class:`Path` is most likely what you need. It instantiates -a :ref:`concrete path ` for the platform the code is running on. - -Pure paths are useful in some special cases; for example: - -#. If you want to manipulate Windows paths on a Unix machine (or vice versa). - You cannot instantiate a :class:`WindowsPath` when running on Unix, but you - can instantiate :class:`PureWindowsPath`. -#. You want to make sure that your code only manipulates paths without actually - accessing the OS. In this case, instantiating one of the pure classes may be - useful since those simply don't have any OS-accessing operations. - -.. seealso:: - :pep:`428`: The pathlib module -- object-oriented filesystem paths. - -.. seealso:: - For low-level path manipulation on strings, you can also use the - :mod:`os.path` module. - - -Basic use ---------- - -Importing the main class:: - - >>> from pathlib import Path - -Listing subdirectories:: - - >>> p = Path('.') - >>> [x for x in p.iterdir() if x.is_dir()] - [PosixPath('.hg'), PosixPath('docs'), PosixPath('dist'), - PosixPath('__pycache__'), PosixPath('build')] - -Listing Python source files in this directory tree:: - - >>> list(p.glob('**/*.py')) - [PosixPath('test_pathlib.py'), PosixPath('setup.py'), - PosixPath('pathlib.py'), PosixPath('docs/conf.py'), - PosixPath('build/lib/pathlib.py')] - -Navigating inside a directory tree:: - - >>> p = Path('/etc') - >>> q = p / 'init.d' / 'reboot' - >>> q - PosixPath('/etc/init.d/reboot') - >>> q.resolve() - PosixPath('/etc/rc.d/init.d/halt') - -Querying path properties:: - - >>> q.exists() - True - >>> q.is_dir() - False - -Opening a file:: - - >>> with q.open() as f: f.readline() - ... - '#!/bin/bash\n' - - -.. _pure-paths: - -Pure paths ----------- - -Pure path objects provide path-handling operations which don't actually -access a filesystem. There are three ways to access these classes, which -we also call *flavours*: - -.. class:: PurePath(*pathsegments) - - A generic class that represents the system's path flavour (instantiating - it creates either a :class:`PurePosixPath` or a :class:`PureWindowsPath`):: - - >>> PurePath('setup.py') # Running on a Unix machine - PurePosixPath('setup.py') - - Each element of *pathsegments* can be either a string representing a - path segment, an object implementing the :class:`os.PathLike` interface - which returns a string, or another path object:: - - >>> PurePath('foo', 'some/path', 'bar') - PurePosixPath('foo/some/path/bar') - >>> PurePath(Path('foo'), Path('bar')) - PurePosixPath('foo/bar') - - When *pathsegments* is empty, the current directory is assumed:: - - >>> PurePath() - PurePosixPath('.') - - If a segment is an absolute path, all previous segments are ignored - (like :func:`os.path.join`):: - - >>> PurePath('/etc', '/usr', 'lib64') - PurePosixPath('/usr/lib64') - >>> PureWindowsPath('c:/Windows', 'd:bar') - PureWindowsPath('d:bar') - - On Windows, the drive is not reset when a rooted relative path - segment (e.g., ``r'\foo'``) is encountered:: - - >>> PureWindowsPath('c:/Windows', '/Program Files') - PureWindowsPath('c:/Program Files') - - Spurious slashes and single dots are collapsed, but double dots (``'..'``) - and leading double slashes (``'//'``) are not, since this would change the - meaning of a path for various reasons (e.g. symbolic links, UNC paths):: - - >>> PurePath('foo//bar') - PurePosixPath('foo/bar') - >>> PurePath('//foo/bar') - PurePosixPath('//foo/bar') - >>> PurePath('foo/./bar') - PurePosixPath('foo/bar') - >>> PurePath('foo/../bar') - PurePosixPath('foo/../bar') - - (a naïve approach would make ``PurePosixPath('foo/../bar')`` equivalent - to ``PurePosixPath('bar')``, which is wrong if ``foo`` is a symbolic link - to another directory) - - Pure path objects implement the :class:`os.PathLike` interface, allowing them - to be used anywhere the interface is accepted. - - .. versionchanged:: 3.6 - Added support for the :class:`os.PathLike` interface. - -.. class:: PurePosixPath(*pathsegments) - - A subclass of :class:`PurePath`, this path flavour represents non-Windows - filesystem paths:: - - >>> PurePosixPath('/etc') - PurePosixPath('/etc') - - *pathsegments* is specified similarly to :class:`PurePath`. - -.. class:: PureWindowsPath(*pathsegments) - - A subclass of :class:`PurePath`, this path flavour represents Windows - filesystem paths, including `UNC paths`_:: - - >>> PureWindowsPath('c:/Program Files/') - PureWindowsPath('c:/Program Files') - >>> PureWindowsPath('//server/share/file') - PureWindowsPath('//server/share/file') - - *pathsegments* is specified similarly to :class:`PurePath`. - - .. _unc paths: https://en.wikipedia.org/wiki/Path_(computing)#UNC - -Regardless of the system you're running on, you can instantiate all of -these classes, since they don't provide any operation that does system calls. - - -General properties -^^^^^^^^^^^^^^^^^^ - -Paths are immutable and :term:`hashable`. Paths of a same flavour are comparable -and orderable. These properties respect the flavour's case-folding -semantics:: - - >>> PurePosixPath('foo') == PurePosixPath('FOO') - False - >>> PureWindowsPath('foo') == PureWindowsPath('FOO') - True - >>> PureWindowsPath('FOO') in { PureWindowsPath('foo') } - True - >>> PureWindowsPath('C:') < PureWindowsPath('d:') - True - -Paths of a different flavour compare unequal and cannot be ordered:: - - >>> PureWindowsPath('foo') == PurePosixPath('foo') - False - >>> PureWindowsPath('foo') < PurePosixPath('foo') - Traceback (most recent call last): - File "", line 1, in - TypeError: '<' not supported between instances of 'PureWindowsPath' and 'PurePosixPath' - - -Operators -^^^^^^^^^ - -The slash operator helps create child paths, like :func:`os.path.join`. -If the argument is an absolute path, the previous path is ignored. -On Windows, the drive is not reset when the argument is a rooted -relative path (e.g., ``r'\foo'``):: - - >>> p = PurePath('/etc') - >>> p - PurePosixPath('/etc') - >>> p / 'init.d' / 'apache2' - PurePosixPath('/etc/init.d/apache2') - >>> q = PurePath('bin') - >>> '/usr' / q - PurePosixPath('/usr/bin') - >>> p / '/an_absolute_path' - PurePosixPath('/an_absolute_path') - >>> PureWindowsPath('c:/Windows', '/Program Files') - PureWindowsPath('c:/Program Files') - -A path object can be used anywhere an object implementing :class:`os.PathLike` -is accepted:: - - >>> import os - >>> p = PurePath('/etc') - >>> os.fspath(p) - '/etc' - -The string representation of a path is the raw filesystem path itself -(in native form, e.g. with backslashes under Windows), which you can -pass to any function taking a file path as a string:: - - >>> p = PurePath('/etc') - >>> str(p) - '/etc' - >>> p = PureWindowsPath('c:/Program Files') - >>> str(p) - 'c:\\Program Files' - -Similarly, calling :class:`bytes` on a path gives the raw filesystem path as a -bytes object, as encoded by :func:`os.fsencode`:: - - >>> bytes(p) - b'/etc' - -.. note:: - Calling :class:`bytes` is only recommended under Unix. Under Windows, - the unicode form is the canonical representation of filesystem paths. - - -Accessing individual parts -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To access the individual "parts" (components) of a path, use the following -property: - -.. attribute:: PurePath.parts - - A tuple giving access to the path's various components:: - - >>> p = PurePath('/usr/bin/python3') - >>> p.parts - ('/', 'usr', 'bin', 'python3') - - >>> p = PureWindowsPath('c:/Program Files/PSF') - >>> p.parts - ('c:\\', 'Program Files', 'PSF') - - (note how the drive and local root are regrouped in a single part) - - -Methods and properties -^^^^^^^^^^^^^^^^^^^^^^ - -.. testsetup:: - - from pathlib import PurePath, PurePosixPath, PureWindowsPath - -Pure paths provide the following methods and properties: - -.. attribute:: PurePath.drive - - A string representing the drive letter or name, if any:: - - >>> PureWindowsPath('c:/Program Files/').drive - 'c:' - >>> PureWindowsPath('/Program Files/').drive - '' - >>> PurePosixPath('/etc').drive - '' - - UNC shares are also considered drives:: - - >>> PureWindowsPath('//host/share/foo.txt').drive - '\\\\host\\share' - -.. attribute:: PurePath.root - - A string representing the (local or global) root, if any:: - - >>> PureWindowsPath('c:/Program Files/').root - '\\' - >>> PureWindowsPath('c:Program Files/').root - '' - >>> PurePosixPath('/etc').root - '/' - - UNC shares always have a root:: - - >>> PureWindowsPath('//host/share').root - '\\' - - If the path starts with more than two successive slashes, - :class:`~pathlib.PurePosixPath` collapses them:: - - >>> PurePosixPath('//etc').root - '//' - >>> PurePosixPath('///etc').root - '/' - >>> PurePosixPath('////etc').root - '/' - - .. note:: - - This behavior conforms to *The Open Group Base Specifications Issue 6*, - paragraph `4.11 Pathname Resolution - `_: - - *"A pathname that begins with two successive slashes may be interpreted in - an implementation-defined manner, although more than two leading slashes - shall be treated as a single slash."* - -.. attribute:: PurePath.anchor - - The concatenation of the drive and root:: - - >>> PureWindowsPath('c:/Program Files/').anchor - 'c:\\' - >>> PureWindowsPath('c:Program Files/').anchor - 'c:' - >>> PurePosixPath('/etc').anchor - '/' - >>> PureWindowsPath('//host/share').anchor - '\\\\host\\share\\' - - -.. attribute:: PurePath.parents - - An immutable sequence providing access to the logical ancestors of - the path:: - - >>> p = PureWindowsPath('c:/foo/bar/setup.py') - >>> p.parents[0] - PureWindowsPath('c:/foo/bar') - >>> p.parents[1] - PureWindowsPath('c:/foo') - >>> p.parents[2] - PureWindowsPath('c:/') - - .. versionchanged:: 3.10 - The parents sequence now supports :term:`slices ` and negative index values. - -.. attribute:: PurePath.parent - - The logical parent of the path:: - - >>> p = PurePosixPath('/a/b/c/d') - >>> p.parent - PurePosixPath('/a/b/c') - - You cannot go past an anchor, or empty path:: - - >>> p = PurePosixPath('/') - >>> p.parent - PurePosixPath('/') - >>> p = PurePosixPath('.') - >>> p.parent - PurePosixPath('.') - - .. note:: - This is a purely lexical operation, hence the following behaviour:: - - >>> p = PurePosixPath('foo/..') - >>> p.parent - PurePosixPath('foo') - - If you want to walk an arbitrary filesystem path upwards, it is - recommended to first call :meth:`Path.resolve` so as to resolve - symlinks and eliminate ``".."`` components. - - -.. attribute:: PurePath.name - - A string representing the final path component, excluding the drive and - root, if any:: - - >>> PurePosixPath('my/library/setup.py').name - 'setup.py' - - UNC drive names are not considered:: - - >>> PureWindowsPath('//some/share/setup.py').name - 'setup.py' - >>> PureWindowsPath('//some/share').name - '' - - -.. attribute:: PurePath.suffix - - The file extension of the final component, if any:: - - >>> PurePosixPath('my/library/setup.py').suffix - '.py' - >>> PurePosixPath('my/library.tar.gz').suffix - '.gz' - >>> PurePosixPath('my/library').suffix - '' - - -.. attribute:: PurePath.suffixes - - A list of the path's file extensions:: - - >>> PurePosixPath('my/library.tar.gar').suffixes - ['.tar', '.gar'] - >>> PurePosixPath('my/library.tar.gz').suffixes - ['.tar', '.gz'] - >>> PurePosixPath('my/library').suffixes - [] - - -.. attribute:: PurePath.stem - - The final path component, without its suffix:: - - >>> PurePosixPath('my/library.tar.gz').stem - 'library.tar' - >>> PurePosixPath('my/library.tar').stem - 'library' - >>> PurePosixPath('my/library').stem - 'library' - - -.. method:: PurePath.as_posix() - - Return a string representation of the path with forward slashes (``/``):: - - >>> p = PureWindowsPath('c:\\windows') - >>> str(p) - 'c:\\windows' - >>> p.as_posix() - 'c:/windows' - - -.. method:: PurePath.as_uri() - - Represent the path as a ``file`` URI. :exc:`ValueError` is raised if - the path isn't absolute. - - >>> p = PurePosixPath('/etc/passwd') - >>> p.as_uri() - 'file:///etc/passwd' - >>> p = PureWindowsPath('c:/Windows') - >>> p.as_uri() - 'file:///c:/Windows' - - -.. method:: PurePath.is_absolute() - - Return whether the path is absolute or not. A path is considered absolute - if it has both a root and (if the flavour allows) a drive:: - - >>> PurePosixPath('/a/b').is_absolute() - True - >>> PurePosixPath('a/b').is_absolute() - False - - >>> PureWindowsPath('c:/a/b').is_absolute() - True - >>> PureWindowsPath('/a/b').is_absolute() - False - >>> PureWindowsPath('c:').is_absolute() - False - >>> PureWindowsPath('//some/share').is_absolute() - True - - -.. method:: PurePath.is_relative_to(*other) - - Return whether or not this path is relative to the *other* path. - - >>> p = PurePath('/etc/passwd') - >>> p.is_relative_to('/etc') - True - >>> p.is_relative_to('/usr') - False - - .. versionadded:: 3.9 - - -.. method:: PurePath.is_reserved() - - With :class:`PureWindowsPath`, return ``True`` if the path is considered - reserved under Windows, ``False`` otherwise. With :class:`PurePosixPath`, - ``False`` is always returned. - - >>> PureWindowsPath('nul').is_reserved() - True - >>> PurePosixPath('nul').is_reserved() - False - - File system calls on reserved paths can fail mysteriously or have - unintended effects. - - -.. method:: PurePath.joinpath(*other) - - Calling this method is equivalent to combining the path with each of - the *other* arguments in turn:: - - >>> PurePosixPath('/etc').joinpath('passwd') - PurePosixPath('/etc/passwd') - >>> PurePosixPath('/etc').joinpath(PurePosixPath('passwd')) - PurePosixPath('/etc/passwd') - >>> PurePosixPath('/etc').joinpath('init.d', 'apache2') - PurePosixPath('/etc/init.d/apache2') - >>> PureWindowsPath('c:').joinpath('/Program Files') - PureWindowsPath('c:/Program Files') - - -.. method:: PurePath.match(pattern) - - Match this path against the provided glob-style pattern. Return ``True`` - if matching is successful, ``False`` otherwise. - - If *pattern* is relative, the path can be either relative or absolute, - and matching is done from the right:: - - >>> PurePath('a/b.py').match('*.py') - True - >>> PurePath('/a/b/c.py').match('b/*.py') - True - >>> PurePath('/a/b/c.py').match('a/*.py') - False - - If *pattern* is absolute, the path must be absolute, and the whole path - must match:: - - >>> PurePath('/a.py').match('/*.py') - True - >>> PurePath('a/b.py').match('/*.py') - False - - As with other methods, case-sensitivity follows platform defaults:: - - >>> PurePosixPath('b.py').match('*.PY') - False - >>> PureWindowsPath('b.py').match('*.PY') - True - - -.. method:: PurePath.relative_to(*other) - - Compute a version of this path relative to the path represented by - *other*. If it's impossible, ValueError is raised:: - - >>> p = PurePosixPath('/etc/passwd') - >>> p.relative_to('/') - PurePosixPath('etc/passwd') - >>> p.relative_to('/etc') - PurePosixPath('passwd') - >>> p.relative_to('/usr') - Traceback (most recent call last): - File "", line 1, in - File "pathlib.py", line 694, in relative_to - .format(str(self), str(formatted))) - ValueError: '/etc/passwd' is not in the subpath of '/usr' OR one path is relative and the other absolute. - - NOTE: This function is part of :class:`PurePath` and works with strings. It does not check or access the underlying file structure. - - -.. method:: PurePath.with_name(name) - - Return a new path with the :attr:`name` changed. If the original path - doesn't have a name, ValueError is raised:: - - >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') - >>> p.with_name('setup.py') - PureWindowsPath('c:/Downloads/setup.py') - >>> p = PureWindowsPath('c:/') - >>> p.with_name('setup.py') - Traceback (most recent call last): - File "", line 1, in - File "/home/antoine/cpython/default/Lib/pathlib.py", line 751, in with_name - raise ValueError("%r has an empty name" % (self,)) - ValueError: PureWindowsPath('c:/') has an empty name - - -.. method:: PurePath.with_stem(stem) - - Return a new path with the :attr:`stem` changed. If the original path - doesn't have a name, ValueError is raised:: - - >>> p = PureWindowsPath('c:/Downloads/draft.txt') - >>> p.with_stem('final') - PureWindowsPath('c:/Downloads/final.txt') - >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') - >>> p.with_stem('lib') - PureWindowsPath('c:/Downloads/lib.gz') - >>> p = PureWindowsPath('c:/') - >>> p.with_stem('') - Traceback (most recent call last): - File "", line 1, in - File "/home/antoine/cpython/default/Lib/pathlib.py", line 861, in with_stem - return self.with_name(stem + self.suffix) - File "/home/antoine/cpython/default/Lib/pathlib.py", line 851, in with_name - raise ValueError("%r has an empty name" % (self,)) - ValueError: PureWindowsPath('c:/') has an empty name - - .. versionadded:: 3.9 - - -.. method:: PurePath.with_suffix(suffix) - - Return a new path with the :attr:`suffix` changed. If the original path - doesn't have a suffix, the new *suffix* is appended instead. If the - *suffix* is an empty string, the original suffix is removed:: - - >>> p = PureWindowsPath('c:/Downloads/pathlib.tar.gz') - >>> p.with_suffix('.bz2') - PureWindowsPath('c:/Downloads/pathlib.tar.bz2') - >>> p = PureWindowsPath('README') - >>> p.with_suffix('.txt') - PureWindowsPath('README.txt') - >>> p = PureWindowsPath('README.txt') - >>> p.with_suffix('') - PureWindowsPath('README') - - -.. _concrete-paths: - - -Concrete paths --------------- - -Concrete paths are subclasses of the pure path classes. In addition to -operations provided by the latter, they also provide methods to do system -calls on path objects. There are three ways to instantiate concrete paths: - -.. class:: Path(*pathsegments) - - A subclass of :class:`PurePath`, this class represents concrete paths of - the system's path flavour (instantiating it creates either a - :class:`PosixPath` or a :class:`WindowsPath`):: - - >>> Path('setup.py') - PosixPath('setup.py') - - *pathsegments* is specified similarly to :class:`PurePath`. - -.. class:: PosixPath(*pathsegments) - - A subclass of :class:`Path` and :class:`PurePosixPath`, this class - represents concrete non-Windows filesystem paths:: - - >>> PosixPath('/etc') - PosixPath('/etc') - - *pathsegments* is specified similarly to :class:`PurePath`. - -.. class:: WindowsPath(*pathsegments) - - A subclass of :class:`Path` and :class:`PureWindowsPath`, this class - represents concrete Windows filesystem paths:: - - >>> WindowsPath('c:/Program Files/') - WindowsPath('c:/Program Files') - - *pathsegments* is specified similarly to :class:`PurePath`. - -You can only instantiate the class flavour that corresponds to your system -(allowing system calls on non-compatible path flavours could lead to -bugs or failures in your application):: - - >>> import os - >>> os.name - 'posix' - >>> Path('setup.py') - PosixPath('setup.py') - >>> PosixPath('setup.py') - PosixPath('setup.py') - >>> WindowsPath('setup.py') - Traceback (most recent call last): - File "", line 1, in - File "pathlib.py", line 798, in __new__ - % (cls.__name__,)) - NotImplementedError: cannot instantiate 'WindowsPath' on your system - - -Methods -^^^^^^^ - -Concrete paths provide the following methods in addition to pure paths -methods. Many of these methods can raise an :exc:`OSError` if a system -call fails (for example because the path doesn't exist). - -.. versionchanged:: 3.8 - - :meth:`~Path.exists()`, :meth:`~Path.is_dir()`, :meth:`~Path.is_file()`, - :meth:`~Path.is_mount()`, :meth:`~Path.is_symlink()`, - :meth:`~Path.is_block_device()`, :meth:`~Path.is_char_device()`, - :meth:`~Path.is_fifo()`, :meth:`~Path.is_socket()` now return ``False`` - instead of raising an exception for paths that contain characters - unrepresentable at the OS level. - - -.. classmethod:: Path.cwd() - - Return a new path object representing the current directory (as returned - by :func:`os.getcwd`):: - - >>> Path.cwd() - PosixPath('/home/antoine/pathlib') - - -.. classmethod:: Path.home() - - Return a new path object representing the user's home directory (as - returned by :func:`os.path.expanduser` with ``~`` construct). If the home - directory can't be resolved, :exc:`RuntimeError` is raised. - - :: - - >>> Path.home() - PosixPath('/home/antoine') - - .. versionadded:: 3.5 - - -.. method:: Path.stat(*, follow_symlinks=True) - - Return a :class:`os.stat_result` object containing information about this path, like :func:`os.stat`. - The result is looked up at each call to this method. - - This method normally follows symlinks; to stat a symlink add the argument - ``follow_symlinks=False``, or use :meth:`~Path.lstat`. - - :: - - >>> p = Path('setup.py') - >>> p.stat().st_size - 956 - >>> p.stat().st_mtime - 1327883547.852554 - - .. versionchanged:: 3.10 - The *follow_symlinks* parameter was added. - -.. method:: Path.chmod(mode, *, follow_symlinks=True) - - Change the file mode and permissions, like :func:`os.chmod`. - - This method normally follows symlinks. Some Unix flavours support changing - permissions on the symlink itself; on these platforms you may add the - argument ``follow_symlinks=False``, or use :meth:`~Path.lchmod`. - - :: - - >>> p = Path('setup.py') - >>> p.stat().st_mode - 33277 - >>> p.chmod(0o444) - >>> p.stat().st_mode - 33060 - - .. versionchanged:: 3.10 - The *follow_symlinks* parameter was added. - -.. method:: Path.exists() - - Whether the path points to an existing file or directory:: - - >>> Path('.').exists() - True - >>> Path('setup.py').exists() - True - >>> Path('/etc').exists() - True - >>> Path('nonexistentfile').exists() - False - - .. note:: - If the path points to a symlink, :meth:`exists` returns whether the - symlink *points to* an existing file or directory. - - -.. method:: Path.expanduser() - - Return a new path with expanded ``~`` and ``~user`` constructs, - as returned by :meth:`os.path.expanduser`. If a home directory can't be - resolved, :exc:`RuntimeError` is raised. - - :: - - >>> p = PosixPath('~/films/Monty Python') - >>> p.expanduser() - PosixPath('/home/eric/films/Monty Python') - - .. versionadded:: 3.5 - - -.. method:: Path.glob(pattern) - - Glob the given relative *pattern* in the directory represented by this path, - yielding all matching files (of any kind):: - - >>> sorted(Path('.').glob('*.py')) - [PosixPath('pathlib.py'), PosixPath('setup.py'), PosixPath('test_pathlib.py')] - >>> sorted(Path('.').glob('*/*.py')) - [PosixPath('docs/conf.py')] - - Patterns are the same as for :mod:`fnmatch`, with the addition of "``**``" - which means "this directory and all subdirectories, recursively". In other - words, it enables recursive globbing:: - - >>> sorted(Path('.').glob('**/*.py')) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - .. note:: - Using the "``**``" pattern in large directory trees may consume - an inordinate amount of time. - - .. audit-event:: pathlib.Path.glob self,pattern pathlib.Path.glob - - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). - -.. method:: Path.group() - - Return the name of the group owning the file. :exc:`KeyError` is raised - if the file's gid isn't found in the system database. - - -.. method:: Path.is_dir() - - Return ``True`` if the path points to a directory (or a symbolic link - pointing to a directory), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_file() - - Return ``True`` if the path points to a regular file (or a symbolic link - pointing to a regular file), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_mount() - - Return ``True`` if the path is a :dfn:`mount point`: a point in a - file system where a different file system has been mounted. On POSIX, the - function checks whether *path*'s parent, :file:`path/..`, is on a different - device than *path*, or whether :file:`path/..` and *path* point to the same - i-node on the same device --- this should detect mount points for all Unix - and POSIX variants. Not implemented on Windows. - - .. versionadded:: 3.7 - - -.. method:: Path.is_symlink() - - Return ``True`` if the path points to a symbolic link, ``False`` otherwise. - - ``False`` is also returned if the path doesn't exist; other errors (such - as permission errors) are propagated. - - -.. method:: Path.is_socket() - - Return ``True`` if the path points to a Unix socket (or a symbolic link - pointing to a Unix socket), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_fifo() - - Return ``True`` if the path points to a FIFO (or a symbolic link - pointing to a FIFO), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_block_device() - - Return ``True`` if the path points to a block device (or a symbolic link - pointing to a block device), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.is_char_device() - - Return ``True`` if the path points to a character device (or a symbolic link - pointing to a character device), ``False`` if it points to another kind of file. - - ``False`` is also returned if the path doesn't exist or is a broken symlink; - other errors (such as permission errors) are propagated. - - -.. method:: Path.iterdir() - - When the path points to a directory, yield path objects of the directory - contents:: - - >>> p = Path('docs') - >>> for child in p.iterdir(): child - ... - PosixPath('docs/conf.py') - PosixPath('docs/_templates') - PosixPath('docs/make.bat') - PosixPath('docs/index.rst') - PosixPath('docs/_build') - PosixPath('docs/_static') - PosixPath('docs/Makefile') - - The children are yielded in arbitrary order, and the special entries - ``'.'`` and ``'..'`` are not included. If a file is removed from or added - to the directory after creating the iterator, whether a path object for - that file be included is unspecified. - -.. method:: Path.lchmod(mode) - - Like :meth:`Path.chmod` but, if the path points to a symbolic link, the - symbolic link's mode is changed rather than its target's. - - -.. method:: Path.lstat() - - Like :meth:`Path.stat` but, if the path points to a symbolic link, return - the symbolic link's information rather than its target's. - - -.. method:: Path.mkdir(mode=0o777, parents=False, exist_ok=False) - - Create a new directory at this given path. If *mode* is given, it is - combined with the process' ``umask`` value to determine the file mode - and access flags. If the path already exists, :exc:`FileExistsError` - is raised. - - If *parents* is true, any missing parents of this path are created - as needed; they are created with the default permissions without taking - *mode* into account (mimicking the POSIX ``mkdir -p`` command). - - If *parents* is false (the default), a missing parent raises - :exc:`FileNotFoundError`. - - If *exist_ok* is false (the default), :exc:`FileExistsError` is - raised if the target directory already exists. - - If *exist_ok* is true, :exc:`FileExistsError` exceptions will be - ignored (same behavior as the POSIX ``mkdir -p`` command), but only if the - last path component is not an existing non-directory file. - - .. versionchanged:: 3.5 - The *exist_ok* parameter was added. - - -.. method:: Path.open(mode='r', buffering=-1, encoding=None, errors=None, newline=None) - - Open the file pointed to by the path, like the built-in :func:`open` - function does:: - - >>> p = Path('setup.py') - >>> with p.open() as f: - ... f.readline() - ... - '#!/usr/bin/env python3\n' - - -.. method:: Path.owner() - - Return the name of the user owning the file. :exc:`KeyError` is raised - if the file's uid isn't found in the system database. - - -.. method:: Path.read_bytes() - - Return the binary contents of the pointed-to file as a bytes object:: - - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' - - .. versionadded:: 3.5 - - -.. method:: Path.read_text(encoding=None, errors=None) - - Return the decoded contents of the pointed-to file as a string:: - - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' - - The file is opened and then closed. The optional parameters have the same - meaning as in :func:`open`. - - .. versionadded:: 3.5 - - -.. method:: Path.readlink() - - Return the path to which the symbolic link points (as returned by - :func:`os.readlink`):: - - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.readlink() - PosixPath('setup.py') - - .. versionadded:: 3.9 - - -.. method:: Path.rename(target) - - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. On Unix, if *target* exists and is a file, - it will be replaced silently if the user has permission. - On Windows, if *target* exists, :exc:`FileExistsError` will be raised. - *target* can be either a string or another path object:: - - >>> p = Path('foo') - >>> p.open('w').write('some text') - 9 - >>> target = Path('bar') - >>> p.rename(target) - PosixPath('bar') - >>> target.open().read() - 'some text' - - The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. - - It is implemented in terms of :func:`os.rename` and gives the same guarantees. - - .. versionchanged:: 3.8 - Added return value, return the new Path instance. - - -.. method:: Path.replace(target) - - Rename this file or directory to the given *target*, and return a new Path - instance pointing to *target*. If *target* points to an existing file or - empty directory, it will be unconditionally replaced. - - The target path may be absolute or relative. Relative paths are interpreted - relative to the current working directory, *not* the directory of the Path - object. - - .. versionchanged:: 3.8 - Added return value, return the new Path instance. - - -.. method:: Path.absolute() - - Make the path absolute, without normalization or resolving symlinks. - Returns a new path object:: - - >>> p = Path('tests') - >>> p - PosixPath('tests') - >>> p.absolute() - PosixPath('/home/antoine/pathlib/tests') - - -.. method:: Path.resolve(strict=False) - - Make the path absolute, resolving any symlinks. A new path object is - returned:: - - >>> p = Path() - >>> p - PosixPath('.') - >>> p.resolve() - PosixPath('/home/antoine/pathlib') - - "``..``" components are also eliminated (this is the only method to do so):: - - >>> p = Path('docs/../setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - - If the path doesn't exist and *strict* is ``True``, :exc:`FileNotFoundError` - is raised. If *strict* is ``False``, the path is resolved as far as possible - and any remainder is appended without checking whether it exists. If an - infinite loop is encountered along the resolution path, :exc:`RuntimeError` - is raised. - - .. versionadded:: 3.6 - The *strict* argument (pre-3.6 behavior is strict). - -.. method:: Path.rglob(pattern) - - This is like calling :func:`Path.glob` with "``**/``" added in front of the - given relative *pattern*:: - - >>> sorted(Path().rglob("*.py")) - [PosixPath('build/lib/pathlib.py'), - PosixPath('docs/conf.py'), - PosixPath('pathlib.py'), - PosixPath('setup.py'), - PosixPath('test_pathlib.py')] - - .. audit-event:: pathlib.Path.rglob self,pattern pathlib.Path.rglob - - .. versionchanged:: 3.11 - Return only directories if *pattern* ends with a pathname components - separator (:data:`~os.sep` or :data:`~os.altsep`). - -.. method:: Path.rmdir() - - Remove this directory. The directory must be empty. - - -.. method:: Path.samefile(other_path) - - Return whether this path points to the same file as *other_path*, which - can be either a Path object, or a string. The semantics are similar - to :func:`os.path.samefile` and :func:`os.path.samestat`. - - An :exc:`OSError` can be raised if either file cannot be accessed for some - reason. - - :: - - >>> p = Path('spam') - >>> q = Path('eggs') - >>> p.samefile(q) - False - >>> p.samefile('spam') - True - - .. versionadded:: 3.5 - - -.. method:: Path.symlink_to(target, target_is_directory=False) - - Make this path a symbolic link to *target*. Under Windows, - *target_is_directory* must be true (default ``False``) if the link's target - is a directory. Under POSIX, *target_is_directory*'s value is ignored. - - :: - - >>> p = Path('mylink') - >>> p.symlink_to('setup.py') - >>> p.resolve() - PosixPath('/home/antoine/pathlib/setup.py') - >>> p.stat().st_size - 956 - >>> p.lstat().st_size - 8 - - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.symlink`'s. - -.. method:: Path.hardlink_to(target) - - Make this path a hard link to the same file as *target*. - - .. note:: - The order of arguments (link, target) is the reverse - of :func:`os.link`'s. - - .. versionadded:: 3.10 - -.. method:: Path.link_to(target) - - Make *target* a hard link to this path. - - .. warning:: - - This function does not make this path a hard link to *target*, despite - the implication of the function and argument names. The argument order - (target, link) is the reverse of :func:`Path.symlink_to` and - :func:`Path.hardlink_to`, but matches that of :func:`os.link`. - - .. versionadded:: 3.8 - - .. deprecated:: 3.10 - - This method is deprecated in favor of :meth:`Path.hardlink_to`, as the - argument order of :meth:`Path.link_to` does not match that of - :meth:`Path.symlink_to`. - - -.. method:: Path.touch(mode=0o666, exist_ok=True) - - Create a file at this given path. If *mode* is given, it is combined - with the process' ``umask`` value to determine the file mode and access - flags. If the file already exists, the function succeeds if *exist_ok* - is true (and its modification time is updated to the current time), - otherwise :exc:`FileExistsError` is raised. - - -.. method:: Path.unlink(missing_ok=False) - - Remove this file or symbolic link. If the path points to a directory, - use :func:`Path.rmdir` instead. - - If *missing_ok* is false (the default), :exc:`FileNotFoundError` is - raised if the path does not exist. - - If *missing_ok* is true, :exc:`FileNotFoundError` exceptions will be - ignored (same behavior as the POSIX ``rm -f`` command). - - .. versionchanged:: 3.8 - The *missing_ok* parameter was added. - - -.. method:: Path.write_bytes(data) - - Open the file pointed to in bytes mode, write *data* to it, and close the - file:: - - >>> p = Path('my_binary_file') - >>> p.write_bytes(b'Binary file contents') - 20 - >>> p.read_bytes() - b'Binary file contents' - - An existing file of the same name is overwritten. - - .. versionadded:: 3.5 - - -.. method:: Path.write_text(data, encoding=None, errors=None, newline=None) - - Open the file pointed to in text mode, write *data* to it, and close the - file:: - - >>> p = Path('my_text_file') - >>> p.write_text('Text file contents') - 18 - >>> p.read_text() - 'Text file contents' - - An existing file of the same name is overwritten. The optional parameters - have the same meaning as in :func:`open`. - - .. versionadded:: 3.5 - - .. versionchanged:: 3.10 - The *newline* parameter was added. - -Correspondence to tools in the :mod:`os` module ------------------------------------------------ - -Below is a table mapping various :mod:`os` functions to their corresponding -:class:`PurePath`/:class:`Path` equivalent. - -.. note:: - - Not all pairs of functions/methods below are equivalent. Some of them, - despite having some overlapping use-cases, have different semantics. They - include :func:`os.path.abspath` and :meth:`Path.absolute`, - :func:`os.path.relpath` and :meth:`PurePath.relative_to`. - -==================================== ============================== -:mod:`os` and :mod:`os.path` :mod:`pathlib` -==================================== ============================== -:func:`os.path.abspath` :meth:`Path.absolute` [#]_ -:func:`os.path.realpath` :meth:`Path.resolve` -:func:`os.chmod` :meth:`Path.chmod` -:func:`os.mkdir` :meth:`Path.mkdir` -:func:`os.makedirs` :meth:`Path.mkdir` -:func:`os.rename` :meth:`Path.rename` -:func:`os.replace` :meth:`Path.replace` -:func:`os.rmdir` :meth:`Path.rmdir` -:func:`os.remove`, :func:`os.unlink` :meth:`Path.unlink` -:func:`os.getcwd` :func:`Path.cwd` -:func:`os.path.exists` :meth:`Path.exists` -:func:`os.path.expanduser` :meth:`Path.expanduser` and - :meth:`Path.home` -:func:`os.listdir` :meth:`Path.iterdir` -:func:`os.path.isdir` :meth:`Path.is_dir` -:func:`os.path.isfile` :meth:`Path.is_file` -:func:`os.path.islink` :meth:`Path.is_symlink` -:func:`os.link` :meth:`Path.hardlink_to` -:func:`os.symlink` :meth:`Path.symlink_to` -:func:`os.readlink` :meth:`Path.readlink` -:func:`os.path.relpath` :meth:`PurePath.relative_to` [#]_ -:func:`os.stat` :meth:`Path.stat`, - :meth:`Path.owner`, - :meth:`Path.group` -:func:`os.path.isabs` :meth:`PurePath.is_absolute` -:func:`os.path.join` :func:`PurePath.joinpath` -:func:`os.path.basename` :attr:`PurePath.name` -:func:`os.path.dirname` :attr:`PurePath.parent` -:func:`os.path.samefile` :meth:`Path.samefile` -:func:`os.path.splitext` :attr:`PurePath.stem` and - :attr:`PurePath.suffix` -==================================== ============================== - -.. rubric:: Footnotes - -.. [#] :func:`os.path.abspath` normalizes the resulting path, which may change its meaning in the presence of symlinks, while :meth:`Path.absolute` does not. -.. [#] :meth:`PurePath.relative_to` requires ``self`` to be the subpath of the argument, but :func:`os.path.relpath` does not. diff --git a/Doc/library/pdb.rst b/Doc/library/pdb.rst deleted file mode 100644 index 8798155764b1ca..00000000000000 --- a/Doc/library/pdb.rst +++ /dev/null @@ -1,622 +0,0 @@ -.. _debugger: - -:mod:`pdb` --- The Python Debugger -================================== - -.. module:: pdb - :synopsis: The Python debugger for interactive interpreters. - -**Source code:** :source:`Lib/pdb.py` - -.. index:: single: debugging - --------------- - -The module :mod:`pdb` defines an interactive source code debugger for Python -programs. It supports setting (conditional) breakpoints and single stepping at -the source line level, inspection of stack frames, source code listing, and -evaluation of arbitrary Python code in the context of any stack frame. It also -supports post-mortem debugging and can be called under program control. - -.. index:: - single: Pdb (class in pdb) - pair: module; bdb - pair: module; cmd - -The debugger is extensible -- it is actually defined as the class :class:`Pdb`. -This is currently undocumented but easily understood by reading the source. The -extension interface uses the modules :mod:`bdb` and :mod:`cmd`. - -.. seealso:: - - Module :mod:`faulthandler` - Used to dump Python tracebacks explicitly, on a fault, after a timeout, - or on a user signal. - - Module :mod:`traceback` - Standard interface to extract, format and print stack traces of Python programs. - -The typical usage to break into the debugger is to insert:: - - import pdb; pdb.set_trace() - -Or:: - - breakpoint() - -at the location you want to break into the debugger, and then run the program. -You can then step through the code following this statement, and continue -running without the debugger using the :pdbcmd:`continue` command. - -.. versionadded:: 3.7 - The built-in :func:`breakpoint()`, when called with defaults, can be used - instead of ``import pdb; pdb.set_trace()``. - -:: - - def double(x): - breakpoint() - return x * 2 - val = 3 - print(f"{val} * 2 is {double(val)}") - -The debugger's prompt is ``(Pdb)``, which is the indicator that you are in debug mode:: - - > ...(3)double() - -> return x * 2 - (Pdb) p x - 3 - (Pdb) continue - 3 * 2 is 6 - -.. versionchanged:: 3.3 - Tab-completion via the :mod:`readline` module is available for commands and - command arguments, e.g. the current global and local names are offered as - arguments of the ``p`` command. - - -You can also invoke :mod:`pdb` from the command line to debug other scripts. For -example:: - - python -m pdb myscript.py - -When invoked as a module, pdb will automatically enter post-mortem debugging if -the program being debugged exits abnormally. After post-mortem debugging (or -after normal exit of the program), pdb will restart the program. Automatic -restarting preserves pdb's state (such as breakpoints) and in most cases is more -useful than quitting the debugger upon program's exit. - -.. versionadded:: 3.2 - ``-c`` option is introduced to execute commands as if given - in a :file:`.pdbrc` file, see :ref:`debugger-commands`. - -.. versionadded:: 3.7 - ``-m`` option is introduced to execute modules similar to the way - ``python -m`` does. As with a script, the debugger will pause execution just - before the first line of the module. - -Typical usage to execute a statement under control of the debugger is:: - - >>> import pdb - >>> def f(x): - ... print(1 / x) - >>> pdb.run("f(2)") - > (1)() - (Pdb) continue - 0.5 - >>> - -The typical usage to inspect a crashed program is:: - - >>> import pdb - >>> def f(x): - ... print(1 / x) - ... - >>> f(0) - Traceback (most recent call last): - File "", line 1, in - File "", line 2, in f - ZeroDivisionError: division by zero - >>> pdb.pm() - > (2)f() - (Pdb) p x - 0 - (Pdb) - - -The module defines the following functions; each enters the debugger in a -slightly different way: - -.. function:: run(statement, globals=None, locals=None) - - Execute the *statement* (given as a string or a code object) under debugger - control. The debugger prompt appears before any code is executed; you can - set breakpoints and type :pdbcmd:`continue`, or you can step through the - statement using :pdbcmd:`step` or :pdbcmd:`next` (all these commands are - explained below). The optional *globals* and *locals* arguments specify the - environment in which the code is executed; by default the dictionary of the - module :mod:`__main__` is used. (See the explanation of the built-in - :func:`exec` or :func:`eval` functions.) - - -.. function:: runeval(expression, globals=None, locals=None) - - Evaluate the *expression* (given as a string or a code object) under debugger - control. When :func:`runeval` returns, it returns the value of the - *expression*. Otherwise this function is similar to :func:`run`. - - -.. function:: runcall(function, *args, **kwds) - - Call the *function* (a function or method object, not a string) with the - given arguments. When :func:`runcall` returns, it returns whatever the - function call returned. The debugger prompt appears as soon as the function - is entered. - - -.. function:: set_trace(*, header=None) - - Enter the debugger at the calling stack frame. This is useful to hard-code - a breakpoint at a given point in a program, even if the code is not - otherwise being debugged (e.g. when an assertion fails). If given, - *header* is printed to the console just before debugging begins. - - .. versionchanged:: 3.7 - The keyword-only argument *header*. - - -.. function:: post_mortem(traceback=None) - - Enter post-mortem debugging of the given *traceback* object. If no - *traceback* is given, it uses the one of the exception that is currently - being handled (an exception must be being handled if the default is to be - used). - - -.. function:: pm() - - Enter post-mortem debugging of the traceback found in - :data:`sys.last_traceback`. - - -The ``run*`` functions and :func:`set_trace` are aliases for instantiating the -:class:`Pdb` class and calling the method of the same name. If you want to -access further features, you have to do this yourself: - -.. class:: Pdb(completekey='tab', stdin=None, stdout=None, skip=None, \ - nosigint=False, readrc=True) - - :class:`Pdb` is the debugger class. - - The *completekey*, *stdin* and *stdout* arguments are passed to the - underlying :class:`cmd.Cmd` class; see the description there. - - The *skip* argument, if given, must be an iterable of glob-style module name - patterns. The debugger will not step into frames that originate in a module - that matches one of these patterns. [1]_ - - By default, Pdb sets a handler for the SIGINT signal (which is sent when the - user presses :kbd:`Ctrl-C` on the console) when you give a :pdbcmd:`continue` command. - This allows you to break into the debugger again by pressing :kbd:`Ctrl-C`. If you - want Pdb not to touch the SIGINT handler, set *nosigint* to true. - - The *readrc* argument defaults to true and controls whether Pdb will load - .pdbrc files from the filesystem. - - Example call to enable tracing with *skip*:: - - import pdb; pdb.Pdb(skip=['django.*']).set_trace() - - .. audit-event:: pdb.Pdb "" pdb.Pdb - - .. versionadded:: 3.1 - The *skip* argument. - - .. versionadded:: 3.2 - The *nosigint* argument. Previously, a SIGINT handler was never set by - Pdb. - - .. versionchanged:: 3.6 - The *readrc* argument. - - .. method:: run(statement, globals=None, locals=None) - runeval(expression, globals=None, locals=None) - runcall(function, *args, **kwds) - set_trace() - - See the documentation for the functions explained above. - - -.. _debugger-commands: - -Debugger Commands ------------------ - -The commands recognized by the debugger are listed below. Most commands can be -abbreviated to one or two letters as indicated; e.g. ``h(elp)`` means that -either ``h`` or ``help`` can be used to enter the help command (but not ``he`` -or ``hel``, nor ``H`` or ``Help`` or ``HELP``). Arguments to commands must be -separated by whitespace (spaces or tabs). Optional arguments are enclosed in -square brackets (``[]``) in the command syntax; the square brackets must not be -typed. Alternatives in the command syntax are separated by a vertical bar -(``|``). - -Entering a blank line repeats the last command entered. Exception: if the last -command was a :pdbcmd:`list` command, the next 11 lines are listed. - -Commands that the debugger doesn't recognize are assumed to be Python statements -and are executed in the context of the program being debugged. Python -statements can also be prefixed with an exclamation point (``!``). This is a -powerful way to inspect the program being debugged; it is even possible to -change a variable or call a function. When an exception occurs in such a -statement, the exception name is printed but the debugger's state is not -changed. - -The debugger supports :ref:`aliases `. Aliases can have -parameters which allows one a certain level of adaptability to the context under -examination. - -Multiple commands may be entered on a single line, separated by ``;;``. (A -single ``;`` is not used as it is the separator for multiple commands in a line -that is passed to the Python parser.) No intelligence is applied to separating -the commands; the input is split at the first ``;;`` pair, even if it is in the -middle of a quoted string. A workaround for strings with double semicolons -is to use implicit string concatenation ``';'';'`` or ``";"";"``. - -.. index:: - pair: .pdbrc; file - triple: debugger; configuration; file - -If a file :file:`.pdbrc` exists in the user's home directory or in the current -directory, it is read with ``'utf-8'`` encoding and executed as if it had been -typed at the debugger prompt. This is particularly useful for aliases. If both -files exist, the one in the home directory is read first and aliases defined there -can be overridden by the local file. - -.. versionchanged:: 3.11 - :file:`.pdbrc` is now read with ``'utf-8'`` encoding. Previously, it was read - with the system locale encoding. - -.. versionchanged:: 3.2 - :file:`.pdbrc` can now contain commands that continue debugging, such as - :pdbcmd:`continue` or :pdbcmd:`next`. Previously, these commands had no - effect. - - -.. pdbcommand:: h(elp) [command] - - Without argument, print the list of available commands. With a *command* as - argument, print help about that command. ``help pdb`` displays the full - documentation (the docstring of the :mod:`pdb` module). Since the *command* - argument must be an identifier, ``help exec`` must be entered to get help on - the ``!`` command. - -.. pdbcommand:: w(here) - - Print a stack trace, with the most recent frame at the bottom. An arrow (``>``) - indicates the current frame, which determines the context of most commands. - -.. pdbcommand:: d(own) [count] - - Move the current frame *count* (default one) levels down in the stack trace - (to a newer frame). - -.. pdbcommand:: u(p) [count] - - Move the current frame *count* (default one) levels up in the stack trace (to - an older frame). - -.. pdbcommand:: b(reak) [([filename:]lineno | function) [, condition]] - - With a *lineno* argument, set a break there in the current file. With a - *function* argument, set a break at the first executable statement within - that function. The line number may be prefixed with a filename and a colon, - to specify a breakpoint in another file (probably one that hasn't been loaded - yet). The file is searched on :data:`sys.path`. Note that each breakpoint - is assigned a number to which all the other breakpoint commands refer. - - If a second argument is present, it is an expression which must evaluate to - true before the breakpoint is honored. - - Without argument, list all breaks, including for each breakpoint, the number - of times that breakpoint has been hit, the current ignore count, and the - associated condition if any. - -.. pdbcommand:: tbreak [([filename:]lineno | function) [, condition]] - - Temporary breakpoint, which is removed automatically when it is first hit. - The arguments are the same as for :pdbcmd:`break`. - -.. pdbcommand:: cl(ear) [filename:lineno | bpnumber ...] - - With a *filename:lineno* argument, clear all the breakpoints at this line. - With a space separated list of breakpoint numbers, clear those breakpoints. - Without argument, clear all breaks (but first ask confirmation). - -.. pdbcommand:: disable [bpnumber ...] - - Disable the breakpoints given as a space separated list of breakpoint - numbers. Disabling a breakpoint means it cannot cause the program to stop - execution, but unlike clearing a breakpoint, it remains in the list of - breakpoints and can be (re-)enabled. - -.. pdbcommand:: enable [bpnumber ...] - - Enable the breakpoints specified. - -.. pdbcommand:: ignore bpnumber [count] - - Set the ignore count for the given breakpoint number. If *count* is omitted, - the ignore count is set to 0. A breakpoint becomes active when the ignore - count is zero. When non-zero, the *count* is decremented each time the - breakpoint is reached and the breakpoint is not disabled and any associated - condition evaluates to true. - -.. pdbcommand:: condition bpnumber [condition] - - Set a new *condition* for the breakpoint, an expression which must evaluate - to true before the breakpoint is honored. If *condition* is absent, any - existing condition is removed; i.e., the breakpoint is made unconditional. - -.. pdbcommand:: commands [bpnumber] - - Specify a list of commands for breakpoint number *bpnumber*. The commands - themselves appear on the following lines. Type a line containing just - ``end`` to terminate the commands. An example:: - - (Pdb) commands 1 - (com) p some_variable - (com) end - (Pdb) - - To remove all commands from a breakpoint, type ``commands`` and follow it - immediately with ``end``; that is, give no commands. - - With no *bpnumber* argument, ``commands`` refers to the last breakpoint set. - - You can use breakpoint commands to start your program up again. Simply use - the :pdbcmd:`continue` command, or :pdbcmd:`step`, - or any other command that resumes execution. - - Specifying any command resuming execution - (currently :pdbcmd:`continue`, :pdbcmd:`step`, :pdbcmd:`next`, - :pdbcmd:`return`, :pdbcmd:`jump`, :pdbcmd:`quit` and their abbreviations) - terminates the command list (as if - that command was immediately followed by end). This is because any time you - resume execution (even with a simple next or step), you may encounter another - breakpoint—which could have its own command list, leading to ambiguities about - which list to execute. - - If you use the ``silent`` command in the command list, the usual message about - stopping at a breakpoint is not printed. This may be desirable for breakpoints - that are to print a specific message and then continue. If none of the other - commands print anything, you see no sign that the breakpoint was reached. - -.. pdbcommand:: s(tep) - - Execute the current line, stop at the first possible occasion (either in a - function that is called or on the next line in the current function). - -.. pdbcommand:: n(ext) - - Continue execution until the next line in the current function is reached or - it returns. (The difference between :pdbcmd:`next` and :pdbcmd:`step` is - that :pdbcmd:`step` stops inside a called function, while :pdbcmd:`next` - executes called functions at (nearly) full speed, only stopping at the next - line in the current function.) - -.. pdbcommand:: unt(il) [lineno] - - Without argument, continue execution until the line with a number greater - than the current one is reached. - - With *lineno*, continue execution until a line with a number greater or - equal to *lineno* is reached. In both cases, also stop when the current frame - returns. - - .. versionchanged:: 3.2 - Allow giving an explicit line number. - -.. pdbcommand:: r(eturn) - - Continue execution until the current function returns. - -.. pdbcommand:: c(ont(inue)) - - Continue execution, only stop when a breakpoint is encountered. - -.. pdbcommand:: j(ump) lineno - - Set the next line that will be executed. Only available in the bottom-most - frame. This lets you jump back and execute code again, or jump forward to - skip code that you don't want to run. - - It should be noted that not all jumps are allowed -- for instance it is not - possible to jump into the middle of a :keyword:`for` loop or out of a - :keyword:`finally` clause. - -.. pdbcommand:: l(ist) [first[, last]] - - List source code for the current file. Without arguments, list 11 lines - around the current line or continue the previous listing. With ``.`` as - argument, list 11 lines around the current line. With one argument, - list 11 lines around at that line. With two arguments, list the given range; - if the second argument is less than the first, it is interpreted as a count. - - The current line in the current frame is indicated by ``->``. If an - exception is being debugged, the line where the exception was originally - raised or propagated is indicated by ``>>``, if it differs from the current - line. - - .. versionadded:: 3.2 - The ``>>`` marker. - -.. pdbcommand:: ll | longlist - - List all source code for the current function or frame. Interesting lines - are marked as for :pdbcmd:`list`. - - .. versionadded:: 3.2 - -.. pdbcommand:: a(rgs) - - Print the arguments of the current function and their current values. - -.. pdbcommand:: p expression - - Evaluate *expression* in the current context and print its value. - - .. note:: - - ``print()`` can also be used, but is not a debugger command --- this executes the - Python :func:`print` function. - - -.. pdbcommand:: pp expression - - Like the :pdbcmd:`p` command, except the value of *expression* is - pretty-printed using the :mod:`pprint` module. - -.. pdbcommand:: whatis expression - - Print the type of *expression*. - -.. pdbcommand:: source expression - - Try to get source code of *expression* and display it. - - .. versionadded:: 3.2 - -.. pdbcommand:: display [expression] - - Display the value of *expression* if it changed, each time execution stops - in the current frame. - - Without *expression*, list all display expressions for the current frame. - - .. note:: - - Display evaluates *expression* and compares to the result of the previous - evaluation of *expression*, so when the result is mutable, display may not - be able to pick up the changes. - - Example:: - - lst = [] - breakpoint() - pass - lst.append(1) - print(lst) - - Display won't realize ``lst`` has been changed because the result of evaluation - is modified in place by ``lst.append(1)`` before being compared:: - - > example.py(3)() - -> pass - (Pdb) display lst - display lst: [] - (Pdb) n - > example.py(4)() - -> lst.append(1) - (Pdb) n - > example.py(5)() - -> print(lst) - (Pdb) - - You can do some tricks with copy mechanism to make it work:: - - > example.py(3)() - -> pass - (Pdb) display lst[:] - display lst[:]: [] - (Pdb) n - > example.py(4)() - -> lst.append(1) - (Pdb) n - > example.py(5)() - -> print(lst) - display lst[:]: [1] [old: []] - (Pdb) - - .. versionadded:: 3.2 - -.. pdbcommand:: undisplay [expression] - - Do not display *expression* anymore in the current frame. Without - *expression*, clear all display expressions for the current frame. - - .. versionadded:: 3.2 - -.. pdbcommand:: interact - - Start an interactive interpreter (using the :mod:`code` module) whose global - namespace contains all the (global and local) names found in the current - scope. - - .. versionadded:: 3.2 - -.. _debugger-aliases: - -.. pdbcommand:: alias [name [command]] - - Create an alias called *name* that executes *command*. The *command* must - *not* be enclosed in quotes. Replaceable parameters can be indicated by - ``%1``, ``%2``, and so on, while ``%*`` is replaced by all the parameters. - If *command* is omitted, the current alias for *name* is shown. If no - arguments are given, all aliases are listed. - - Aliases may be nested and can contain anything that can be legally typed at - the pdb prompt. Note that internal pdb commands *can* be overridden by - aliases. Such a command is then hidden until the alias is removed. Aliasing - is recursively applied to the first word of the command line; all other words - in the line are left alone. - - As an example, here are two useful aliases (especially when placed in the - :file:`.pdbrc` file):: - - # Print instance variables (usage "pi classInst") - alias pi for k in %1.__dict__.keys(): print(f"%1.{k} = {%1.__dict__[k]}") - # Print instance variables in self - alias ps pi self - -.. pdbcommand:: unalias name - - Delete the specified alias *name*. - -.. pdbcommand:: ! statement - - Execute the (one-line) *statement* in the context of the current stack frame. - The exclamation point can be omitted unless the first word of the statement - resembles a debugger command. To set a global variable, you can prefix the - assignment command with a :keyword:`global` statement on the same line, - e.g.:: - - (Pdb) global list_options; list_options = ['-l'] - (Pdb) - -.. pdbcommand:: run [args ...] - restart [args ...] - - Restart the debugged Python program. If *args* is supplied, it is split - with :mod:`shlex` and the result is used as the new :data:`sys.argv`. - History, breakpoints, actions and debugger options are preserved. - :pdbcmd:`restart` is an alias for :pdbcmd:`run`. - -.. pdbcommand:: q(uit) - - Quit from the debugger. The program being executed is aborted. - -.. pdbcommand:: debug code - - Enter a recursive debugger that steps through *code* - (which is an arbitrary expression or statement to be - executed in the current environment). - -.. pdbcommand:: retval - - Print the return value for the last return of the current function. - -.. rubric:: Footnotes - -.. [1] Whether a frame is considered to originate in a certain module - is determined by the ``__name__`` in the frame globals. diff --git a/Doc/library/persistence.rst b/Doc/library/persistence.rst deleted file mode 100644 index d5bb193fa3aa09..00000000000000 --- a/Doc/library/persistence.rst +++ /dev/null @@ -1,23 +0,0 @@ -.. _persistence: - -**************** -Data Persistence -**************** - -The modules described in this chapter support storing Python data in a -persistent form on disk. The :mod:`pickle` and :mod:`marshal` modules can turn -many Python data types into a stream of bytes and then recreate the objects from -the bytes. The various DBM-related modules support a family of hash-based file -formats that store a mapping of strings to other strings. - -The list of modules described in this chapter is: - - -.. toctree:: - - pickle.rst - copyreg.rst - shelve.rst - marshal.rst - dbm.rst - sqlite3.rst diff --git a/Doc/library/pickle.rst b/Doc/library/pickle.rst deleted file mode 100644 index d6be4ba33fa80b..00000000000000 --- a/Doc/library/pickle.rst +++ /dev/null @@ -1,1222 +0,0 @@ -:mod:`pickle` --- Python object serialization -============================================= - -.. module:: pickle - :synopsis: Convert Python objects to streams of bytes and back. - -.. sectionauthor:: Jim Kerr . -.. sectionauthor:: Barry Warsaw - -**Source code:** :source:`Lib/pickle.py` - -.. index:: - single: persistence - pair: persistent; objects - pair: serializing; objects - pair: marshalling; objects - pair: flattening; objects - pair: pickling; objects - --------------- - -The :mod:`pickle` module implements binary protocols for serializing and -de-serializing a Python object structure. *"Pickling"* is the process -whereby a Python object hierarchy is converted into a byte stream, and -*"unpickling"* is the inverse operation, whereby a byte stream -(from a :term:`binary file` or :term:`bytes-like object`) is converted -back into an object hierarchy. Pickling (and unpickling) is alternatively -known as "serialization", "marshalling," [#]_ or "flattening"; however, to -avoid confusion, the terms used here are "pickling" and "unpickling". - -.. warning:: - - The ``pickle`` module **is not secure**. Only unpickle data you trust. - - It is possible to construct malicious pickle data which will **execute - arbitrary code during unpickling**. Never unpickle data that could have come - from an untrusted source, or that could have been tampered with. - - Consider signing data with :mod:`hmac` if you need to ensure that it has not - been tampered with. - - Safer serialization formats such as :mod:`json` may be more appropriate if - you are processing untrusted data. See :ref:`comparison-with-json`. - - -Relationship to other Python modules ------------------------------------- - -Comparison with ``marshal`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Python has a more primitive serialization module called :mod:`marshal`, but in -general :mod:`pickle` should always be the preferred way to serialize Python -objects. :mod:`marshal` exists primarily to support Python's :file:`.pyc` -files. - -The :mod:`pickle` module differs from :mod:`marshal` in several significant ways: - -* The :mod:`pickle` module keeps track of the objects it has already serialized, - so that later references to the same object won't be serialized again. - :mod:`marshal` doesn't do this. - - This has implications both for recursive objects and object sharing. Recursive - objects are objects that contain references to themselves. These are not - handled by marshal, and in fact, attempting to marshal recursive objects will - crash your Python interpreter. Object sharing happens when there are multiple - references to the same object in different places in the object hierarchy being - serialized. :mod:`pickle` stores such objects only once, and ensures that all - other references point to the master copy. Shared objects remain shared, which - can be very important for mutable objects. - -* :mod:`marshal` cannot be used to serialize user-defined classes and their - instances. :mod:`pickle` can save and restore class instances transparently, - however the class definition must be importable and live in the same module as - when the object was stored. - -* The :mod:`marshal` serialization format is not guaranteed to be portable - across Python versions. Because its primary job in life is to support - :file:`.pyc` files, the Python implementers reserve the right to change the - serialization format in non-backwards compatible ways should the need arise. - The :mod:`pickle` serialization format is guaranteed to be backwards compatible - across Python releases provided a compatible pickle protocol is chosen and - pickling and unpickling code deals with Python 2 to Python 3 type differences - if your data is crossing that unique breaking change language boundary. - - -.. _comparison-with-json: - -Comparison with ``json`` -^^^^^^^^^^^^^^^^^^^^^^^^ - -There are fundamental differences between the pickle protocols and -`JSON (JavaScript Object Notation) `_: - -* JSON is a text serialization format (it outputs unicode text, although - most of the time it is then encoded to ``utf-8``), while pickle is - a binary serialization format; - -* JSON is human-readable, while pickle is not; - -* JSON is interoperable and widely used outside of the Python ecosystem, - while pickle is Python-specific; - -* JSON, by default, can only represent a subset of the Python built-in - types, and no custom classes; pickle can represent an extremely large - number of Python types (many of them automatically, by clever usage - of Python's introspection facilities; complex cases can be tackled by - implementing :ref:`specific object APIs `); - -* Unlike pickle, deserializing untrusted JSON does not in itself create an - arbitrary code execution vulnerability. - -.. seealso:: - The :mod:`json` module: a standard library module allowing JSON - serialization and deserialization. - - -.. _pickle-protocols: - -Data stream format ------------------- - -.. index:: - single: External Data Representation - -The data format used by :mod:`pickle` is Python-specific. This has the -advantage that there are no restrictions imposed by external standards such as -JSON or XDR (which can't represent pointer sharing); however it means that -non-Python programs may not be able to reconstruct pickled Python objects. - -By default, the :mod:`pickle` data format uses a relatively compact binary -representation. If you need optimal size characteristics, you can efficiently -:doc:`compress ` pickled data. - -The module :mod:`pickletools` contains tools for analyzing data streams -generated by :mod:`pickle`. :mod:`pickletools` source code has extensive -comments about opcodes used by pickle protocols. - -There are currently 6 different protocols which can be used for pickling. -The higher the protocol used, the more recent the version of Python needed -to read the pickle produced. - -* Protocol version 0 is the original "human-readable" protocol and is - backwards compatible with earlier versions of Python. - -* Protocol version 1 is an old binary format which is also compatible with - earlier versions of Python. - -* Protocol version 2 was introduced in Python 2.3. It provides much more - efficient pickling of :term:`new-style classes `. Refer to :pep:`307` for - information about improvements brought by protocol 2. - -* Protocol version 3 was added in Python 3.0. It has explicit support for - :class:`bytes` objects and cannot be unpickled by Python 2.x. This was - the default protocol in Python 3.0--3.7. - -* Protocol version 4 was added in Python 3.4. It adds support for very large - objects, pickling more kinds of objects, and some data format - optimizations. It is the default protocol starting with Python 3.8. - Refer to :pep:`3154` for information about improvements brought by - protocol 4. - -* Protocol version 5 was added in Python 3.8. It adds support for out-of-band - data and speedup for in-band data. Refer to :pep:`574` for information about - improvements brought by protocol 5. - -.. note:: - Serialization is a more primitive notion than persistence; although - :mod:`pickle` reads and writes file objects, it does not handle the issue of - naming persistent objects, nor the (even more complicated) issue of concurrent - access to persistent objects. The :mod:`pickle` module can transform a complex - object into a byte stream and it can transform the byte stream into an object - with the same internal structure. Perhaps the most obvious thing to do with - these byte streams is to write them onto a file, but it is also conceivable to - send them across a network or store them in a database. The :mod:`shelve` - module provides a simple interface to pickle and unpickle objects on - DBM-style database files. - - -Module Interface ----------------- - -To serialize an object hierarchy, you simply call the :func:`dumps` function. -Similarly, to de-serialize a data stream, you call the :func:`loads` function. -However, if you want more control over serialization and de-serialization, -you can create a :class:`Pickler` or an :class:`Unpickler` object, respectively. - -The :mod:`pickle` module provides the following constants: - - -.. data:: HIGHEST_PROTOCOL - - An integer, the highest :ref:`protocol version ` - available. This value can be passed as a *protocol* value to functions - :func:`dump` and :func:`dumps` as well as the :class:`Pickler` - constructor. - -.. data:: DEFAULT_PROTOCOL - - An integer, the default :ref:`protocol version ` used - for pickling. May be less than :data:`HIGHEST_PROTOCOL`. Currently the - default protocol is 4, first introduced in Python 3.4 and incompatible - with previous versions. - - .. versionchanged:: 3.0 - - The default protocol is 3. - - .. versionchanged:: 3.8 - - The default protocol is 4. - -The :mod:`pickle` module provides the following functions to make the pickling -process more convenient: - -.. function:: dump(obj, file, protocol=None, *, fix_imports=True, buffer_callback=None) - - Write the pickled representation of the object *obj* to the open - :term:`file object` *file*. This is equivalent to - ``Pickler(file, protocol).dump(obj)``. - - Arguments *file*, *protocol*, *fix_imports* and *buffer_callback* have - the same meaning as in the :class:`Pickler` constructor. - - .. versionchanged:: 3.8 - The *buffer_callback* argument was added. - -.. function:: dumps(obj, protocol=None, *, fix_imports=True, buffer_callback=None) - - Return the pickled representation of the object *obj* as a :class:`bytes` object, - instead of writing it to a file. - - Arguments *protocol*, *fix_imports* and *buffer_callback* have the same - meaning as in the :class:`Pickler` constructor. - - .. versionchanged:: 3.8 - The *buffer_callback* argument was added. - -.. function:: load(file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) - - Read the pickled representation of an object from the open :term:`file object` - *file* and return the reconstituted object hierarchy specified therein. - This is equivalent to ``Unpickler(file).load()``. - - The protocol version of the pickle is detected automatically, so no - protocol argument is needed. Bytes past the pickled representation - of the object are ignored. - - Arguments *file*, *fix_imports*, *encoding*, *errors*, *strict* and *buffers* - have the same meaning as in the :class:`Unpickler` constructor. - - .. versionchanged:: 3.8 - The *buffers* argument was added. - -.. function:: loads(data, /, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) - - Return the reconstituted object hierarchy of the pickled representation - *data* of an object. *data* must be a :term:`bytes-like object`. - - The protocol version of the pickle is detected automatically, so no - protocol argument is needed. Bytes past the pickled representation - of the object are ignored. - - Arguments *fix_imports*, *encoding*, *errors*, *strict* and *buffers* - have the same meaning as in the :class:`Unpickler` constructor. - - .. versionchanged:: 3.8 - The *buffers* argument was added. - - -The :mod:`pickle` module defines three exceptions: - -.. exception:: PickleError - - Common base class for the other pickling exceptions. It inherits - :exc:`Exception`. - -.. exception:: PicklingError - - Error raised when an unpicklable object is encountered by :class:`Pickler`. - It inherits :exc:`PickleError`. - - Refer to :ref:`pickle-picklable` to learn what kinds of objects can be - pickled. - -.. exception:: UnpicklingError - - Error raised when there is a problem unpickling an object, such as a data - corruption or a security violation. It inherits :exc:`PickleError`. - - Note that other exceptions may also be raised during unpickling, including - (but not necessarily limited to) AttributeError, EOFError, ImportError, and - IndexError. - - -The :mod:`pickle` module exports three classes, :class:`Pickler`, -:class:`Unpickler` and :class:`PickleBuffer`: - -.. class:: Pickler(file, protocol=None, *, fix_imports=True, buffer_callback=None) - - This takes a binary file for writing a pickle data stream. - - The optional *protocol* argument, an integer, tells the pickler to use - the given protocol; supported protocols are 0 to :data:`HIGHEST_PROTOCOL`. - If not specified, the default is :data:`DEFAULT_PROTOCOL`. If a negative - number is specified, :data:`HIGHEST_PROTOCOL` is selected. - - The *file* argument must have a write() method that accepts a single bytes - argument. It can thus be an on-disk file opened for binary writing, an - :class:`io.BytesIO` instance, or any other custom object that meets this - interface. - - If *fix_imports* is true and *protocol* is less than 3, pickle will try to - map the new Python 3 names to the old module names used in Python 2, so - that the pickle data stream is readable with Python 2. - - If *buffer_callback* is None (the default), buffer views are - serialized into *file* as part of the pickle stream. - - If *buffer_callback* is not None, then it can be called any number - of times with a buffer view. If the callback returns a false value - (such as None), the given buffer is :ref:`out-of-band `; - otherwise the buffer is serialized in-band, i.e. inside the pickle stream. - - It is an error if *buffer_callback* is not None and *protocol* is - None or smaller than 5. - - .. versionchanged:: 3.8 - The *buffer_callback* argument was added. - - .. method:: dump(obj) - - Write the pickled representation of *obj* to the open file object given in - the constructor. - - .. method:: persistent_id(obj) - - Do nothing by default. This exists so a subclass can override it. - - If :meth:`persistent_id` returns ``None``, *obj* is pickled as usual. Any - other value causes :class:`Pickler` to emit the returned value as a - persistent ID for *obj*. The meaning of this persistent ID should be - defined by :meth:`Unpickler.persistent_load`. Note that the value - returned by :meth:`persistent_id` cannot itself have a persistent ID. - - See :ref:`pickle-persistent` for details and examples of uses. - - .. attribute:: dispatch_table - - A pickler object's dispatch table is a registry of *reduction - functions* of the kind which can be declared using - :func:`copyreg.pickle`. It is a mapping whose keys are classes - and whose values are reduction functions. A reduction function - takes a single argument of the associated class and should - conform to the same interface as a :meth:`__reduce__` - method. - - By default, a pickler object will not have a - :attr:`dispatch_table` attribute, and it will instead use the - global dispatch table managed by the :mod:`copyreg` module. - However, to customize the pickling for a specific pickler object - one can set the :attr:`dispatch_table` attribute to a dict-like - object. Alternatively, if a subclass of :class:`Pickler` has a - :attr:`dispatch_table` attribute then this will be used as the - default dispatch table for instances of that class. - - See :ref:`pickle-dispatch` for usage examples. - - .. versionadded:: 3.3 - - .. method:: reducer_override(obj) - - Special reducer that can be defined in :class:`Pickler` subclasses. This - method has priority over any reducer in the :attr:`dispatch_table`. It - should conform to the same interface as a :meth:`__reduce__` method, and - can optionally return ``NotImplemented`` to fallback on - :attr:`dispatch_table`-registered reducers to pickle ``obj``. - - For a detailed example, see :ref:`reducer_override`. - - .. versionadded:: 3.8 - - .. attribute:: fast - - Deprecated. Enable fast mode if set to a true value. The fast mode - disables the usage of memo, therefore speeding the pickling process by not - generating superfluous PUT opcodes. It should not be used with - self-referential objects, doing otherwise will cause :class:`Pickler` to - recurse infinitely. - - Use :func:`pickletools.optimize` if you need more compact pickles. - - -.. class:: Unpickler(file, *, fix_imports=True, encoding="ASCII", errors="strict", buffers=None) - - This takes a binary file for reading a pickle data stream. - - The protocol version of the pickle is detected automatically, so no - protocol argument is needed. - - The argument *file* must have three methods, a read() method that takes an - integer argument, a readinto() method that takes a buffer argument - and a readline() method that requires no arguments, as in the - :class:`io.BufferedIOBase` interface. Thus *file* can be an on-disk file - opened for binary reading, an :class:`io.BytesIO` object, or any other - custom object that meets this interface. - - The optional arguments *fix_imports*, *encoding* and *errors* are used - to control compatibility support for pickle stream generated by Python 2. - If *fix_imports* is true, pickle will try to map the old Python 2 names - to the new names used in Python 3. The *encoding* and *errors* tell - pickle how to decode 8-bit string instances pickled by Python 2; - these default to 'ASCII' and 'strict', respectively. The *encoding* can - be 'bytes' to read these 8-bit string instances as bytes objects. - Using ``encoding='latin1'`` is required for unpickling NumPy arrays and - instances of :class:`~datetime.datetime`, :class:`~datetime.date` and - :class:`~datetime.time` pickled by Python 2. - - If *buffers* is None (the default), then all data necessary for - deserialization must be contained in the pickle stream. This means - that the *buffer_callback* argument was None when a :class:`Pickler` - was instantiated (or when :func:`dump` or :func:`dumps` was called). - - If *buffers* is not None, it should be an iterable of buffer-enabled - objects that is consumed each time the pickle stream references - an :ref:`out-of-band ` buffer view. Such buffers have been - given in order to the *buffer_callback* of a Pickler object. - - .. versionchanged:: 3.8 - The *buffers* argument was added. - - .. method:: load() - - Read the pickled representation of an object from the open file object - given in the constructor, and return the reconstituted object hierarchy - specified therein. Bytes past the pickled representation of the object - are ignored. - - .. method:: persistent_load(pid) - - Raise an :exc:`UnpicklingError` by default. - - If defined, :meth:`persistent_load` should return the object specified by - the persistent ID *pid*. If an invalid persistent ID is encountered, an - :exc:`UnpicklingError` should be raised. - - See :ref:`pickle-persistent` for details and examples of uses. - - .. method:: find_class(module, name) - - Import *module* if necessary and return the object called *name* from it, - where the *module* and *name* arguments are :class:`str` objects. Note, - unlike its name suggests, :meth:`find_class` is also used for finding - functions. - - Subclasses may override this to gain control over what type of objects and - how they can be loaded, potentially reducing security risks. Refer to - :ref:`pickle-restrict` for details. - - .. audit-event:: pickle.find_class module,name pickle.Unpickler.find_class - -.. class:: PickleBuffer(buffer) - - A wrapper for a buffer representing picklable data. *buffer* must be a - :ref:`buffer-providing ` object, such as a - :term:`bytes-like object` or a N-dimensional array. - - :class:`PickleBuffer` is itself a buffer provider, therefore it is - possible to pass it to other APIs expecting a buffer-providing object, - such as :class:`memoryview`. - - :class:`PickleBuffer` objects can only be serialized using pickle - protocol 5 or higher. They are eligible for - :ref:`out-of-band serialization `. - - .. versionadded:: 3.8 - - .. method:: raw() - - Return a :class:`memoryview` of the memory area underlying this buffer. - The returned object is a one-dimensional, C-contiguous memoryview - with format ``B`` (unsigned bytes). :exc:`BufferError` is raised if - the buffer is neither C- nor Fortran-contiguous. - - .. method:: release() - - Release the underlying buffer exposed by the PickleBuffer object. - - -.. _pickle-picklable: - -What can be pickled and unpickled? ----------------------------------- - -The following types can be pickled: - -* built-in constants (``None``, ``True``, ``False``, ``Ellipsis``, and - ``NotImplemented``); - -* integers, floating-point numbers, complex numbers; - -* strings, bytes, bytearrays; - -* tuples, lists, sets, and dictionaries containing only picklable objects; - -* functions (built-in and user-defined) accessible from the top level of a - module (using :keyword:`def`, not :keyword:`lambda`); - -* classes accessible from the top level of a module; - -* instances of such classes whose the result of calling :meth:`__getstate__` - is picklable (see section :ref:`pickle-inst` for details). - -Attempts to pickle unpicklable objects will raise the :exc:`PicklingError` -exception; when this happens, an unspecified number of bytes may have already -been written to the underlying file. Trying to pickle a highly recursive data -structure may exceed the maximum recursion depth, a :exc:`RecursionError` will be -raised in this case. You can carefully raise this limit with -:func:`sys.setrecursionlimit`. - -Note that functions (built-in and user-defined) are pickled by fully -:term:`qualified name`, not by value. [#]_ This means that only the function name is -pickled, along with the name of the containing module and classes. Neither -the function's code, nor any of its function attributes are pickled. Thus the -defining module must be importable in the unpickling environment, and the module -must contain the named object, otherwise an exception will be raised. [#]_ - -Similarly, classes are pickled by fully qualified name, so the same restrictions in -the unpickling environment apply. Note that none of the class's code or data is -pickled, so in the following example the class attribute ``attr`` is not -restored in the unpickling environment:: - - class Foo: - attr = 'A class attribute' - - picklestring = pickle.dumps(Foo) - -These restrictions are why picklable functions and classes must be defined at -the top level of a module. - -Similarly, when class instances are pickled, their class's code and data are not -pickled along with them. Only the instance data are pickled. This is done on -purpose, so you can fix bugs in a class or add methods to the class and still -load objects that were created with an earlier version of the class. If you -plan to have long-lived objects that will see many versions of a class, it may -be worthwhile to put a version number in the objects so that suitable -conversions can be made by the class's :meth:`__setstate__` method. - - -.. _pickle-inst: - -Pickling Class Instances ------------------------- - -.. currentmodule:: None - -In this section, we describe the general mechanisms available to you to define, -customize, and control how class instances are pickled and unpickled. - -In most cases, no additional code is needed to make instances picklable. By -default, pickle will retrieve the class and the attributes of an instance via -introspection. When a class instance is unpickled, its :meth:`__init__` method -is usually *not* invoked. The default behaviour first creates an uninitialized -instance and then restores the saved attributes. The following code shows an -implementation of this behaviour:: - - def save(obj): - return (obj.__class__, obj.__dict__) - - def restore(cls, attributes): - obj = cls.__new__(cls) - obj.__dict__.update(attributes) - return obj - -Classes can alter the default behaviour by providing one or several special -methods: - -.. method:: object.__getnewargs_ex__() - - In protocols 2 and newer, classes that implements the - :meth:`__getnewargs_ex__` method can dictate the values passed to the - :meth:`__new__` method upon unpickling. The method must return a pair - ``(args, kwargs)`` where *args* is a tuple of positional arguments - and *kwargs* a dictionary of named arguments for constructing the - object. Those will be passed to the :meth:`__new__` method upon - unpickling. - - You should implement this method if the :meth:`__new__` method of your - class requires keyword-only arguments. Otherwise, it is recommended for - compatibility to implement :meth:`__getnewargs__`. - - .. versionchanged:: 3.6 - :meth:`__getnewargs_ex__` is now used in protocols 2 and 3. - - -.. method:: object.__getnewargs__() - - This method serves a similar purpose as :meth:`__getnewargs_ex__`, but - supports only positional arguments. It must return a tuple of arguments - ``args`` which will be passed to the :meth:`__new__` method upon unpickling. - - :meth:`__getnewargs__` will not be called if :meth:`__getnewargs_ex__` is - defined. - - .. versionchanged:: 3.6 - Before Python 3.6, :meth:`__getnewargs__` was called instead of - :meth:`__getnewargs_ex__` in protocols 2 and 3. - - -.. method:: object.__getstate__() - - Classes can further influence how their instances are pickled by overriding - the method :meth:`__getstate__`. It is called and the returned object - is pickled as the contents for the instance, instead of a default state. - There are several cases: - - * For a class that has no instance :attr:`~object.__dict__` and no - :attr:`~object.__slots__`, the default state is ``None``. - - * For a class that has an instance :attr:`~object.__dict__` and no - :attr:`~object.__slots__`, the default state is ``self.__dict__``. - - * For a class that has an instance :attr:`~object.__dict__` and - :attr:`~object.__slots__`, the default state is a tuple consisting of two - dictionaries: ``self.__dict__``, and a dictionary mapping slot - names to slot values. Only slots that have a value are - included in the latter. - - * For a class that has :attr:`~object.__slots__` and no instance - :attr:`~object.__dict__`, the default state is a tuple whose first item - is ``None`` and whose second item is a dictionary mapping slot names - to slot values described in the previous bullet. - - .. versionchanged:: 3.11 - Added the default implementation of the ``__getstate__()`` method in the - :class:`object` class. - - -.. method:: object.__setstate__(state) - - Upon unpickling, if the class defines :meth:`__setstate__`, it is called with - the unpickled state. In that case, there is no requirement for the state - object to be a dictionary. Otherwise, the pickled state must be a dictionary - and its items are assigned to the new instance's dictionary. - - .. note:: - - If :meth:`__getstate__` returns a false value, the :meth:`__setstate__` - method will not be called upon unpickling. - - -Refer to the section :ref:`pickle-state` for more information about how to use -the methods :meth:`__getstate__` and :meth:`__setstate__`. - -.. note:: - - At unpickling time, some methods like :meth:`__getattr__`, - :meth:`__getattribute__`, or :meth:`__setattr__` may be called upon the - instance. In case those methods rely on some internal invariant being - true, the type should implement :meth:`__new__` to establish such an - invariant, as :meth:`__init__` is not called when unpickling an - instance. - -.. index:: pair: copy; protocol - -As we shall see, pickle does not use directly the methods described above. In -fact, these methods are part of the copy protocol which implements the -:meth:`__reduce__` special method. The copy protocol provides a unified -interface for retrieving the data necessary for pickling and copying -objects. [#]_ - -Although powerful, implementing :meth:`__reduce__` directly in your classes is -error prone. For this reason, class designers should use the high-level -interface (i.e., :meth:`__getnewargs_ex__`, :meth:`__getstate__` and -:meth:`__setstate__`) whenever possible. We will show, however, cases where -using :meth:`__reduce__` is the only option or leads to more efficient pickling -or both. - -.. method:: object.__reduce__() - - The interface is currently defined as follows. The :meth:`__reduce__` method - takes no argument and shall return either a string or preferably a tuple (the - returned object is often referred to as the "reduce value"). - - If a string is returned, the string should be interpreted as the name of a - global variable. It should be the object's local name relative to its - module; the pickle module searches the module namespace to determine the - object's module. This behaviour is typically useful for singletons. - - When a tuple is returned, it must be between two and six items long. - Optional items can either be omitted, or ``None`` can be provided as their - value. The semantics of each item are in order: - - .. XXX Mention __newobj__ special-case? - - * A callable object that will be called to create the initial version of the - object. - - * A tuple of arguments for the callable object. An empty tuple must be given - if the callable does not accept any argument. - - * Optionally, the object's state, which will be passed to the object's - :meth:`__setstate__` method as previously described. If the object has no - such method then, the value must be a dictionary and it will be added to - the object's :attr:`~object.__dict__` attribute. - - * Optionally, an iterator (and not a sequence) yielding successive items. - These items will be appended to the object either using - ``obj.append(item)`` or, in batch, using ``obj.extend(list_of_items)``. - This is primarily used for list subclasses, but may be used by other - classes as long as they have :meth:`append` and :meth:`extend` methods with - the appropriate signature. (Whether :meth:`append` or :meth:`extend` is - used depends on which pickle protocol version is used as well as the number - of items to append, so both must be supported.) - - * Optionally, an iterator (not a sequence) yielding successive key-value - pairs. These items will be stored to the object using ``obj[key] = - value``. This is primarily used for dictionary subclasses, but may be used - by other classes as long as they implement :meth:`__setitem__`. - - * Optionally, a callable with a ``(obj, state)`` signature. This - callable allows the user to programmatically control the state-updating - behavior of a specific object, instead of using ``obj``'s static - :meth:`__setstate__` method. If not ``None``, this callable will have - priority over ``obj``'s :meth:`__setstate__`. - - .. versionadded:: 3.8 - The optional sixth tuple item, ``(obj, state)``, was added. - - -.. method:: object.__reduce_ex__(protocol) - - Alternatively, a :meth:`__reduce_ex__` method may be defined. The only - difference is this method should take a single integer argument, the protocol - version. When defined, pickle will prefer it over the :meth:`__reduce__` - method. In addition, :meth:`__reduce__` automatically becomes a synonym for - the extended version. The main use for this method is to provide - backwards-compatible reduce values for older Python releases. - -.. currentmodule:: pickle - -.. _pickle-persistent: - -Persistence of External Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: persistent_id (pickle protocol) - single: persistent_load (pickle protocol) - -For the benefit of object persistence, the :mod:`pickle` module supports the -notion of a reference to an object outside the pickled data stream. Such -objects are referenced by a persistent ID, which should be either a string of -alphanumeric characters (for protocol 0) [#]_ or just an arbitrary object (for -any newer protocol). - -The resolution of such persistent IDs is not defined by the :mod:`pickle` -module; it will delegate this resolution to the user-defined methods on the -pickler and unpickler, :meth:`~Pickler.persistent_id` and -:meth:`~Unpickler.persistent_load` respectively. - -To pickle objects that have an external persistent ID, the pickler must have a -custom :meth:`~Pickler.persistent_id` method that takes an object as an -argument and returns either ``None`` or the persistent ID for that object. -When ``None`` is returned, the pickler simply pickles the object as normal. -When a persistent ID string is returned, the pickler will pickle that object, -along with a marker so that the unpickler will recognize it as a persistent ID. - -To unpickle external objects, the unpickler must have a custom -:meth:`~Unpickler.persistent_load` method that takes a persistent ID object and -returns the referenced object. - -Here is a comprehensive example presenting how persistent ID can be used to -pickle external objects by reference. - -.. literalinclude:: ../includes/dbpickle.py - -.. _pickle-dispatch: - -Dispatch Tables -^^^^^^^^^^^^^^^ - -If one wants to customize pickling of some classes without disturbing -any other code which depends on pickling, then one can create a -pickler with a private dispatch table. - -The global dispatch table managed by the :mod:`copyreg` module is -available as :data:`copyreg.dispatch_table`. Therefore, one may -choose to use a modified copy of :data:`copyreg.dispatch_table` as a -private dispatch table. - -For example :: - - f = io.BytesIO() - p = pickle.Pickler(f) - p.dispatch_table = copyreg.dispatch_table.copy() - p.dispatch_table[SomeClass] = reduce_SomeClass - -creates an instance of :class:`pickle.Pickler` with a private dispatch -table which handles the ``SomeClass`` class specially. Alternatively, -the code :: - - class MyPickler(pickle.Pickler): - dispatch_table = copyreg.dispatch_table.copy() - dispatch_table[SomeClass] = reduce_SomeClass - f = io.BytesIO() - p = MyPickler(f) - -does the same but all instances of ``MyPickler`` will by default -share the private dispatch table. On the other hand, the code :: - - copyreg.pickle(SomeClass, reduce_SomeClass) - f = io.BytesIO() - p = pickle.Pickler(f) - -modifies the global dispatch table shared by all users of the :mod:`copyreg` module. - -.. _pickle-state: - -Handling Stateful Objects -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: __getstate__() (copy protocol) - single: __setstate__() (copy protocol) - -Here's an example that shows how to modify pickling behavior for a class. -The :class:`TextReader` class opens a text file, and returns the line number and -line contents each time its :meth:`!readline` method is called. If a -:class:`TextReader` instance is pickled, all attributes *except* the file object -member are saved. When the instance is unpickled, the file is reopened, and -reading resumes from the last location. The :meth:`__setstate__` and -:meth:`__getstate__` methods are used to implement this behavior. :: - - class TextReader: - """Print and number lines in a text file.""" - - def __init__(self, filename): - self.filename = filename - self.file = open(filename) - self.lineno = 0 - - def readline(self): - self.lineno += 1 - line = self.file.readline() - if not line: - return None - if line.endswith('\n'): - line = line[:-1] - return "%i: %s" % (self.lineno, line) - - def __getstate__(self): - # Copy the object's state from self.__dict__ which contains - # all our instance attributes. Always use the dict.copy() - # method to avoid modifying the original state. - state = self.__dict__.copy() - # Remove the unpicklable entries. - del state['file'] - return state - - def __setstate__(self, state): - # Restore instance attributes (i.e., filename and lineno). - self.__dict__.update(state) - # Restore the previously opened file's state. To do so, we need to - # reopen it and read from it until the line count is restored. - file = open(self.filename) - for _ in range(self.lineno): - file.readline() - # Finally, save the file. - self.file = file - - -A sample usage might be something like this:: - - >>> reader = TextReader("hello.txt") - >>> reader.readline() - '1: Hello world!' - >>> reader.readline() - '2: I am line number two.' - >>> new_reader = pickle.loads(pickle.dumps(reader)) - >>> new_reader.readline() - '3: Goodbye!' - -.. _reducer_override: - -Custom Reduction for Types, Functions, and Other Objects --------------------------------------------------------- - -.. versionadded:: 3.8 - -Sometimes, :attr:`~Pickler.dispatch_table` may not be flexible enough. -In particular we may want to customize pickling based on another criterion -than the object's type, or we may want to customize the pickling of -functions and classes. - -For those cases, it is possible to subclass from the :class:`Pickler` class and -implement a :meth:`~Pickler.reducer_override` method. This method can return an -arbitrary reduction tuple (see :meth:`__reduce__`). It can alternatively return -``NotImplemented`` to fallback to the traditional behavior. - -If both the :attr:`~Pickler.dispatch_table` and -:meth:`~Pickler.reducer_override` are defined, then -:meth:`~Pickler.reducer_override` method takes priority. - -.. Note:: - For performance reasons, :meth:`~Pickler.reducer_override` may not be - called for the following objects: ``None``, ``True``, ``False``, and - exact instances of :class:`int`, :class:`float`, :class:`bytes`, - :class:`str`, :class:`dict`, :class:`set`, :class:`frozenset`, :class:`list` - and :class:`tuple`. - -Here is a simple example where we allow pickling and reconstructing -a given class:: - - import io - import pickle - - class MyClass: - my_attribute = 1 - - class MyPickler(pickle.Pickler): - def reducer_override(self, obj): - """Custom reducer for MyClass.""" - if getattr(obj, "__name__", None) == "MyClass": - return type, (obj.__name__, obj.__bases__, - {'my_attribute': obj.my_attribute}) - else: - # For any other object, fallback to usual reduction - return NotImplemented - - f = io.BytesIO() - p = MyPickler(f) - p.dump(MyClass) - - del MyClass - - unpickled_class = pickle.loads(f.getvalue()) - - assert isinstance(unpickled_class, type) - assert unpickled_class.__name__ == "MyClass" - assert unpickled_class.my_attribute == 1 - - -.. _pickle-oob: - -Out-of-band Buffers -------------------- - -.. versionadded:: 3.8 - -In some contexts, the :mod:`pickle` module is used to transfer massive amounts -of data. Therefore, it can be important to minimize the number of memory -copies, to preserve performance and resource consumption. However, normal -operation of the :mod:`pickle` module, as it transforms a graph-like structure -of objects into a sequential stream of bytes, intrinsically involves copying -data to and from the pickle stream. - -This constraint can be eschewed if both the *provider* (the implementation -of the object types to be transferred) and the *consumer* (the implementation -of the communications system) support the out-of-band transfer facilities -provided by pickle protocol 5 and higher. - -Provider API -^^^^^^^^^^^^ - -The large data objects to be pickled must implement a :meth:`__reduce_ex__` -method specialized for protocol 5 and higher, which returns a -:class:`PickleBuffer` instance (instead of e.g. a :class:`bytes` object) -for any large data. - -A :class:`PickleBuffer` object *signals* that the underlying buffer is -eligible for out-of-band data transfer. Those objects remain compatible -with normal usage of the :mod:`pickle` module. However, consumers can also -opt-in to tell :mod:`pickle` that they will handle those buffers by -themselves. - -Consumer API -^^^^^^^^^^^^ - -A communications system can enable custom handling of the :class:`PickleBuffer` -objects generated when serializing an object graph. - -On the sending side, it needs to pass a *buffer_callback* argument to -:class:`Pickler` (or to the :func:`dump` or :func:`dumps` function), which -will be called with each :class:`PickleBuffer` generated while pickling -the object graph. Buffers accumulated by the *buffer_callback* will not -see their data copied into the pickle stream, only a cheap marker will be -inserted. - -On the receiving side, it needs to pass a *buffers* argument to -:class:`Unpickler` (or to the :func:`load` or :func:`loads` function), -which is an iterable of the buffers which were passed to *buffer_callback*. -That iterable should produce buffers in the same order as they were passed -to *buffer_callback*. Those buffers will provide the data expected by the -reconstructors of the objects whose pickling produced the original -:class:`PickleBuffer` objects. - -Between the sending side and the receiving side, the communications system -is free to implement its own transfer mechanism for out-of-band buffers. -Potential optimizations include the use of shared memory or datatype-dependent -compression. - -Example -^^^^^^^ - -Here is a trivial example where we implement a :class:`bytearray` subclass -able to participate in out-of-band buffer pickling:: - - class ZeroCopyByteArray(bytearray): - - def __reduce_ex__(self, protocol): - if protocol >= 5: - return type(self)._reconstruct, (PickleBuffer(self),), None - else: - # PickleBuffer is forbidden with pickle protocols <= 4. - return type(self)._reconstruct, (bytearray(self),) - - @classmethod - def _reconstruct(cls, obj): - with memoryview(obj) as m: - # Get a handle over the original buffer object - obj = m.obj - if type(obj) is cls: - # Original buffer object is a ZeroCopyByteArray, return it - # as-is. - return obj - else: - return cls(obj) - -The reconstructor (the ``_reconstruct`` class method) returns the buffer's -providing object if it has the right type. This is an easy way to simulate -zero-copy behaviour on this toy example. - -On the consumer side, we can pickle those objects the usual way, which -when unserialized will give us a copy of the original object:: - - b = ZeroCopyByteArray(b"abc") - data = pickle.dumps(b, protocol=5) - new_b = pickle.loads(data) - print(b == new_b) # True - print(b is new_b) # False: a copy was made - -But if we pass a *buffer_callback* and then give back the accumulated -buffers when unserializing, we are able to get back the original object:: - - b = ZeroCopyByteArray(b"abc") - buffers = [] - data = pickle.dumps(b, protocol=5, buffer_callback=buffers.append) - new_b = pickle.loads(data, buffers=buffers) - print(b == new_b) # True - print(b is new_b) # True: no copy was made - -This example is limited by the fact that :class:`bytearray` allocates its -own memory: you cannot create a :class:`bytearray` instance that is backed -by another object's memory. However, third-party datatypes such as NumPy -arrays do not have this limitation, and allow use of zero-copy pickling -(or making as few copies as possible) when transferring between distinct -processes or systems. - -.. seealso:: :pep:`574` -- Pickle protocol 5 with out-of-band data - - -.. _pickle-restrict: - -Restricting Globals -------------------- - -.. index:: - single: find_class() (pickle protocol) - -By default, unpickling will import any class or function that it finds in the -pickle data. For many applications, this behaviour is unacceptable as it -permits the unpickler to import and invoke arbitrary code. Just consider what -this hand-crafted pickle data stream does when loaded:: - - >>> import pickle - >>> pickle.loads(b"cos\nsystem\n(S'echo hello world'\ntR.") - hello world - 0 - -In this example, the unpickler imports the :func:`os.system` function and then -apply the string argument "echo hello world". Although this example is -inoffensive, it is not difficult to imagine one that could damage your system. - -For this reason, you may want to control what gets unpickled by customizing -:meth:`Unpickler.find_class`. Unlike its name suggests, -:meth:`Unpickler.find_class` is called whenever a global (i.e., a class or -a function) is requested. Thus it is possible to either completely forbid -globals or restrict them to a safe subset. - -Here is an example of an unpickler allowing only few safe classes from the -:mod:`builtins` module to be loaded:: - - import builtins - import io - import pickle - - safe_builtins = { - 'range', - 'complex', - 'set', - 'frozenset', - 'slice', - } - - class RestrictedUnpickler(pickle.Unpickler): - - def find_class(self, module, name): - # Only allow safe classes from builtins. - if module == "builtins" and name in safe_builtins: - return getattr(builtins, name) - # Forbid everything else. - raise pickle.UnpicklingError("global '%s.%s' is forbidden" % - (module, name)) - - def restricted_loads(s): - """Helper function analogous to pickle.loads().""" - return RestrictedUnpickler(io.BytesIO(s)).load() - -A sample usage of our unpickler working as intended:: - - >>> restricted_loads(pickle.dumps([1, 2, range(15)])) - [1, 2, range(0, 15)] - >>> restricted_loads(b"cos\nsystem\n(S'echo hello world'\ntR.") - Traceback (most recent call last): - ... - pickle.UnpicklingError: global 'os.system' is forbidden - >>> restricted_loads(b'cbuiltins\neval\n' - ... b'(S\'getattr(__import__("os"), "system")' - ... b'("echo hello world")\'\ntR.') - Traceback (most recent call last): - ... - pickle.UnpicklingError: global 'builtins.eval' is forbidden - - -.. XXX Add note about how extension codes could evade our protection - mechanism (e.g. cached classes do not invokes find_class()). - -As our examples shows, you have to be careful with what you allow to be -unpickled. Therefore if security is a concern, you may want to consider -alternatives such as the marshalling API in :mod:`xmlrpc.client` or -third-party solutions. - - -Performance ------------ - -Recent versions of the pickle protocol (from protocol 2 and upwards) feature -efficient binary encodings for several common features and built-in types. -Also, the :mod:`pickle` module has a transparent optimizer written in C. - - -.. _pickle-example: - -Examples --------- - -For the simplest code, use the :func:`dump` and :func:`load` functions. :: - - import pickle - - # An arbitrary collection of objects supported by pickle. - data = { - 'a': [1, 2.0, 3+4j], - 'b': ("character string", b"byte string"), - 'c': {None, True, False} - } - - with open('data.pickle', 'wb') as f: - # Pickle the 'data' dictionary using the highest protocol available. - pickle.dump(data, f, pickle.HIGHEST_PROTOCOL) - - -The following example reads the resulting pickled data. :: - - import pickle - - with open('data.pickle', 'rb') as f: - # The protocol version used is detected automatically, so we do not - # have to specify it. - data = pickle.load(f) - - -.. XXX: Add examples showing how to optimize pickles for size (like using -.. pickletools.optimize() or the gzip module). - - -.. seealso:: - - Module :mod:`copyreg` - Pickle interface constructor registration for extension types. - - Module :mod:`pickletools` - Tools for working with and analyzing pickled data. - - Module :mod:`shelve` - Indexed databases of objects; uses :mod:`pickle`. - - Module :mod:`copy` - Shallow and deep object copying. - - Module :mod:`marshal` - High-performance serialization of built-in types. - - -.. rubric:: Footnotes - -.. [#] Don't confuse this with the :mod:`marshal` module - -.. [#] This is why :keyword:`lambda` functions cannot be pickled: all - :keyword:`!lambda` functions share the same name: ````. - -.. [#] The exception raised will likely be an :exc:`ImportError` or an - :exc:`AttributeError` but it could be something else. - -.. [#] The :mod:`copy` module uses this protocol for shallow and deep copying - operations. - -.. [#] The limitation on alphanumeric characters is due to the fact - that persistent IDs in protocol 0 are delimited by the newline - character. Therefore if any kind of newline characters occurs in - persistent IDs, the resulting pickled data will become unreadable. diff --git a/Doc/library/pickletools.rst b/Doc/library/pickletools.rst deleted file mode 100644 index 41930f8cbe8412..00000000000000 --- a/Doc/library/pickletools.rst +++ /dev/null @@ -1,112 +0,0 @@ -:mod:`pickletools` --- Tools for pickle developers -================================================== - -.. module:: pickletools - :synopsis: Contains extensive comments about the pickle protocols and - pickle-machine opcodes, as well as some useful functions. - -**Source code:** :source:`Lib/pickletools.py` - --------------- - - -This module contains various constants relating to the intimate details of the -:mod:`pickle` module, some lengthy comments about the implementation, and a -few useful functions for analyzing pickled data. The contents of this module -are useful for Python core developers who are working on the :mod:`pickle`; -ordinary users of the :mod:`pickle` module probably won't find the -:mod:`pickletools` module relevant. - -.. _pickletools-cli: - -Command line usage ------------------- - -.. versionadded:: 3.2 - -When invoked from the command line, ``python -m pickletools`` will -disassemble the contents of one or more pickle files. Note that if -you want to see the Python object stored in the pickle rather than the -details of pickle format, you may want to use ``-m pickle`` instead. -However, when the pickle file that you want to examine comes from an -untrusted source, ``-m pickletools`` is a safer option because it does -not execute pickle bytecode. - -For example, with a tuple ``(1, 2)`` pickled in file ``x.pickle``: - -.. code-block:: shell-session - - $ python -m pickle x.pickle - (1, 2) - - $ python -m pickletools x.pickle - 0: \x80 PROTO 3 - 2: K BININT1 1 - 4: K BININT1 2 - 6: \x86 TUPLE2 - 7: q BINPUT 0 - 9: . STOP - highest protocol among opcodes = 2 - -Command line options -^^^^^^^^^^^^^^^^^^^^ - -.. program:: pickletools - -.. option:: -a, --annotate - - Annotate each line with a short opcode description. - -.. option:: -o, --output= - - Name of a file where the output should be written. - -.. option:: -l, --indentlevel= - - The number of blanks by which to indent a new MARK level. - -.. option:: -m, --memo - - When multiple objects are disassembled, preserve memo between - disassemblies. - -.. option:: -p, --preamble= - - When more than one pickle file are specified, print given preamble - before each disassembly. - - - -Programmatic Interface ----------------------- - - -.. function:: dis(pickle, out=None, memo=None, indentlevel=4, annotate=0) - - Outputs a symbolic disassembly of the pickle to the file-like - object *out*, defaulting to ``sys.stdout``. *pickle* can be a - string or a file-like object. *memo* can be a Python dictionary - that will be used as the pickle's memo; it can be used to perform - disassemblies across multiple pickles created by the same - pickler. Successive levels, indicated by ``MARK`` opcodes in the - stream, are indented by *indentlevel* spaces. If a nonzero value - is given to *annotate*, each opcode in the output is annotated with - a short description. The value of *annotate* is used as a hint for - the column where annotation should start. - - .. versionadded:: 3.2 - The *annotate* argument. - -.. function:: genops(pickle) - - Provides an :term:`iterator` over all of the opcodes in a pickle, returning a - sequence of ``(opcode, arg, pos)`` triples. *opcode* is an instance of an - :class:`OpcodeInfo` class; *arg* is the decoded value, as a Python object, of - the opcode's argument; *pos* is the position at which this opcode is located. - *pickle* can be a string or a file-like object. - -.. function:: optimize(picklestring) - - Returns a new equivalent pickle string after eliminating unused ``PUT`` - opcodes. The optimized pickle is shorter, takes less transmission time, - requires less storage space, and unpickles more efficiently. diff --git a/Doc/library/pipes.rst b/Doc/library/pipes.rst deleted file mode 100644 index 471ae0dbc9768a..00000000000000 --- a/Doc/library/pipes.rst +++ /dev/null @@ -1,103 +0,0 @@ -:mod:`pipes` --- Interface to shell pipelines -============================================= - -.. module:: pipes - :platform: Unix - :synopsis: A Python interface to Unix shell pipelines. - :deprecated: - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/pipes.py` - -.. deprecated-removed:: 3.11 3.13 - The :mod:`pipes` module is deprecated - (see :pep:`PEP 594 <594#pipes>` for details). - Please use the :mod:`subprocess` module instead. - --------------- - -The :mod:`pipes` module defines a class to abstract the concept of a *pipeline* ---- a sequence of converters from one file to another. - -Because the module uses :program:`/bin/sh` command lines, a POSIX or compatible -shell for :func:`os.system` and :func:`os.popen` is required. - -.. availability:: Unix, not VxWorks. - -The :mod:`pipes` module defines the following class: - - -.. class:: Template() - - An abstraction of a pipeline. - -Example:: - - >>> import pipes - >>> t = pipes.Template() - >>> t.append('tr a-z A-Z', '--') - >>> f = t.open('pipefile', 'w') - >>> f.write('hello world') - >>> f.close() - >>> open('pipefile').read() - 'HELLO WORLD' - - -.. _template-objects: - -Template Objects ----------------- - -Template objects following methods: - - -.. method:: Template.reset() - - Restore a pipeline template to its initial state. - - -.. method:: Template.clone() - - Return a new, equivalent, pipeline template. - - -.. method:: Template.debug(flag) - - If *flag* is true, turn debugging on. Otherwise, turn debugging off. When - debugging is on, commands to be executed are printed, and the shell is given - ``set -x`` command to be more verbose. - - -.. method:: Template.append(cmd, kind) - - Append a new action at the end. The *cmd* variable must be a valid bourne shell - command. The *kind* variable consists of two letters. - - The first letter can be either of ``'-'`` (which means the command reads its - standard input), ``'f'`` (which means the commands reads a given file on the - command line) or ``'.'`` (which means the commands reads no input, and hence - must be first.) - - Similarly, the second letter can be either of ``'-'`` (which means the command - writes to standard output), ``'f'`` (which means the command writes a file on - the command line) or ``'.'`` (which means the command does not write anything, - and hence must be last.) - - -.. method:: Template.prepend(cmd, kind) - - Add a new action at the beginning. See :meth:`append` for explanations of the - arguments. - - -.. method:: Template.open(file, mode) - - Return a file-like object, open to *file*, but read from or written to by the - pipeline. Note that only one of ``'r'``, ``'w'`` may be given. - - -.. method:: Template.copy(infile, outfile) - - Copy *infile* to *outfile* through the pipe. - diff --git a/Doc/library/pkgutil.rst b/Doc/library/pkgutil.rst deleted file mode 100644 index 64e617b82b48bc..00000000000000 --- a/Doc/library/pkgutil.rst +++ /dev/null @@ -1,270 +0,0 @@ -:mod:`pkgutil` --- Package extension utility -============================================ - -.. module:: pkgutil - :synopsis: Utilities for the import system. - -**Source code:** :source:`Lib/pkgutil.py` - --------------- - -This module provides utilities for the import system, in particular package -support. - -.. class:: ModuleInfo(module_finder, name, ispkg) - - A namedtuple that holds a brief summary of a module's info. - - .. versionadded:: 3.6 - -.. function:: extend_path(path, name) - - Extend the search path for the modules which comprise a package. Intended - use is to place the following code in a package's :file:`__init__.py`:: - - from pkgutil import extend_path - __path__ = extend_path(__path__, __name__) - - For each directory on :data:`sys.path` that has a subdirectory that matches the - package name, add the subdirectory to the package's :attr:`__path__`. This is useful - if one wants to distribute different parts of a single logical package as multiple - directories. - - It also looks for :file:`\*.pkg` files beginning where ``*`` matches the - *name* argument. This feature is similar to :file:`\*.pth` files (see the - :mod:`site` module for more information), except that it doesn't special-case - lines starting with ``import``. A :file:`\*.pkg` file is trusted at face - value: apart from checking for duplicates, all entries found in a - :file:`\*.pkg` file are added to the path, regardless of whether they exist - on the filesystem. (This is a feature.) - - If the input path is not a list (as is the case for frozen packages) it is - returned unchanged. The input path is not modified; an extended copy is - returned. Items are only appended to the copy at the end. - - It is assumed that :data:`sys.path` is a sequence. Items of :data:`sys.path` - that are not strings referring to existing directories are ignored. Unicode - items on :data:`sys.path` that cause errors when used as filenames may cause - this function to raise an exception (in line with :func:`os.path.isdir` - behavior). - - -.. class:: ImpImporter(dirname=None) - - :pep:`302` Finder that wraps Python's "classic" import algorithm. - - If *dirname* is a string, a :pep:`302` finder is created that searches that - directory. If *dirname* is ``None``, a :pep:`302` finder is created that - searches the current :data:`sys.path`, plus any modules that are frozen or - built-in. - - Note that :class:`ImpImporter` does not currently support being used by - placement on :data:`sys.meta_path`. - - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully :pep:`302` compliant and available in :mod:`importlib`. - - -.. class:: ImpLoader(fullname, file, filename, etc) - - :term:`Loader ` that wraps Python's "classic" import algorithm. - - .. deprecated:: 3.3 - This emulation is no longer needed, as the standard import mechanism - is now fully :pep:`302` compliant and available in :mod:`importlib`. - - -.. function:: find_loader(fullname) - - Retrieve a module :term:`loader` for the given *fullname*. - - This is a backwards compatibility wrapper around - :func:`importlib.util.find_spec` that converts most failures to - :exc:`ImportError` and only returns the loader rather than the full - :class:`importlib.machinery.ModuleSpec`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - -.. function:: get_importer(path_item) - - Retrieve a :term:`finder` for the given *path_item*. - - The returned finder is cached in :data:`sys.path_importer_cache` if it was - newly created by a path hook. - - The cache (or part of it) can be cleared manually if a rescan of - :data:`sys.path_hooks` is necessary. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - -.. function:: get_loader(module_or_name) - - Get a :term:`loader` object for *module_or_name*. - - If the module or package is accessible via the normal import mechanism, a - wrapper around the relevant part of that machinery is returned. Returns - ``None`` if the module cannot be found or imported. If the named module is - not already imported, its containing package (if any) is imported, in order - to establish the package ``__path__``. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - .. versionchanged:: 3.4 - Updated to be based on :pep:`451` - - -.. function:: iter_importers(fullname='') - - Yield :term:`finder` objects for the given module name. - - If fullname contains a ``'.'``, the finders will be for the package - containing fullname, otherwise they will be all registered top level - finders (i.e. those on both :data:`sys.meta_path` and :data:`sys.path_hooks`). - - If the named module is in a package, that package is imported as a side - effect of invoking this function. - - If no module name is specified, all top level finders are produced. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - -.. function:: iter_modules(path=None, prefix='') - - Yields :class:`ModuleInfo` for all submodules on *path*, or, if - *path* is ``None``, all top-level modules on :data:`sys.path`. - - *path* should be either ``None`` or a list of paths to look for modules in. - - *prefix* is a string to output on the front of every module name on output. - - .. note:: - - Only works for a :term:`finder` which defines an ``iter_modules()`` - method. This interface is non-standard, so the module also provides - implementations for :class:`importlib.machinery.FileFinder` and - :class:`zipimport.zipimporter`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - -.. function:: walk_packages(path=None, prefix='', onerror=None) - - Yields :class:`ModuleInfo` for all modules recursively on - *path*, or, if *path* is ``None``, all accessible modules. - - *path* should be either ``None`` or a list of paths to look for modules in. - - *prefix* is a string to output on the front of every module name on output. - - Note that this function must import all *packages* (*not* all modules!) on - the given *path*, in order to access the ``__path__`` attribute to find - submodules. - - *onerror* is a function which gets called with one argument (the name of the - package which was being imported) if any exception occurs while trying to - import a package. If no *onerror* function is supplied, :exc:`ImportError`\s - are caught and ignored, while all other exceptions are propagated, - terminating the search. - - Examples:: - - # list all modules python can access - walk_packages() - - # list all submodules of ctypes - walk_packages(ctypes.__path__, ctypes.__name__ + '.') - - .. note:: - - Only works for a :term:`finder` which defines an ``iter_modules()`` - method. This interface is non-standard, so the module also provides - implementations for :class:`importlib.machinery.FileFinder` and - :class:`zipimport.zipimporter`. - - .. versionchanged:: 3.3 - Updated to be based directly on :mod:`importlib` rather than relying - on the package internal :pep:`302` import emulation. - - -.. function:: get_data(package, resource) - - Get a resource from a package. - - This is a wrapper for the :term:`loader` - :meth:`get_data ` API. The - *package* argument should be the name of a package, in standard module format - (``foo.bar``). The *resource* argument should be in the form of a relative - filename, using ``/`` as the path separator. The parent directory name - ``..`` is not allowed, and nor is a rooted name (starting with a ``/``). - - The function returns a binary string that is the contents of the specified - resource. - - For packages located in the filesystem, which have already been imported, - this is the rough equivalent of:: - - d = os.path.dirname(sys.modules[package].__file__) - data = open(os.path.join(d, resource), 'rb').read() - - If the package cannot be located or loaded, or it uses a :term:`loader` - which does not support :meth:`get_data `, - then ``None`` is returned. In particular, the :term:`loader` for - :term:`namespace packages ` does not support - :meth:`get_data `. - - -.. function:: resolve_name(name) - - Resolve a name to an object. - - This functionality is used in numerous places in the standard library (see - :issue:`12915`) - and equivalent functionality is also in widely used - third-party packages such as setuptools, Django and Pyramid. - - It is expected that *name* will be a string in one of the following - formats, where W is shorthand for a valid Python identifier and dot stands - for a literal period in these pseudo-regexes: - - * ``W(.W)*`` - * ``W(.W)*:(W(.W)*)?`` - - The first form is intended for backward compatibility only. It assumes that - some part of the dotted name is a package, and the rest is an object - somewhere within that package, possibly nested inside other objects. - Because the place where the package stops and the object hierarchy starts - can't be inferred by inspection, repeated attempts to import must be done - with this form. - - In the second form, the caller makes the division point clear through the - provision of a single colon: the dotted name to the left of the colon is a - package to be imported, and the dotted name to the right is the object - hierarchy within that package. Only one import is needed in this form. If - it ends with the colon, then a module object is returned. - - The function will return an object (which might be a module), or raise one - of the following exceptions: - - :exc:`ValueError` -- if *name* isn't in a recognised format. - - :exc:`ImportError` -- if an import failed when it shouldn't have. - - :exc:`AttributeError` -- If a failure occurred when traversing the object - hierarchy within the imported package to get to the desired object. - - .. versionadded:: 3.9 diff --git a/Doc/library/platform.rst b/Doc/library/platform.rst deleted file mode 100644 index 60cd6b880847ef..00000000000000 --- a/Doc/library/platform.rst +++ /dev/null @@ -1,295 +0,0 @@ -:mod:`platform` --- Access to underlying platform's identifying data -===================================================================== - -.. module:: platform - :synopsis: Retrieves as much platform identifying data as possible. - -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Bjorn Pettersen - -**Source code:** :source:`Lib/platform.py` - --------------- - -.. note:: - - Specific platforms listed alphabetically, with Linux included in the Unix - section. - - -Cross Platform --------------- - - -.. function:: architecture(executable=sys.executable, bits='', linkage='') - - Queries the given executable (defaults to the Python interpreter binary) for - various architecture information. - - Returns a tuple ``(bits, linkage)`` which contain information about the bit - architecture and the linkage format used for the executable. Both values are - returned as strings. - - Values that cannot be determined are returned as given by the parameter presets. - If bits is given as ``''``, the ``sizeof(pointer)`` (or - ``sizeof(long)`` on Python version < 1.5.2) is used as indicator for the - supported pointer size. - - The function relies on the system's :file:`file` command to do the actual work. - This is available on most if not all Unix platforms and some non-Unix platforms - and then only if the executable points to the Python interpreter. Reasonable - defaults are used when the above needs are not met. - - .. note:: - - On macOS (and perhaps other platforms), executable files may be - universal files containing multiple architectures. - - To get at the "64-bitness" of the current interpreter, it is more - reliable to query the :data:`sys.maxsize` attribute:: - - is_64bits = sys.maxsize > 2**32 - - -.. function:: machine() - - Returns the machine type, e.g. ``'AMD64'``. An empty string is returned if the - value cannot be determined. - - -.. function:: node() - - Returns the computer's network name (may not be fully qualified!). An empty - string is returned if the value cannot be determined. - - -.. function:: platform(aliased=0, terse=0) - - Returns a single string identifying the underlying platform with as much useful - information as possible. - - The output is intended to be *human readable* rather than machine parseable. It - may look different on different platforms and this is intended. - - If *aliased* is true, the function will use aliases for various platforms that - report system names which differ from their common names, for example SunOS will - be reported as Solaris. The :func:`system_alias` function is used to implement - this. - - Setting *terse* to true causes the function to return only the absolute minimum - information needed to identify the platform. - - .. versionchanged:: 3.8 - On macOS, the function now uses :func:`mac_ver`, if it returns a - non-empty release string, to get the macOS version rather than the darwin - version. - - -.. function:: processor() - - Returns the (real) processor name, e.g. ``'amdk6'``. - - An empty string is returned if the value cannot be determined. Note that many - platforms do not provide this information or simply return the same value as for - :func:`machine`. NetBSD does this. - - -.. function:: python_build() - - Returns a tuple ``(buildno, builddate)`` stating the Python build number and - date as strings. - - -.. function:: python_compiler() - - Returns a string identifying the compiler used for compiling Python. - - -.. function:: python_branch() - - Returns a string identifying the Python implementation SCM branch. - - -.. function:: python_implementation() - - Returns a string identifying the Python implementation. Possible return values - are: 'CPython', 'IronPython', 'Jython', 'PyPy'. - - -.. function:: python_revision() - - Returns a string identifying the Python implementation SCM revision. - - -.. function:: python_version() - - Returns the Python version as string ``'major.minor.patchlevel'``. - - Note that unlike the Python ``sys.version``, the returned value will always - include the patchlevel (it defaults to 0). - - -.. function:: python_version_tuple() - - Returns the Python version as tuple ``(major, minor, patchlevel)`` of strings. - - Note that unlike the Python ``sys.version``, the returned value will always - include the patchlevel (it defaults to ``'0'``). - - -.. function:: release() - - Returns the system's release, e.g. ``'2.2.0'`` or ``'NT'``. An empty string is - returned if the value cannot be determined. - - -.. function:: system() - - Returns the system/OS name, such as ``'Linux'``, ``'Darwin'``, ``'Java'``, - ``'Windows'``. An empty string is returned if the value cannot be determined. - - -.. function:: system_alias(system, release, version) - - Returns ``(system, release, version)`` aliased to common marketing names used - for some systems. It also does some reordering of the information in some cases - where it would otherwise cause confusion. - - -.. function:: version() - - Returns the system's release version, e.g. ``'#3 on degas'``. An empty string is - returned if the value cannot be determined. - - -.. function:: uname() - - Fairly portable uname interface. Returns a :func:`~collections.namedtuple` - containing six attributes: :attr:`system`, :attr:`node`, :attr:`release`, - :attr:`version`, :attr:`machine`, and :attr:`processor`. - - Note that this adds a sixth attribute (:attr:`processor`) not present - in the :func:`os.uname` result. Also, the attribute names are different - for the first two attributes; :func:`os.uname` names them - :attr:`sysname` and :attr:`nodename`. - - Entries which cannot be determined are set to ``''``. - - .. versionchanged:: 3.3 - Result changed from a tuple to a :func:`~collections.namedtuple`. - - -Java Platform -------------- - - -.. function:: java_ver(release='', vendor='', vminfo=('','',''), osinfo=('','','')) - - Version interface for Jython. - - Returns a tuple ``(release, vendor, vminfo, osinfo)`` with *vminfo* being a - tuple ``(vm_name, vm_release, vm_vendor)`` and *osinfo* being a tuple - ``(os_name, os_version, os_arch)``. Values which cannot be determined are set to - the defaults given as parameters (which all default to ``''``). - - -Windows Platform ----------------- - - -.. function:: win32_ver(release='', version='', csd='', ptype='') - - Get additional version information from the Windows Registry and return a tuple - ``(release, version, csd, ptype)`` referring to OS release, version number, - CSD level (service pack) and OS type (multi/single processor). Values which - cannot be determined are set to the defaults given as parameters (which all - default to an empty string). - - As a hint: *ptype* is ``'Uniprocessor Free'`` on single processor NT machines - and ``'Multiprocessor Free'`` on multi processor machines. The *'Free'* refers - to the OS version being free of debugging code. It could also state *'Checked'* - which means the OS version uses debugging code, i.e. code that checks arguments, - ranges, etc. - -.. function:: win32_edition() - - Returns a string representing the current Windows edition, or ``None`` if the - value cannot be determined. Possible values include but are not limited to - ``'Enterprise'``, ``'IoTUAP'``, ``'ServerStandard'``, and ``'nanoserver'``. - - .. versionadded:: 3.8 - -.. function:: win32_is_iot() - - Return ``True`` if the Windows edition returned by :func:`win32_edition` - is recognized as an IoT edition. - - .. versionadded:: 3.8 - - -macOS Platform --------------- - - -.. function:: mac_ver(release='', versioninfo=('','',''), machine='') - - Get macOS version information and return it as tuple ``(release, versioninfo, - machine)`` with *versioninfo* being a tuple ``(version, dev_stage, - non_release_version)``. - - Entries which cannot be determined are set to ``''``. All tuple entries are - strings. - - -Unix Platforms --------------- - -.. function:: libc_ver(executable=sys.executable, lib='', version='', chunksize=16384) - - Tries to determine the libc version against which the file executable (defaults - to the Python interpreter) is linked. Returns a tuple of strings ``(lib, - version)`` which default to the given parameters in case the lookup fails. - - Note that this function has intimate knowledge of how different libc versions - add symbols to the executable is probably only usable for executables compiled - using :program:`gcc`. - - The file is read and scanned in chunks of *chunksize* bytes. - - -Linux Platforms ---------------- - -.. function:: freedesktop_os_release() - - Get operating system identification from ``os-release`` file and return - it as a dict. The ``os-release`` file is a `freedesktop.org standard - `_ and - is available in most Linux distributions. A noticeable exception is - Android and Android-based distributions. - - Raises :exc:`OSError` or subclass when neither ``/etc/os-release`` nor - ``/usr/lib/os-release`` can be read. - - On success, the function returns a dictionary where keys and values are - strings. Values have their special characters like ``"`` and ``$`` - unquoted. The fields ``NAME``, ``ID``, and ``PRETTY_NAME`` are always - defined according to the standard. All other fields are optional. Vendors - may include additional fields. - - Note that fields like ``NAME``, ``VERSION``, and ``VARIANT`` are strings - suitable for presentation to users. Programs should use fields like - ``ID``, ``ID_LIKE``, ``VERSION_ID``, or ``VARIANT_ID`` to identify - Linux distributions. - - Example:: - - def get_like_distro(): - info = platform.freedesktop_os_release() - ids = [info["ID"]] - if "ID_LIKE" in info: - # ids are space separated and ordered by precedence - ids.extend(info["ID_LIKE"].split()) - return ids - - .. versionadded:: 3.10 diff --git a/Doc/library/plistlib.rst b/Doc/library/plistlib.rst deleted file mode 100644 index 732ef3536863cc..00000000000000 --- a/Doc/library/plistlib.rst +++ /dev/null @@ -1,193 +0,0 @@ -:mod:`plistlib` --- Generate and parse Apple ``.plist`` files -============================================================= - -.. module:: plistlib - :synopsis: Generate and parse Apple plist files. - -.. moduleauthor:: Jack Jansen -.. sectionauthor:: Georg Brandl -.. (harvested from docstrings in the original file) - -**Source code:** :source:`Lib/plistlib.py` - -.. index:: - pair: plist; file - single: property list - --------------- - -This module provides an interface for reading and writing the "property list" -files used by Apple, primarily on macOS and iOS. This module supports both binary -and XML plist files. - -The property list (``.plist``) file format is a simple serialization supporting -basic object types, like dictionaries, lists, numbers and strings. Usually the -top level object is a dictionary. - -To write out and to parse a plist file, use the :func:`dump` and -:func:`load` functions. - -To work with plist data in bytes objects, use :func:`dumps` -and :func:`loads`. - -Values can be strings, integers, floats, booleans, tuples, lists, dictionaries -(but only with string keys), :class:`bytes`, :class:`bytearray` -or :class:`datetime.datetime` objects. - -.. versionchanged:: 3.4 - New API, old API deprecated. Support for binary format plists added. - -.. versionchanged:: 3.8 - Support added for reading and writing :class:`UID` tokens in binary plists as used - by NSKeyedArchiver and NSKeyedUnarchiver. - -.. versionchanged:: 3.9 - Old API removed. - -.. seealso:: - - `PList manual page `_ - Apple's documentation of the file format. - - -This module defines the following functions: - -.. function:: load(fp, *, fmt=None, dict_type=dict) - - Read a plist file. *fp* should be a readable and binary file object. - Return the unpacked root object (which usually is a - dictionary). - - The *fmt* is the format of the file and the following values are valid: - - * :data:`None`: Autodetect the file format - - * :data:`FMT_XML`: XML file format - - * :data:`FMT_BINARY`: Binary plist format - - The *dict_type* is the type used for dictionaries that are read from the - plist file. - - XML data for the :data:`FMT_XML` format is parsed using the Expat parser - from :mod:`xml.parsers.expat` -- see its documentation for possible - exceptions on ill-formed XML. Unknown elements will simply be ignored - by the plist parser. - - The parser for the binary format raises :exc:`InvalidFileException` - when the file cannot be parsed. - - .. versionadded:: 3.4 - - -.. function:: loads(data, *, fmt=None, dict_type=dict) - - Load a plist from a bytes object. See :func:`load` for an explanation of - the keyword arguments. - - .. versionadded:: 3.4 - - -.. function:: dump(value, fp, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) - - Write *value* to a plist file. *Fp* should be a writable, binary - file object. - - The *fmt* argument specifies the format of the plist file and can be - one of the following values: - - * :data:`FMT_XML`: XML formatted plist file - - * :data:`FMT_BINARY`: Binary formatted plist file - - When *sort_keys* is true (the default) the keys for dictionaries will be - written to the plist in sorted order, otherwise they will be written in - the iteration order of the dictionary. - - When *skipkeys* is false (the default) the function raises :exc:`TypeError` - when a key of a dictionary is not a string, otherwise such keys are skipped. - - A :exc:`TypeError` will be raised if the object is of an unsupported type or - a container that contains objects of unsupported types. - - An :exc:`OverflowError` will be raised for integer values that cannot - be represented in (binary) plist files. - - .. versionadded:: 3.4 - - -.. function:: dumps(value, *, fmt=FMT_XML, sort_keys=True, skipkeys=False) - - Return *value* as a plist-formatted bytes object. See - the documentation for :func:`dump` for an explanation of the keyword - arguments of this function. - - .. versionadded:: 3.4 - - -The following classes are available: - -.. class:: UID(data) - - Wraps an :class:`int`. This is used when reading or writing NSKeyedArchiver - encoded data, which contains UID (see PList manual). - - It has one attribute, :attr:`data`, which can be used to retrieve the int value - of the UID. :attr:`data` must be in the range ``0 <= data < 2**64``. - - .. versionadded:: 3.8 - - -The following constants are available: - -.. data:: FMT_XML - - The XML format for plist files. - - .. versionadded:: 3.4 - - -.. data:: FMT_BINARY - - The binary format for plist files - - .. versionadded:: 3.4 - - -Examples --------- - -Generating a plist:: - - import datetime - import plistlib - - pl = dict( - aString = "Doodah", - aList = ["A", "B", 12, 32.1, [1, 2, 3]], - aFloat = 0.1, - anInt = 728, - aDict = dict( - anotherString = "", - aThirdString = "M\xe4ssig, Ma\xdf", - aTrueValue = True, - aFalseValue = False, - ), - someData = b"", - someMoreData = b"" * 10, - aDate = datetime.datetime.now() - ) - print(plistlib.dumps(pl).decode()) - -Parsing a plist:: - - import plistlib - - plist = b""" - - foo - bar - - """ - pl = plistlib.loads(plist) - print(pl["foo"]) diff --git a/Doc/library/poplib.rst b/Doc/library/poplib.rst deleted file mode 100644 index 5e0371592ecb96..00000000000000 --- a/Doc/library/poplib.rst +++ /dev/null @@ -1,279 +0,0 @@ -:mod:`poplib` --- POP3 protocol client -====================================== - -.. module:: poplib - :synopsis: POP3 protocol client (requires sockets). - -.. sectionauthor:: Andrew T. Csillag -.. revised by ESR, January 2000 - -**Source code:** :source:`Lib/poplib.py` - -.. index:: pair: POP3; protocol - --------------- - -This module defines a class, :class:`POP3`, which encapsulates a connection to a -POP3 server and implements the protocol as defined in :rfc:`1939`. The -:class:`POP3` class supports both the minimal and optional command sets from -:rfc:`1939`. The :class:`POP3` class also supports the ``STLS`` command introduced -in :rfc:`2595` to enable encrypted communication on an already established connection. - -Additionally, this module provides a class :class:`POP3_SSL`, which provides -support for connecting to POP3 servers that use SSL as an underlying protocol -layer. - -Note that POP3, though widely supported, is obsolescent. The implementation -quality of POP3 servers varies widely, and too many are quite poor. If your -mailserver supports IMAP, you would be better off using the -:class:`imaplib.IMAP4` class, as IMAP servers tend to be better implemented. - -.. include:: ../includes/wasm-notavail.rst - -The :mod:`poplib` module provides two classes: - - -.. class:: POP3(host, port=POP3_PORT[, timeout]) - - This class implements the actual POP3 protocol. The connection is created when - the instance is initialized. If *port* is omitted, the standard POP3 port (110) - is used. The optional *timeout* parameter specifies a timeout in seconds for the - connection attempt (if not specified, the global default timeout setting will - be used). - - .. audit-event:: poplib.connect self,host,port poplib.POP3 - - .. audit-event:: poplib.putline self,line poplib.POP3 - - All commands will raise an :ref:`auditing event ` - ``poplib.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - -.. class:: POP3_SSL(host, port=POP3_SSL_PORT, keyfile=None, certfile=None, timeout=None, context=None) - - This is a subclass of :class:`POP3` that connects to the server over an SSL - encrypted socket. If *port* is not specified, 995, the standard POP3-over-SSL - port is used. *timeout* works as in the :class:`POP3` constructor. - *context* is an optional :class:`ssl.SSLContext` object which allows - bundling SSL configuration options, certificates and private keys into a - single (potentially long-lived) structure. Please read :ref:`ssl-security` - for best practices. - - *keyfile* and *certfile* are a legacy alternative to *context* - they can - point to PEM-formatted private key and certificate chain files, - respectively, for the SSL connection. - - .. audit-event:: poplib.connect self,host,port poplib.POP3_SSL - - .. audit-event:: poplib.putline self,line poplib.POP3_SSL - - All commands will raise an :ref:`auditing event ` - ``poplib.putline`` with arguments ``self`` and ``line``, - where ``line`` is the bytes about to be sent to the remote host. - - .. versionchanged:: 3.2 - *context* parameter added. - - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :const:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket. - -One exception is defined as an attribute of the :mod:`poplib` module: - - -.. exception:: error_proto - - Exception raised on any errors from this module (errors from :mod:`socket` - module are not caught). The reason for the exception is passed to the - constructor as a string. - - -.. seealso:: - - Module :mod:`imaplib` - The standard Python IMAP module. - - `Frequently Asked Questions About Fetchmail `_ - The FAQ for the :program:`fetchmail` POP/IMAP client collects information on - POP3 server variations and RFC noncompliance that may be useful if you need to - write an application based on the POP protocol. - - -.. _pop3-objects: - -POP3 Objects ------------- - -All POP3 commands are represented by methods of the same name, in lowercase; -most return the response text sent by the server. - -A :class:`POP3` instance has the following methods: - - -.. method:: POP3.set_debuglevel(level) - - Set the instance's debugging level. This controls the amount of debugging - output printed. The default, ``0``, produces no debugging output. A value of - ``1`` produces a moderate amount of debugging output, generally a single line - per request. A value of ``2`` or higher produces the maximum amount of - debugging output, logging each line sent and received on the control connection. - - -.. method:: POP3.getwelcome() - - Returns the greeting string sent by the POP3 server. - - -.. method:: POP3.capa() - - Query the server's capabilities as specified in :rfc:`2449`. - Returns a dictionary in the form ``{'name': ['param'...]}``. - - .. versionadded:: 3.4 - - -.. method:: POP3.user(username) - - Send user command, response should indicate that a password is required. - - -.. method:: POP3.pass_(password) - - Send password, response includes message count and mailbox size. Note: the - mailbox on the server is locked until :meth:`~POP3.quit` is called. - - -.. method:: POP3.apop(user, secret) - - Use the more secure APOP authentication to log into the POP3 server. - - -.. method:: POP3.rpop(user) - - Use RPOP authentication (similar to UNIX r-commands) to log into POP3 server. - - -.. method:: POP3.stat() - - Get mailbox status. The result is a tuple of 2 integers: ``(message count, - mailbox size)``. - - -.. method:: POP3.list([which]) - - Request message list, result is in the form ``(response, ['mesg_num octets', - ...], octets)``. If *which* is set, it is the message to list. - - -.. method:: POP3.retr(which) - - Retrieve whole message number *which*, and set its seen flag. Result is in form - ``(response, ['line', ...], octets)``. - - -.. method:: POP3.dele(which) - - Flag message number *which* for deletion. On most servers deletions are not - actually performed until QUIT (the major exception is Eudora QPOP, which - deliberately violates the RFCs by doing pending deletes on any disconnect). - - -.. method:: POP3.rset() - - Remove any deletion marks for the mailbox. - - -.. method:: POP3.noop() - - Do nothing. Might be used as a keep-alive. - - -.. method:: POP3.quit() - - Signoff: commit changes, unlock mailbox, drop connection. - - -.. method:: POP3.top(which, howmuch) - - Retrieves the message header plus *howmuch* lines of the message after the - header of message number *which*. Result is in form ``(response, ['line', ...], - octets)``. - - The POP3 TOP command this method uses, unlike the RETR command, doesn't set the - message's seen flag; unfortunately, TOP is poorly specified in the RFCs and is - frequently broken in off-brand servers. Test this method by hand against the - POP3 servers you will use before trusting it. - - -.. method:: POP3.uidl(which=None) - - Return message digest (unique id) list. If *which* is specified, result contains - the unique id for that message in the form ``'response mesgnum uid``, otherwise - result is list ``(response, ['mesgnum uid', ...], octets)``. - - -.. method:: POP3.utf8() - - Try to switch to UTF-8 mode. Returns the server response if successful, - raises :class:`error_proto` if not. Specified in :RFC:`6856`. - - .. versionadded:: 3.5 - - -.. method:: POP3.stls(context=None) - - Start a TLS session on the active connection as specified in :rfc:`2595`. - This is only allowed before user authentication - - *context* parameter is a :class:`ssl.SSLContext` object which allows - bundling SSL configuration options, certificates and private keys into - a single (potentially long-lived) structure. Please read :ref:`ssl-security` - for best practices. - - This method supports hostname checking via - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :const:`ssl.HAS_SNI`). - - .. versionadded:: 3.4 - - -Instances of :class:`POP3_SSL` have no additional methods. The interface of this -subclass is identical to its parent. - - -.. _pop3-example: - -POP3 Example ------------- - -Here is a minimal example (without error checking) that opens a mailbox and -retrieves and prints all messages:: - - import getpass, poplib - - M = poplib.POP3('localhost') - M.user(getpass.getuser()) - M.pass_(getpass.getpass()) - numMessages = len(M.list()[1]) - for i in range(numMessages): - for j in M.retr(i+1)[1]: - print(j) - -At the end of the module, there is a test section that contains a more extensive -example of usage. diff --git a/Doc/library/posix.rst b/Doc/library/posix.rst deleted file mode 100644 index 5871574b442667..00000000000000 --- a/Doc/library/posix.rst +++ /dev/null @@ -1,94 +0,0 @@ -:mod:`posix` --- The most common POSIX system calls -=================================================== - -.. module:: posix - :platform: Unix - :synopsis: The most common POSIX system calls (normally used via module os). - --------------- - -This module provides access to operating system functionality that is -standardized by the C Standard and the POSIX standard (a thinly disguised Unix -interface). - -.. availability:: Unix. - -.. index:: pair: module; os - -**Do not import this module directly.** Instead, import the module :mod:`os`, -which provides a *portable* version of this interface. On Unix, the :mod:`os` -module provides a superset of the :mod:`posix` interface. On non-Unix operating -systems the :mod:`posix` module is not available, but a subset is always -available through the :mod:`os` interface. Once :mod:`os` is imported, there is -*no* performance penalty in using it instead of :mod:`posix`. In addition, -:mod:`os` provides some additional functionality, such as automatically calling -:func:`~os.putenv` when an entry in ``os.environ`` is changed. - -Errors are reported as exceptions; the usual exceptions are given for type -errors, while errors reported by the system calls raise :exc:`OSError`. - - -.. _posix-large-files: - -Large File Support ------------------- - -.. index:: - single: large files - single: file; large files - -.. sectionauthor:: Steve Clift - -Several operating systems (including AIX and Solaris) provide -support for files that are larger than 2 GiB from a C programming model where -:c:expr:`int` and :c:expr:`long` are 32-bit values. This is typically accomplished -by defining the relevant size and offset types as 64-bit values. Such files are -sometimes referred to as :dfn:`large files`. - -Large file support is enabled in Python when the size of an :c:type:`off_t` is -larger than a :c:expr:`long` and the :c:expr:`long long` is at least as large -as an :c:type:`off_t`. -It may be necessary to configure and compile Python with certain compiler flags -to enable this mode. For example, with Solaris 2.6 and 2.7 you need to do -something like:: - - CFLAGS="`getconf LFS_CFLAGS`" OPT="-g -O2 $CFLAGS" \ - ./configure - -On large-file-capable Linux systems, this might work:: - - CFLAGS='-D_LARGEFILE64_SOURCE -D_FILE_OFFSET_BITS=64' OPT="-g -O2 $CFLAGS" \ - ./configure - - -.. _posix-contents: - -Notable Module Contents ------------------------ - -In addition to many functions described in the :mod:`os` module documentation, -:mod:`posix` defines the following data item: - -.. data:: environ - - A dictionary representing the string environment at the time the interpreter - was started. Keys and values are bytes on Unix and str on Windows. For - example, ``environ[b'HOME']`` (``environ['HOME']`` on Windows) is the - pathname of your home directory, equivalent to ``getenv("HOME")`` in C. - - Modifying this dictionary does not affect the string environment passed on by - :func:`~os.execv`, :func:`~os.popen` or :func:`~os.system`; if you need to - change the environment, pass ``environ`` to :func:`~os.execve` or add - variable assignments and export statements to the command string for - :func:`~os.system` or :func:`~os.popen`. - - .. versionchanged:: 3.2 - On Unix, keys and values are bytes. - - .. note:: - - The :mod:`os` module provides an alternate implementation of ``environ`` - which updates the environment on modification. Note also that updating - :data:`os.environ` will render this dictionary obsolete. Use of the - :mod:`os` module version of this is recommended over direct access to the - :mod:`posix` module. diff --git a/Doc/library/pprint.rst b/Doc/library/pprint.rst deleted file mode 100644 index fa5153284a2aab..00000000000000 --- a/Doc/library/pprint.rst +++ /dev/null @@ -1,415 +0,0 @@ -:mod:`pprint` --- Data pretty printer -===================================== - -.. module:: pprint - :synopsis: Data pretty printer. - -.. moduleauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/pprint.py` - --------------- - -The :mod:`pprint` module provides a capability to "pretty-print" arbitrary -Python data structures in a form which can be used as input to the interpreter. -If the formatted structures include objects which are not fundamental Python -types, the representation may not be loadable. This may be the case if objects -such as files, sockets or classes are included, as well as many other -objects which are not representable as Python literals. - -The formatted representation keeps objects on a single line if it can, and -breaks them onto multiple lines if they don't fit within the allowed width. -Construct :class:`PrettyPrinter` objects explicitly if you need to adjust the -width constraint. - -Dictionaries are sorted by key before the display is computed. - -.. versionchanged:: 3.9 - Added support for pretty-printing :class:`types.SimpleNamespace`. - -.. versionchanged:: 3.10 - Added support for pretty-printing :class:`dataclasses.dataclass`. - -The :mod:`pprint` module defines one class: - -.. First the implementation class: - - -.. index:: single: ...; placeholder - -.. class:: PrettyPrinter(indent=1, width=80, depth=None, stream=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Construct a :class:`PrettyPrinter` instance. This constructor understands - several keyword parameters. - - *stream* (default ``sys.stdout``) is a :term:`file-like object` to - which the output will be written by calling its :meth:`!write` method. - If both *stream* and ``sys.stdout`` are ``None``, then - :meth:`~PrettyPrinter.pprint` silently returns. - - Other values configure the manner in which nesting of complex data - structures is displayed. - - *indent* (default 1) specifies the amount of indentation added for - each nesting level. - - *depth* controls the number of nesting levels which may be printed; if - the data structure being printed is too deep, the next contained level - is replaced by ``...``. By default, there is no constraint on the - depth of the objects being formatted. - - *width* (default 80) specifies the desired maximum number of characters per - line in the output. If a structure cannot be formatted within the width - constraint, a best effort will be made. - - *compact* impacts the way that long sequences (lists, tuples, sets, etc) - are formatted. If *compact* is false (the default) then each item of a - sequence will be formatted on a separate line. If *compact* is true, as - many items as will fit within the *width* will be formatted on each output - line. - - If *sort_dicts* is true (the default), dictionaries will be formatted with - their keys sorted, otherwise they will display in insertion order. - - If *underscore_numbers* is true, integers will be formatted with the - ``_`` character for a thousands separator, otherwise underscores are not - displayed (the default). - - .. versionchanged:: 3.4 - Added the *compact* parameter. - - .. versionchanged:: 3.8 - Added the *sort_dicts* parameter. - - .. versionchanged:: 3.10 - Added the *underscore_numbers* parameter. - - .. versionchanged:: 3.11 - No longer attempts to write to ``sys.stdout`` if it is ``None``. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff[:]) - >>> pp = pprint.PrettyPrinter(indent=4) - >>> pp.pprint(stuff) - [ ['spam', 'eggs', 'lumberjack', 'knights', 'ni'], - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - >>> pp = pprint.PrettyPrinter(width=41, compact=True) - >>> pp.pprint(stuff) - [['spam', 'eggs', 'lumberjack', - 'knights', 'ni'], - 'spam', 'eggs', 'lumberjack', 'knights', - 'ni'] - >>> tup = ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', - ... ('parrot', ('fresh fruit',)))))))) - >>> pp = pprint.PrettyPrinter(depth=6) - >>> pp.pprint(tup) - ('spam', ('eggs', ('lumberjack', ('knights', ('ni', ('dead', (...))))))) - -.. function:: pformat(object, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Return the formatted representation of *object* as a string. *indent*, - *width*, *depth*, *compact*, *sort_dicts* and *underscore_numbers* are - passed to the :class:`PrettyPrinter` constructor as formatting parameters - and their meanings are as described in its documentation above. - - -.. function:: pp(object, *args, sort_dicts=False, **kwargs) - - Prints the formatted representation of *object* followed by a newline. - If *sort_dicts* is false (the default), dictionaries will be displayed with - their keys in insertion order, otherwise the dict keys will be sorted. - *args* and *kwargs* will be passed to :func:`pprint` as formatting - parameters. - - .. versionadded:: 3.8 - - -.. function:: pprint(object, stream=None, indent=1, width=80, depth=None, *, \ - compact=False, sort_dicts=True, underscore_numbers=False) - - Prints the formatted representation of *object* on *stream*, followed by a - newline. If *stream* is ``None``, ``sys.stdout`` is used. This may be used - in the interactive interpreter instead of the :func:`print` function for - inspecting values (you can even reassign ``print = pprint.pprint`` for use - within a scope). - - The configuration parameters *stream*, *indent*, *width*, *depth*, - *compact*, *sort_dicts* and *underscore_numbers* are passed to the - :class:`PrettyPrinter` constructor and their meanings are as - described in its documentation above. - - >>> import pprint - >>> stuff = ['spam', 'eggs', 'lumberjack', 'knights', 'ni'] - >>> stuff.insert(0, stuff) - >>> pprint.pprint(stuff) - [, - 'spam', - 'eggs', - 'lumberjack', - 'knights', - 'ni'] - -.. function:: isreadable(object) - - .. index:: pair: built-in function; eval - - Determine if the formatted representation of *object* is "readable", or can be - used to reconstruct the value using :func:`eval`. This always returns ``False`` - for recursive objects. - - >>> pprint.isreadable(stuff) - False - - -.. function:: isrecursive(object) - - Determine if *object* requires a recursive representation. - - -One more support function is also defined: - -.. function:: saferepr(object) - - Return a string representation of *object*, protected against recursive data - structures. If the representation of *object* exposes a recursive entry, the - recursive reference will be represented as ````. The representation is not otherwise formatted. - - >>> pprint.saferepr(stuff) - "[, 'spam', 'eggs', 'lumberjack', 'knights', 'ni']" - - -.. _prettyprinter-objects: - -PrettyPrinter Objects ---------------------- - -:class:`PrettyPrinter` instances have the following methods: - - -.. method:: PrettyPrinter.pformat(object) - - Return the formatted representation of *object*. This takes into account the - options passed to the :class:`PrettyPrinter` constructor. - - -.. method:: PrettyPrinter.pprint(object) - - Print the formatted representation of *object* on the configured stream, - followed by a newline. - -The following methods provide the implementations for the corresponding -functions of the same names. Using these methods on an instance is slightly -more efficient since new :class:`PrettyPrinter` objects don't need to be -created. - - -.. method:: PrettyPrinter.isreadable(object) - - .. index:: pair: built-in function; eval - - Determine if the formatted representation of the object is "readable," or can be - used to reconstruct the value using :func:`eval`. Note that this returns - ``False`` for recursive objects. If the *depth* parameter of the - :class:`PrettyPrinter` is set and the object is deeper than allowed, this - returns ``False``. - - -.. method:: PrettyPrinter.isrecursive(object) - - Determine if the object requires a recursive representation. - -This method is provided as a hook to allow subclasses to modify the way objects -are converted to strings. The default implementation uses the internals of the -:func:`saferepr` implementation. - - -.. method:: PrettyPrinter.format(object, context, maxlevels, level) - - Returns three values: the formatted version of *object* as a string, a flag - indicating whether the result is readable, and a flag indicating whether - recursion was detected. The first argument is the object to be presented. The - second is a dictionary which contains the :func:`id` of objects that are part of - the current presentation context (direct and indirect containers for *object* - that are affecting the presentation) as the keys; if an object needs to be - presented which is already represented in *context*, the third return value - should be ``True``. Recursive calls to the :meth:`.format` method should add - additional entries for containers to this dictionary. The third argument, - *maxlevels*, gives the requested limit to recursion; this will be ``0`` if there - is no requested limit. This argument should be passed unmodified to recursive - calls. The fourth argument, *level*, gives the current level; recursive calls - should be passed a value less than that of the current call. - - -.. _pprint-example: - -Example -------- - -To demonstrate several uses of the :func:`pprint` function and its parameters, -let's fetch information about a project from `PyPI `_:: - - >>> import json - >>> import pprint - >>> from urllib.request import urlopen - >>> with urlopen('https://pypi.org/pypi/sampleproject/json') as resp: - ... project_info = json.load(resp)['info'] - -In its basic form, :func:`pprint` shows the whole object:: - - >>> pprint.pprint(project_info) - {'author': 'The Python Packaging Authority', - 'author_email': 'pypa-dev@googlegroups.com', - 'bugtrack_url': None, - 'classifiers': ['Development Status :: 3 - Alpha', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: MIT License', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.6', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.2', - 'Programming Language :: Python :: 3.3', - 'Programming Language :: Python :: 3.4', - 'Topic :: Software Development :: Build Tools'], - 'description': 'A sample Python project\n' - '=======================\n' - '\n' - 'This is the description file for the project.\n' - '\n' - 'The file should use UTF-8 encoding and be written using ' - 'ReStructured Text. It\n' - 'will be used to generate the project webpage on PyPI, and ' - 'should be written for\n' - 'that purpose.\n' - '\n' - 'Typical contents for this file would include an overview of ' - 'the project, basic\n' - 'usage examples, etc. Generally, including the project ' - 'changelog in here is not\n' - 'a good idea, although a simple "What\'s New" section for the ' - 'most recent version\n' - 'may be appropriate.', - 'description_content_type': None, - 'docs_url': None, - 'download_url': 'UNKNOWN', - 'downloads': {'last_day': -1, 'last_month': -1, 'last_week': -1}, - 'home_page': 'https://github.com/pypa/sampleproject', - 'keywords': 'sample setuptools development', - 'license': 'MIT', - 'maintainer': None, - 'maintainer_email': None, - 'name': 'sampleproject', - 'package_url': 'https://pypi.org/project/sampleproject/', - 'platform': 'UNKNOWN', - 'project_url': 'https://pypi.org/project/sampleproject/', - 'project_urls': {'Download': 'UNKNOWN', - 'Homepage': 'https://github.com/pypa/sampleproject'}, - 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', - 'requires_dist': None, - 'requires_python': None, - 'summary': 'A sample Python project', - 'version': '1.2.0'} - -The result can be limited to a certain *depth* (ellipsis is used for deeper -contents):: - - >>> pprint.pprint(project_info, depth=1) - {'author': 'The Python Packaging Authority', - 'author_email': 'pypa-dev@googlegroups.com', - 'bugtrack_url': None, - 'classifiers': [...], - 'description': 'A sample Python project\n' - '=======================\n' - '\n' - 'This is the description file for the project.\n' - '\n' - 'The file should use UTF-8 encoding and be written using ' - 'ReStructured Text. It\n' - 'will be used to generate the project webpage on PyPI, and ' - 'should be written for\n' - 'that purpose.\n' - '\n' - 'Typical contents for this file would include an overview of ' - 'the project, basic\n' - 'usage examples, etc. Generally, including the project ' - 'changelog in here is not\n' - 'a good idea, although a simple "What\'s New" section for the ' - 'most recent version\n' - 'may be appropriate.', - 'description_content_type': None, - 'docs_url': None, - 'download_url': 'UNKNOWN', - 'downloads': {...}, - 'home_page': 'https://github.com/pypa/sampleproject', - 'keywords': 'sample setuptools development', - 'license': 'MIT', - 'maintainer': None, - 'maintainer_email': None, - 'name': 'sampleproject', - 'package_url': 'https://pypi.org/project/sampleproject/', - 'platform': 'UNKNOWN', - 'project_url': 'https://pypi.org/project/sampleproject/', - 'project_urls': {...}, - 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', - 'requires_dist': None, - 'requires_python': None, - 'summary': 'A sample Python project', - 'version': '1.2.0'} - -Additionally, maximum character *width* can be suggested. If a long object -cannot be split, the specified width will be exceeded:: - - >>> pprint.pprint(project_info, depth=1, width=60) - {'author': 'The Python Packaging Authority', - 'author_email': 'pypa-dev@googlegroups.com', - 'bugtrack_url': None, - 'classifiers': [...], - 'description': 'A sample Python project\n' - '=======================\n' - '\n' - 'This is the description file for the ' - 'project.\n' - '\n' - 'The file should use UTF-8 encoding and be ' - 'written using ReStructured Text. It\n' - 'will be used to generate the project ' - 'webpage on PyPI, and should be written ' - 'for\n' - 'that purpose.\n' - '\n' - 'Typical contents for this file would ' - 'include an overview of the project, ' - 'basic\n' - 'usage examples, etc. Generally, including ' - 'the project changelog in here is not\n' - 'a good idea, although a simple "What\'s ' - 'New" section for the most recent version\n' - 'may be appropriate.', - 'description_content_type': None, - 'docs_url': None, - 'download_url': 'UNKNOWN', - 'downloads': {...}, - 'home_page': 'https://github.com/pypa/sampleproject', - 'keywords': 'sample setuptools development', - 'license': 'MIT', - 'maintainer': None, - 'maintainer_email': None, - 'name': 'sampleproject', - 'package_url': 'https://pypi.org/project/sampleproject/', - 'platform': 'UNKNOWN', - 'project_url': 'https://pypi.org/project/sampleproject/', - 'project_urls': {...}, - 'release_url': 'https://pypi.org/project/sampleproject/1.2.0/', - 'requires_dist': None, - 'requires_python': None, - 'summary': 'A sample Python project', - 'version': '1.2.0'} diff --git a/Doc/library/profile.rst b/Doc/library/profile.rst deleted file mode 100644 index 4c60a1e0d781b0..00000000000000 --- a/Doc/library/profile.rst +++ /dev/null @@ -1,699 +0,0 @@ -.. _profile: - -******************** -The Python Profilers -******************** - -**Source code:** :source:`Lib/profile.py` and :source:`Lib/pstats.py` - --------------- - -.. _profiler-introduction: - -Introduction to the profilers -============================= - -.. index:: - single: deterministic profiling - single: profiling, deterministic - -:mod:`cProfile` and :mod:`profile` provide :dfn:`deterministic profiling` of -Python programs. A :dfn:`profile` is a set of statistics that describes how -often and for how long various parts of the program executed. These statistics -can be formatted into reports via the :mod:`pstats` module. - -The Python standard library provides two different implementations of the same -profiling interface: - -1. :mod:`cProfile` is recommended for most users; it's a C extension with - reasonable overhead that makes it suitable for profiling long-running - programs. Based on :mod:`lsprof`, contributed by Brett Rosen and Ted - Czotter. - -2. :mod:`profile`, a pure Python module whose interface is imitated by - :mod:`cProfile`, but which adds significant overhead to profiled programs. - If you're trying to extend the profiler in some way, the task might be easier - with this module. Originally designed and written by Jim Roskind. - -.. note:: - - The profiler modules are designed to provide an execution profile for a given - program, not for benchmarking purposes (for that, there is :mod:`timeit` for - reasonably accurate results). This particularly applies to benchmarking - Python code against C code: the profilers introduce overhead for Python code, - but not for C-level functions, and so the C code would seem faster than any - Python one. - - -.. _profile-instant: - -Instant User's Manual -===================== - -This section is provided for users that "don't want to read the manual." It -provides a very brief overview, and allows a user to rapidly perform profiling -on an existing application. - -To profile a function that takes a single argument, you can do:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")') - -(Use :mod:`profile` instead of :mod:`cProfile` if the latter is not available on -your system.) - -The above action would run :func:`re.compile` and print profile results like -the following:: - - 214 function calls (207 primitive calls) in 0.002 seconds - - Ordered by: cumulative time - - ncalls tottime percall cumtime percall filename:lineno(function) - 1 0.000 0.000 0.002 0.002 {built-in method builtins.exec} - 1 0.000 0.000 0.001 0.001 :1() - 1 0.000 0.000 0.001 0.001 __init__.py:250(compile) - 1 0.000 0.000 0.001 0.001 __init__.py:289(_compile) - 1 0.000 0.000 0.000 0.000 _compiler.py:759(compile) - 1 0.000 0.000 0.000 0.000 _parser.py:937(parse) - 1 0.000 0.000 0.000 0.000 _compiler.py:598(_code) - 1 0.000 0.000 0.000 0.000 _parser.py:435(_parse_sub) - -The first line indicates that 214 calls were monitored. Of those calls, 207 -were :dfn:`primitive`, meaning that the call was not induced via recursion. The -next line: ``Ordered by: cumulative time``, indicates that the text string in the -far right column was used to sort the output. The column headings include: - -ncalls - for the number of calls. - -tottime - for the total time spent in the given function (and excluding time made in - calls to sub-functions) - -percall - is the quotient of ``tottime`` divided by ``ncalls`` - -cumtime - is the cumulative time spent in this and all subfunctions (from invocation - till exit). This figure is accurate *even* for recursive functions. - -percall - is the quotient of ``cumtime`` divided by primitive calls - -filename:lineno(function) - provides the respective data of each function - -When there are two numbers in the first column (for example ``3/1``), it means -that the function recursed. The second value is the number of primitive calls -and the former is the total number of calls. Note that when the function does -not recurse, these two values are the same, and only the single figure is -printed. - -Instead of printing the output at the end of the profile run, you can save the -results to a file by specifying a filename to the :func:`run` function:: - - import cProfile - import re - cProfile.run('re.compile("foo|bar")', 'restats') - -The :class:`pstats.Stats` class reads profile results from a file and formats -them in various ways. - -.. _profile-cli: - -The files :mod:`cProfile` and :mod:`profile` can also be invoked as a script to -profile another script. For example:: - - python -m cProfile [-o output_file] [-s sort_order] (-m module | myscript.py) - -``-o`` writes the profile results to a file instead of to stdout - -``-s`` specifies one of the :func:`~pstats.Stats.sort_stats` sort values to sort -the output by. This only applies when ``-o`` is not supplied. - -``-m`` specifies that a module is being profiled instead of a script. - -.. versionadded:: 3.7 - Added the ``-m`` option to :mod:`cProfile`. - -.. versionadded:: 3.8 - Added the ``-m`` option to :mod:`profile`. - -The :mod:`pstats` module's :class:`~pstats.Stats` class has a variety of methods -for manipulating and printing the data saved into a profile results file:: - - import pstats - from pstats import SortKey - p = pstats.Stats('restats') - p.strip_dirs().sort_stats(-1).print_stats() - -The :meth:`~pstats.Stats.strip_dirs` method removed the extraneous path from all -the module names. The :meth:`~pstats.Stats.sort_stats` method sorted all the -entries according to the standard module/line/name string that is printed. The -:meth:`~pstats.Stats.print_stats` method printed out all the statistics. You -might try the following sort calls:: - - p.sort_stats(SortKey.NAME) - p.print_stats() - -The first call will actually sort the list by function name, and the second call -will print out the statistics. The following are some interesting calls to -experiment with:: - - p.sort_stats(SortKey.CUMULATIVE).print_stats(10) - -This sorts the profile by cumulative time in a function, and then only prints -the ten most significant lines. If you want to understand what algorithms are -taking time, the above line is what you would use. - -If you were looking to see what functions were looping a lot, and taking a lot -of time, you would do:: - - p.sort_stats(SortKey.TIME).print_stats(10) - -to sort according to time spent within each function, and then print the -statistics for the top ten functions. - -You might also try:: - - p.sort_stats(SortKey.FILENAME).print_stats('__init__') - -This will sort all the statistics by file name, and then print out statistics -for only the class init methods (since they are spelled with ``__init__`` in -them). As one final example, you could try:: - - p.sort_stats(SortKey.TIME, SortKey.CUMULATIVE).print_stats(.5, 'init') - -This line sorts statistics with a primary key of time, and a secondary key of -cumulative time, and then prints out some of the statistics. To be specific, the -list is first culled down to 50% (re: ``.5``) of its original size, then only -lines containing ``init`` are maintained, and that sub-sub-list is printed. - -If you wondered what functions called the above functions, you could now (``p`` -is still sorted according to the last criteria) do:: - - p.print_callers(.5, 'init') - -and you would get a list of callers for each of the listed functions. - -If you want more functionality, you're going to have to read the manual, or -guess what the following functions do:: - - p.print_callees() - p.add('restats') - -Invoked as a script, the :mod:`pstats` module is a statistics browser for -reading and examining profile dumps. It has a simple line-oriented interface -(implemented using :mod:`cmd`) and interactive help. - -:mod:`profile` and :mod:`cProfile` Module Reference -======================================================= - -.. module:: cProfile -.. module:: profile - :synopsis: Python source profiler. - -Both the :mod:`profile` and :mod:`cProfile` modules provide the following -functions: - -.. function:: run(command, filename=None, sort=-1) - - This function takes a single argument that can be passed to the :func:`exec` - function, and an optional file name. In all cases this routine executes:: - - exec(command, __main__.__dict__, __main__.__dict__) - - and gathers profiling statistics from the execution. If no file name is - present, then this function automatically creates a :class:`~pstats.Stats` - instance and prints a simple profiling report. If the sort value is specified, - it is passed to this :class:`~pstats.Stats` instance to control how the - results are sorted. - -.. function:: runctx(command, globals, locals, filename=None, sort=-1) - - This function is similar to :func:`run`, with added arguments to supply the - globals and locals dictionaries for the *command* string. This routine - executes:: - - exec(command, globals, locals) - - and gathers profiling statistics as in the :func:`run` function above. - -.. class:: Profile(timer=None, timeunit=0.0, subcalls=True, builtins=True) - - This class is normally only used if more precise control over profiling is - needed than what the :func:`cProfile.run` function provides. - - A custom timer can be supplied for measuring how long code takes to run via - the *timer* argument. This must be a function that returns a single number - representing the current time. If the number is an integer, the *timeunit* - specifies a multiplier that specifies the duration of each unit of time. For - example, if the timer returns times measured in thousands of seconds, the - time unit would be ``.001``. - - Directly using the :class:`Profile` class allows formatting profile results - without writing the profile data to a file:: - - import cProfile, pstats, io - from pstats import SortKey - pr = cProfile.Profile() - pr.enable() - # ... do something ... - pr.disable() - s = io.StringIO() - sortby = SortKey.CUMULATIVE - ps = pstats.Stats(pr, stream=s).sort_stats(sortby) - ps.print_stats() - print(s.getvalue()) - - The :class:`Profile` class can also be used as a context manager (supported - only in :mod:`cProfile` module. see :ref:`typecontextmanager`):: - - import cProfile - - with cProfile.Profile() as pr: - # ... do something ... - - pr.print_stats() - - .. versionchanged:: 3.8 - Added context manager support. - - .. method:: enable() - - Start collecting profiling data. Only in :mod:`cProfile`. - - .. method:: disable() - - Stop collecting profiling data. Only in :mod:`cProfile`. - - .. method:: create_stats() - - Stop collecting profiling data and record the results internally - as the current profile. - - .. method:: print_stats(sort=-1) - - Create a :class:`~pstats.Stats` object based on the current - profile and print the results to stdout. - - .. method:: dump_stats(filename) - - Write the results of the current profile to *filename*. - - .. method:: run(cmd) - - Profile the cmd via :func:`exec`. - - .. method:: runctx(cmd, globals, locals) - - Profile the cmd via :func:`exec` with the specified global and - local environment. - - .. method:: runcall(func, /, *args, **kwargs) - - Profile ``func(*args, **kwargs)`` - -Note that profiling will only work if the called command/function actually -returns. If the interpreter is terminated (e.g. via a :func:`sys.exit` call -during the called command/function execution) no profiling results will be -printed. - -.. _profile-stats: - -The :class:`Stats` Class -======================== - -Analysis of the profiler data is done using the :class:`~pstats.Stats` class. - -.. module:: pstats - :synopsis: Statistics object for use with the profiler. - -.. class:: Stats(*filenames or profile, stream=sys.stdout) - - This class constructor creates an instance of a "statistics object" from a - *filename* (or list of filenames) or from a :class:`Profile` instance. Output - will be printed to the stream specified by *stream*. - - The file selected by the above constructor must have been created by the - corresponding version of :mod:`profile` or :mod:`cProfile`. To be specific, - there is *no* file compatibility guaranteed with future versions of this - profiler, and there is no compatibility with files produced by other - profilers, or the same profiler run on a different operating system. If - several files are provided, all the statistics for identical functions will - be coalesced, so that an overall view of several processes can be considered - in a single report. If additional files need to be combined with data in an - existing :class:`~pstats.Stats` object, the :meth:`~pstats.Stats.add` method - can be used. - - Instead of reading the profile data from a file, a :class:`cProfile.Profile` - or :class:`profile.Profile` object can be used as the profile data source. - - :class:`Stats` objects have the following methods: - - .. method:: strip_dirs() - - This method for the :class:`Stats` class removes all leading path - information from file names. It is very useful in reducing the size of - the printout to fit within (close to) 80 columns. This method modifies - the object, and the stripped information is lost. After performing a - strip operation, the object is considered to have its entries in a - "random" order, as it was just after object initialization and loading. - If :meth:`~pstats.Stats.strip_dirs` causes two function names to be - indistinguishable (they are on the same line of the same filename, and - have the same function name), then the statistics for these two entries - are accumulated into a single entry. - - - .. method:: add(*filenames) - - This method of the :class:`Stats` class accumulates additional profiling - information into the current profiling object. Its arguments should refer - to filenames created by the corresponding version of :func:`profile.run` - or :func:`cProfile.run`. Statistics for identically named (re: file, line, - name) functions are automatically accumulated into single function - statistics. - - - .. method:: dump_stats(filename) - - Save the data loaded into the :class:`Stats` object to a file named - *filename*. The file is created if it does not exist, and is overwritten - if it already exists. This is equivalent to the method of the same name - on the :class:`profile.Profile` and :class:`cProfile.Profile` classes. - - - .. method:: sort_stats(*keys) - - This method modifies the :class:`Stats` object by sorting it according to - the supplied criteria. The argument can be either a string or a SortKey - enum identifying the basis of a sort (example: ``'time'``, ``'name'``, - ``SortKey.TIME`` or ``SortKey.NAME``). The SortKey enums argument have - advantage over the string argument in that it is more robust and less - error prone. - - When more than one key is provided, then additional keys are used as - secondary criteria when there is equality in all keys selected before - them. For example, ``sort_stats(SortKey.NAME, SortKey.FILE)`` will sort - all the entries according to their function name, and resolve all ties - (identical function names) by sorting by file name. - - For the string argument, abbreviations can be used for any key names, as - long as the abbreviation is unambiguous. - - The following are the valid string and SortKey: - - +------------------+---------------------+----------------------+ - | Valid String Arg | Valid enum Arg | Meaning | - +==================+=====================+======================+ - | ``'calls'`` | SortKey.CALLS | call count | - +------------------+---------------------+----------------------+ - | ``'cumulative'`` | SortKey.CUMULATIVE | cumulative time | - +------------------+---------------------+----------------------+ - | ``'cumtime'`` | N/A | cumulative time | - +------------------+---------------------+----------------------+ - | ``'file'`` | N/A | file name | - +------------------+---------------------+----------------------+ - | ``'filename'`` | SortKey.FILENAME | file name | - +------------------+---------------------+----------------------+ - | ``'module'`` | N/A | file name | - +------------------+---------------------+----------------------+ - | ``'ncalls'`` | N/A | call count | - +------------------+---------------------+----------------------+ - | ``'pcalls'`` | SortKey.PCALLS | primitive call count | - +------------------+---------------------+----------------------+ - | ``'line'`` | SortKey.LINE | line number | - +------------------+---------------------+----------------------+ - | ``'name'`` | SortKey.NAME | function name | - +------------------+---------------------+----------------------+ - | ``'nfl'`` | SortKey.NFL | name/file/line | - +------------------+---------------------+----------------------+ - | ``'stdname'`` | SortKey.STDNAME | standard name | - +------------------+---------------------+----------------------+ - | ``'time'`` | SortKey.TIME | internal time | - +------------------+---------------------+----------------------+ - | ``'tottime'`` | N/A | internal time | - +------------------+---------------------+----------------------+ - - Note that all sorts on statistics are in descending order (placing most - time consuming items first), where as name, file, and line number searches - are in ascending order (alphabetical). The subtle distinction between - ``SortKey.NFL`` and ``SortKey.STDNAME`` is that the standard name is a - sort of the name as printed, which means that the embedded line numbers - get compared in an odd way. For example, lines 3, 20, and 40 would (if - the file names were the same) appear in the string order 20, 3 and 40. - In contrast, ``SortKey.NFL`` does a numeric compare of the line numbers. - In fact, ``sort_stats(SortKey.NFL)`` is the same as - ``sort_stats(SortKey.NAME, SortKey.FILENAME, SortKey.LINE)``. - - For backward-compatibility reasons, the numeric arguments ``-1``, ``0``, - ``1``, and ``2`` are permitted. They are interpreted as ``'stdname'``, - ``'calls'``, ``'time'``, and ``'cumulative'`` respectively. If this old - style format (numeric) is used, only one sort key (the numeric key) will - be used, and additional arguments will be silently ignored. - - .. For compatibility with the old profiler. - - .. versionadded:: 3.7 - Added the SortKey enum. - - .. method:: reverse_order() - - This method for the :class:`Stats` class reverses the ordering of the - basic list within the object. Note that by default ascending vs - descending order is properly selected based on the sort key of choice. - - .. This method is provided primarily for compatibility with the old - profiler. - - - .. method:: print_stats(*restrictions) - - This method for the :class:`Stats` class prints out a report as described - in the :func:`profile.run` definition. - - The order of the printing is based on the last - :meth:`~pstats.Stats.sort_stats` operation done on the object (subject to - caveats in :meth:`~pstats.Stats.add` and - :meth:`~pstats.Stats.strip_dirs`). - - The arguments provided (if any) can be used to limit the list down to the - significant entries. Initially, the list is taken to be the complete set - of profiled functions. Each restriction is either an integer (to select a - count of lines), or a decimal fraction between 0.0 and 1.0 inclusive (to - select a percentage of lines), or a string that will interpreted as a - regular expression (to pattern match the standard name that is printed). - If several restrictions are provided, then they are applied sequentially. - For example:: - - print_stats(.1, 'foo:') - - would first limit the printing to first 10% of list, and then only print - functions that were part of filename :file:`.\*foo:`. In contrast, the - command:: - - print_stats('foo:', .1) - - would limit the list to all functions having file names :file:`.\*foo:`, - and then proceed to only print the first 10% of them. - - - .. method:: print_callers(*restrictions) - - This method for the :class:`Stats` class prints a list of all functions - that called each function in the profiled database. The ordering is - identical to that provided by :meth:`~pstats.Stats.print_stats`, and the - definition of the restricting argument is also identical. Each caller is - reported on its own line. The format differs slightly depending on the - profiler that produced the stats: - - * With :mod:`profile`, a number is shown in parentheses after each caller - to show how many times this specific call was made. For convenience, a - second non-parenthesized number repeats the cumulative time spent in the - function at the right. - - * With :mod:`cProfile`, each caller is preceded by three numbers: the - number of times this specific call was made, and the total and - cumulative times spent in the current function while it was invoked by - this specific caller. - - - .. method:: print_callees(*restrictions) - - This method for the :class:`Stats` class prints a list of all function - that were called by the indicated function. Aside from this reversal of - direction of calls (re: called vs was called by), the arguments and - ordering are identical to the :meth:`~pstats.Stats.print_callers` method. - - - .. method:: get_stats_profile() - - This method returns an instance of StatsProfile, which contains a mapping - of function names to instances of FunctionProfile. Each FunctionProfile - instance holds information related to the function's profile such as how - long the function took to run, how many times it was called, etc... - - .. versionadded:: 3.9 - Added the following dataclasses: StatsProfile, FunctionProfile. - Added the following function: get_stats_profile. - -.. _deterministic-profiling: - -What Is Deterministic Profiling? -================================ - -:dfn:`Deterministic profiling` is meant to reflect the fact that all *function -call*, *function return*, and *exception* events are monitored, and precise -timings are made for the intervals between these events (during which time the -user's code is executing). In contrast, :dfn:`statistical profiling` (which is -not done by this module) randomly samples the effective instruction pointer, and -deduces where time is being spent. The latter technique traditionally involves -less overhead (as the code does not need to be instrumented), but provides only -relative indications of where time is being spent. - -In Python, since there is an interpreter active during execution, the presence -of instrumented code is not required in order to do deterministic profiling. -Python automatically provides a :dfn:`hook` (optional callback) for each event. -In addition, the interpreted nature of Python tends to add so much overhead to -execution, that deterministic profiling tends to only add small processing -overhead in typical applications. The result is that deterministic profiling is -not that expensive, yet provides extensive run time statistics about the -execution of a Python program. - -Call count statistics can be used to identify bugs in code (surprising counts), -and to identify possible inline-expansion points (high call counts). Internal -time statistics can be used to identify "hot loops" that should be carefully -optimized. Cumulative time statistics should be used to identify high level -errors in the selection of algorithms. Note that the unusual handling of -cumulative times in this profiler allows statistics for recursive -implementations of algorithms to be directly compared to iterative -implementations. - - -.. _profile-limitations: - -Limitations -=========== - -One limitation has to do with accuracy of timing information. There is a -fundamental problem with deterministic profilers involving accuracy. The most -obvious restriction is that the underlying "clock" is only ticking at a rate -(typically) of about .001 seconds. Hence no measurements will be more accurate -than the underlying clock. If enough measurements are taken, then the "error" -will tend to average out. Unfortunately, removing this first error induces a -second source of error. - -The second problem is that it "takes a while" from when an event is dispatched -until the profiler's call to get the time actually *gets* the state of the -clock. Similarly, there is a certain lag when exiting the profiler event -handler from the time that the clock's value was obtained (and then squirreled -away), until the user's code is once again executing. As a result, functions -that are called many times, or call many functions, will typically accumulate -this error. The error that accumulates in this fashion is typically less than -the accuracy of the clock (less than one clock tick), but it *can* accumulate -and become very significant. - -The problem is more important with :mod:`profile` than with the lower-overhead -:mod:`cProfile`. For this reason, :mod:`profile` provides a means of -calibrating itself for a given platform so that this error can be -probabilistically (on the average) removed. After the profiler is calibrated, it -will be more accurate (in a least square sense), but it will sometimes produce -negative numbers (when call counts are exceptionally low, and the gods of -probability work against you :-). ) Do *not* be alarmed by negative numbers in -the profile. They should *only* appear if you have calibrated your profiler, -and the results are actually better than without calibration. - - -.. _profile-calibration: - -Calibration -=========== - -The profiler of the :mod:`profile` module subtracts a constant from each event -handling time to compensate for the overhead of calling the time function, and -socking away the results. By default, the constant is 0. The following -procedure can be used to obtain a better constant for a given platform (see -:ref:`profile-limitations`). :: - - import profile - pr = profile.Profile() - for i in range(5): - print(pr.calibrate(10000)) - -The method executes the number of Python calls given by the argument, directly -and again under the profiler, measuring the time for both. It then computes the -hidden overhead per profiler event, and returns that as a float. For example, -on a 1.8Ghz Intel Core i5 running macOS, and using Python's time.process_time() as -the timer, the magical number is about 4.04e-6. - -The object of this exercise is to get a fairly consistent result. If your -computer is *very* fast, or your timer function has poor resolution, you might -have to pass 100000, or even 1000000, to get consistent results. - -When you have a consistent answer, there are three ways you can use it:: - - import profile - - # 1. Apply computed bias to all Profile instances created hereafter. - profile.Profile.bias = your_computed_bias - - # 2. Apply computed bias to a specific Profile instance. - pr = profile.Profile() - pr.bias = your_computed_bias - - # 3. Specify computed bias in instance constructor. - pr = profile.Profile(bias=your_computed_bias) - -If you have a choice, you are better off choosing a smaller constant, and then -your results will "less often" show up as negative in profile statistics. - -.. _profile-timers: - -Using a custom timer -==================== - -If you want to change how current time is determined (for example, to force use -of wall-clock time or elapsed process time), pass the timing function you want -to the :class:`Profile` class constructor:: - - pr = profile.Profile(your_time_func) - -The resulting profiler will then call ``your_time_func``. Depending on whether -you are using :class:`profile.Profile` or :class:`cProfile.Profile`, -``your_time_func``'s return value will be interpreted differently: - -:class:`profile.Profile` - ``your_time_func`` should return a single number, or a list of numbers whose - sum is the current time (like what :func:`os.times` returns). If the - function returns a single time number, or the list of returned numbers has - length 2, then you will get an especially fast version of the dispatch - routine. - - Be warned that you should calibrate the profiler class for the timer function - that you choose (see :ref:`profile-calibration`). For most machines, a timer - that returns a lone integer value will provide the best results in terms of - low overhead during profiling. (:func:`os.times` is *pretty* bad, as it - returns a tuple of floating point values). If you want to substitute a - better timer in the cleanest fashion, derive a class and hardwire a - replacement dispatch method that best handles your timer call, along with the - appropriate calibration constant. - -:class:`cProfile.Profile` - ``your_time_func`` should return a single number. If it returns integers, - you can also invoke the class constructor with a second argument specifying - the real duration of one unit of time. For example, if - ``your_integer_time_func`` returns times measured in thousands of seconds, - you would construct the :class:`Profile` instance as follows:: - - pr = cProfile.Profile(your_integer_time_func, 0.001) - - As the :class:`cProfile.Profile` class cannot be calibrated, custom timer - functions should be used with care and should be as fast as possible. For - the best results with a custom timer, it might be necessary to hard-code it - in the C source of the internal :mod:`_lsprof` module. - -Python 3.3 adds several new functions in :mod:`time` that can be used to make -precise measurements of process or wall-clock time. For example, see -:func:`time.perf_counter`. diff --git a/Doc/library/pty.rst b/Doc/library/pty.rst deleted file mode 100644 index af9378464edb9f..00000000000000 --- a/Doc/library/pty.rst +++ /dev/null @@ -1,122 +0,0 @@ -:mod:`pty` --- Pseudo-terminal utilities -======================================== - -.. module:: pty - :platform: Unix - :synopsis: Pseudo-Terminal Handling for Unix. - -.. moduleauthor:: Steen Lumholt -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/pty.py` - --------------- - -The :mod:`pty` module defines operations for handling the pseudo-terminal -concept: starting another process and being able to write to and read from its -controlling terminal programmatically. - -.. availability:: Unix. - -Pseudo-terminal handling is highly platform dependent. This code is mainly -tested on Linux, FreeBSD, and macOS (it is supposed to work on other POSIX -platforms but it's not been thoroughly tested). - -The :mod:`pty` module defines the following functions: - - -.. function:: fork() - - Fork. Connect the child's controlling terminal to a pseudo-terminal. Return - value is ``(pid, fd)``. Note that the child gets *pid* 0, and the *fd* is - *invalid*. The parent's return value is the *pid* of the child, and *fd* is a - file descriptor connected to the child's controlling terminal (and also to the - child's standard input and output). - - -.. function:: openpty() - - Open a new pseudo-terminal pair, using :func:`os.openpty` if possible, or - emulation code for generic Unix systems. Return a pair of file descriptors - ``(master, slave)``, for the master and the slave end, respectively. - - -.. function:: spawn(argv[, master_read[, stdin_read]]) - - Spawn a process, and connect its controlling terminal with the current - process's standard io. This is often used to baffle programs which insist on - reading from the controlling terminal. It is expected that the process - spawned behind the pty will eventually terminate, and when it does *spawn* - will return. - - A loop copies STDIN of the current process to the child and data received - from the child to STDOUT of the current process. It is not signaled to the - child if STDIN of the current process closes down. - - The functions *master_read* and *stdin_read* are passed a file descriptor - which they should read from, and they should always return a byte string. In - order to force spawn to return before the child process exits an - empty byte array should be returned to signal end of file. - - The default implementation for both functions will read and return up to 1024 - bytes each time the function is called. The *master_read* callback is passed - the pseudoterminal’s master file descriptor to read output from the child - process, and *stdin_read* is passed file descriptor 0, to read from the - parent process's standard input. - - Returning an empty byte string from either callback is interpreted as an - end-of-file (EOF) condition, and that callback will not be called after - that. If *stdin_read* signals EOF the controlling terminal can no longer - communicate with the parent process OR the child process. Unless the child - process will quit without any input, *spawn* will then loop forever. If - *master_read* signals EOF the same behavior results (on linux at least). - - Return the exit status value from :func:`os.waitpid` on the child process. - - :func:`os.waitstatus_to_exitcode` can be used to convert the exit status into - an exit code. - - .. audit-event:: pty.spawn argv pty.spawn - - .. versionchanged:: 3.4 - :func:`spawn` now returns the status value from :func:`os.waitpid` - on the child process. - -Example -------- - -.. sectionauthor:: Steen Lumholt - -The following program acts like the Unix command :manpage:`script(1)`, using a -pseudo-terminal to record all input and output of a terminal session in a -"typescript". :: - - import argparse - import os - import pty - import sys - import time - - parser = argparse.ArgumentParser() - parser.add_argument('-a', dest='append', action='store_true') - parser.add_argument('-p', dest='use_python', action='store_true') - parser.add_argument('filename', nargs='?', default='typescript') - options = parser.parse_args() - - shell = sys.executable if options.use_python else os.environ.get('SHELL', 'sh') - filename = options.filename - mode = 'ab' if options.append else 'wb' - - with open(filename, mode) as script: - def read(fd): - data = os.read(fd, 1024) - script.write(data) - return data - - print('Script started, file is', filename) - script.write(('Script started on %s\n' % time.asctime()).encode()) - - pty.spawn(shell, read) - - script.write(('Script done on %s\n' % time.asctime()).encode()) - print('Script done, file is', filename) diff --git a/Doc/library/pwd.rst b/Doc/library/pwd.rst deleted file mode 100644 index 755f0d29ac7345..00000000000000 --- a/Doc/library/pwd.rst +++ /dev/null @@ -1,78 +0,0 @@ -:mod:`pwd` --- The password database -==================================== - -.. module:: pwd - :platform: Unix - :synopsis: The password database (getpwnam() and friends). - --------------- - -This module provides access to the Unix user account and password database. It -is available on all Unix versions. - -.. availability:: Unix, not Emscripten, not WASI. - -Password database entries are reported as a tuple-like object, whose attributes -correspond to the members of the ``passwd`` structure (Attribute field below, -see ````): - -+-------+---------------+-----------------------------+ -| Index | Attribute | Meaning | -+=======+===============+=============================+ -| 0 | ``pw_name`` | Login name | -+-------+---------------+-----------------------------+ -| 1 | ``pw_passwd`` | Optional encrypted password | -+-------+---------------+-----------------------------+ -| 2 | ``pw_uid`` | Numerical user ID | -+-------+---------------+-----------------------------+ -| 3 | ``pw_gid`` | Numerical group ID | -+-------+---------------+-----------------------------+ -| 4 | ``pw_gecos`` | User name or comment field | -+-------+---------------+-----------------------------+ -| 5 | ``pw_dir`` | User home directory | -+-------+---------------+-----------------------------+ -| 6 | ``pw_shell`` | User command interpreter | -+-------+---------------+-----------------------------+ - -The uid and gid items are integers, all others are strings. :exc:`KeyError` is -raised if the entry asked for cannot be found. - -.. note:: - - .. index:: pair: module; crypt - - In traditional Unix the field ``pw_passwd`` usually contains a password - encrypted with a DES derived algorithm (see module :mod:`crypt`). However most - modern unices use a so-called *shadow password* system. On those unices the - *pw_passwd* field only contains an asterisk (``'*'``) or the letter ``'x'`` - where the encrypted password is stored in a file :file:`/etc/shadow` which is - not world readable. Whether the *pw_passwd* field contains anything useful is - system-dependent. If available, the :mod:`spwd` module should be used where - access to the encrypted password is required. - -It defines the following items: - - -.. function:: getpwuid(uid) - - Return the password database entry for the given numeric user ID. - - -.. function:: getpwnam(name) - - Return the password database entry for the given user name. - - -.. function:: getpwall() - - Return a list of all available password database entries, in arbitrary order. - - -.. seealso:: - - Module :mod:`grp` - An interface to the group database, similar to this. - - Module :mod:`spwd` - An interface to the shadow password database, similar to this. - diff --git a/Doc/library/py_compile.rst b/Doc/library/py_compile.rst deleted file mode 100644 index 38c416f9ad0305..00000000000000 --- a/Doc/library/py_compile.rst +++ /dev/null @@ -1,162 +0,0 @@ -:mod:`py_compile` --- Compile Python source files -================================================= - -.. module:: py_compile - :synopsis: Generate byte-code files from Python source files. - -.. sectionauthor:: Fred L. Drake, Jr. -.. documentation based on module docstrings - -**Source code:** :source:`Lib/py_compile.py` - -.. index:: pair: file; byte-code - --------------- - -The :mod:`py_compile` module provides a function to generate a byte-code file -from a source file, and another function used when the module source file is -invoked as a script. - -Though not often needed, this function can be useful when installing modules for -shared use, especially if some of the users may not have permission to write the -byte-code cache files in the directory containing the source code. - - -.. exception:: PyCompileError - - Exception raised when an error occurs while attempting to compile the file. - - -.. function:: compile(file, cfile=None, dfile=None, doraise=False, optimize=-1, invalidation_mode=PycInvalidationMode.TIMESTAMP, quiet=0) - - Compile a source file to byte-code and write out the byte-code cache file. - The source code is loaded from the file named *file*. The byte-code is - written to *cfile*, which defaults to the :pep:`3147`/:pep:`488` path, ending - in ``.pyc``. - For example, if *file* is ``/foo/bar/baz.py`` *cfile* will default to - ``/foo/bar/__pycache__/baz.cpython-32.pyc`` for Python 3.2. If *dfile* is - specified, it is used instead of *file* as the name of the source file from - which source lines are obtained for display in exception tracebacks. - If *doraise* is true, a :exc:`PyCompileError` is raised - when an error is encountered while compiling *file*. If *doraise* is false - (the default), an error string is written to ``sys.stderr``, but no exception - is raised. This function returns the path to byte-compiled file, i.e. - whatever *cfile* value was used. - - The *doraise* and *quiet* arguments determine how errors are handled while - compiling file. If *quiet* is 0 or 1, and *doraise* is false, the default - behaviour is enabled: an error string is written to ``sys.stderr``, and the - function returns ``None`` instead of a path. If *doraise* is true, - a :exc:`PyCompileError` is raised instead. However if *quiet* is 2, - no message is written, and *doraise* has no effect. - - If the path that *cfile* becomes (either explicitly specified or computed) - is a symlink or non-regular file, :exc:`FileExistsError` will be raised. - This is to act as a warning that import will turn those paths into regular - files if it is allowed to write byte-compiled files to those paths. This is - a side-effect of import using file renaming to place the final byte-compiled - file into place to prevent concurrent file writing issues. - - *optimize* controls the optimization level and is passed to the built-in - :func:`compile` function. The default of ``-1`` selects the optimization - level of the current interpreter. - - *invalidation_mode* should be a member of the :class:`PycInvalidationMode` - enum and controls how the generated bytecode cache is invalidated at - runtime. The default is :attr:`PycInvalidationMode.CHECKED_HASH` if - the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, otherwise - the default is :attr:`PycInvalidationMode.TIMESTAMP`. - - .. versionchanged:: 3.2 - Changed default value of *cfile* to be :PEP:`3147`-compliant. Previous - default was *file* + ``'c'`` (``'o'`` if optimization was enabled). - Also added the *optimize* parameter. - - .. versionchanged:: 3.4 - Changed code to use :mod:`importlib` for the byte-code cache file writing. - This means file creation/writing semantics now match what :mod:`importlib` - does, e.g. permissions, write-and-move semantics, etc. Also added the - caveat that :exc:`FileExistsError` is raised if *cfile* is a symlink or - non-regular file. - - .. versionchanged:: 3.7 - The *invalidation_mode* parameter was added as specified in :pep:`552`. - If the :envvar:`SOURCE_DATE_EPOCH` environment variable is set, - *invalidation_mode* will be forced to - :attr:`PycInvalidationMode.CHECKED_HASH`. - - .. versionchanged:: 3.7.2 - The :envvar:`SOURCE_DATE_EPOCH` environment variable no longer - overrides the value of the *invalidation_mode* argument, and determines - its default value instead. - - .. versionchanged:: 3.8 - The *quiet* parameter was added. - - -.. class:: PycInvalidationMode - - A enumeration of possible methods the interpreter can use to determine - whether a bytecode file is up to date with a source file. The ``.pyc`` file - indicates the desired invalidation mode in its header. See - :ref:`pyc-invalidation` for more information on how Python invalidates - ``.pyc`` files at runtime. - - .. versionadded:: 3.7 - - .. attribute:: TIMESTAMP - - The ``.pyc`` file includes the timestamp and size of the source file, - which Python will compare against the metadata of the source file at - runtime to determine if the ``.pyc`` file needs to be regenerated. - - .. attribute:: CHECKED_HASH - - The ``.pyc`` file includes a hash of the source file content, which Python - will compare against the source at runtime to determine if the ``.pyc`` - file needs to be regenerated. - - .. attribute:: UNCHECKED_HASH - - Like :attr:`CHECKED_HASH`, the ``.pyc`` file includes a hash of the source - file content. However, Python will at runtime assume the ``.pyc`` file is - up to date and not validate the ``.pyc`` against the source file at all. - - This option is useful when the ``.pycs`` are kept up to date by some - system external to Python like a build system. - -.. _py_compile-cli: - -Command-Line Interface ----------------------- - -This module can be invoked as a script to compile several source -files. The files named in *filenames* are compiled and the resulting -bytecode is cached in the normal manner. This program does not search -a directory structure to locate source files; it only compiles files -named explicitly. The exit status is nonzero if one of the files could -not be compiled. - -.. program:: python -m py_compile - -.. option:: ... - - - - Positional arguments are files to compile. If ``-`` is the only - parameter, the list of files is taken from standard input. - -.. option:: -q, --quiet - - Suppress errors output. - -.. versionchanged:: 3.2 - Added support for ``-``. - -.. versionchanged:: 3.10 - Added support for :option:`-q`. - - -.. seealso:: - - Module :mod:`compileall` - Utilities to compile all Python source files in a directory tree. diff --git a/Doc/library/pyclbr.rst b/Doc/library/pyclbr.rst deleted file mode 100644 index 1c40ba4838ca75..00000000000000 --- a/Doc/library/pyclbr.rst +++ /dev/null @@ -1,163 +0,0 @@ -:mod:`pyclbr` --- Python module browser support -=============================================== - -.. module:: pyclbr - :synopsis: Supports information extraction for a Python module browser. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/pyclbr.py` - --------------- - -The :mod:`pyclbr` module provides limited information about the -functions, classes, and methods defined in a Python-coded module. The -information is sufficient to implement a module browser. The -information is extracted from the Python source code rather than by -importing the module, so this module is safe to use with untrusted code. -This restriction makes it impossible to use this module with modules not -implemented in Python, including all standard and optional extension -modules. - - -.. function:: readmodule(module, path=None) - - Return a dictionary mapping module-level class names to class - descriptors. If possible, descriptors for imported base classes are - included. Parameter *module* is a string with the name of the module - to read; it may be the name of a module within a package. If given, - *path* is a sequence of directory paths prepended to ``sys.path``, - which is used to locate the module source code. - - This function is the original interface and is only kept for back - compatibility. It returns a filtered version of the following. - - -.. function:: readmodule_ex(module, path=None) - - Return a dictionary-based tree containing a function or class - descriptors for each function and class defined in the module with a - ``def`` or ``class`` statement. The returned dictionary maps - module-level function and class names to their descriptors. Nested - objects are entered into the children dictionary of their parent. As - with readmodule, *module* names the module to be read and *path* is - prepended to sys.path. If the module being read is a package, the - returned dictionary has a key ``'__path__'`` whose value is a list - containing the package search path. - -.. versionadded:: 3.7 - Descriptors for nested definitions. They are accessed through the - new children attribute. Each has a new parent attribute. - -The descriptors returned by these functions are instances of -Function and Class classes. Users are not expected to create instances -of these classes. - - -.. _pyclbr-function-objects: - -Function Objects ----------------- -Class :class:`Function` instances describe functions defined by def -statements. They have the following attributes: - - -.. attribute:: Function.file - - Name of the file in which the function is defined. - - -.. attribute:: Function.module - - The name of the module defining the function described. - - -.. attribute:: Function.name - - The name of the function. - - -.. attribute:: Function.lineno - - The line number in the file where the definition starts. - - -.. attribute:: Function.parent - - For top-level functions, None. For nested functions, the parent. - - .. versionadded:: 3.7 - - -.. attribute:: Function.children - - A dictionary mapping names to descriptors for nested functions and - classes. - - .. versionadded:: 3.7 - - -.. attribute:: Function.is_async - - ``True`` for functions that are defined with the ``async`` prefix, ``False`` otherwise. - - .. versionadded:: 3.10 - - -.. _pyclbr-class-objects: - -Class Objects -------------- -Class :class:`Class` instances describe classes defined by class -statements. They have the same attributes as Functions and two more. - - -.. attribute:: Class.file - - Name of the file in which the class is defined. - - -.. attribute:: Class.module - - The name of the module defining the class described. - - -.. attribute:: Class.name - - The name of the class. - - -.. attribute:: Class.lineno - - The line number in the file where the definition starts. - - -.. attribute:: Class.parent - - For top-level classes, None. For nested classes, the parent. - - .. versionadded:: 3.7 - - -.. attribute:: Class.children - - A dictionary mapping names to descriptors for nested functions and - classes. - - .. versionadded:: 3.7 - - -.. attribute:: Class.super - - A list of :class:`Class` objects which describe the immediate base - classes of the class being described. Classes which are named as - superclasses but which are not discoverable by :func:`readmodule_ex` - are listed as a string with the class name instead of as - :class:`Class` objects. - - -.. attribute:: Class.methods - - A dictionary mapping method names to line numbers. This can be - derived from the newer children dictionary, but remains for - back-compatibility. diff --git a/Doc/library/pydoc.rst b/Doc/library/pydoc.rst deleted file mode 100644 index 03e0915bf6d135..00000000000000 --- a/Doc/library/pydoc.rst +++ /dev/null @@ -1,109 +0,0 @@ -:mod:`pydoc` --- Documentation generator and online help system -=============================================================== - -.. module:: pydoc - :synopsis: Documentation generator and online help system. - -.. moduleauthor:: Ka-Ping Yee -.. sectionauthor:: Ka-Ping Yee - -**Source code:** :source:`Lib/pydoc.py` - -.. index:: - single: documentation; generation - single: documentation; online - single: help; online - --------------- - -The :mod:`pydoc` module automatically generates documentation from Python -modules. The documentation can be presented as pages of text on the console, -served to a web browser, or saved to HTML files. - -For modules, classes, functions and methods, the displayed documentation is -derived from the docstring (i.e. the :attr:`__doc__` attribute) of the object, -and recursively of its documentable members. If there is no docstring, -:mod:`pydoc` tries to obtain a description from the block of comment lines just -above the definition of the class, function or method in the source file, or at -the top of the module (see :func:`inspect.getcomments`). - -The built-in function :func:`help` invokes the online help system in the -interactive interpreter, which uses :mod:`pydoc` to generate its documentation -as text on the console. The same text documentation can also be viewed from -outside the Python interpreter by running :program:`pydoc` as a script at the -operating system's command prompt. For example, running :: - - python -m pydoc sys - -at a shell prompt will display documentation on the :mod:`sys` module, in a -style similar to the manual pages shown by the Unix :program:`man` command. The -argument to :program:`pydoc` can be the name of a function, module, or package, -or a dotted reference to a class, method, or function within a module or module -in a package. If the argument to :program:`pydoc` looks like a path (that is, -it contains the path separator for your operating system, such as a slash in -Unix), and refers to an existing Python source file, then documentation is -produced for that file. - -.. note:: - - In order to find objects and their documentation, :mod:`pydoc` imports the - module(s) to be documented. Therefore, any code on module level will be - executed on that occasion. Use an ``if __name__ == '__main__':`` guard to - only execute code when a file is invoked as a script and not just imported. - -When printing output to the console, :program:`pydoc` attempts to paginate the -output for easier reading. If the :envvar:`PAGER` environment variable is set, -:program:`pydoc` will use its value as a pagination program. - -Specifying a ``-w`` flag before the argument will cause HTML documentation -to be written out to a file in the current directory, instead of displaying text -on the console. - -Specifying a ``-k`` flag before the argument will search the synopsis -lines of all available modules for the keyword given as the argument, again in a -manner similar to the Unix :program:`man` command. The synopsis line of a -module is the first line of its documentation string. - -You can also use :program:`pydoc` to start an HTTP server on the local machine -that will serve documentation to visiting web browsers. :program:`python -m pydoc -p 1234` -will start a HTTP server on port 1234, allowing you to browse the -documentation at ``http://localhost:1234/`` in your preferred web browser. -Specifying ``0`` as the port number will select an arbitrary unused port. - -:program:`python -m pydoc -n ` will start the server listening at the given -hostname. By default the hostname is 'localhost' but if you want the server to -be reached from other machines, you may want to change the host name that the -server responds to. During development this is especially useful if you want -to run pydoc from within a container. - -:program:`python -m pydoc -b` will start the server and additionally open a web -browser to a module index page. Each served page has a navigation bar at the -top where you can *Get* help on an individual item, *Search* all modules with a -keyword in their synopsis line, and go to the *Module index*, *Topics* and -*Keywords* pages. - -When :program:`pydoc` generates documentation, it uses the current environment -and path to locate modules. Thus, invoking :program:`pydoc spam` -documents precisely the version of the module you would get if you started the -Python interpreter and typed ``import spam``. - -Module docs for core modules are assumed to reside in -``https://docs.python.org/X.Y/library/`` where ``X`` and ``Y`` are the -major and minor version numbers of the Python interpreter. This can -be overridden by setting the :envvar:`PYTHONDOCS` environment variable -to a different URL or to a local directory containing the Library -Reference Manual pages. - -.. versionchanged:: 3.2 - Added the ``-b`` option. - -.. versionchanged:: 3.3 - The ``-g`` command line option was removed. - -.. versionchanged:: 3.4 - :mod:`pydoc` now uses :func:`inspect.signature` rather than - :func:`inspect.getfullargspec` to extract signature information from - callables. - -.. versionchanged:: 3.7 - Added the ``-n`` option. diff --git a/Doc/library/pyexpat.rst b/Doc/library/pyexpat.rst deleted file mode 100644 index 935e872480efda..00000000000000 --- a/Doc/library/pyexpat.rst +++ /dev/null @@ -1,910 +0,0 @@ -:mod:`xml.parsers.expat` --- Fast XML parsing using Expat -========================================================= - -.. module:: xml.parsers.expat - :synopsis: An interface to the Expat non-validating XML parser. - -.. moduleauthor:: Paul Prescod - --------------- - -.. Markup notes: - - Many of the attributes of the XMLParser objects are callbacks. Since - signature information must be presented, these are described using the method - directive. Since they are attributes which are set by client code, in-text - references to these attributes should be marked using the :member: role. - - -.. warning:: - - The :mod:`pyexpat` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - - -.. index:: single: Expat - -The :mod:`xml.parsers.expat` module is a Python interface to the Expat -non-validating XML parser. The module provides a single extension type, -:class:`xmlparser`, that represents the current state of an XML parser. After -an :class:`xmlparser` object has been created, various attributes of the object -can be set to handler functions. When an XML document is then fed to the -parser, the handler functions are called for the character data and markup in -the XML document. - -.. index:: pair: module; pyexpat - -This module uses the :mod:`pyexpat` module to provide access to the Expat -parser. Direct use of the :mod:`pyexpat` module is deprecated. - -This module provides one exception and one type object: - - -.. exception:: ExpatError - - The exception raised when Expat reports an error. See section - :ref:`expaterror-objects` for more information on interpreting Expat errors. - - -.. exception:: error - - Alias for :exc:`ExpatError`. - - -.. data:: XMLParserType - - The type of the return values from the :func:`ParserCreate` function. - -The :mod:`xml.parsers.expat` module contains two functions: - - -.. function:: ErrorString(errno) - - Returns an explanatory string for a given error number *errno*. - - -.. function:: ParserCreate(encoding=None, namespace_separator=None) - - Creates and returns a new :class:`xmlparser` object. *encoding*, if specified, - must be a string naming the encoding used by the XML data. Expat doesn't - support as many encodings as Python does, and its repertoire of encodings can't - be extended; it supports UTF-8, UTF-16, ISO-8859-1 (Latin1), and ASCII. If - *encoding* [1]_ is given it will override the implicit or explicit encoding of the - document. - - Expat can optionally do XML namespace processing for you, enabled by providing a - value for *namespace_separator*. The value must be a one-character string; a - :exc:`ValueError` will be raised if the string has an illegal length (``None`` - is considered the same as omission). When namespace processing is enabled, - element type names and attribute names that belong to a namespace will be - expanded. The element name passed to the element handlers - :attr:`StartElementHandler` and :attr:`EndElementHandler` will be the - concatenation of the namespace URI, the namespace separator character, and the - local part of the name. If the namespace separator is a zero byte (``chr(0)``) - then the namespace URI and the local part will be concatenated without any - separator. - - For example, if *namespace_separator* is set to a space character (``' '``) and - the following document is parsed: - - .. code-block:: xml - - - - - - - - :attr:`StartElementHandler` will receive the following strings for each - element:: - - http://default-namespace.org/ root - http://www.python.org/ns/ elem1 - elem2 - - Due to limitations in the ``Expat`` library used by :mod:`pyexpat`, - the :class:`xmlparser` instance returned can only be used to parse a single - XML document. Call ``ParserCreate`` for each document to provide unique - parser instances. - - -.. seealso:: - - `The Expat XML Parser `_ - Home page of the Expat project. - - -.. _xmlparser-objects: - -XMLParser Objects ------------------ - -:class:`xmlparser` objects have the following methods: - - -.. method:: xmlparser.Parse(data[, isfinal]) - - Parses the contents of the string *data*, calling the appropriate handler - functions to process the parsed data. *isfinal* must be true on the final call - to this method; it allows the parsing of a single file in fragments, - not the submission of multiple files. - *data* can be the empty string at any time. - - -.. method:: xmlparser.ParseFile(file) - - Parse XML data reading from the object *file*. *file* only needs to provide - the ``read(nbytes)`` method, returning the empty string when there's no more - data. - - -.. method:: xmlparser.SetBase(base) - - Sets the base to be used for resolving relative URIs in system identifiers in - declarations. Resolving relative identifiers is left to the application: this - value will be passed through as the *base* argument to the - :func:`ExternalEntityRefHandler`, :func:`NotationDeclHandler`, and - :func:`UnparsedEntityDeclHandler` functions. - - -.. method:: xmlparser.GetBase() - - Returns a string containing the base set by a previous call to :meth:`SetBase`, - or ``None`` if :meth:`SetBase` hasn't been called. - - -.. method:: xmlparser.GetInputContext() - - Returns the input data that generated the current event as a string. The data is - in the encoding of the entity which contains the text. When called while an - event handler is not active, the return value is ``None``. - - -.. method:: xmlparser.ExternalEntityParserCreate(context[, encoding]) - - Create a "child" parser which can be used to parse an external parsed entity - referred to by content parsed by the parent parser. The *context* parameter - should be the string passed to the :meth:`ExternalEntityRefHandler` handler - function, described below. The child parser is created with the - :attr:`ordered_attributes` and :attr:`specified_attributes` set to the values of - this parser. - -.. method:: xmlparser.SetParamEntityParsing(flag) - - Control parsing of parameter entities (including the external DTD subset). - Possible *flag* values are :const:`XML_PARAM_ENTITY_PARSING_NEVER`, - :const:`XML_PARAM_ENTITY_PARSING_UNLESS_STANDALONE` and - :const:`XML_PARAM_ENTITY_PARSING_ALWAYS`. Return true if setting the flag - was successful. - -.. method:: xmlparser.UseForeignDTD([flag]) - - Calling this with a true value for *flag* (the default) will cause Expat to call - the :attr:`ExternalEntityRefHandler` with :const:`None` for all arguments to - allow an alternate DTD to be loaded. If the document does not contain a - document type declaration, the :attr:`ExternalEntityRefHandler` will still be - called, but the :attr:`StartDoctypeDeclHandler` and - :attr:`EndDoctypeDeclHandler` will not be called. - - Passing a false value for *flag* will cancel a previous call that passed a true - value, but otherwise has no effect. - - This method can only be called before the :meth:`Parse` or :meth:`ParseFile` - methods are called; calling it after either of those have been called causes - :exc:`ExpatError` to be raised with the :attr:`code` attribute set to - ``errors.codes[errors.XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING]``. - -:class:`xmlparser` objects have the following attributes: - - -.. attribute:: xmlparser.buffer_size - - The size of the buffer used when :attr:`buffer_text` is true. - A new buffer size can be set by assigning a new integer value - to this attribute. - When the size is changed, the buffer will be flushed. - - -.. attribute:: xmlparser.buffer_text - - Setting this to true causes the :class:`xmlparser` object to buffer textual - content returned by Expat to avoid multiple calls to the - :meth:`CharacterDataHandler` callback whenever possible. This can improve - performance substantially since Expat normally breaks character data into chunks - at every line ending. This attribute is false by default, and may be changed at - any time. - - -.. attribute:: xmlparser.buffer_used - - If :attr:`buffer_text` is enabled, the number of bytes stored in the buffer. - These bytes represent UTF-8 encoded text. This attribute has no meaningful - interpretation when :attr:`buffer_text` is false. - - -.. attribute:: xmlparser.ordered_attributes - - Setting this attribute to a non-zero integer causes the attributes to be - reported as a list rather than a dictionary. The attributes are presented in - the order found in the document text. For each attribute, two list entries are - presented: the attribute name and the attribute value. (Older versions of this - module also used this format.) By default, this attribute is false; it may be - changed at any time. - - -.. attribute:: xmlparser.specified_attributes - - If set to a non-zero integer, the parser will report only those attributes which - were specified in the document instance and not those which were derived from - attribute declarations. Applications which set this need to be especially - careful to use what additional information is available from the declarations as - needed to comply with the standards for the behavior of XML processors. By - default, this attribute is false; it may be changed at any time. - - -The following attributes contain values relating to the most recent error -encountered by an :class:`xmlparser` object, and will only have correct values -once a call to :meth:`Parse` or :meth:`ParseFile` has raised an -:exc:`xml.parsers.expat.ExpatError` exception. - - -.. attribute:: xmlparser.ErrorByteIndex - - Byte index at which an error occurred. - - -.. attribute:: xmlparser.ErrorCode - - Numeric code specifying the problem. This value can be passed to the - :func:`ErrorString` function, or compared to one of the constants defined in the - ``errors`` object. - - -.. attribute:: xmlparser.ErrorColumnNumber - - Column number at which an error occurred. - - -.. attribute:: xmlparser.ErrorLineNumber - - Line number at which an error occurred. - -The following attributes contain values relating to the current parse location -in an :class:`xmlparser` object. During a callback reporting a parse event they -indicate the location of the first of the sequence of characters that generated -the event. When called outside of a callback, the position indicated will be -just past the last parse event (regardless of whether there was an associated -callback). - - -.. attribute:: xmlparser.CurrentByteIndex - - Current byte index in the parser input. - - -.. attribute:: xmlparser.CurrentColumnNumber - - Current column number in the parser input. - - -.. attribute:: xmlparser.CurrentLineNumber - - Current line number in the parser input. - -Here is the list of handlers that can be set. To set a handler on an -:class:`xmlparser` object *o*, use ``o.handlername = func``. *handlername* must -be taken from the following list, and *func* must be a callable object accepting -the correct number of arguments. The arguments are all strings, unless -otherwise stated. - - -.. method:: xmlparser.XmlDeclHandler(version, encoding, standalone) - - Called when the XML declaration is parsed. The XML declaration is the - (optional) declaration of the applicable version of the XML recommendation, the - encoding of the document text, and an optional "standalone" declaration. - *version* and *encoding* will be strings, and *standalone* will be ``1`` if the - document is declared standalone, ``0`` if it is declared not to be standalone, - or ``-1`` if the standalone clause was omitted. This is only available with - Expat version 1.95.0 or newer. - - -.. method:: xmlparser.StartDoctypeDeclHandler(doctypeName, systemId, publicId, has_internal_subset) - - Called when Expat begins parsing the document type declaration (``'``. - - -.. method:: xmlparser.StartCdataSectionHandler() - - Called at the start of a CDATA section. This and :attr:`EndCdataSectionHandler` - are needed to be able to identify the syntactical start and end for CDATA - sections. - - -.. method:: xmlparser.EndCdataSectionHandler() - - Called at the end of a CDATA section. - - -.. method:: xmlparser.DefaultHandler(data) - - Called for any characters in the XML document for which no applicable handler - has been specified. This means characters that are part of a construct which - could be reported, but for which no handler has been supplied. - - -.. method:: xmlparser.DefaultHandlerExpand(data) - - This is the same as the :func:`DefaultHandler`, but doesn't inhibit expansion - of internal entities. The entity reference will not be passed to the default - handler. - - -.. method:: xmlparser.NotStandaloneHandler() - - Called if the XML document hasn't been declared as being a standalone document. - This happens when there is an external subset or a reference to a parameter - entity, but the XML declaration does not set standalone to ``yes`` in an XML - declaration. If this handler returns ``0``, then the parser will raise an - :const:`XML_ERROR_NOT_STANDALONE` error. If this handler is not set, no - exception is raised by the parser for this condition. - - -.. method:: xmlparser.ExternalEntityRefHandler(context, base, systemId, publicId) - - Called for references to external entities. *base* is the current base, as set - by a previous call to :meth:`SetBase`. The public and system identifiers, - *systemId* and *publicId*, are strings if given; if the public identifier is not - given, *publicId* will be ``None``. The *context* value is opaque and should - only be used as described below. - - For external entities to be parsed, this handler must be implemented. It is - responsible for creating the sub-parser using - ``ExternalEntityParserCreate(context)``, initializing it with the appropriate - callbacks, and parsing the entity. This handler should return an integer; if it - returns ``0``, the parser will raise an - :const:`XML_ERROR_EXTERNAL_ENTITY_HANDLING` error, otherwise parsing will - continue. - - If this handler is not provided, external entities are reported by the - :attr:`DefaultHandler` callback, if provided. - - -.. _expaterror-objects: - -ExpatError Exceptions ---------------------- - -.. sectionauthor:: Fred L. Drake, Jr. - - -:exc:`ExpatError` exceptions have a number of interesting attributes: - - -.. attribute:: ExpatError.code - - Expat's internal error number for the specific error. The - :data:`errors.messages ` dictionary maps - these error numbers to Expat's error messages. For example:: - - from xml.parsers.expat import ParserCreate, ExpatError, errors - - p = ParserCreate() - try: - p.Parse(some_xml_document) - except ExpatError as err: - print("Error:", errors.messages[err.code]) - - The :mod:`~xml.parsers.expat.errors` module also provides error message - constants and a dictionary :data:`~xml.parsers.expat.errors.codes` mapping - these messages back to the error codes, see below. - - -.. attribute:: ExpatError.lineno - - Line number on which the error was detected. The first line is numbered ``1``. - - -.. attribute:: ExpatError.offset - - Character offset into the line where the error occurred. The first column is - numbered ``0``. - - -.. _expat-example: - -Example -------- - -The following program defines three handlers that just print out their -arguments. :: - - import xml.parsers.expat - - # 3 handler functions - def start_element(name, attrs): - print('Start element:', name, attrs) - def end_element(name): - print('End element:', name) - def char_data(data): - print('Character data:', repr(data)) - - p = xml.parsers.expat.ParserCreate() - - p.StartElementHandler = start_element - p.EndElementHandler = end_element - p.CharacterDataHandler = char_data - - p.Parse(""" - Text goes here - More text - """, 1) - -The output from this program is:: - - Start element: parent {'id': 'top'} - Start element: child1 {'name': 'paul'} - Character data: 'Text goes here' - End element: child1 - Character data: '\n' - Start element: child2 {'name': 'fred'} - Character data: 'More text' - End element: child2 - Character data: '\n' - End element: parent - - -.. _expat-content-models: - -Content Model Descriptions --------------------------- - -.. module:: xml.parsers.expat.model - -.. sectionauthor:: Fred L. Drake, Jr. - -Content models are described using nested tuples. Each tuple contains four -values: the type, the quantifier, the name, and a tuple of children. Children -are simply additional content model descriptions. - -The values of the first two fields are constants defined in the -:mod:`xml.parsers.expat.model` module. These constants can be collected in two -groups: the model type group and the quantifier group. - -The constants in the model type group are: - - -.. data:: XML_CTYPE_ANY - :noindex: - - The element named by the model name was declared to have a content model of - ``ANY``. - - -.. data:: XML_CTYPE_CHOICE - :noindex: - - The named element allows a choice from a number of options; this is used for - content models such as ``(A | B | C)``. - - -.. data:: XML_CTYPE_EMPTY - :noindex: - - Elements which are declared to be ``EMPTY`` have this model type. - - -.. data:: XML_CTYPE_MIXED - :noindex: - - -.. data:: XML_CTYPE_NAME - :noindex: - - -.. data:: XML_CTYPE_SEQ - :noindex: - - Models which represent a series of models which follow one after the other are - indicated with this model type. This is used for models such as ``(A, B, C)``. - -The constants in the quantifier group are: - - -.. data:: XML_CQUANT_NONE - :noindex: - - No modifier is given, so it can appear exactly once, as for ``A``. - - -.. data:: XML_CQUANT_OPT - :noindex: - - The model is optional: it can appear once or not at all, as for ``A?``. - - -.. data:: XML_CQUANT_PLUS - :noindex: - - The model must occur one or more times (like ``A+``). - - -.. data:: XML_CQUANT_REP - :noindex: - - The model must occur zero or more times, as for ``A*``. - - -.. _expat-errors: - -Expat error constants ---------------------- - -.. module:: xml.parsers.expat.errors - -The following constants are provided in the :mod:`xml.parsers.expat.errors` -module. These constants are useful in interpreting some of the attributes of -the :exc:`ExpatError` exception objects raised when an error has occurred. -Since for backwards compatibility reasons, the constants' value is the error -*message* and not the numeric error *code*, you do this by comparing its -:attr:`code` attribute with -:samp:`errors.codes[errors.XML_ERROR_{CONSTANT_NAME}]`. - -The ``errors`` module has the following attributes: - -.. data:: codes - - A dictionary mapping string descriptions to their error codes. - - .. versionadded:: 3.2 - - -.. data:: messages - - A dictionary mapping numeric error codes to their string descriptions. - - .. versionadded:: 3.2 - - -.. data:: XML_ERROR_ASYNC_ENTITY - - -.. data:: XML_ERROR_ATTRIBUTE_EXTERNAL_ENTITY_REF - - An entity reference in an attribute value referred to an external entity instead - of an internal entity. - - -.. data:: XML_ERROR_BAD_CHAR_REF - - A character reference referred to a character which is illegal in XML (for - example, character ``0``, or '``�``'). - - -.. data:: XML_ERROR_BINARY_ENTITY_REF - - An entity reference referred to an entity which was declared with a notation, so - cannot be parsed. - - -.. data:: XML_ERROR_DUPLICATE_ATTRIBUTE - - An attribute was used more than once in a start tag. - - -.. data:: XML_ERROR_INCORRECT_ENCODING - - -.. data:: XML_ERROR_INVALID_TOKEN - - Raised when an input byte could not properly be assigned to a character; for - example, a NUL byte (value ``0``) in a UTF-8 input stream. - - -.. data:: XML_ERROR_JUNK_AFTER_DOC_ELEMENT - - Something other than whitespace occurred after the document element. - - -.. data:: XML_ERROR_MISPLACED_XML_PI - - An XML declaration was found somewhere other than the start of the input data. - - -.. data:: XML_ERROR_NO_ELEMENTS - - The document contains no elements (XML requires all documents to contain exactly - one top-level element).. - - -.. data:: XML_ERROR_NO_MEMORY - - Expat was not able to allocate memory internally. - - -.. data:: XML_ERROR_PARAM_ENTITY_REF - - A parameter entity reference was found where it was not allowed. - - -.. data:: XML_ERROR_PARTIAL_CHAR - - An incomplete character was found in the input. - - -.. data:: XML_ERROR_RECURSIVE_ENTITY_REF - - An entity reference contained another reference to the same entity; possibly via - a different name, and possibly indirectly. - - -.. data:: XML_ERROR_SYNTAX - - Some unspecified syntax error was encountered. - - -.. data:: XML_ERROR_TAG_MISMATCH - - An end tag did not match the innermost open start tag. - - -.. data:: XML_ERROR_UNCLOSED_TOKEN - - Some token (such as a start tag) was not closed before the end of the stream or - the next token was encountered. - - -.. data:: XML_ERROR_UNDEFINED_ENTITY - - A reference was made to an entity which was not defined. - - -.. data:: XML_ERROR_UNKNOWN_ENCODING - - The document encoding is not supported by Expat. - - -.. data:: XML_ERROR_UNCLOSED_CDATA_SECTION - - A CDATA marked section was not closed. - - -.. data:: XML_ERROR_EXTERNAL_ENTITY_HANDLING - - -.. data:: XML_ERROR_NOT_STANDALONE - - The parser determined that the document was not "standalone" though it declared - itself to be in the XML declaration, and the :attr:`NotStandaloneHandler` was - set and returned ``0``. - - -.. data:: XML_ERROR_UNEXPECTED_STATE - - -.. data:: XML_ERROR_ENTITY_DECLARED_IN_PE - - -.. data:: XML_ERROR_FEATURE_REQUIRES_XML_DTD - - An operation was requested that requires DTD support to be compiled in, but - Expat was configured without DTD support. This should never be reported by a - standard build of the :mod:`xml.parsers.expat` module. - - -.. data:: XML_ERROR_CANT_CHANGE_FEATURE_ONCE_PARSING - - A behavioral change was requested after parsing started that can only be changed - before parsing has started. This is (currently) only raised by - :meth:`UseForeignDTD`. - - -.. data:: XML_ERROR_UNBOUND_PREFIX - - An undeclared prefix was found when namespace processing was enabled. - - -.. data:: XML_ERROR_UNDECLARING_PREFIX - - The document attempted to remove the namespace declaration associated with a - prefix. - - -.. data:: XML_ERROR_INCOMPLETE_PE - - A parameter entity contained incomplete markup. - - -.. data:: XML_ERROR_XML_DECL - - The document contained no document element at all. - - -.. data:: XML_ERROR_TEXT_DECL - - There was an error parsing a text declaration in an external entity. - - -.. data:: XML_ERROR_PUBLICID - - Characters were found in the public id that are not allowed. - - -.. data:: XML_ERROR_SUSPENDED - - The requested operation was made on a suspended parser, but isn't allowed. This - includes attempts to provide additional input or to stop the parser. - - -.. data:: XML_ERROR_NOT_SUSPENDED - - An attempt to resume the parser was made when the parser had not been suspended. - - -.. data:: XML_ERROR_ABORTED - - This should not be reported to Python applications. - - -.. data:: XML_ERROR_FINISHED - - The requested operation was made on a parser which was finished parsing input, - but isn't allowed. This includes attempts to provide additional input or to - stop the parser. - - -.. data:: XML_ERROR_SUSPEND_PE - - -.. data:: XML_ERROR_RESERVED_PREFIX_XML - - An attempt was made to - undeclare reserved namespace prefix ``xml`` - or to bind it to another namespace URI. - - -.. data:: XML_ERROR_RESERVED_PREFIX_XMLNS - - An attempt was made to declare or undeclare reserved namespace prefix ``xmlns``. - - -.. data:: XML_ERROR_RESERVED_NAMESPACE_URI - - An attempt was made to bind the URI of one the reserved namespace - prefixes ``xml`` and ``xmlns`` to another namespace prefix. - - -.. data:: XML_ERROR_INVALID_ARGUMENT - - This should not be reported to Python applications. - - -.. data:: XML_ERROR_NO_BUFFER - - This should not be reported to Python applications. - - -.. data:: XML_ERROR_AMPLIFICATION_LIMIT_BREACH - - The limit on input amplification factor (from DTD and entities) - has been breached. - - -.. rubric:: Footnotes - -.. [1] The encoding string included in XML output should conform to the - appropriate standards. For example, "UTF-8" is valid, but "UTF8" is - not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl - and https://www.iana.org/assignments/character-sets/character-sets.xhtml. - diff --git a/Doc/library/python.rst b/Doc/library/python.rst deleted file mode 100644 index f39613f572884f..00000000000000 --- a/Doc/library/python.rst +++ /dev/null @@ -1,27 +0,0 @@ -.. _python: - -*********************** -Python Runtime Services -*********************** - -The modules described in this chapter provide a wide range of services related -to the Python interpreter and its interaction with its environment. Here's an -overview: - - -.. toctree:: - - sys.rst - sysconfig.rst - builtins.rst - __main__.rst - warnings.rst - dataclasses.rst - contextlib.rst - abc.rst - atexit.rst - traceback.rst - __future__.rst - gc.rst - inspect.rst - site.rst diff --git a/Doc/library/queue.rst b/Doc/library/queue.rst deleted file mode 100644 index b2b787c5a8260c..00000000000000 --- a/Doc/library/queue.rst +++ /dev/null @@ -1,280 +0,0 @@ -:mod:`queue` --- A synchronized queue class -=========================================== - -.. module:: queue - :synopsis: A synchronized queue class. - -**Source code:** :source:`Lib/queue.py` - --------------- - -The :mod:`queue` module implements multi-producer, multi-consumer queues. -It is especially useful in threaded programming when information must be -exchanged safely between multiple threads. The :class:`Queue` class in this -module implements all the required locking semantics. - -The module implements three types of queue, which differ only in the order in -which the entries are retrieved. In a :abbr:`FIFO (first-in, first-out)` -queue, the first tasks added are the first retrieved. In a -:abbr:`LIFO (last-in, first-out)` queue, the most recently added entry is -the first retrieved (operating like a stack). With a priority queue, -the entries are kept sorted (using the :mod:`heapq` module) and the -lowest valued entry is retrieved first. - -Internally, those three types of queues use locks to temporarily block -competing threads; however, they are not designed to handle reentrancy -within a thread. - -In addition, the module implements a "simple" -:abbr:`FIFO (first-in, first-out)` queue type, :class:`SimpleQueue`, whose -specific implementation provides additional guarantees -in exchange for the smaller functionality. - -The :mod:`queue` module defines the following classes and exceptions: - -.. class:: Queue(maxsize=0) - - Constructor for a :abbr:`FIFO (first-in, first-out)` queue. *maxsize* is - an integer that sets the upperbound - limit on the number of items that can be placed in the queue. Insertion will - block once this size has been reached, until queue items are consumed. If - *maxsize* is less than or equal to zero, the queue size is infinite. - -.. class:: LifoQueue(maxsize=0) - - Constructor for a :abbr:`LIFO (last-in, first-out)` queue. *maxsize* is - an integer that sets the upperbound - limit on the number of items that can be placed in the queue. Insertion will - block once this size has been reached, until queue items are consumed. If - *maxsize* is less than or equal to zero, the queue size is infinite. - - -.. class:: PriorityQueue(maxsize=0) - - Constructor for a priority queue. *maxsize* is an integer that sets the upperbound - limit on the number of items that can be placed in the queue. Insertion will - block once this size has been reached, until queue items are consumed. If - *maxsize* is less than or equal to zero, the queue size is infinite. - - The lowest valued entries are retrieved first (the lowest valued entry is the - one that would be returned by ``min(entries)``). A typical pattern for - entries is a tuple in the form: ``(priority_number, data)``. - - If the *data* elements are not comparable, the data can be wrapped in a class - that ignores the data item and only compares the priority number:: - - from dataclasses import dataclass, field - from typing import Any - - @dataclass(order=True) - class PrioritizedItem: - priority: int - item: Any=field(compare=False) - -.. class:: SimpleQueue() - - Constructor for an unbounded :abbr:`FIFO (first-in, first-out)` queue. - Simple queues lack advanced functionality such as task tracking. - - .. versionadded:: 3.7 - - -.. exception:: Empty - - Exception raised when non-blocking :meth:`~Queue.get` (or - :meth:`~Queue.get_nowait`) is called - on a :class:`Queue` object which is empty. - - -.. exception:: Full - - Exception raised when non-blocking :meth:`~Queue.put` (or - :meth:`~Queue.put_nowait`) is called - on a :class:`Queue` object which is full. - - -.. _queueobjects: - -Queue Objects -------------- - -Queue objects (:class:`Queue`, :class:`LifoQueue`, or :class:`PriorityQueue`) -provide the public methods described below. - - -.. method:: Queue.qsize() - - Return the approximate size of the queue. Note, qsize() > 0 doesn't - guarantee that a subsequent get() will not block, nor will qsize() < maxsize - guarantee that put() will not block. - - -.. method:: Queue.empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. If empty() - returns ``True`` it doesn't guarantee that a subsequent call to put() - will not block. Similarly, if empty() returns ``False`` it doesn't - guarantee that a subsequent call to get() will not block. - - -.. method:: Queue.full() - - Return ``True`` if the queue is full, ``False`` otherwise. If full() - returns ``True`` it doesn't guarantee that a subsequent call to get() - will not block. Similarly, if full() returns ``False`` it doesn't - guarantee that a subsequent call to put() will not block. - - -.. method:: Queue.put(item, block=True, timeout=None) - - Put *item* into the queue. If optional args *block* is true and *timeout* is - ``None`` (the default), block if necessary until a free slot is available. If - *timeout* is a positive number, it blocks at most *timeout* seconds and raises - the :exc:`Full` exception if no free slot was available within that time. - Otherwise (*block* is false), put an item on the queue if a free slot is - immediately available, else raise the :exc:`Full` exception (*timeout* is - ignored in that case). - - -.. method:: Queue.put_nowait(item) - - Equivalent to ``put(item, block=False)``. - - -.. method:: Queue.get(block=True, timeout=None) - - Remove and return an item from the queue. If optional args *block* is true and - *timeout* is ``None`` (the default), block if necessary until an item is available. - If *timeout* is a positive number, it blocks at most *timeout* seconds and - raises the :exc:`Empty` exception if no item was available within that time. - Otherwise (*block* is false), return an item if one is immediately available, - else raise the :exc:`Empty` exception (*timeout* is ignored in that case). - - Prior to 3.0 on POSIX systems, and for all versions on Windows, if - *block* is true and *timeout* is ``None``, this operation goes into - an uninterruptible wait on an underlying lock. This means that no exceptions - can occur, and in particular a SIGINT will not trigger a :exc:`KeyboardInterrupt`. - - -.. method:: Queue.get_nowait() - - Equivalent to ``get(False)``. - -Two methods are offered to support tracking whether enqueued tasks have been -fully processed by daemon consumer threads. - - -.. method:: Queue.task_done() - - Indicate that a formerly enqueued task is complete. Used by queue consumer - threads. For each :meth:`get` used to fetch a task, a subsequent call to - :meth:`task_done` tells the queue that the processing on the task is complete. - - If a :meth:`join` is currently blocking, it will resume when all items have been - processed (meaning that a :meth:`task_done` call was received for every item - that had been :meth:`put` into the queue). - - Raises a :exc:`ValueError` if called more times than there were items placed in - the queue. - - -.. method:: Queue.join() - - Blocks until all items in the queue have been gotten and processed. - - The count of unfinished tasks goes up whenever an item is added to the queue. - The count goes down whenever a consumer thread calls :meth:`task_done` to - indicate that the item was retrieved and all work on it is complete. When the - count of unfinished tasks drops to zero, :meth:`join` unblocks. - - -Example of how to wait for enqueued tasks to be completed:: - - import threading - import queue - - q = queue.Queue() - - def worker(): - while True: - item = q.get() - print(f'Working on {item}') - print(f'Finished {item}') - q.task_done() - - # Turn-on the worker thread. - threading.Thread(target=worker, daemon=True).start() - - # Send thirty task requests to the worker. - for item in range(30): - q.put(item) - - # Block until all tasks are done. - q.join() - print('All work completed') - - -SimpleQueue Objects -------------------- - -:class:`SimpleQueue` objects provide the public methods described below. - -.. method:: SimpleQueue.qsize() - - Return the approximate size of the queue. Note, qsize() > 0 doesn't - guarantee that a subsequent get() will not block. - - -.. method:: SimpleQueue.empty() - - Return ``True`` if the queue is empty, ``False`` otherwise. If empty() - returns ``False`` it doesn't guarantee that a subsequent call to get() - will not block. - - -.. method:: SimpleQueue.put(item, block=True, timeout=None) - - Put *item* into the queue. The method never blocks and always succeeds - (except for potential low-level errors such as failure to allocate memory). - The optional args *block* and *timeout* are ignored and only provided - for compatibility with :meth:`Queue.put`. - - .. impl-detail:: - This method has a C implementation which is reentrant. That is, a - ``put()`` or ``get()`` call can be interrupted by another ``put()`` - call in the same thread without deadlocking or corrupting internal - state inside the queue. This makes it appropriate for use in - destructors such as ``__del__`` methods or :mod:`weakref` callbacks. - - -.. method:: SimpleQueue.put_nowait(item) - - Equivalent to ``put(item, block=False)``, provided for compatibility with - :meth:`Queue.put_nowait`. - - -.. method:: SimpleQueue.get(block=True, timeout=None) - - Remove and return an item from the queue. If optional args *block* is true and - *timeout* is ``None`` (the default), block if necessary until an item is available. - If *timeout* is a positive number, it blocks at most *timeout* seconds and - raises the :exc:`Empty` exception if no item was available within that time. - Otherwise (*block* is false), return an item if one is immediately available, - else raise the :exc:`Empty` exception (*timeout* is ignored in that case). - - -.. method:: SimpleQueue.get_nowait() - - Equivalent to ``get(False)``. - - -.. seealso:: - - Class :class:`multiprocessing.Queue` - A queue class for use in a multi-processing (rather than multi-threading) - context. - - :class:`collections.deque` is an alternative implementation of unbounded - queues with fast atomic :meth:`~collections.deque.append` and - :meth:`~collections.deque.popleft` operations that do not require locking - and also support indexing. diff --git a/Doc/library/quopri.rst b/Doc/library/quopri.rst deleted file mode 100644 index 86717c00c3c136..00000000000000 --- a/Doc/library/quopri.rst +++ /dev/null @@ -1,63 +0,0 @@ -:mod:`quopri` --- Encode and decode MIME quoted-printable data -============================================================== - -.. module:: quopri - :synopsis: Encode and decode files using the MIME quoted-printable encoding. - -**Source code:** :source:`Lib/quopri.py` - -.. index:: - pair: quoted-printable; encoding - single: MIME; quoted-printable encoding - --------------- - -This module performs quoted-printable transport encoding and decoding, as -defined in :rfc:`1521`: "MIME (Multipurpose Internet Mail Extensions) Part One: -Mechanisms for Specifying and Describing the Format of Internet Message Bodies". -The quoted-printable encoding is designed for data where there are relatively -few nonprintable characters; the base64 encoding scheme available via the -:mod:`base64` module is more compact if there are many such characters, as when -sending a graphics file. - -.. function:: decode(input, output, header=False) - - Decode the contents of the *input* file and write the resulting decoded binary - data to the *output* file. *input* and *output* must be :term:`binary file objects - `. If the optional argument *header* is present and true, underscore - will be decoded as space. This is used to decode "Q"-encoded headers as - described in :rfc:`1522`: "MIME (Multipurpose Internet Mail Extensions) - Part Two: Message Header Extensions for Non-ASCII Text". - - -.. function:: encode(input, output, quotetabs, header=False) - - Encode the contents of the *input* file and write the resulting quoted-printable - data to the *output* file. *input* and *output* must be - :term:`binary file objects `. *quotetabs*, a - non-optional flag which controls whether to encode embedded spaces - and tabs; when true it encodes such embedded whitespace, and when - false it leaves them unencoded. - Note that spaces and tabs appearing at the end of lines are always encoded, - as per :rfc:`1521`. *header* is a flag which controls if spaces are encoded - as underscores as per :rfc:`1522`. - - -.. function:: decodestring(s, header=False) - - Like :func:`decode`, except that it accepts a source :class:`bytes` and - returns the corresponding decoded :class:`bytes`. - - -.. function:: encodestring(s, quotetabs=False, header=False) - - Like :func:`encode`, except that it accepts a source :class:`bytes` and - returns the corresponding encoded :class:`bytes`. By default, it sends a - ``False`` value to *quotetabs* parameter of the :func:`encode` function. - - - -.. seealso:: - - Module :mod:`base64` - Encode and decode MIME base64 data diff --git a/Doc/library/random.rst b/Doc/library/random.rst deleted file mode 100644 index 4af4f83e280ca7..00000000000000 --- a/Doc/library/random.rst +++ /dev/null @@ -1,651 +0,0 @@ -:mod:`random` --- Generate pseudo-random numbers -================================================ - -.. module:: random - :synopsis: Generate pseudo-random numbers with various common distributions. - -**Source code:** :source:`Lib/random.py` - --------------- - -This module implements pseudo-random number generators for various -distributions. - -For integers, there is uniform selection from a range. For sequences, there is -uniform selection of a random element, a function to generate a random -permutation of a list in-place, and a function for random sampling without -replacement. - -On the real line, there are functions to compute uniform, normal (Gaussian), -lognormal, negative exponential, gamma, and beta distributions. For generating -distributions of angles, the von Mises distribution is available. - -Almost all module functions depend on the basic function :func:`.random`, which -generates a random float uniformly in the half-open range ``0.0 <= X < 1.0``. -Python uses the Mersenne Twister as the core generator. It produces 53-bit precision -floats and has a period of 2\*\*19937-1. The underlying implementation in C is -both fast and threadsafe. The Mersenne Twister is one of the most extensively -tested random number generators in existence. However, being completely -deterministic, it is not suitable for all purposes, and is completely unsuitable -for cryptographic purposes. - -The functions supplied by this module are actually bound methods of a hidden -instance of the :class:`random.Random` class. You can instantiate your own -instances of :class:`Random` to get generators that don't share state. - -Class :class:`Random` can also be subclassed if you want to use a different -basic generator of your own devising: in that case, override the :meth:`~Random.random`, -:meth:`~Random.seed`, :meth:`~Random.getstate`, and :meth:`~Random.setstate` methods. -Optionally, a new generator can supply a :meth:`~Random.getrandbits` method --- this -allows :meth:`randrange` to produce selections over an arbitrarily large range. - -The :mod:`random` module also provides the :class:`SystemRandom` class which -uses the system function :func:`os.urandom` to generate random numbers -from sources provided by the operating system. - -.. warning:: - - The pseudo-random generators of this module should not be used for - security purposes. For security or cryptographic uses, see the - :mod:`secrets` module. - -.. seealso:: - - M. Matsumoto and T. Nishimura, "Mersenne Twister: A 623-dimensionally - equidistributed uniform pseudorandom number generator", ACM Transactions on - Modeling and Computer Simulation Vol. 8, No. 1, January pp.3--30 1998. - - - `Complementary-Multiply-with-Carry recipe - `_ for a compatible alternative - random number generator with a long period and comparatively simple update - operations. - - -Bookkeeping functions ---------------------- - -.. function:: seed(a=None, version=2) - - Initialize the random number generator. - - If *a* is omitted or ``None``, the current system time is used. If - randomness sources are provided by the operating system, they are used - instead of the system time (see the :func:`os.urandom` function for details - on availability). - - If *a* is an int, it is used directly. - - With version 2 (the default), a :class:`str`, :class:`bytes`, or :class:`bytearray` - object gets converted to an :class:`int` and all of its bits are used. - - With version 1 (provided for reproducing random sequences from older versions - of Python), the algorithm for :class:`str` and :class:`bytes` generates a - narrower range of seeds. - - .. versionchanged:: 3.2 - Moved to the version 2 scheme which uses all of the bits in a string seed. - - .. versionchanged:: 3.11 - The *seed* must be one of the following types: - *NoneType*, :class:`int`, :class:`float`, :class:`str`, - :class:`bytes`, or :class:`bytearray`. - -.. function:: getstate() - - Return an object capturing the current internal state of the generator. This - object can be passed to :func:`setstate` to restore the state. - - -.. function:: setstate(state) - - *state* should have been obtained from a previous call to :func:`getstate`, and - :func:`setstate` restores the internal state of the generator to what it was at - the time :func:`getstate` was called. - - -Functions for bytes -------------------- - -.. function:: randbytes(n) - - Generate *n* random bytes. - - This method should not be used for generating security tokens. - Use :func:`secrets.token_bytes` instead. - - .. versionadded:: 3.9 - - -Functions for integers ----------------------- - -.. function:: randrange(stop) - randrange(start, stop[, step]) - - Return a randomly selected element from ``range(start, stop, step)``. This is - equivalent to ``choice(range(start, stop, step))``, but doesn't actually build a - range object. - - The positional argument pattern matches that of :func:`range`. Keyword arguments - should not be used because the function may use them in unexpected ways. - - .. versionchanged:: 3.2 - :meth:`randrange` is more sophisticated about producing equally distributed - values. Formerly it used a style like ``int(random()*n)`` which could produce - slightly uneven distributions. - - .. deprecated:: 3.10 - The automatic conversion of non-integer types to equivalent integers is - deprecated. Currently ``randrange(10.0)`` is losslessly converted to - ``randrange(10)``. In the future, this will raise a :exc:`TypeError`. - - .. deprecated:: 3.10 - The exception raised for non-integer values such as ``randrange(10.5)`` - or ``randrange('10')`` will be changed from :exc:`ValueError` to - :exc:`TypeError`. - -.. function:: randint(a, b) - - Return a random integer *N* such that ``a <= N <= b``. Alias for - ``randrange(a, b+1)``. - -.. function:: getrandbits(k) - - Returns a non-negative Python integer with *k* random bits. This method - is supplied with the MersenneTwister generator and some other generators - may also provide it as an optional part of the API. When available, - :meth:`getrandbits` enables :meth:`randrange` to handle arbitrarily large - ranges. - - .. versionchanged:: 3.9 - This method now accepts zero for *k*. - - -Functions for sequences ------------------------ - -.. function:: choice(seq) - - Return a random element from the non-empty sequence *seq*. If *seq* is empty, - raises :exc:`IndexError`. - -.. function:: choices(population, weights=None, *, cum_weights=None, k=1) - - Return a *k* sized list of elements chosen from the *population* with replacement. - If the *population* is empty, raises :exc:`IndexError`. - - If a *weights* sequence is specified, selections are made according to the - relative weights. Alternatively, if a *cum_weights* sequence is given, the - selections are made according to the cumulative weights (perhaps computed - using :func:`itertools.accumulate`). For example, the relative weights - ``[10, 5, 30, 5]`` are equivalent to the cumulative weights - ``[10, 15, 45, 50]``. Internally, the relative weights are converted to - cumulative weights before making selections, so supplying the cumulative - weights saves work. - - If neither *weights* nor *cum_weights* are specified, selections are made - with equal probability. If a weights sequence is supplied, it must be - the same length as the *population* sequence. It is a :exc:`TypeError` - to specify both *weights* and *cum_weights*. - - The *weights* or *cum_weights* can use any numeric type that interoperates - with the :class:`float` values returned by :func:`random` (that includes - integers, floats, and fractions but excludes decimals). Weights are assumed - to be non-negative and finite. A :exc:`ValueError` is raised if all - weights are zero. - - For a given seed, the :func:`choices` function with equal weighting - typically produces a different sequence than repeated calls to - :func:`choice`. The algorithm used by :func:`choices` uses floating - point arithmetic for internal consistency and speed. The algorithm used - by :func:`choice` defaults to integer arithmetic with repeated selections - to avoid small biases from round-off error. - - .. versionadded:: 3.6 - - .. versionchanged:: 3.9 - Raises a :exc:`ValueError` if all weights are zero. - - -.. function:: shuffle(x) - - Shuffle the sequence *x* in place. - - To shuffle an immutable sequence and return a new shuffled list, use - ``sample(x, k=len(x))`` instead. - - Note that even for small ``len(x)``, the total number of permutations of *x* - can quickly grow larger than the period of most random number generators. - This implies that most permutations of a long sequence can never be - generated. For example, a sequence of length 2080 is the largest that - can fit within the period of the Mersenne Twister random number generator. - - .. deprecated-removed:: 3.9 3.11 - The optional parameter *random*. - - -.. function:: sample(population, k, *, counts=None) - - Return a *k* length list of unique elements chosen from the population - sequence. Used for random sampling without replacement. - - Returns a new list containing elements from the population while leaving the - original population unchanged. The resulting list is in selection order so that - all sub-slices will also be valid random samples. This allows raffle winners - (the sample) to be partitioned into grand prize and second place winners (the - subslices). - - Members of the population need not be :term:`hashable` or unique. If the population - contains repeats, then each occurrence is a possible selection in the sample. - - Repeated elements can be specified one at a time or with the optional - keyword-only *counts* parameter. For example, ``sample(['red', 'blue'], - counts=[4, 2], k=5)`` is equivalent to ``sample(['red', 'red', 'red', 'red', - 'blue', 'blue'], k=5)``. - - To choose a sample from a range of integers, use a :func:`range` object as an - argument. This is especially fast and space efficient for sampling from a large - population: ``sample(range(10000000), k=60)``. - - If the sample size is larger than the population size, a :exc:`ValueError` - is raised. - - .. versionchanged:: 3.9 - Added the *counts* parameter. - - .. versionchanged:: 3.11 - - The *population* must be a sequence. Automatic conversion of sets - to lists is no longer supported. - - -.. _real-valued-distributions: - -Real-valued distributions -------------------------- - -The following functions generate specific real-valued distributions. Function -parameters are named after the corresponding variables in the distribution's -equation, as used in common mathematical practice; most of these equations can -be found in any statistics text. - - -.. function:: random() - - Return the next random floating point number in the range ``0.0 <= X < 1.0`` - - -.. function:: uniform(a, b) - - Return a random floating point number *N* such that ``a <= N <= b`` for - ``a <= b`` and ``b <= N <= a`` for ``b < a``. - - The end-point value ``b`` may or may not be included in the range - depending on floating-point rounding in the equation ``a + (b-a) * random()``. - - -.. function:: triangular(low, high, mode) - - Return a random floating point number *N* such that ``low <= N <= high`` and - with the specified *mode* between those bounds. The *low* and *high* bounds - default to zero and one. The *mode* argument defaults to the midpoint - between the bounds, giving a symmetric distribution. - - -.. function:: betavariate(alpha, beta) - - Beta distribution. Conditions on the parameters are ``alpha > 0`` and - ``beta > 0``. Returned values range between 0 and 1. - - -.. function:: expovariate(lambd) - - Exponential distribution. *lambd* is 1.0 divided by the desired - mean. It should be nonzero. (The parameter would be called - "lambda", but that is a reserved word in Python.) Returned values - range from 0 to positive infinity if *lambd* is positive, and from - negative infinity to 0 if *lambd* is negative. - - -.. function:: gammavariate(alpha, beta) - - Gamma distribution. (*Not* the gamma function!) The shape and - scale parameters, *alpha* and *beta*, must have positive values. - (Calling conventions vary and some sources define 'beta' - as the inverse of the scale). - - The probability distribution function is:: - - x ** (alpha - 1) * math.exp(-x / beta) - pdf(x) = -------------------------------------- - math.gamma(alpha) * beta ** alpha - - -.. function:: gauss(mu=0.0, sigma=1.0) - - Normal distribution, also called the Gaussian distribution. - *mu* is the mean, - and *sigma* is the standard deviation. This is slightly faster than - the :func:`normalvariate` function defined below. - - Multithreading note: When two threads call this function - simultaneously, it is possible that they will receive the - same return value. This can be avoided in three ways. - 1) Have each thread use a different instance of the random - number generator. 2) Put locks around all calls. 3) Use the - slower, but thread-safe :func:`normalvariate` function instead. - - .. versionchanged:: 3.11 - *mu* and *sigma* now have default arguments. - - -.. function:: lognormvariate(mu, sigma) - - Log normal distribution. If you take the natural logarithm of this - distribution, you'll get a normal distribution with mean *mu* and standard - deviation *sigma*. *mu* can have any value, and *sigma* must be greater than - zero. - - -.. function:: normalvariate(mu=0.0, sigma=1.0) - - Normal distribution. *mu* is the mean, and *sigma* is the standard deviation. - - .. versionchanged:: 3.11 - *mu* and *sigma* now have default arguments. - - -.. function:: vonmisesvariate(mu, kappa) - - *mu* is the mean angle, expressed in radians between 0 and 2\*\ *pi*, and *kappa* - is the concentration parameter, which must be greater than or equal to zero. If - *kappa* is equal to zero, this distribution reduces to a uniform random angle - over the range 0 to 2\*\ *pi*. - - -.. function:: paretovariate(alpha) - - Pareto distribution. *alpha* is the shape parameter. - - -.. function:: weibullvariate(alpha, beta) - - Weibull distribution. *alpha* is the scale parameter and *beta* is the shape - parameter. - - -Alternative Generator ---------------------- - -.. class:: Random([seed]) - - Class that implements the default pseudo-random number generator used by the - :mod:`random` module. - - .. deprecated:: 3.9 - In the future, the *seed* must be one of the following types: - :class:`NoneType`, :class:`int`, :class:`float`, :class:`str`, - :class:`bytes`, or :class:`bytearray`. - -.. class:: SystemRandom([seed]) - - Class that uses the :func:`os.urandom` function for generating random numbers - from sources provided by the operating system. Not available on all systems. - Does not rely on software state, and sequences are not reproducible. Accordingly, - the :meth:`seed` method has no effect and is ignored. - The :meth:`getstate` and :meth:`setstate` methods raise - :exc:`NotImplementedError` if called. - - -Notes on Reproducibility ------------------------- - -Sometimes it is useful to be able to reproduce the sequences given by a -pseudo-random number generator. By re-using a seed value, the same sequence should be -reproducible from run to run as long as multiple threads are not running. - -Most of the random module's algorithms and seeding functions are subject to -change across Python versions, but two aspects are guaranteed not to change: - -* If a new seeding method is added, then a backward compatible seeder will be - offered. - -* The generator's :meth:`~Random.random` method will continue to produce the same - sequence when the compatible seeder is given the same seed. - -.. _random-examples: - -Examples --------- - -Basic examples:: - - >>> random() # Random float: 0.0 <= x < 1.0 - 0.37444887175646646 - - >>> uniform(2.5, 10.0) # Random float: 2.5 <= x <= 10.0 - 3.1800146073117523 - - >>> expovariate(1 / 5) # Interval between arrivals averaging 5 seconds - 5.148957571865031 - - >>> randrange(10) # Integer from 0 to 9 inclusive - 7 - - >>> randrange(0, 101, 2) # Even integer from 0 to 100 inclusive - 26 - - >>> choice(['win', 'lose', 'draw']) # Single random element from a sequence - 'draw' - - >>> deck = 'ace two three four'.split() - >>> shuffle(deck) # Shuffle a list - >>> deck - ['four', 'two', 'ace', 'three'] - - >>> sample([10, 20, 30, 40, 50], k=4) # Four samples without replacement - [40, 10, 50, 30] - -Simulations:: - - >>> # Six roulette wheel spins (weighted sampling with replacement) - >>> choices(['red', 'black', 'green'], [18, 18, 2], k=6) - ['red', 'green', 'black', 'black', 'red', 'black'] - - >>> # Deal 20 cards without replacement from a deck - >>> # of 52 playing cards, and determine the proportion of cards - >>> # with a ten-value: ten, jack, queen, or king. - >>> dealt = sample(['tens', 'low cards'], counts=[16, 36], k=20) - >>> dealt.count('tens') / 20 - 0.15 - - >>> # Estimate the probability of getting 5 or more heads from 7 spins - >>> # of a biased coin that settles on heads 60% of the time. - >>> def trial(): - ... return choices('HT', cum_weights=(0.60, 1.00), k=7).count('H') >= 5 - ... - >>> sum(trial() for i in range(10_000)) / 10_000 - 0.4169 - - >>> # Probability of the median of 5 samples being in middle two quartiles - >>> def trial(): - ... return 2_500 <= sorted(choices(range(10_000), k=5))[2] < 7_500 - ... - >>> sum(trial() for i in range(10_000)) / 10_000 - 0.7958 - -Example of `statistical bootstrapping -`_ using resampling -with replacement to estimate a confidence interval for the mean of a sample:: - - # https://www.thoughtco.com/example-of-bootstrapping-3126155 - from statistics import fmean as mean - from random import choices - - data = [41, 50, 29, 37, 81, 30, 73, 63, 20, 35, 68, 22, 60, 31, 95] - means = sorted(mean(choices(data, k=len(data))) for i in range(100)) - print(f'The sample mean of {mean(data):.1f} has a 90% confidence ' - f'interval from {means[5]:.1f} to {means[94]:.1f}') - -Example of a `resampling permutation test -`_ -to determine the statistical significance or `p-value -`_ of an observed difference -between the effects of a drug versus a placebo:: - - # Example from "Statistics is Easy" by Dennis Shasha and Manda Wilson - from statistics import fmean as mean - from random import shuffle - - drug = [54, 73, 53, 70, 73, 68, 52, 65, 65] - placebo = [54, 51, 58, 44, 55, 52, 42, 47, 58, 46] - observed_diff = mean(drug) - mean(placebo) - - n = 10_000 - count = 0 - combined = drug + placebo - for i in range(n): - shuffle(combined) - new_diff = mean(combined[:len(drug)]) - mean(combined[len(drug):]) - count += (new_diff >= observed_diff) - - print(f'{n} label reshufflings produced only {count} instances with a difference') - print(f'at least as extreme as the observed difference of {observed_diff:.1f}.') - print(f'The one-sided p-value of {count / n:.4f} leads us to reject the null') - print(f'hypothesis that there is no difference between the drug and the placebo.') - -Simulation of arrival times and service deliveries for a multiserver queue:: - - from heapq import heapify, heapreplace - from random import expovariate, gauss - from statistics import mean, quantiles - - average_arrival_interval = 5.6 - average_service_time = 15.0 - stdev_service_time = 3.5 - num_servers = 3 - - waits = [] - arrival_time = 0.0 - servers = [0.0] * num_servers # time when each server becomes available - heapify(servers) - for i in range(1_000_000): - arrival_time += expovariate(1.0 / average_arrival_interval) - next_server_available = servers[0] - wait = max(0.0, next_server_available - arrival_time) - waits.append(wait) - service_duration = max(0.0, gauss(average_service_time, stdev_service_time)) - service_completed = arrival_time + wait + service_duration - heapreplace(servers, service_completed) - - print(f'Mean wait: {mean(waits):.1f} Max wait: {max(waits):.1f}') - print('Quartiles:', [round(q, 1) for q in quantiles(waits)]) - -.. seealso:: - - `Statistics for Hackers `_ - a video tutorial by - `Jake Vanderplas `_ - on statistical analysis using just a few fundamental concepts - including simulation, sampling, shuffling, and cross-validation. - - `Economics Simulation - `_ - a simulation of a marketplace by - `Peter Norvig `_ that shows effective - use of many of the tools and distributions provided by this module - (gauss, uniform, sample, betavariate, choice, triangular, and randrange). - - `A Concrete Introduction to Probability (using Python) - `_ - a tutorial by `Peter Norvig `_ covering - the basics of probability theory, how to write simulations, and - how to perform data analysis using Python. - - -Recipes -------- - -These recipes show how to efficiently make random selections -from the combinatoric iterators in the :mod:`itertools` module: - -.. testcode:: - import random - - def random_product(*args, repeat=1): - "Random selection from itertools.product(*args, **kwds)" - pools = [tuple(pool) for pool in args] * repeat - return tuple(map(random.choice, pools)) - - def random_permutation(iterable, r=None): - "Random selection from itertools.permutations(iterable, r)" - pool = tuple(iterable) - r = len(pool) if r is None else r - return tuple(random.sample(pool, r)) - - def random_combination(iterable, r): - "Random selection from itertools.combinations(iterable, r)" - pool = tuple(iterable) - n = len(pool) - indices = sorted(random.sample(range(n), r)) - return tuple(pool[i] for i in indices) - - def random_combination_with_replacement(iterable, r): - "Choose r elements with replacement. Order the result to match the iterable." - # Result will be in set(itertools.combinations_with_replacement(iterable, r)). - pool = tuple(iterable) - n = len(pool) - indices = sorted(random.choices(range(n), k=r)) - return tuple(pool[i] for i in indices) - -The default :func:`.random` returns multiples of 2⁻⁵³ in the range -*0.0 ≤ x < 1.0*. All such numbers are evenly spaced and are exactly -representable as Python floats. However, many other representable -floats in that interval are not possible selections. For example, -``0.05954861408025609`` isn't an integer multiple of 2⁻⁵³. - -The following recipe takes a different approach. All floats in the -interval are possible selections. The mantissa comes from a uniform -distribution of integers in the range *2⁵² ≤ mantissa < 2⁵³*. The -exponent comes from a geometric distribution where exponents smaller -than *-53* occur half as often as the next larger exponent. - -:: - - from random import Random - from math import ldexp - - class FullRandom(Random): - - def random(self): - mantissa = 0x10_0000_0000_0000 | self.getrandbits(52) - exponent = -53 - x = 0 - while not x: - x = self.getrandbits(32) - exponent += x.bit_length() - 32 - return ldexp(mantissa, exponent) - -All :ref:`real valued distributions ` -in the class will use the new method:: - - >>> fr = FullRandom() - >>> fr.random() - 0.05954861408025609 - >>> fr.expovariate(0.25) - 8.87925541791544 - -The recipe is conceptually equivalent to an algorithm that chooses from -all the multiples of 2⁻¹⁰⁷⁴ in the range *0.0 ≤ x < 1.0*. All such -numbers are evenly spaced, but most have to be rounded down to the -nearest representable Python float. (The value 2⁻¹⁰⁷⁴ is the smallest -positive unnormalized float and is equal to ``math.ulp(0.0)``.) - - -.. seealso:: - - `Generating Pseudo-random Floating-Point Values - `_ a - paper by Allen B. Downey describing ways to generate more - fine-grained floats than normally generated by :func:`.random`. diff --git a/Doc/library/re.rst b/Doc/library/re.rst deleted file mode 100644 index 83fff79d2884e5..00000000000000 --- a/Doc/library/re.rst +++ /dev/null @@ -1,1845 +0,0 @@ -:mod:`re` --- Regular expression operations -=========================================== - -.. module:: re - :synopsis: Regular expression operations. - -.. moduleauthor:: Fredrik Lundh -.. sectionauthor:: Andrew M. Kuchling - -**Source code:** :source:`Lib/re/` - --------------- - -This module provides regular expression matching operations similar to -those found in Perl. - -Both patterns and strings to be searched can be Unicode strings (:class:`str`) -as well as 8-bit strings (:class:`bytes`). -However, Unicode strings and 8-bit strings cannot be mixed: -that is, you cannot match a Unicode string with a byte pattern or -vice-versa; similarly, when asking for a substitution, the replacement -string must be of the same type as both the pattern and the search string. - -Regular expressions use the backslash character (``'\'``) to indicate -special forms or to allow special characters to be used without invoking -their special meaning. This collides with Python's usage of the same -character for the same purpose in string literals; for example, to match -a literal backslash, one might have to write ``'\\\\'`` as the pattern -string, because the regular expression must be ``\\``, and each -backslash must be expressed as ``\\`` inside a regular Python string -literal. Also, please note that any invalid escape sequences in Python's -usage of the backslash in string literals now generate a :exc:`DeprecationWarning` -and in the future this will become a :exc:`SyntaxError`. This behaviour -will happen even if it is a valid escape sequence for a regular expression. - -The solution is to use Python's raw string notation for regular expression -patterns; backslashes are not handled in any special way in a string literal -prefixed with ``'r'``. So ``r"\n"`` is a two-character string containing -``'\'`` and ``'n'``, while ``"\n"`` is a one-character string containing a -newline. Usually patterns will be expressed in Python code using this raw -string notation. - -It is important to note that most regular expression operations are available as -module-level functions and methods on -:ref:`compiled regular expressions `. The functions are shortcuts -that don't require you to compile a regex object first, but miss some -fine-tuning parameters. - -.. seealso:: - - The third-party `regex `_ module, - which has an API compatible with the standard library :mod:`re` module, - but offers additional functionality and a more thorough Unicode support. - - -.. _re-syntax: - -Regular Expression Syntax -------------------------- - -A regular expression (or RE) specifies a set of strings that matches it; the -functions in this module let you check if a particular string matches a given -regular expression (or if a given regular expression matches a particular -string, which comes down to the same thing). - -Regular expressions can be concatenated to form new regular expressions; if *A* -and *B* are both regular expressions, then *AB* is also a regular expression. -In general, if a string *p* matches *A* and another string *q* matches *B*, the -string *pq* will match AB. This holds unless *A* or *B* contain low precedence -operations; boundary conditions between *A* and *B*; or have numbered group -references. Thus, complex expressions can easily be constructed from simpler -primitive expressions like the ones described here. For details of the theory -and implementation of regular expressions, consult the Friedl book [Frie09]_, -or almost any textbook about compiler construction. - -A brief explanation of the format of regular expressions follows. For further -information and a gentler presentation, consult the :ref:`regex-howto`. - -Regular expressions can contain both special and ordinary characters. Most -ordinary characters, like ``'A'``, ``'a'``, or ``'0'``, are the simplest regular -expressions; they simply match themselves. You can concatenate ordinary -characters, so ``last`` matches the string ``'last'``. (In the rest of this -section, we'll write RE's in ``this special style``, usually without quotes, and -strings to be matched ``'in single quotes'``.) - -Some characters, like ``'|'`` or ``'('``, are special. Special -characters either stand for classes of ordinary characters, or affect -how the regular expressions around them are interpreted. - -Repetition operators or quantifiers (``*``, ``+``, ``?``, ``{m,n}``, etc) cannot be -directly nested. This avoids ambiguity with the non-greedy modifier suffix -``?``, and with other modifiers in other implementations. To apply a second -repetition to an inner repetition, parentheses may be used. For example, -the expression ``(?:a{6})*`` matches any multiple of six ``'a'`` characters. - - -The special characters are: - -.. index:: single: . (dot); in regular expressions - -``.`` - (Dot.) In the default mode, this matches any character except a newline. If - the :const:`DOTALL` flag has been specified, this matches any character - including a newline. - -.. index:: single: ^ (caret); in regular expressions - -``^`` - (Caret.) Matches the start of the string, and in :const:`MULTILINE` mode also - matches immediately after each newline. - -.. index:: single: $ (dollar); in regular expressions - -``$`` - Matches the end of the string or just before the newline at the end of the - string, and in :const:`MULTILINE` mode also matches before a newline. ``foo`` - matches both 'foo' and 'foobar', while the regular expression ``foo$`` matches - only 'foo'. More interestingly, searching for ``foo.$`` in ``'foo1\nfoo2\n'`` - matches 'foo2' normally, but 'foo1' in :const:`MULTILINE` mode; searching for - a single ``$`` in ``'foo\n'`` will find two (empty) matches: one just before - the newline, and one at the end of the string. - -.. index:: single: * (asterisk); in regular expressions - -``*`` - Causes the resulting RE to match 0 or more repetitions of the preceding RE, as - many repetitions as are possible. ``ab*`` will match 'a', 'ab', or 'a' followed - by any number of 'b's. - -.. index:: single: + (plus); in regular expressions - -``+`` - Causes the resulting RE to match 1 or more repetitions of the preceding RE. - ``ab+`` will match 'a' followed by any non-zero number of 'b's; it will not - match just 'a'. - -.. index:: single: ? (question mark); in regular expressions - -``?`` - Causes the resulting RE to match 0 or 1 repetitions of the preceding RE. - ``ab?`` will match either 'a' or 'ab'. - -.. index:: - single: *?; in regular expressions - single: +?; in regular expressions - single: ??; in regular expressions - -``*?``, ``+?``, ``??`` - The ``'*'``, ``'+'``, and ``'?'`` quantifiers are all :dfn:`greedy`; they match - as much text as possible. Sometimes this behaviour isn't desired; if the RE - ``<.*>`` is matched against ``' b '``, it will match the entire - string, and not just ``''``. Adding ``?`` after the quantifier makes it - perform the match in :dfn:`non-greedy` or :dfn:`minimal` fashion; as *few* - characters as possible will be matched. Using the RE ``<.*?>`` will match - only ``''``. - -.. index:: - single: *+; in regular expressions - single: ++; in regular expressions - single: ?+; in regular expressions - -``*+``, ``++``, ``?+`` - Like the ``'*'``, ``'+'``, and ``'?'`` quantifiers, those where ``'+'`` is - appended also match as many times as possible. - However, unlike the true greedy quantifiers, these do not allow - back-tracking when the expression following it fails to match. - These are known as :dfn:`possessive` quantifiers. - For example, ``a*a`` will match ``'aaaa'`` because the ``a*`` will match - all 4 ``'a'``\ s, but, when the final ``'a'`` is encountered, the - expression is backtracked so that in the end the ``a*`` ends up matching - 3 ``'a'``\ s total, and the fourth ``'a'`` is matched by the final ``'a'``. - However, when ``a*+a`` is used to match ``'aaaa'``, the ``a*+`` will - match all 4 ``'a'``, but when the final ``'a'`` fails to find any more - characters to match, the expression cannot be backtracked and will thus - fail to match. - ``x*+``, ``x++`` and ``x?+`` are equivalent to ``(?>x*)``, ``(?>x+)`` - and ``(?>x?)`` correspondingly. - - .. versionadded:: 3.11 - -.. index:: - single: {} (curly brackets); in regular expressions - -``{m}`` - Specifies that exactly *m* copies of the previous RE should be matched; fewer - matches cause the entire RE not to match. For example, ``a{6}`` will match - exactly six ``'a'`` characters, but not five. - -``{m,n}`` - Causes the resulting RE to match from *m* to *n* repetitions of the preceding - RE, attempting to match as many repetitions as possible. For example, - ``a{3,5}`` will match from 3 to 5 ``'a'`` characters. Omitting *m* specifies a - lower bound of zero, and omitting *n* specifies an infinite upper bound. As an - example, ``a{4,}b`` will match ``'aaaab'`` or a thousand ``'a'`` characters - followed by a ``'b'``, but not ``'aaab'``. The comma may not be omitted or the - modifier would be confused with the previously described form. - -``{m,n}?`` - Causes the resulting RE to match from *m* to *n* repetitions of the preceding - RE, attempting to match as *few* repetitions as possible. This is the - non-greedy version of the previous quantifier. For example, on the - 6-character string ``'aaaaaa'``, ``a{3,5}`` will match 5 ``'a'`` characters, - while ``a{3,5}?`` will only match 3 characters. - -``{m,n}+`` - Causes the resulting RE to match from *m* to *n* repetitions of the - preceding RE, attempting to match as many repetitions as possible - *without* establishing any backtracking points. - This is the possessive version of the quantifier above. - For example, on the 6-character string ``'aaaaaa'``, ``a{3,5}+aa`` - attempt to match 5 ``'a'`` characters, then, requiring 2 more ``'a'``\ s, - will need more characters than available and thus fail, while - ``a{3,5}aa`` will match with ``a{3,5}`` capturing 5, then 4 ``'a'``\ s - by backtracking and then the final 2 ``'a'``\ s are matched by the final - ``aa`` in the pattern. - ``x{m,n}+`` is equivalent to ``(?>x{m,n})``. - - .. versionadded:: 3.11 - -.. index:: single: \ (backslash); in regular expressions - -``\`` - Either escapes special characters (permitting you to match characters like - ``'*'``, ``'?'``, and so forth), or signals a special sequence; special - sequences are discussed below. - - If you're not using a raw string to express the pattern, remember that Python - also uses the backslash as an escape sequence in string literals; if the escape - sequence isn't recognized by Python's parser, the backslash and subsequent - character are included in the resulting string. However, if Python would - recognize the resulting sequence, the backslash should be repeated twice. This - is complicated and hard to understand, so it's highly recommended that you use - raw strings for all but the simplest expressions. - -.. index:: - single: [] (square brackets); in regular expressions - -``[]`` - Used to indicate a set of characters. In a set: - - * Characters can be listed individually, e.g. ``[amk]`` will match ``'a'``, - ``'m'``, or ``'k'``. - - .. index:: single: - (minus); in regular expressions - - * Ranges of characters can be indicated by giving two characters and separating - them by a ``'-'``, for example ``[a-z]`` will match any lowercase ASCII letter, - ``[0-5][0-9]`` will match all the two-digits numbers from ``00`` to ``59``, and - ``[0-9A-Fa-f]`` will match any hexadecimal digit. If ``-`` is escaped (e.g. - ``[a\-z]``) or if it's placed as the first or last character - (e.g. ``[-a]`` or ``[a-]``), it will match a literal ``'-'``. - - * Special characters lose their special meaning inside sets. For example, - ``[(+*)]`` will match any of the literal characters ``'('``, ``'+'``, - ``'*'``, or ``')'``. - - .. index:: single: \ (backslash); in regular expressions - - * Character classes such as ``\w`` or ``\S`` (defined below) are also accepted - inside a set, although the characters they match depends on whether - :const:`ASCII` or :const:`LOCALE` mode is in force. - - .. index:: single: ^ (caret); in regular expressions - - * Characters that are not within a range can be matched by :dfn:`complementing` - the set. If the first character of the set is ``'^'``, all the characters - that are *not* in the set will be matched. For example, ``[^5]`` will match - any character except ``'5'``, and ``[^^]`` will match any character except - ``'^'``. ``^`` has no special meaning if it's not the first character in - the set. - - * To match a literal ``']'`` inside a set, precede it with a backslash, or - place it at the beginning of the set. For example, both ``[()[\]{}]`` and - ``[]()[{}]`` will match a right bracket, as well as left bracket, braces, - and parentheses. - - .. .. index:: single: --; in regular expressions - .. .. index:: single: &&; in regular expressions - .. .. index:: single: ~~; in regular expressions - .. .. index:: single: ||; in regular expressions - - * Support of nested sets and set operations as in `Unicode Technical - Standard #18`_ might be added in the future. This would change the - syntax, so to facilitate this change a :exc:`FutureWarning` will be raised - in ambiguous cases for the time being. - That includes sets starting with a literal ``'['`` or containing literal - character sequences ``'--'``, ``'&&'``, ``'~~'``, and ``'||'``. To - avoid a warning escape them with a backslash. - - .. _Unicode Technical Standard #18: https://unicode.org/reports/tr18/ - - .. versionchanged:: 3.7 - :exc:`FutureWarning` is raised if a character set contains constructs - that will change semantically in the future. - -.. index:: single: | (vertical bar); in regular expressions - -``|`` - ``A|B``, where *A* and *B* can be arbitrary REs, creates a regular expression that - will match either *A* or *B*. An arbitrary number of REs can be separated by the - ``'|'`` in this way. This can be used inside groups (see below) as well. As - the target string is scanned, REs separated by ``'|'`` are tried from left to - right. When one pattern completely matches, that branch is accepted. This means - that once *A* matches, *B* will not be tested further, even if it would - produce a longer overall match. In other words, the ``'|'`` operator is never - greedy. To match a literal ``'|'``, use ``\|``, or enclose it inside a - character class, as in ``[|]``. - -.. index:: - single: () (parentheses); in regular expressions - -``(...)`` - Matches whatever regular expression is inside the parentheses, and indicates the - start and end of a group; the contents of a group can be retrieved after a match - has been performed, and can be matched later in the string with the ``\number`` - special sequence, described below. To match the literals ``'('`` or ``')'``, - use ``\(`` or ``\)``, or enclose them inside a character class: ``[(]``, ``[)]``. - -.. index:: single: (?; in regular expressions - -``(?...)`` - This is an extension notation (a ``'?'`` following a ``'('`` is not meaningful - otherwise). The first character after the ``'?'`` determines what the meaning - and further syntax of the construct is. Extensions usually do not create a new - group; ``(?P...)`` is the only exception to this rule. Following are the - currently supported extensions. - -``(?aiLmsux)`` - (One or more letters from the set ``'a'``, ``'i'``, ``'L'``, ``'m'``, - ``'s'``, ``'u'``, ``'x'``.) The group matches the empty string; the - letters set the corresponding flags: :const:`re.A` (ASCII-only matching), - :const:`re.I` (ignore case), :const:`re.L` (locale dependent), - :const:`re.M` (multi-line), :const:`re.S` (dot matches all), - :const:`re.U` (Unicode matching), and :const:`re.X` (verbose), - for the entire regular expression. - (The flags are described in :ref:`contents-of-module-re`.) - This is useful if you wish to include the flags as part of the - regular expression, instead of passing a *flag* argument to the - :func:`re.compile` function. Flags should be used first in the - expression string. - - .. versionchanged:: 3.11 - This construction can only be used at the start of the expression. - -.. index:: single: (?:; in regular expressions - -``(?:...)`` - A non-capturing version of regular parentheses. Matches whatever regular - expression is inside the parentheses, but the substring matched by the group - *cannot* be retrieved after performing a match or referenced later in the - pattern. - -``(?aiLmsux-imsx:...)`` - (Zero or more letters from the set ``'a'``, ``'i'``, ``'L'``, ``'m'``, - ``'s'``, ``'u'``, ``'x'``, optionally followed by ``'-'`` followed by - one or more letters from the ``'i'``, ``'m'``, ``'s'``, ``'x'``.) - The letters set or remove the corresponding flags: - :const:`re.A` (ASCII-only matching), :const:`re.I` (ignore case), - :const:`re.L` (locale dependent), :const:`re.M` (multi-line), - :const:`re.S` (dot matches all), :const:`re.U` (Unicode matching), - and :const:`re.X` (verbose), for the part of the expression. - (The flags are described in :ref:`contents-of-module-re`.) - - The letters ``'a'``, ``'L'`` and ``'u'`` are mutually exclusive when used - as inline flags, so they can't be combined or follow ``'-'``. Instead, - when one of them appears in an inline group, it overrides the matching mode - in the enclosing group. In Unicode patterns ``(?a:...)`` switches to - ASCII-only matching, and ``(?u:...)`` switches to Unicode matching - (default). In byte pattern ``(?L:...)`` switches to locale depending - matching, and ``(?a:...)`` switches to ASCII-only matching (default). - This override is only in effect for the narrow inline group, and the - original matching mode is restored outside of the group. - - .. versionadded:: 3.6 - - .. versionchanged:: 3.7 - The letters ``'a'``, ``'L'`` and ``'u'`` also can be used in a group. - -``(?>...)`` - Attempts to match ``...`` as if it was a separate regular expression, and - if successful, continues to match the rest of the pattern following it. - If the subsequent pattern fails to match, the stack can only be unwound - to a point *before* the ``(?>...)`` because once exited, the expression, - known as an :dfn:`atomic group`, has thrown away all stack points within - itself. - Thus, ``(?>.*).`` would never match anything because first the ``.*`` - would match all characters possible, then, having nothing left to match, - the final ``.`` would fail to match. - Since there are no stack points saved in the Atomic Group, and there is - no stack point before it, the entire expression would thus fail to match. - - .. versionadded:: 3.11 - -.. index:: single: (?P<; in regular expressions - -``(?P...)`` - Similar to regular parentheses, but the substring matched by the group is - accessible via the symbolic group name *name*. Group names must be valid - Python identifiers, and each group name must be defined only once within a - regular expression. A symbolic group is also a numbered group, just as if - the group were not named. - - Named groups can be referenced in three contexts. If the pattern is - ``(?P['"]).*?(?P=quote)`` (i.e. matching a string quoted with either - single or double quotes): - - +---------------------------------------+----------------------------------+ - | Context of reference to group "quote" | Ways to reference it | - +=======================================+==================================+ - | in the same pattern itself | * ``(?P=quote)`` (as shown) | - | | * ``\1`` | - +---------------------------------------+----------------------------------+ - | when processing match object *m* | * ``m.group('quote')`` | - | | * ``m.end('quote')`` (etc.) | - +---------------------------------------+----------------------------------+ - | in a string passed to the *repl* | * ``\g`` | - | argument of ``re.sub()`` | * ``\g<1>`` | - | | * ``\1`` | - +---------------------------------------+----------------------------------+ - - .. deprecated:: 3.11 - Group *name* containing characters outside the ASCII range - (``b'\x00'``-``b'\x7f'``) in :class:`bytes` patterns. - -.. index:: single: (?P=; in regular expressions - -``(?P=name)`` - A backreference to a named group; it matches whatever text was matched by the - earlier group named *name*. - -.. index:: single: (?#; in regular expressions - -``(?#...)`` - A comment; the contents of the parentheses are simply ignored. - -.. index:: single: (?=; in regular expressions - -``(?=...)`` - Matches if ``...`` matches next, but doesn't consume any of the string. This is - called a :dfn:`lookahead assertion`. For example, ``Isaac (?=Asimov)`` will match - ``'Isaac '`` only if it's followed by ``'Asimov'``. - -.. index:: single: (?!; in regular expressions - -``(?!...)`` - Matches if ``...`` doesn't match next. This is a :dfn:`negative lookahead assertion`. - For example, ``Isaac (?!Asimov)`` will match ``'Isaac '`` only if it's *not* - followed by ``'Asimov'``. - -.. index:: single: (?<=; in regular expressions - -``(?<=...)`` - Matches if the current position in the string is preceded by a match for ``...`` - that ends at the current position. This is called a :dfn:`positive lookbehind - assertion`. ``(?<=abc)def`` will find a match in ``'abcdef'``, since the - lookbehind will back up 3 characters and check if the contained pattern matches. - The contained pattern must only match strings of some fixed length, meaning that - ``abc`` or ``a|b`` are allowed, but ``a*`` and ``a{3,4}`` are not. Note that - patterns which start with positive lookbehind assertions will not match at the - beginning of the string being searched; you will most likely want to use the - :func:`search` function rather than the :func:`match` function: - - >>> import re - >>> m = re.search('(?<=abc)def', 'abcdef') - >>> m.group(0) - 'def' - - This example looks for a word following a hyphen: - - >>> m = re.search(r'(?<=-)\w+', 'spam-egg') - >>> m.group(0) - 'egg' - - .. versionchanged:: 3.5 - Added support for group references of fixed length. - -.. index:: single: (?|$)`` is a poor email matching pattern, which - will match with ``''`` as well as ``'user@host.com'``, but - not with ``''``. - - .. deprecated:: 3.11 - Group *id* containing anything except ASCII digits. - Group *name* containing characters outside the ASCII range - (``b'\x00'``-``b'\x7f'``) in :class:`bytes` replacement strings. - - -.. _re-special-sequences: - -The special sequences consist of ``'\'`` and a character from the list below. -If the ordinary character is not an ASCII digit or an ASCII letter, then the -resulting RE will match the second character. For example, ``\$`` matches the -character ``'$'``. - -.. index:: single: \ (backslash); in regular expressions - -``\number`` - Matches the contents of the group of the same number. Groups are numbered - starting from 1. For example, ``(.+) \1`` matches ``'the the'`` or ``'55 55'``, - but not ``'thethe'`` (note the space after the group). This special sequence - can only be used to match one of the first 99 groups. If the first digit of - *number* is 0, or *number* is 3 octal digits long, it will not be interpreted as - a group match, but as the character with octal value *number*. Inside the - ``'['`` and ``']'`` of a character class, all numeric escapes are treated as - characters. - -.. index:: single: \A; in regular expressions - -``\A`` - Matches only at the start of the string. - -.. index:: single: \b; in regular expressions - -``\b`` - Matches the empty string, but only at the beginning or end of a word. - A word is defined as a sequence of word characters. Note that formally, - ``\b`` is defined as the boundary between a ``\w`` and a ``\W`` character - (or vice versa), or between ``\w`` and the beginning/end of the string. - This means that ``r'\bfoo\b'`` matches ``'foo'``, ``'foo.'``, ``'(foo)'``, - ``'bar foo baz'`` but not ``'foobar'`` or ``'foo3'``. - - By default Unicode alphanumerics are the ones used in Unicode patterns, but - this can be changed by using the :const:`ASCII` flag. Word boundaries are - determined by the current locale if the :const:`LOCALE` flag is used. - Inside a character range, ``\b`` represents the backspace character, for - compatibility with Python's string literals. - -.. index:: single: \B; in regular expressions - -``\B`` - Matches the empty string, but only when it is *not* at the beginning or end - of a word. This means that ``r'py\B'`` matches ``'python'``, ``'py3'``, - ``'py2'``, but not ``'py'``, ``'py.'``, or ``'py!'``. - ``\B`` is just the opposite of ``\b``, so word characters in Unicode - patterns are Unicode alphanumerics or the underscore, although this can - be changed by using the :const:`ASCII` flag. Word boundaries are - determined by the current locale if the :const:`LOCALE` flag is used. - -.. index:: single: \d; in regular expressions - -``\d`` - For Unicode (str) patterns: - Matches any Unicode decimal digit (that is, any character in - Unicode character category [Nd]). This includes ``[0-9]``, and - also many other digit characters. If the :const:`ASCII` flag is - used only ``[0-9]`` is matched. - - For 8-bit (bytes) patterns: - Matches any decimal digit; this is equivalent to ``[0-9]``. - -.. index:: single: \D; in regular expressions - -``\D`` - Matches any character which is not a decimal digit. This is - the opposite of ``\d``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^0-9]``. - -.. index:: single: \s; in regular expressions - -``\s`` - For Unicode (str) patterns: - Matches Unicode whitespace characters (which includes - ``[ \t\n\r\f\v]``, and also many other characters, for example the - non-breaking spaces mandated by typography rules in many - languages). If the :const:`ASCII` flag is used, only - ``[ \t\n\r\f\v]`` is matched. - - For 8-bit (bytes) patterns: - Matches characters considered whitespace in the ASCII character set; - this is equivalent to ``[ \t\n\r\f\v]``. - -.. index:: single: \S; in regular expressions - -``\S`` - Matches any character which is not a whitespace character. This is - the opposite of ``\s``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^ \t\n\r\f\v]``. - -.. index:: single: \w; in regular expressions - -``\w`` - For Unicode (str) patterns: - Matches Unicode word characters; this includes alphanumeric characters (as defined by :meth:`str.isalnum`) - as well as the underscore (``_``). - If the :const:`ASCII` flag is used, only ``[a-zA-Z0-9_]`` is matched. - - For 8-bit (bytes) patterns: - Matches characters considered alphanumeric in the ASCII character set; - this is equivalent to ``[a-zA-Z0-9_]``. If the :const:`LOCALE` flag is - used, matches characters considered alphanumeric in the current locale - and the underscore. - -.. index:: single: \W; in regular expressions - -``\W`` - Matches any character which is not a word character. This is - the opposite of ``\w``. If the :const:`ASCII` flag is used this - becomes the equivalent of ``[^a-zA-Z0-9_]``. If the :const:`LOCALE` flag is - used, matches characters which are neither alphanumeric in the current locale - nor the underscore. - -.. index:: single: \Z; in regular expressions - -``\Z`` - Matches only at the end of the string. - -.. index:: - single: \a; in regular expressions - single: \b; in regular expressions - single: \f; in regular expressions - single: \n; in regular expressions - single: \N; in regular expressions - single: \r; in regular expressions - single: \t; in regular expressions - single: \u; in regular expressions - single: \U; in regular expressions - single: \v; in regular expressions - single: \x; in regular expressions - single: \\; in regular expressions - -Most of the standard escapes supported by Python string literals are also -accepted by the regular expression parser:: - - \a \b \f \n - \N \r \t \u - \U \v \x \\ - -(Note that ``\b`` is used to represent word boundaries, and means "backspace" -only inside character classes.) - -``'\u'``, ``'\U'``, and ``'\N'`` escape sequences are only recognized in Unicode -patterns. In bytes patterns they are errors. Unknown escapes of ASCII -letters are reserved for future use and treated as errors. - -Octal escapes are included in a limited form. If the first digit is a 0, or if -there are three octal digits, it is considered an octal escape. Otherwise, it is -a group reference. As for string literals, octal escapes are always at most -three digits in length. - -.. versionchanged:: 3.3 - The ``'\u'`` and ``'\U'`` escape sequences have been added. - -.. versionchanged:: 3.6 - Unknown escapes consisting of ``'\'`` and an ASCII letter now are errors. - -.. versionchanged:: 3.8 - The :samp:`'\\N\\{{name}\\}'` escape sequence has been added. As in string literals, - it expands to the named Unicode character (e.g. ``'\N{EM DASH}'``). - - -.. _contents-of-module-re: - -Module Contents ---------------- - -The module defines several functions, constants, and an exception. Some of the -functions are simplified versions of the full featured methods for compiled -regular expressions. Most non-trivial applications always use the compiled -form. - - -Flags -^^^^^ - -.. versionchanged:: 3.6 - Flag constants are now instances of :class:`RegexFlag`, which is a subclass of - :class:`enum.IntFlag`. - - -.. class:: RegexFlag - - An :class:`enum.IntFlag` class containing the regex options listed below. - - .. versionadded:: 3.11 - added to ``__all__`` - -.. data:: A - ASCII - - Make ``\w``, ``\W``, ``\b``, ``\B``, ``\d``, ``\D``, ``\s`` and ``\S`` - perform ASCII-only matching instead of full Unicode matching. This is only - meaningful for Unicode patterns, and is ignored for byte patterns. - Corresponds to the inline flag ``(?a)``. - - Note that for backward compatibility, the :const:`re.U` flag still - exists (as well as its synonym :const:`re.UNICODE` and its embedded - counterpart ``(?u)``), but these are redundant in Python 3 since - matches are Unicode by default for strings (and Unicode matching - isn't allowed for bytes). - - -.. data:: DEBUG - - Display debug information about compiled expression. - No corresponding inline flag. - - -.. data:: I - IGNORECASE - - Perform case-insensitive matching; expressions like ``[A-Z]`` will also - match lowercase letters. Full Unicode matching (such as ``Ü`` matching - ``ü``) also works unless the :const:`re.ASCII` flag is used to disable - non-ASCII matches. The current locale does not change the effect of this - flag unless the :const:`re.LOCALE` flag is also used. - Corresponds to the inline flag ``(?i)``. - - Note that when the Unicode patterns ``[a-z]`` or ``[A-Z]`` are used in - combination with the :const:`IGNORECASE` flag, they will match the 52 ASCII - letters and 4 additional non-ASCII letters: 'İ' (U+0130, Latin capital - letter I with dot above), 'ı' (U+0131, Latin small letter dotless i), - 'ſ' (U+017F, Latin small letter long s) and 'K' (U+212A, Kelvin sign). - If the :const:`ASCII` flag is used, only letters 'a' to 'z' - and 'A' to 'Z' are matched. - -.. data:: L - LOCALE - - Make ``\w``, ``\W``, ``\b``, ``\B`` and case-insensitive matching - dependent on the current locale. This flag can be used only with bytes - patterns. The use of this flag is discouraged as the locale mechanism - is very unreliable, it only handles one "culture" at a time, and it only - works with 8-bit locales. Unicode matching is already enabled by default - in Python 3 for Unicode (str) patterns, and it is able to handle different - locales/languages. - Corresponds to the inline flag ``(?L)``. - - .. versionchanged:: 3.6 - :const:`re.LOCALE` can be used only with bytes patterns and is - not compatible with :const:`re.ASCII`. - - .. versionchanged:: 3.7 - Compiled regular expression objects with the :const:`re.LOCALE` flag no - longer depend on the locale at compile time. Only the locale at - matching time affects the result of matching. - - -.. data:: M - MULTILINE - - When specified, the pattern character ``'^'`` matches at the beginning of the - string and at the beginning of each line (immediately following each newline); - and the pattern character ``'$'`` matches at the end of the string and at the - end of each line (immediately preceding each newline). By default, ``'^'`` - matches only at the beginning of the string, and ``'$'`` only at the end of the - string and immediately before the newline (if any) at the end of the string. - Corresponds to the inline flag ``(?m)``. - -.. data:: NOFLAG - - Indicates no flag being applied, the value is ``0``. This flag may be used - as a default value for a function keyword argument or as a base value that - will be conditionally ORed with other flags. Example of use as a default - value:: - - def myfunc(text, flag=re.NOFLAG): - return re.match(text, flag) - - .. versionadded:: 3.11 - -.. data:: S - DOTALL - - Make the ``'.'`` special character match any character at all, including a - newline; without this flag, ``'.'`` will match anything *except* a newline. - Corresponds to the inline flag ``(?s)``. - - -.. data:: U - UNICODE - - In Python 2, this flag made :ref:`special sequences ` - include Unicode characters in matches. Since Python 3, Unicode characters - are matched by default. - - See :const:`A` for restricting matching on ASCII characters instead. - - This flag is only kept for backward compatibility. - -.. data:: X - VERBOSE - - .. index:: single: # (hash); in regular expressions - - This flag allows you to write regular expressions that look nicer and are - more readable by allowing you to visually separate logical sections of the - pattern and add comments. Whitespace within the pattern is ignored, except - when in a character class, or when preceded by an unescaped backslash, - or within tokens like ``*?``, ``(?:`` or ``(?P<...>``. For example, ``(? :`` - and ``* ?`` are not allowed. - When a line contains a ``#`` that is not in a character class and is not - preceded by an unescaped backslash, all characters from the leftmost such - ``#`` through the end of the line are ignored. - - This means that the two following regular expression objects that match a - decimal number are functionally equal:: - - a = re.compile(r"""\d + # the integral part - \. # the decimal point - \d * # some fractional digits""", re.X) - b = re.compile(r"\d+\.\d*") - - Corresponds to the inline flag ``(?x)``. - - -Functions -^^^^^^^^^ - -.. function:: compile(pattern, flags=0) - - Compile a regular expression pattern into a :ref:`regular expression object - `, which can be used for matching using its - :func:`~Pattern.match`, :func:`~Pattern.search` and other methods, described - below. - - The expression's behaviour can be modified by specifying a *flags* value. - Values can be any of the following variables, combined using bitwise OR (the - ``|`` operator). - - The sequence :: - - prog = re.compile(pattern) - result = prog.match(string) - - is equivalent to :: - - result = re.match(pattern, string) - - but using :func:`re.compile` and saving the resulting regular expression - object for reuse is more efficient when the expression will be used several - times in a single program. - - .. note:: - - The compiled versions of the most recent patterns passed to - :func:`re.compile` and the module-level matching functions are cached, so - programs that use only a few regular expressions at a time needn't worry - about compiling regular expressions. - - -.. function:: search(pattern, string, flags=0) - - Scan through *string* looking for the first location where the regular expression - *pattern* produces a match, and return a corresponding :class:`~re.Match`. Return - ``None`` if no position in the string matches the pattern; note that this is - different from finding a zero-length match at some point in the string. - - -.. function:: match(pattern, string, flags=0) - - If zero or more characters at the beginning of *string* match the regular - expression *pattern*, return a corresponding :class:`~re.Match`. Return - ``None`` if the string does not match the pattern; note that this is - different from a zero-length match. - - Note that even in :const:`MULTILINE` mode, :func:`re.match` will only match - at the beginning of the string and not at the beginning of each line. - - If you want to locate a match anywhere in *string*, use :func:`search` - instead (see also :ref:`search-vs-match`). - - -.. function:: fullmatch(pattern, string, flags=0) - - If the whole *string* matches the regular expression *pattern*, return a - corresponding :class:`~re.Match`. Return ``None`` if the string does not match - the pattern; note that this is different from a zero-length match. - - .. versionadded:: 3.4 - - -.. function:: split(pattern, string, maxsplit=0, flags=0) - - Split *string* by the occurrences of *pattern*. If capturing parentheses are - used in *pattern*, then the text of all groups in the pattern are also returned - as part of the resulting list. If *maxsplit* is nonzero, at most *maxsplit* - splits occur, and the remainder of the string is returned as the final element - of the list. :: - - >>> re.split(r'\W+', 'Words, words, words.') - ['Words', 'words', 'words', ''] - >>> re.split(r'(\W+)', 'Words, words, words.') - ['Words', ', ', 'words', ', ', 'words', '.', ''] - >>> re.split(r'\W+', 'Words, words, words.', 1) - ['Words', 'words, words.'] - >>> re.split('[a-f]+', '0a3B9', flags=re.IGNORECASE) - ['0', '3', '9'] - - If there are capturing groups in the separator and it matches at the start of - the string, the result will start with an empty string. The same holds for - the end of the string:: - - >>> re.split(r'(\W+)', '...words, words...') - ['', '...', 'words', ', ', 'words', '...', ''] - - That way, separator components are always found at the same relative - indices within the result list. - - Empty matches for the pattern split the string only when not adjacent - to a previous empty match. - - >>> re.split(r'\b', 'Words, words, words.') - ['', 'Words', ', ', 'words', ', ', 'words', '.'] - >>> re.split(r'\W*', '...words...') - ['', '', 'w', 'o', 'r', 'd', 's', '', ''] - >>> re.split(r'(\W*)', '...words...') - ['', '...', '', '', 'w', '', 'o', '', 'r', '', 'd', '', 's', '...', '', '', ''] - - .. versionchanged:: 3.1 - Added the optional flags argument. - - .. versionchanged:: 3.7 - Added support of splitting on a pattern that could match an empty string. - - -.. function:: findall(pattern, string, flags=0) - - Return all non-overlapping matches of *pattern* in *string*, as a list of - strings or tuples. The *string* is scanned left-to-right, and matches - are returned in the order found. Empty matches are included in the result. - - The result depends on the number of capturing groups in the pattern. - If there are no groups, return a list of strings matching the whole - pattern. If there is exactly one group, return a list of strings - matching that group. If multiple groups are present, return a list - of tuples of strings matching the groups. Non-capturing groups do not - affect the form of the result. - - >>> re.findall(r'\bf[a-z]*', 'which foot or hand fell fastest') - ['foot', 'fell', 'fastest'] - >>> re.findall(r'(\w+)=(\d+)', 'set width=20 and height=10') - [('width', '20'), ('height', '10')] - - .. versionchanged:: 3.7 - Non-empty matches can now start just after a previous empty match. - - -.. function:: finditer(pattern, string, flags=0) - - Return an :term:`iterator` yielding :class:`~re.Match` objects over - all non-overlapping matches for the RE *pattern* in *string*. The *string* - is scanned left-to-right, and matches are returned in the order found. Empty - matches are included in the result. - - .. versionchanged:: 3.7 - Non-empty matches can now start just after a previous empty match. - - -.. function:: sub(pattern, repl, string, count=0, flags=0) - - Return the string obtained by replacing the leftmost non-overlapping occurrences - of *pattern* in *string* by the replacement *repl*. If the pattern isn't found, - *string* is returned unchanged. *repl* can be a string or a function; if it is - a string, any backslash escapes in it are processed. That is, ``\n`` is - converted to a single newline character, ``\r`` is converted to a carriage return, and - so forth. Unknown escapes of ASCII letters are reserved for future use and - treated as errors. Other unknown escapes such as ``\&`` are left alone. - Backreferences, such - as ``\6``, are replaced with the substring matched by group 6 in the pattern. - For example:: - - >>> re.sub(r'def\s+([a-zA-Z_][a-zA-Z_0-9]*)\s*\(\s*\):', - ... r'static PyObject*\npy_\1(void)\n{', - ... 'def myfunc():') - 'static PyObject*\npy_myfunc(void)\n{' - - If *repl* is a function, it is called for every non-overlapping occurrence of - *pattern*. The function takes a single :class:`~re.Match` argument, and returns - the replacement string. For example:: - - >>> def dashrepl(matchobj): - ... if matchobj.group(0) == '-': return ' ' - ... else: return '-' - >>> re.sub('-{1,2}', dashrepl, 'pro----gram-files') - 'pro--gram files' - >>> re.sub(r'\sAND\s', ' & ', 'Baked Beans And Spam', flags=re.IGNORECASE) - 'Baked Beans & Spam' - - The pattern may be a string or a :class:`~re.Pattern`. - - The optional argument *count* is the maximum number of pattern occurrences to be - replaced; *count* must be a non-negative integer. If omitted or zero, all - occurrences will be replaced. Empty matches for the pattern are replaced only - when not adjacent to a previous empty match, so ``sub('x*', '-', 'abxd')`` returns - ``'-a-b--d-'``. - - .. index:: single: \g; in regular expressions - - In string-type *repl* arguments, in addition to the character escapes and - backreferences described above, - ``\g`` will use the substring matched by the group named ``name``, as - defined by the ``(?P...)`` syntax. ``\g`` uses the corresponding - group number; ``\g<2>`` is therefore equivalent to ``\2``, but isn't ambiguous - in a replacement such as ``\g<2>0``. ``\20`` would be interpreted as a - reference to group 20, not a reference to group 2 followed by the literal - character ``'0'``. The backreference ``\g<0>`` substitutes in the entire - substring matched by the RE. - - .. versionchanged:: 3.1 - Added the optional flags argument. - - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - - .. versionchanged:: 3.6 - Unknown escapes in *pattern* consisting of ``'\'`` and an ASCII letter - now are errors. - - .. versionchanged:: 3.7 - Unknown escapes in *repl* consisting of ``'\'`` and an ASCII letter - now are errors. - - .. versionchanged:: 3.7 - Empty matches for the pattern are replaced when adjacent to a previous - non-empty match. - - .. deprecated:: 3.11 - Group *id* containing anything except ASCII digits. - Group *name* containing characters outside the ASCII range - (``b'\x00'``-``b'\x7f'``) in :class:`bytes` replacement strings. - - -.. function:: subn(pattern, repl, string, count=0, flags=0) - - Perform the same operation as :func:`sub`, but return a tuple ``(new_string, - number_of_subs_made)``. - - .. versionchanged:: 3.1 - Added the optional flags argument. - - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - - -.. function:: escape(pattern) - - Escape special characters in *pattern*. - This is useful if you want to match an arbitrary literal string that may - have regular expression metacharacters in it. For example:: - - >>> print(re.escape('https://www.python.org')) - https://www\.python\.org - - >>> legal_chars = string.ascii_lowercase + string.digits + "!#$%&'*+-.^_`|~:" - >>> print('[%s]+' % re.escape(legal_chars)) - [abcdefghijklmnopqrstuvwxyz0123456789!\#\$%\&'\*\+\-\.\^_`\|\~:]+ - - >>> operators = ['+', '-', '*', '/', '**'] - >>> print('|'.join(map(re.escape, sorted(operators, reverse=True)))) - /|\-|\+|\*\*|\* - - This function must not be used for the replacement string in :func:`sub` - and :func:`subn`, only backslashes should be escaped. For example:: - - >>> digits_re = r'\d+' - >>> sample = '/usr/sbin/sendmail - 0 errors, 12 warnings' - >>> print(re.sub(digits_re, digits_re.replace('\\', r'\\'), sample)) - /usr/sbin/sendmail - \d+ errors, \d+ warnings - - .. versionchanged:: 3.3 - The ``'_'`` character is no longer escaped. - - .. versionchanged:: 3.7 - Only characters that can have special meaning in a regular expression - are escaped. As a result, ``'!'``, ``'"'``, ``'%'``, ``"'"``, ``','``, - ``'/'``, ``':'``, ``';'``, ``'<'``, ``'='``, ``'>'``, ``'@'``, and - ``"`"`` are no longer escaped. - - -.. function:: purge() - - Clear the regular expression cache. - - -Exceptions -^^^^^^^^^^ - -.. exception:: error(msg, pattern=None, pos=None) - - Exception raised when a string passed to one of the functions here is not a - valid regular expression (for example, it might contain unmatched parentheses) - or when some other error occurs during compilation or matching. It is never an - error if a string contains no match for a pattern. The error instance has - the following additional attributes: - - .. attribute:: msg - - The unformatted error message. - - .. attribute:: pattern - - The regular expression pattern. - - .. attribute:: pos - - The index in *pattern* where compilation failed (may be ``None``). - - .. attribute:: lineno - - The line corresponding to *pos* (may be ``None``). - - .. attribute:: colno - - The column corresponding to *pos* (may be ``None``). - - .. versionchanged:: 3.5 - Added additional attributes. - -.. _re-objects: - -Regular Expression Objects --------------------------- - -.. class:: Pattern - - Compiled regular expression object returned by :func:`re.compile`. - - .. versionchanged:: 3.9 - :py:class:`re.Pattern` supports ``[]`` to indicate a Unicode (str) or bytes pattern. - See :ref:`types-genericalias`. - -.. method:: Pattern.search(string[, pos[, endpos]]) - - Scan through *string* looking for the first location where this regular - expression produces a match, and return a corresponding :class:`~re.Match`. - Return ``None`` if no position in the string matches the pattern; note that - this is different from finding a zero-length match at some point in the string. - - The optional second parameter *pos* gives an index in the string where the - search is to start; it defaults to ``0``. This is not completely equivalent to - slicing the string; the ``'^'`` pattern character matches at the real beginning - of the string and at positions just after a newline, but not necessarily at the - index where the search is to start. - - The optional parameter *endpos* limits how far the string will be searched; it - will be as if the string is *endpos* characters long, so only the characters - from *pos* to ``endpos - 1`` will be searched for a match. If *endpos* is less - than *pos*, no match will be found; otherwise, if *rx* is a compiled regular - expression object, ``rx.search(string, 0, 50)`` is equivalent to - ``rx.search(string[:50], 0)``. :: - - >>> pattern = re.compile("d") - >>> pattern.search("dog") # Match at index 0 - - >>> pattern.search("dog", 1) # No match; search doesn't include the "d" - - -.. method:: Pattern.match(string[, pos[, endpos]]) - - If zero or more characters at the *beginning* of *string* match this regular - expression, return a corresponding :class:`~re.Match`. Return ``None`` if the - string does not match the pattern; note that this is different from a - zero-length match. - - The optional *pos* and *endpos* parameters have the same meaning as for the - :meth:`~Pattern.search` method. :: - - >>> pattern = re.compile("o") - >>> pattern.match("dog") # No match as "o" is not at the start of "dog". - >>> pattern.match("dog", 1) # Match as "o" is the 2nd character of "dog". - - - If you want to locate a match anywhere in *string*, use - :meth:`~Pattern.search` instead (see also :ref:`search-vs-match`). - - -.. method:: Pattern.fullmatch(string[, pos[, endpos]]) - - If the whole *string* matches this regular expression, return a corresponding - :class:`~re.Match`. Return ``None`` if the string does not match the pattern; - note that this is different from a zero-length match. - - The optional *pos* and *endpos* parameters have the same meaning as for the - :meth:`~Pattern.search` method. :: - - >>> pattern = re.compile("o[gh]") - >>> pattern.fullmatch("dog") # No match as "o" is not at the start of "dog". - >>> pattern.fullmatch("ogre") # No match as not the full string matches. - >>> pattern.fullmatch("doggie", 1, 3) # Matches within given limits. - - - .. versionadded:: 3.4 - - -.. method:: Pattern.split(string, maxsplit=0) - - Identical to the :func:`split` function, using the compiled pattern. - - -.. method:: Pattern.findall(string[, pos[, endpos]]) - - Similar to the :func:`findall` function, using the compiled pattern, but - also accepts optional *pos* and *endpos* parameters that limit the search - region like for :meth:`search`. - - -.. method:: Pattern.finditer(string[, pos[, endpos]]) - - Similar to the :func:`finditer` function, using the compiled pattern, but - also accepts optional *pos* and *endpos* parameters that limit the search - region like for :meth:`search`. - - -.. method:: Pattern.sub(repl, string, count=0) - - Identical to the :func:`sub` function, using the compiled pattern. - - -.. method:: Pattern.subn(repl, string, count=0) - - Identical to the :func:`subn` function, using the compiled pattern. - - -.. attribute:: Pattern.flags - - The regex matching flags. This is a combination of the flags given to - :func:`.compile`, any ``(?...)`` inline flags in the pattern, and implicit - flags such as :data:`UNICODE` if the pattern is a Unicode string. - - -.. attribute:: Pattern.groups - - The number of capturing groups in the pattern. - - -.. attribute:: Pattern.groupindex - - A dictionary mapping any symbolic group names defined by ``(?P)`` to group - numbers. The dictionary is empty if no symbolic groups were used in the - pattern. - - -.. attribute:: Pattern.pattern - - The pattern string from which the pattern object was compiled. - - -.. versionchanged:: 3.7 - Added support of :func:`copy.copy` and :func:`copy.deepcopy`. Compiled - regular expression objects are considered atomic. - - -.. _match-objects: - -Match Objects -------------- - -Match objects always have a boolean value of ``True``. -Since :meth:`~Pattern.match` and :meth:`~Pattern.search` return ``None`` -when there is no match, you can test whether there was a match with a simple -``if`` statement:: - - match = re.search(pattern, string) - if match: - process(match) - -.. class:: Match - - Match object returned by successful ``match``\ es and ``search``\ es. - - .. versionchanged:: 3.9 - :py:class:`re.Match` supports ``[]`` to indicate a Unicode (str) or bytes match. - See :ref:`types-genericalias`. - -.. method:: Match.expand(template) - - Return the string obtained by doing backslash substitution on the template - string *template*, as done by the :meth:`~Pattern.sub` method. - Escapes such as ``\n`` are converted to the appropriate characters, - and numeric backreferences (``\1``, ``\2``) and named backreferences - (``\g<1>``, ``\g``) are replaced by the contents of the - corresponding group. - - .. versionchanged:: 3.5 - Unmatched groups are replaced with an empty string. - -.. method:: Match.group([group1, ...]) - - Returns one or more subgroups of the match. If there is a single argument, the - result is a single string; if there are multiple arguments, the result is a - tuple with one item per argument. Without arguments, *group1* defaults to zero - (the whole match is returned). If a *groupN* argument is zero, the corresponding - return value is the entire matching string; if it is in the inclusive range - [1..99], it is the string matching the corresponding parenthesized group. If a - group number is negative or larger than the number of groups defined in the - pattern, an :exc:`IndexError` exception is raised. If a group is contained in a - part of the pattern that did not match, the corresponding result is ``None``. - If a group is contained in a part of the pattern that matched multiple times, - the last match is returned. :: - - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") - >>> m.group(0) # The entire match - 'Isaac Newton' - >>> m.group(1) # The first parenthesized subgroup. - 'Isaac' - >>> m.group(2) # The second parenthesized subgroup. - 'Newton' - >>> m.group(1, 2) # Multiple arguments give us a tuple. - ('Isaac', 'Newton') - - If the regular expression uses the ``(?P...)`` syntax, the *groupN* - arguments may also be strings identifying groups by their group name. If a - string argument is not used as a group name in the pattern, an :exc:`IndexError` - exception is raised. - - A moderately complicated example:: - - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") - >>> m.group('first_name') - 'Malcolm' - >>> m.group('last_name') - 'Reynolds' - - Named groups can also be referred to by their index:: - - >>> m.group(1) - 'Malcolm' - >>> m.group(2) - 'Reynolds' - - If a group matches multiple times, only the last match is accessible:: - - >>> m = re.match(r"(..)+", "a1b2c3") # Matches 3 times. - >>> m.group(1) # Returns only the last match. - 'c3' - - -.. method:: Match.__getitem__(g) - - This is identical to ``m.group(g)``. This allows easier access to - an individual group from a match:: - - >>> m = re.match(r"(\w+) (\w+)", "Isaac Newton, physicist") - >>> m[0] # The entire match - 'Isaac Newton' - >>> m[1] # The first parenthesized subgroup. - 'Isaac' - >>> m[2] # The second parenthesized subgroup. - 'Newton' - - Named groups are supported as well:: - - >>> m = re.match(r"(?P\w+) (?P\w+)", "Isaac Newton") - >>> m['first_name'] - 'Isaac' - >>> m['last_name'] - 'Newton' - - .. versionadded:: 3.6 - - -.. method:: Match.groups(default=None) - - Return a tuple containing all the subgroups of the match, from 1 up to however - many groups are in the pattern. The *default* argument is used for groups that - did not participate in the match; it defaults to ``None``. - - For example:: - - >>> m = re.match(r"(\d+)\.(\d+)", "24.1632") - >>> m.groups() - ('24', '1632') - - If we make the decimal place and everything after it optional, not all groups - might participate in the match. These groups will default to ``None`` unless - the *default* argument is given:: - - >>> m = re.match(r"(\d+)\.?(\d+)?", "24") - >>> m.groups() # Second group defaults to None. - ('24', None) - >>> m.groups('0') # Now, the second group defaults to '0'. - ('24', '0') - - -.. method:: Match.groupdict(default=None) - - Return a dictionary containing all the *named* subgroups of the match, keyed by - the subgroup name. The *default* argument is used for groups that did not - participate in the match; it defaults to ``None``. For example:: - - >>> m = re.match(r"(?P\w+) (?P\w+)", "Malcolm Reynolds") - >>> m.groupdict() - {'first_name': 'Malcolm', 'last_name': 'Reynolds'} - - -.. method:: Match.start([group]) - Match.end([group]) - - Return the indices of the start and end of the substring matched by *group*; - *group* defaults to zero (meaning the whole matched substring). Return ``-1`` if - *group* exists but did not contribute to the match. For a match object *m*, and - a group *g* that did contribute to the match, the substring matched by group *g* - (equivalent to ``m.group(g)``) is :: - - m.string[m.start(g):m.end(g)] - - Note that ``m.start(group)`` will equal ``m.end(group)`` if *group* matched a - null string. For example, after ``m = re.search('b(c?)', 'cba')``, - ``m.start(0)`` is 1, ``m.end(0)`` is 2, ``m.start(1)`` and ``m.end(1)`` are both - 2, and ``m.start(2)`` raises an :exc:`IndexError` exception. - - An example that will remove *remove_this* from email addresses:: - - >>> email = "tony@tiremove_thisger.net" - >>> m = re.search("remove_this", email) - >>> email[:m.start()] + email[m.end():] - 'tony@tiger.net' - - -.. method:: Match.span([group]) - - For a match *m*, return the 2-tuple ``(m.start(group), m.end(group))``. Note - that if *group* did not contribute to the match, this is ``(-1, -1)``. - *group* defaults to zero, the entire match. - - -.. attribute:: Match.pos - - The value of *pos* which was passed to the :meth:`~Pattern.search` or - :meth:`~Pattern.match` method of a :ref:`regex object `. This is - the index into the string at which the RE engine started looking for a match. - - -.. attribute:: Match.endpos - - The value of *endpos* which was passed to the :meth:`~Pattern.search` or - :meth:`~Pattern.match` method of a :ref:`regex object `. This is - the index into the string beyond which the RE engine will not go. - - -.. attribute:: Match.lastindex - - The integer index of the last matched capturing group, or ``None`` if no group - was matched at all. For example, the expressions ``(a)b``, ``((a)(b))``, and - ``((ab))`` will have ``lastindex == 1`` if applied to the string ``'ab'``, while - the expression ``(a)(b)`` will have ``lastindex == 2``, if applied to the same - string. - - -.. attribute:: Match.lastgroup - - The name of the last matched capturing group, or ``None`` if the group didn't - have a name, or if no group was matched at all. - - -.. attribute:: Match.re - - The :ref:`regular expression object ` whose :meth:`~Pattern.match` or - :meth:`~Pattern.search` method produced this match instance. - - -.. attribute:: Match.string - - The string passed to :meth:`~Pattern.match` or :meth:`~Pattern.search`. - - -.. versionchanged:: 3.7 - Added support of :func:`copy.copy` and :func:`copy.deepcopy`. Match objects - are considered atomic. - - -.. _re-examples: - -Regular Expression Examples ---------------------------- - - -Checking for a Pair -^^^^^^^^^^^^^^^^^^^ - -In this example, we'll use the following helper function to display match -objects a little more gracefully:: - - def displaymatch(match): - if match is None: - return None - return '' % (match.group(), match.groups()) - -Suppose you are writing a poker program where a player's hand is represented as -a 5-character string with each character representing a card, "a" for ace, "k" -for king, "q" for queen, "j" for jack, "t" for 10, and "2" through "9" -representing the card with that value. - -To see if a given string is a valid hand, one could do the following:: - - >>> valid = re.compile(r"^[a2-9tjqk]{5}$") - >>> displaymatch(valid.match("akt5q")) # Valid. - "" - >>> displaymatch(valid.match("akt5e")) # Invalid. - >>> displaymatch(valid.match("akt")) # Invalid. - >>> displaymatch(valid.match("727ak")) # Valid. - "" - -That last hand, ``"727ak"``, contained a pair, or two of the same valued cards. -To match this with a regular expression, one could use backreferences as such:: - - >>> pair = re.compile(r".*(.).*\1") - >>> displaymatch(pair.match("717ak")) # Pair of 7s. - "" - >>> displaymatch(pair.match("718ak")) # No pairs. - >>> displaymatch(pair.match("354aa")) # Pair of aces. - "" - -To find out what card the pair consists of, one could use the -:meth:`~Match.group` method of the match object in the following manner:: - - >>> pair = re.compile(r".*(.).*\1") - >>> pair.match("717ak").group(1) - '7' - - # Error because re.match() returns None, which doesn't have a group() method: - >>> pair.match("718ak").group(1) - Traceback (most recent call last): - File "", line 1, in - re.match(r".*(.).*\1", "718ak").group(1) - AttributeError: 'NoneType' object has no attribute 'group' - - >>> pair.match("354aa").group(1) - 'a' - - -Simulating scanf() -^^^^^^^^^^^^^^^^^^ - -.. index:: single: scanf() - -Python does not currently have an equivalent to :c:func:`!scanf`. Regular -expressions are generally more powerful, though also more verbose, than -:c:func:`!scanf` format strings. The table below offers some more-or-less -equivalent mappings between :c:func:`!scanf` format tokens and regular -expressions. - -+--------------------------------+---------------------------------------------+ -| :c:func:`!scanf` Token | Regular Expression | -+================================+=============================================+ -| ``%c`` | ``.`` | -+--------------------------------+---------------------------------------------+ -| ``%5c`` | ``.{5}`` | -+--------------------------------+---------------------------------------------+ -| ``%d`` | ``[-+]?\d+`` | -+--------------------------------+---------------------------------------------+ -| ``%e``, ``%E``, ``%f``, ``%g`` | ``[-+]?(\d+(\.\d*)?|\.\d+)([eE][-+]?\d+)?`` | -+--------------------------------+---------------------------------------------+ -| ``%i`` | ``[-+]?(0[xX][\dA-Fa-f]+|0[0-7]*|\d+)`` | -+--------------------------------+---------------------------------------------+ -| ``%o`` | ``[-+]?[0-7]+`` | -+--------------------------------+---------------------------------------------+ -| ``%s`` | ``\S+`` | -+--------------------------------+---------------------------------------------+ -| ``%u`` | ``\d+`` | -+--------------------------------+---------------------------------------------+ -| ``%x``, ``%X`` | ``[-+]?(0[xX])?[\dA-Fa-f]+`` | -+--------------------------------+---------------------------------------------+ - -To extract the filename and numbers from a string like :: - - /usr/sbin/sendmail - 0 errors, 4 warnings - -you would use a :c:func:`!scanf` format like :: - - %s - %d errors, %d warnings - -The equivalent regular expression would be :: - - (\S+) - (\d+) errors, (\d+) warnings - - -.. _search-vs-match: - -search() vs. match() -^^^^^^^^^^^^^^^^^^^^ - -.. sectionauthor:: Fred L. Drake, Jr. - -Python offers different primitive operations based on regular expressions: - -+ :func:`re.match` checks for a match only at the beginning of the string -+ :func:`re.search` checks for a match anywhere in the string - (this is what Perl does by default) -+ :func:`re.fullmatch` checks for entire string to be a match - - -For example:: - - >>> re.match("c", "abcdef") # No match - >>> re.search("c", "abcdef") # Match - - >>> re.fullmatch("p.*n", "python") # Match - - >>> re.fullmatch("r.*n", "python") # No match - -Regular expressions beginning with ``'^'`` can be used with :func:`search` to -restrict the match at the beginning of the string:: - - >>> re.match("c", "abcdef") # No match - >>> re.search("^c", "abcdef") # No match - >>> re.search("^a", "abcdef") # Match - - -Note however that in :const:`MULTILINE` mode :func:`match` only matches at the -beginning of the string, whereas using :func:`search` with a regular expression -beginning with ``'^'`` will match at the beginning of each line. :: - - >>> re.match("X", "A\nB\nX", re.MULTILINE) # No match - >>> re.search("^X", "A\nB\nX", re.MULTILINE) # Match - - - -Making a Phonebook -^^^^^^^^^^^^^^^^^^ - -:func:`split` splits a string into a list delimited by the passed pattern. The -method is invaluable for converting textual data into data structures that can be -easily read and modified by Python as demonstrated in the following example that -creates a phonebook. - -First, here is the input. Normally it may come from a file, here we are using -triple-quoted string syntax - -.. doctest:: - - >>> text = """Ross McFluff: 834.345.1254 155 Elm Street - ... - ... Ronald Heathmore: 892.345.3428 436 Finley Avenue - ... Frank Burger: 925.541.7625 662 South Dogwood Way - ... - ... - ... Heather Albrecht: 548.326.4584 919 Park Place""" - -The entries are separated by one or more newlines. Now we convert the string -into a list with each nonempty line having its own entry: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> entries = re.split("\n+", text) - >>> entries - ['Ross McFluff: 834.345.1254 155 Elm Street', - 'Ronald Heathmore: 892.345.3428 436 Finley Avenue', - 'Frank Burger: 925.541.7625 662 South Dogwood Way', - 'Heather Albrecht: 548.326.4584 919 Park Place'] - -Finally, split each entry into a list with first name, last name, telephone -number, and address. We use the ``maxsplit`` parameter of :func:`split` -because the address has spaces, our splitting pattern, in it: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> [re.split(":? ", entry, 3) for entry in entries] - [['Ross', 'McFluff', '834.345.1254', '155 Elm Street'], - ['Ronald', 'Heathmore', '892.345.3428', '436 Finley Avenue'], - ['Frank', 'Burger', '925.541.7625', '662 South Dogwood Way'], - ['Heather', 'Albrecht', '548.326.4584', '919 Park Place']] - -The ``:?`` pattern matches the colon after the last name, so that it does not -occur in the result list. With a ``maxsplit`` of ``4``, we could separate the -house number from the street name: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> [re.split(":? ", entry, 4) for entry in entries] - [['Ross', 'McFluff', '834.345.1254', '155', 'Elm Street'], - ['Ronald', 'Heathmore', '892.345.3428', '436', 'Finley Avenue'], - ['Frank', 'Burger', '925.541.7625', '662', 'South Dogwood Way'], - ['Heather', 'Albrecht', '548.326.4584', '919', 'Park Place']] - - -Text Munging -^^^^^^^^^^^^ - -:func:`sub` replaces every occurrence of a pattern with a string or the -result of a function. This example demonstrates using :func:`sub` with -a function to "munge" text, or randomize the order of all the characters -in each word of a sentence except for the first and last characters:: - - >>> def repl(m): - ... inner_word = list(m.group(2)) - ... random.shuffle(inner_word) - ... return m.group(1) + "".join(inner_word) + m.group(3) - >>> text = "Professor Abdolmalek, please report your absences promptly." - >>> re.sub(r"(\w)(\w+)(\w)", repl, text) - 'Poefsrosr Aealmlobdk, pslaee reorpt your abnseces plmrptoy.' - >>> re.sub(r"(\w)(\w+)(\w)", repl, text) - 'Pofsroser Aodlambelk, plasee reoprt yuor asnebces potlmrpy.' - - -Finding all Adverbs -^^^^^^^^^^^^^^^^^^^ - -:func:`findall` matches *all* occurrences of a pattern, not just the first -one as :func:`search` does. For example, if a writer wanted to -find all of the adverbs in some text, they might use :func:`findall` in -the following manner:: - - >>> text = "He was carefully disguised but captured quickly by police." - >>> re.findall(r"\w+ly\b", text) - ['carefully', 'quickly'] - - -Finding all Adverbs and their Positions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If one wants more information about all matches of a pattern than the matched -text, :func:`finditer` is useful as it provides :class:`~re.Match` objects -instead of strings. Continuing with the previous example, if a writer wanted -to find all of the adverbs *and their positions* in some text, they would use -:func:`finditer` in the following manner:: - - >>> text = "He was carefully disguised but captured quickly by police." - >>> for m in re.finditer(r"\w+ly\b", text): - ... print('%02d-%02d: %s' % (m.start(), m.end(), m.group(0))) - 07-16: carefully - 40-47: quickly - - -Raw String Notation -^^^^^^^^^^^^^^^^^^^ - -Raw string notation (``r"text"``) keeps regular expressions sane. Without it, -every backslash (``'\'``) in a regular expression would have to be prefixed with -another one to escape it. For example, the two following lines of code are -functionally identical:: - - >>> re.match(r"\W(.)\1\W", " ff ") - - >>> re.match("\\W(.)\\1\\W", " ff ") - - -When one wants to match a literal backslash, it must be escaped in the regular -expression. With raw string notation, this means ``r"\\"``. Without raw string -notation, one must use ``"\\\\"``, making the following lines of code -functionally identical:: - - >>> re.match(r"\\", r"\\") - - >>> re.match("\\\\", r"\\") - - - -Writing a Tokenizer -^^^^^^^^^^^^^^^^^^^ - -A `tokenizer or scanner `_ -analyzes a string to categorize groups of characters. This is a useful first -step in writing a compiler or interpreter. - -The text categories are specified with regular expressions. The technique is -to combine those into a single master regular expression and to loop over -successive matches:: - - from typing import NamedTuple - import re - - class Token(NamedTuple): - type: str - value: str - line: int - column: int - - def tokenize(code): - keywords = {'IF', 'THEN', 'ENDIF', 'FOR', 'NEXT', 'GOSUB', 'RETURN'} - token_specification = [ - ('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number - ('ASSIGN', r':='), # Assignment operator - ('END', r';'), # Statement terminator - ('ID', r'[A-Za-z]+'), # Identifiers - ('OP', r'[+\-*/]'), # Arithmetic operators - ('NEWLINE', r'\n'), # Line endings - ('SKIP', r'[ \t]+'), # Skip over spaces and tabs - ('MISMATCH', r'.'), # Any other character - ] - tok_regex = '|'.join('(?P<%s>%s)' % pair for pair in token_specification) - line_num = 1 - line_start = 0 - for mo in re.finditer(tok_regex, code): - kind = mo.lastgroup - value = mo.group() - column = mo.start() - line_start - if kind == 'NUMBER': - value = float(value) if '.' in value else int(value) - elif kind == 'ID' and value in keywords: - kind = value - elif kind == 'NEWLINE': - line_start = mo.end() - line_num += 1 - continue - elif kind == 'SKIP': - continue - elif kind == 'MISMATCH': - raise RuntimeError(f'{value!r} unexpected on line {line_num}') - yield Token(kind, value, line_num, column) - - statements = ''' - IF quantity THEN - total := total + price * quantity; - tax := price * 0.05; - ENDIF; - ''' - - for token in tokenize(statements): - print(token) - -The tokenizer produces the following output:: - - Token(type='IF', value='IF', line=2, column=4) - Token(type='ID', value='quantity', line=2, column=7) - Token(type='THEN', value='THEN', line=2, column=16) - Token(type='ID', value='total', line=3, column=8) - Token(type='ASSIGN', value=':=', line=3, column=14) - Token(type='ID', value='total', line=3, column=17) - Token(type='OP', value='+', line=3, column=23) - Token(type='ID', value='price', line=3, column=25) - Token(type='OP', value='*', line=3, column=31) - Token(type='ID', value='quantity', line=3, column=33) - Token(type='END', value=';', line=3, column=41) - Token(type='ID', value='tax', line=4, column=8) - Token(type='ASSIGN', value=':=', line=4, column=12) - Token(type='ID', value='price', line=4, column=15) - Token(type='OP', value='*', line=4, column=21) - Token(type='NUMBER', value=0.05, line=4, column=23) - Token(type='END', value=';', line=4, column=27) - Token(type='ENDIF', value='ENDIF', line=5, column=4) - Token(type='END', value=';', line=5, column=9) - - -.. [Frie09] Friedl, Jeffrey. Mastering Regular Expressions. 3rd ed., O'Reilly - Media, 2009. The third edition of the book no longer covers Python at all, - but the first edition covered writing good regular expression patterns in - great detail. diff --git a/Doc/library/readline.rst b/Doc/library/readline.rst deleted file mode 100644 index 8fb0eca8df74d8..00000000000000 --- a/Doc/library/readline.rst +++ /dev/null @@ -1,361 +0,0 @@ -:mod:`readline` --- GNU readline interface -========================================== - -.. module:: readline - :platform: Unix - :synopsis: GNU readline support for Python. - -.. sectionauthor:: Skip Montanaro - --------------- - -The :mod:`readline` module defines a number of functions to facilitate -completion and reading/writing of history files from the Python interpreter. -This module can be used directly, or via the :mod:`rlcompleter` module, which -supports completion of Python identifiers at the interactive prompt. Settings -made using this module affect the behaviour of both the interpreter's -interactive prompt and the prompts offered by the built-in :func:`input` -function. - -Readline keybindings may be configured via an initialization file, typically -``.inputrc`` in your home directory. See `Readline Init File -`_ -in the GNU Readline manual for information about the format and -allowable constructs of that file, and the capabilities of the -Readline library in general. - -.. note:: - - The underlying Readline library API may be implemented by - the ``libedit`` library instead of GNU readline. - On macOS the :mod:`readline` module detects which library is being used - at run time. - - The configuration file for ``libedit`` is different from that - of GNU readline. If you programmatically load configuration strings - you can check for the text "libedit" in :const:`readline.__doc__` - to differentiate between GNU readline and libedit. - - If you use *editline*/``libedit`` readline emulation on macOS, the - initialization file located in your home directory is named - ``.editrc``. For example, the following content in ``~/.editrc`` will - turn ON *vi* keybindings and TAB completion:: - - python:bind -v - python:bind ^I rl_complete - - -Init file ---------- - -The following functions relate to the init file and user configuration: - - -.. function:: parse_and_bind(string) - - Execute the init line provided in the *string* argument. This calls - :c:func:`rl_parse_and_bind` in the underlying library. - - -.. function:: read_init_file([filename]) - - Execute a readline initialization file. The default filename is the last filename - used. This calls :c:func:`rl_read_init_file` in the underlying library. - - -Line buffer ------------ - -The following functions operate on the line buffer: - - -.. function:: get_line_buffer() - - Return the current contents of the line buffer (:c:data:`rl_line_buffer` - in the underlying library). - - -.. function:: insert_text(string) - - Insert text into the line buffer at the cursor position. This calls - :c:func:`rl_insert_text` in the underlying library, but ignores - the return value. - - -.. function:: redisplay() - - Change what's displayed on the screen to reflect the current contents of the - line buffer. This calls :c:func:`rl_redisplay` in the underlying library. - - -History file ------------- - -The following functions operate on a history file: - - -.. function:: read_history_file([filename]) - - Load a readline history file, and append it to the history list. - The default filename is :file:`~/.history`. This calls - :c:func:`read_history` in the underlying library. - - -.. function:: write_history_file([filename]) - - Save the history list to a readline history file, overwriting any - existing file. The default filename is :file:`~/.history`. This calls - :c:func:`write_history` in the underlying library. - - -.. function:: append_history_file(nelements[, filename]) - - Append the last *nelements* items of history to a file. The default filename is - :file:`~/.history`. The file must already exist. This calls - :c:func:`append_history` in the underlying library. This function - only exists if Python was compiled for a version of the library - that supports it. - - .. versionadded:: 3.5 - - -.. function:: get_history_length() - set_history_length(length) - - Set or return the desired number of lines to save in the history file. - The :func:`write_history_file` function uses this value to truncate - the history file, by calling :c:func:`history_truncate_file` in - the underlying library. Negative values imply - unlimited history file size. - - -History list ------------- - -The following functions operate on a global history list: - - -.. function:: clear_history() - - Clear the current history. This calls :c:func:`clear_history` in the - underlying library. The Python function only exists if Python was - compiled for a version of the library that supports it. - - -.. function:: get_current_history_length() - - Return the number of items currently in the history. (This is different from - :func:`get_history_length`, which returns the maximum number of lines that will - be written to a history file.) - - -.. function:: get_history_item(index) - - Return the current contents of history item at *index*. The item index - is one-based. This calls :c:func:`history_get` in the underlying library. - - -.. function:: remove_history_item(pos) - - Remove history item specified by its position from the history. - The position is zero-based. This calls :c:func:`remove_history` in - the underlying library. - - -.. function:: replace_history_item(pos, line) - - Replace history item specified by its position with *line*. - The position is zero-based. This calls :c:func:`replace_history_entry` - in the underlying library. - - -.. function:: add_history(line) - - Append *line* to the history buffer, as if it was the last line typed. - This calls :c:func:`add_history` in the underlying library. - - -.. function:: set_auto_history(enabled) - - Enable or disable automatic calls to :c:func:`add_history` when reading - input via readline. The *enabled* argument should be a Boolean value - that when true, enables auto history, and that when false, disables - auto history. - - .. versionadded:: 3.6 - - .. impl-detail:: - Auto history is enabled by default, and changes to this do not persist - across multiple sessions. - - -Startup hooks -------------- - - -.. function:: set_startup_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_startup_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any function - already installed is removed. The hook is called with no - arguments just before readline prints the first prompt. - - -.. function:: set_pre_input_hook([function]) - - Set or remove the function invoked by the :c:data:`rl_pre_input_hook` - callback of the underlying library. If *function* is specified, it will - be used as the new hook function; if omitted or ``None``, any - function already installed is removed. The hook is called - with no arguments after the first prompt has been printed and just before - readline starts reading input characters. This function only exists - if Python was compiled for a version of the library that supports it. - - -Completion ----------- - -The following functions relate to implementing a custom word completion -function. This is typically operated by the Tab key, and can suggest and -automatically complete a word being typed. By default, Readline is set up -to be used by :mod:`rlcompleter` to complete Python identifiers for -the interactive interpreter. If the :mod:`readline` module is to be used -with a custom completer, a different set of word delimiters should be set. - - -.. function:: set_completer([function]) - - Set or remove the completer function. If *function* is specified, it will be - used as the new completer function; if omitted or ``None``, any completer - function already installed is removed. The completer function is called as - ``function(text, state)``, for *state* in ``0``, ``1``, ``2``, ..., until it - returns a non-string value. It should return the next possible completion - starting with *text*. - - The installed completer function is invoked by the *entry_func* callback - passed to :c:func:`rl_completion_matches` in the underlying library. - The *text* string comes from the first parameter to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. - - -.. function:: get_completer() - - Get the completer function, or ``None`` if no completer function has been set. - - -.. function:: get_completion_type() - - Get the type of completion being attempted. This returns the - :c:data:`rl_completion_type` variable in the underlying library as - an integer. - - -.. function:: get_begidx() - get_endidx() - - Get the beginning or ending index of the completion scope. - These indexes are the *start* and *end* arguments passed to the - :c:data:`rl_attempted_completion_function` callback of the - underlying library. The values may be different in the same - input editing scenario based on the underlying C readline implementation. - Ex: libedit is known to behave differently than libreadline. - - -.. function:: set_completer_delims(string) - get_completer_delims() - - Set or get the word delimiters for completion. These determine the - start of the word to be considered for completion (the completion scope). - These functions access the :c:data:`rl_completer_word_break_characters` - variable in the underlying library. - - -.. function:: set_completion_display_matches_hook([function]) - - Set or remove the completion display function. If *function* is - specified, it will be used as the new completion display function; - if omitted or ``None``, any completion display function already - installed is removed. This sets or clears the - :c:data:`rl_completion_display_matches_hook` callback in the - underlying library. The completion display function is called as - ``function(substitution, [matches], longest_match_length)`` once - each time matches need to be displayed. - - -.. _readline-example: - -Example -------- - -The following example demonstrates how to use the :mod:`readline` module's -history reading and writing functions to automatically load and save a history -file named :file:`.python_history` from the user's home directory. The code -below would normally be executed automatically during interactive sessions -from the user's :envvar:`PYTHONSTARTUP` file. :: - - import atexit - import os - import readline - - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - try: - readline.read_history_file(histfile) - # default history len is -1 (infinite), which may grow unruly - readline.set_history_length(1000) - except FileNotFoundError: - pass - - atexit.register(readline.write_history_file, histfile) - -This code is actually automatically run when Python is run in -:ref:`interactive mode ` (see :ref:`rlcompleter-config`). - -The following example achieves the same goal but supports concurrent interactive -sessions, by only appending the new history. :: - - import atexit - import os - import readline - histfile = os.path.join(os.path.expanduser("~"), ".python_history") - - try: - readline.read_history_file(histfile) - h_len = readline.get_current_history_length() - except FileNotFoundError: - open(histfile, 'wb').close() - h_len = 0 - - def save(prev_h_len, histfile): - new_h_len = readline.get_current_history_length() - readline.set_history_length(1000) - readline.append_history_file(new_h_len - prev_h_len, histfile) - atexit.register(save, h_len, histfile) - -The following example extends the :class:`code.InteractiveConsole` class to -support history save/restore. :: - - import atexit - import code - import os - import readline - - class HistoryConsole(code.InteractiveConsole): - def __init__(self, locals=None, filename="", - histfile=os.path.expanduser("~/.console-history")): - code.InteractiveConsole.__init__(self, locals, filename) - self.init_history(histfile) - - def init_history(self, histfile): - readline.parse_and_bind("tab: complete") - if hasattr(readline, "read_history_file"): - try: - readline.read_history_file(histfile) - except FileNotFoundError: - pass - atexit.register(self.save_history, histfile) - - def save_history(self, histfile): - readline.set_history_length(1000) - readline.write_history_file(histfile) diff --git a/Doc/library/reprlib.rst b/Doc/library/reprlib.rst deleted file mode 100644 index 4b37c5ba60f4e6..00000000000000 --- a/Doc/library/reprlib.rst +++ /dev/null @@ -1,171 +0,0 @@ -:mod:`reprlib` --- Alternate :func:`repr` implementation -======================================================== - -.. module:: reprlib - :synopsis: Alternate repr() implementation with size limits. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/reprlib.py` - --------------- - -The :mod:`reprlib` module provides a means for producing object representations -with limits on the size of the resulting strings. This is used in the Python -debugger and may be useful in other contexts as well. - -This module provides a class, an instance, and a function: - - -.. class:: Repr() - - Class which provides formatting services useful in implementing functions - similar to the built-in :func:`repr`; size limits for different object types - are added to avoid the generation of representations which are excessively long. - - -.. data:: aRepr - - This is an instance of :class:`Repr` which is used to provide the - :func:`.repr` function described below. Changing the attributes of this - object will affect the size limits used by :func:`.repr` and the Python - debugger. - - -.. function:: repr(obj) - - This is the :meth:`~Repr.repr` method of ``aRepr``. It returns a string - similar to that returned by the built-in function of the same name, but with - limits on most sizes. - -In addition to size-limiting tools, the module also provides a decorator for -detecting recursive calls to :meth:`__repr__` and substituting a placeholder -string instead. - - -.. index:: single: ...; placeholder - -.. decorator:: recursive_repr(fillvalue="...") - - Decorator for :meth:`__repr__` methods to detect recursive calls within the - same thread. If a recursive call is made, the *fillvalue* is returned, - otherwise, the usual :meth:`__repr__` call is made. For example: - - >>> from reprlib import recursive_repr - >>> class MyList(list): - ... @recursive_repr() - ... def __repr__(self): - ... return '<' + '|'.join(map(repr, self)) + '>' - ... - >>> m = MyList('abc') - >>> m.append(m) - >>> m.append('x') - >>> print(m) - <'a'|'b'|'c'|...|'x'> - - .. versionadded:: 3.2 - - -.. _repr-objects: - -Repr Objects ------------- - -:class:`Repr` instances provide several attributes which can be used to provide -size limits for the representations of different object types, and methods -which format specific object types. - - -.. attribute:: Repr.fillvalue - - This string is displayed for recursive references. It defaults to - ``...``. - - .. versionadded:: 3.11 - - -.. attribute:: Repr.maxlevel - - Depth limit on the creation of recursive representations. The default is ``6``. - - -.. attribute:: Repr.maxdict - Repr.maxlist - Repr.maxtuple - Repr.maxset - Repr.maxfrozenset - Repr.maxdeque - Repr.maxarray - - Limits on the number of entries represented for the named object type. The - default is ``4`` for :attr:`maxdict`, ``5`` for :attr:`maxarray`, and ``6`` for - the others. - - -.. attribute:: Repr.maxlong - - Maximum number of characters in the representation for an integer. Digits - are dropped from the middle. The default is ``40``. - - -.. attribute:: Repr.maxstring - - Limit on the number of characters in the representation of the string. Note - that the "normal" representation of the string is used as the character source: - if escape sequences are needed in the representation, these may be mangled when - the representation is shortened. The default is ``30``. - - -.. attribute:: Repr.maxother - - This limit is used to control the size of object types for which no specific - formatting method is available on the :class:`Repr` object. It is applied in a - similar manner as :attr:`maxstring`. The default is ``20``. - - -.. method:: Repr.repr(obj) - - The equivalent to the built-in :func:`repr` that uses the formatting imposed by - the instance. - - -.. method:: Repr.repr1(obj, level) - - Recursive implementation used by :meth:`.repr`. This uses the type of *obj* to - determine which formatting method to call, passing it *obj* and *level*. The - type-specific methods should call :meth:`repr1` to perform recursive formatting, - with ``level - 1`` for the value of *level* in the recursive call. - - -.. method:: Repr.repr_TYPE(obj, level) - :noindex: - - Formatting methods for specific types are implemented as methods with a name - based on the type name. In the method name, **TYPE** is replaced by - ``'_'.join(type(obj).__name__.split())``. Dispatch to these methods is - handled by :meth:`repr1`. Type-specific methods which need to recursively - format a value should call ``self.repr1(subobj, level - 1)``. - - -.. _subclassing-reprs: - -Subclassing Repr Objects ------------------------- - -The use of dynamic dispatching by :meth:`Repr.repr1` allows subclasses of -:class:`Repr` to add support for additional built-in object types or to modify -the handling of types already supported. This example shows how special support -for file objects could be added:: - - import reprlib - import sys - - class MyRepr(reprlib.Repr): - - def repr_TextIOWrapper(self, obj, level): - if obj.name in {'', '', ''}: - return obj.name - return repr(obj) - - aRepr = MyRepr() - print(aRepr.repr(sys.stdin)) # prints '' diff --git a/Doc/library/resource.rst b/Doc/library/resource.rst deleted file mode 100644 index ef65674d1b0a78..00000000000000 --- a/Doc/library/resource.rst +++ /dev/null @@ -1,383 +0,0 @@ -:mod:`resource` --- Resource usage information -============================================== - -.. module:: resource - :platform: Unix - :synopsis: An interface to provide resource usage information on the current process. - -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Jeremy Hylton - --------------- - -This module provides basic mechanisms for measuring and controlling system -resources utilized by a program. - -.. availability:: Unix, not Emscripten, not WASI. - -Symbolic constants are used to specify particular system resources and to -request usage information about either the current process or its children. - -An :exc:`OSError` is raised on syscall failure. - - -.. exception:: error - - A deprecated alias of :exc:`OSError`. - - .. versionchanged:: 3.3 - Following :pep:`3151`, this class was made an alias of :exc:`OSError`. - - -Resource Limits ---------------- - -Resources usage can be limited using the :func:`setrlimit` function described -below. Each resource is controlled by a pair of limits: a soft limit and a hard -limit. The soft limit is the current limit, and may be lowered or raised by a -process over time. The soft limit can never exceed the hard limit. The hard -limit can be lowered to any value greater than the soft limit, but not raised. -(Only processes with the effective UID of the super-user can raise a hard -limit.) - -The specific resources that can be limited are system dependent. They are -described in the :manpage:`getrlimit(2)` man page. The resources listed below -are supported when the underlying operating system supports them; resources -which cannot be checked or controlled by the operating system are not defined in -this module for those platforms. - - -.. data:: RLIM_INFINITY - - Constant used to represent the limit for an unlimited resource. - - -.. function:: getrlimit(resource) - - Returns a tuple ``(soft, hard)`` with the current soft and hard limits of - *resource*. Raises :exc:`ValueError` if an invalid resource is specified, or - :exc:`error` if the underlying system call fails unexpectedly. - - -.. function:: setrlimit(resource, limits) - - Sets new limits of consumption of *resource*. The *limits* argument must be a - tuple ``(soft, hard)`` of two integers describing the new limits. A value of - :data:`~resource.RLIM_INFINITY` can be used to request a limit that is - unlimited. - - Raises :exc:`ValueError` if an invalid resource is specified, if the new soft - limit exceeds the hard limit, or if a process tries to raise its hard limit. - Specifying a limit of :data:`~resource.RLIM_INFINITY` when the hard or - system limit for that resource is not unlimited will result in a - :exc:`ValueError`. A process with the effective UID of super-user can - request any valid limit value, including unlimited, but :exc:`ValueError` - will still be raised if the requested limit exceeds the system imposed - limit. - - ``setrlimit`` may also raise :exc:`error` if the underlying system call - fails. - - VxWorks only supports setting :data:`RLIMIT_NOFILE`. - - .. audit-event:: resource.setrlimit resource,limits resource.setrlimit - - -.. function:: prlimit(pid, resource[, limits]) - - Combines :func:`setrlimit` and :func:`getrlimit` in one function and - supports to get and set the resources limits of an arbitrary process. If - *pid* is 0, then the call applies to the current process. *resource* and - *limits* have the same meaning as in :func:`setrlimit`, except that - *limits* is optional. - - When *limits* is not given the function returns the *resource* limit of the - process *pid*. When *limits* is given the *resource* limit of the process is - set and the former resource limit is returned. - - Raises :exc:`ProcessLookupError` when *pid* can't be found and - :exc:`PermissionError` when the user doesn't have ``CAP_SYS_RESOURCE`` for - the process. - - .. audit-event:: resource.prlimit pid,resource,limits resource.prlimit - - .. availability:: Linux >= 2.6.36 with glibc >= 2.13. - - .. versionadded:: 3.4 - - -These symbols define resources whose consumption can be controlled using the -:func:`setrlimit` and :func:`getrlimit` functions described below. The values of -these symbols are exactly the constants used by C programs. - -The Unix man page for :manpage:`getrlimit(2)` lists the available resources. -Note that not all systems use the same symbol or same value to denote the same -resource. This module does not attempt to mask platform differences --- symbols -not defined for a platform will not be available from this module on that -platform. - - -.. data:: RLIMIT_CORE - - The maximum size (in bytes) of a core file that the current process can create. - This may result in the creation of a partial core file if a larger core would be - required to contain the entire process image. - - -.. data:: RLIMIT_CPU - - The maximum amount of processor time (in seconds) that a process can use. If - this limit is exceeded, a :const:`SIGXCPU` signal is sent to the process. (See - the :mod:`signal` module documentation for information about how to catch this - signal and do something useful, e.g. flush open files to disk.) - - -.. data:: RLIMIT_FSIZE - - The maximum size of a file which the process may create. - - -.. data:: RLIMIT_DATA - - The maximum size (in bytes) of the process's heap. - - -.. data:: RLIMIT_STACK - - The maximum size (in bytes) of the call stack for the current process. This only - affects the stack of the main thread in a multi-threaded process. - - -.. data:: RLIMIT_RSS - - The maximum resident set size that should be made available to the process. - - -.. data:: RLIMIT_NPROC - - The maximum number of processes the current process may create. - - -.. data:: RLIMIT_NOFILE - - The maximum number of open file descriptors for the current process. - - -.. data:: RLIMIT_OFILE - - The BSD name for :const:`RLIMIT_NOFILE`. - - -.. data:: RLIMIT_MEMLOCK - - The maximum address space which may be locked in memory. - - -.. data:: RLIMIT_VMEM - - The largest area of mapped memory which the process may occupy. - - -.. data:: RLIMIT_AS - - The maximum area (in bytes) of address space which may be taken by the process. - - -.. data:: RLIMIT_MSGQUEUE - - The number of bytes that can be allocated for POSIX message queues. - - .. availability:: Linux >= 2.6.8. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_NICE - - The ceiling for the process's nice level (calculated as 20 - rlim_cur). - - .. availability:: Linux >= 2.6.12. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_RTPRIO - - The ceiling of the real-time priority. - - .. availability:: Linux >= 2.6.12. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_RTTIME - - The time limit (in microseconds) on CPU time that a process can spend - under real-time scheduling without making a blocking syscall. - - .. availability:: Linux >= 2.6.25. - - .. versionadded:: 3.4 - - -.. data:: RLIMIT_SIGPENDING - - The number of signals which the process may queue. - - .. availability:: Linux >= 2.6.8. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_SBSIZE - - The maximum size (in bytes) of socket buffer usage for this user. - This limits the amount of network memory, and hence the amount of mbufs, - that this user may hold at any time. - - .. availability:: FreeBSD. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_SWAP - - The maximum size (in bytes) of the swap space that may be reserved or - used by all of this user id's processes. - This limit is enforced only if bit 1 of the vm.overcommit sysctl is set. - Please see - `tuning(7) `__ - for a complete description of this sysctl. - - .. availability:: FreeBSD. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_NPTS - - The maximum number of pseudo-terminals created by this user id. - - .. availability:: FreeBSD. - - .. versionadded:: 3.4 - -.. data:: RLIMIT_KQUEUES - - The maximum number of kqueues this user id is allowed to create. - - .. availability:: FreeBSD >= 11. - - .. versionadded:: 3.10 - -Resource Usage --------------- - -These functions are used to retrieve resource usage information: - - -.. function:: getrusage(who) - - This function returns an object that describes the resources consumed by either - the current process or its children, as specified by the *who* parameter. The - *who* parameter should be specified using one of the :const:`RUSAGE_\*` - constants described below. - - A simple example:: - - from resource import * - import time - - # a non CPU-bound task - time.sleep(3) - print(getrusage(RUSAGE_SELF)) - - # a CPU-bound task - for i in range(10 ** 8): - _ = 1 + 1 - print(getrusage(RUSAGE_SELF)) - - The fields of the return value each describe how a particular system resource - has been used, e.g. amount of time spent running is user mode or number of times - the process was swapped out of main memory. Some values are dependent on the - clock tick internal, e.g. the amount of memory the process is using. - - For backward compatibility, the return value is also accessible as a tuple of 16 - elements. - - The fields :attr:`ru_utime` and :attr:`ru_stime` of the return value are - floating point values representing the amount of time spent executing in user - mode and the amount of time spent executing in system mode, respectively. The - remaining values are integers. Consult the :manpage:`getrusage(2)` man page for - detailed information about these values. A brief summary is presented here: - - +--------+---------------------+---------------------------------------+ - | Index | Field | Resource | - +========+=====================+=======================================+ - | ``0`` | :attr:`ru_utime` | time in user mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``1`` | :attr:`ru_stime` | time in system mode (float seconds) | - +--------+---------------------+---------------------------------------+ - | ``2`` | :attr:`ru_maxrss` | maximum resident set size | - +--------+---------------------+---------------------------------------+ - | ``3`` | :attr:`ru_ixrss` | shared memory size | - +--------+---------------------+---------------------------------------+ - | ``4`` | :attr:`ru_idrss` | unshared memory size | - +--------+---------------------+---------------------------------------+ - | ``5`` | :attr:`ru_isrss` | unshared stack size | - +--------+---------------------+---------------------------------------+ - | ``6`` | :attr:`ru_minflt` | page faults not requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``7`` | :attr:`ru_majflt` | page faults requiring I/O | - +--------+---------------------+---------------------------------------+ - | ``8`` | :attr:`ru_nswap` | number of swap outs | - +--------+---------------------+---------------------------------------+ - | ``9`` | :attr:`ru_inblock` | block input operations | - +--------+---------------------+---------------------------------------+ - | ``10`` | :attr:`ru_oublock` | block output operations | - +--------+---------------------+---------------------------------------+ - | ``11`` | :attr:`ru_msgsnd` | messages sent | - +--------+---------------------+---------------------------------------+ - | ``12`` | :attr:`ru_msgrcv` | messages received | - +--------+---------------------+---------------------------------------+ - | ``13`` | :attr:`ru_nsignals` | signals received | - +--------+---------------------+---------------------------------------+ - | ``14`` | :attr:`ru_nvcsw` | voluntary context switches | - +--------+---------------------+---------------------------------------+ - | ``15`` | :attr:`ru_nivcsw` | involuntary context switches | - +--------+---------------------+---------------------------------------+ - - This function will raise a :exc:`ValueError` if an invalid *who* parameter is - specified. It may also raise :exc:`error` exception in unusual circumstances. - - -.. function:: getpagesize() - - Returns the number of bytes in a system page. (This need not be the same as the - hardware page size.) - -The following :const:`RUSAGE_\*` symbols are passed to the :func:`getrusage` -function to specify which processes information should be provided for. - - -.. data:: RUSAGE_SELF - - Pass to :func:`getrusage` to request resources consumed by the calling - process, which is the sum of resources used by all threads in the process. - - -.. data:: RUSAGE_CHILDREN - - Pass to :func:`getrusage` to request resources consumed by child processes - of the calling process which have been terminated and waited for. - - -.. data:: RUSAGE_BOTH - - Pass to :func:`getrusage` to request resources consumed by both the current - process and child processes. May not be available on all systems. - - -.. data:: RUSAGE_THREAD - - Pass to :func:`getrusage` to request resources consumed by the current - thread. May not be available on all systems. - - .. versionadded:: 3.2 diff --git a/Doc/library/rlcompleter.rst b/Doc/library/rlcompleter.rst deleted file mode 100644 index 40b09ce897880e..00000000000000 --- a/Doc/library/rlcompleter.rst +++ /dev/null @@ -1,61 +0,0 @@ -:mod:`rlcompleter` --- Completion function for GNU readline -=========================================================== - -.. module:: rlcompleter - :synopsis: Python identifier completion, suitable for the GNU readline library. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/rlcompleter.py` - --------------- - -The :mod:`rlcompleter` module defines a completion function suitable for the -:mod:`readline` module by completing valid Python identifiers and keywords. - -When this module is imported on a Unix platform with the :mod:`readline` module -available, an instance of the :class:`Completer` class is automatically created -and its :meth:`complete` method is set as the :mod:`readline` completer. - -Example:: - - >>> import rlcompleter - >>> import readline - >>> readline.parse_and_bind("tab: complete") - >>> readline. - readline.__doc__ readline.get_line_buffer( readline.read_init_file( - readline.__file__ readline.insert_text( readline.set_completer( - readline.__name__ readline.parse_and_bind( - >>> readline. - -The :mod:`rlcompleter` module is designed for use with Python's -:ref:`interactive mode `. Unless Python is run with the -:option:`-S` option, the module is automatically imported and configured -(see :ref:`rlcompleter-config`). - -On platforms without :mod:`readline`, the :class:`Completer` class defined by -this module can still be used for custom purposes. - - -.. _completer-objects: - -Completer Objects ------------------ - -Completer objects have the following method: - - -.. method:: Completer.complete(text, state) - - Return the *state*\ th completion for *text*. - - If called for *text* that doesn't include a period character (``'.'``), it will - complete from names currently defined in :mod:`__main__`, :mod:`builtins` and - keywords (as defined by the :mod:`keyword` module). - - If called for a dotted name, it will try to evaluate anything without obvious - side-effects (functions will not be evaluated, but it can generate calls to - :meth:`__getattr__`) up to the last part, and find matches for the rest via the - :func:`dir` function. Any exception raised during the evaluation of the - expression is caught, silenced and :const:`None` is returned. - diff --git a/Doc/library/runpy.rst b/Doc/library/runpy.rst deleted file mode 100644 index 4b37e40bfd6b84..00000000000000 --- a/Doc/library/runpy.rst +++ /dev/null @@ -1,180 +0,0 @@ -:mod:`runpy` --- Locating and executing Python modules -====================================================== - -.. module:: runpy - :synopsis: Locate and run Python modules without importing them first. - -.. moduleauthor:: Nick Coghlan - -**Source code:** :source:`Lib/runpy.py` - --------------- - -The :mod:`runpy` module is used to locate and run Python modules without -importing them first. Its main use is to implement the :option:`-m` command -line switch that allows scripts to be located using the Python module -namespace rather than the filesystem. - -Note that this is *not* a sandbox module - all code is executed in the -current process, and any side effects (such as cached imports of other -modules) will remain in place after the functions have returned. - -Furthermore, any functions and classes defined by the executed code are not -guaranteed to work correctly after a :mod:`runpy` function has returned. -If that limitation is not acceptable for a given use case, :mod:`importlib` -is likely to be a more suitable choice than this module. - -The :mod:`runpy` module provides two functions: - - -.. function:: run_module(mod_name, init_globals=None, run_name=None, alter_sys=False) - - .. index:: - pair: module; __main__ - - Execute the code of the specified module and return the resulting module - globals dictionary. The module's code is first located using the standard - import mechanism (refer to :pep:`302` for details) and then executed in a - fresh module namespace. - - The *mod_name* argument should be an absolute module name. - If the module name refers to a package rather than a normal - module, then that package is imported and the :mod:`__main__` submodule within - that package is then executed and the resulting module globals dictionary - returned. - - The optional dictionary argument *init_globals* may be used to pre-populate - the module's globals dictionary before the code is executed. The supplied - dictionary will not be modified. If any of the special global variables - below are defined in the supplied dictionary, those definitions are - overridden by :func:`run_module`. - - The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). - - ``__name__`` is set to *run_name* if this optional argument is not - :const:`None`, to ``mod_name + '.__main__'`` if the named module is a - package and to the *mod_name* argument otherwise. - - ``__spec__`` will be set appropriately for the *actually* imported - module (that is, ``__spec__.name`` will always be *mod_name* or - ``mod_name + '.__main__``, never *run_name*). - - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` are - :ref:`set as normal ` based on the module spec. - - If the argument *alter_sys* is supplied and evaluates to :const:`True`, - then ``sys.argv[0]`` is updated with the value of ``__file__`` and - ``sys.modules[__name__]`` is updated with a temporary module object for the - module being executed. Both ``sys.argv[0]`` and ``sys.modules[__name__]`` - are restored to their original values before the function returns. - - Note that this manipulation of :mod:`sys` is not thread-safe. Other threads - may see the partially initialised module, as well as the altered list of - arguments. It is recommended that the ``sys`` module be left alone when - invoking this function from threaded code. - - .. seealso:: - The :option:`-m` option offering equivalent functionality from the - command line. - - .. versionchanged:: 3.1 - Added ability to execute packages by looking for a :mod:`__main__` submodule. - - .. versionchanged:: 3.2 - Added ``__cached__`` global variable (see :pep:`3147`). - - .. versionchanged:: 3.4 - Updated to take advantage of the module spec feature added by - :pep:`451`. This allows ``__cached__`` to be set correctly for modules - run this way, as well as ensuring the real module name is always - accessible as ``__spec__.name``. - -.. function:: run_path(path_name, init_globals=None, run_name=None) - - .. index:: - pair: module; __main__ - - Execute the code at the named filesystem location and return the resulting - module globals dictionary. As with a script name supplied to the CPython - command line, the supplied path may refer to a Python source file, a - compiled bytecode file or a valid :data:`sys.path` entry containing a - :mod:`__main__` module - (e.g. a zipfile containing a top-level ``__main__.py`` file). - - For a simple script, the specified code is simply executed in a fresh - module namespace. For a valid :data:`sys.path` entry (typically a zipfile or - directory), the entry is first added to the beginning of ``sys.path``. The - function then looks for and executes a :mod:`__main__` module using the - updated path. Note that there is no special protection against invoking - an existing ``__main__`` entry located elsewhere on ``sys.path`` if - there is no such module at the specified location. - - The optional dictionary argument *init_globals* may be used to pre-populate - the module's globals dictionary before the code is executed. The supplied - dictionary will not be modified. If any of the special global variables - below are defined in the supplied dictionary, those definitions are - overridden by :func:`run_path`. - - The special global variables ``__name__``, ``__spec__``, ``__file__``, - ``__cached__``, ``__loader__`` and ``__package__`` are set in the globals - dictionary before the module code is executed (Note that this is a - minimal set of variables - other variables may be set implicitly as an - interpreter implementation detail). - - ``__name__`` is set to *run_name* if this optional argument is not - :const:`None` and to ``''`` otherwise. - - If the supplied path directly references a script file (whether as source - or as precompiled byte code), then ``__file__`` will be set to the - supplied path, and ``__spec__``, ``__cached__``, ``__loader__`` and - ``__package__`` will all be set to :const:`None`. - - If the supplied path is a reference to a valid :data:`sys.path` entry, then - ``__spec__`` will be set appropriately for the imported :mod:`__main__` - module (that is, ``__spec__.name`` will always be ``__main__``). - ``__file__``, ``__cached__``, ``__loader__`` and ``__package__`` will be - :ref:`set as normal ` based on the module spec. - - A number of alterations are also made to the :mod:`sys` module. Firstly, - :data:`sys.path` may be altered as described above. ``sys.argv[0]`` is updated - with the value of ``path_name`` and ``sys.modules[__name__]`` is updated - with a temporary module object for the module being executed. All - modifications to items in :mod:`sys` are reverted before the function - returns. - - Note that, unlike :func:`run_module`, the alterations made to :mod:`sys` - are not optional in this function as these adjustments are essential to - allowing the execution of :data:`sys.path` entries. As the thread-safety - limitations still apply, use of this function in threaded code should be - either serialised with the import lock or delegated to a separate process. - - .. seealso:: - :ref:`using-on-interface-options` for equivalent functionality on the - command line (``python path/to/script``). - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Updated to take advantage of the module spec feature added by - :pep:`451`. This allows ``__cached__`` to be set correctly in the - case where ``__main__`` is imported from a valid :data:`sys.path` entry rather - than being executed directly. - -.. seealso:: - - :pep:`338` -- Executing modules as scripts - PEP written and implemented by Nick Coghlan. - - :pep:`366` -- Main module explicit relative imports - PEP written and implemented by Nick Coghlan. - - :pep:`451` -- A ModuleSpec Type for the Import System - PEP written and implemented by Eric Snow - - :ref:`using-on-general` - CPython command line details - - The :func:`importlib.import_module` function diff --git a/Doc/library/sched.rst b/Doc/library/sched.rst deleted file mode 100644 index 01bac5afd0b9b3..00000000000000 --- a/Doc/library/sched.rst +++ /dev/null @@ -1,143 +0,0 @@ -:mod:`sched` --- Event scheduler -================================ - -.. module:: sched - :synopsis: General purpose event scheduler. - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/sched.py` - -.. index:: single: event scheduling - --------------- - -The :mod:`sched` module defines a class which implements a general purpose event -scheduler: - -.. class:: scheduler(timefunc=time.monotonic, delayfunc=time.sleep) - - The :class:`scheduler` class defines a generic interface to scheduling events. - It needs two functions to actually deal with the "outside world" --- *timefunc* - should be callable without arguments, and return a number (the "time", in any - units whatsoever). The *delayfunc* function should be callable with one - argument, compatible with the output of *timefunc*, and should delay that many - time units. *delayfunc* will also be called with the argument ``0`` after each - event is run to allow other threads an opportunity to run in multi-threaded - applications. - - .. versionchanged:: 3.3 - *timefunc* and *delayfunc* parameters are optional. - - .. versionchanged:: 3.3 - :class:`scheduler` class can be safely used in multi-threaded - environments. - -Example:: - - >>> import sched, time - >>> s = sched.scheduler(time.monotonic, time.sleep) - >>> def print_time(a='default'): - ... print("From print_time", time.time(), a) - ... - >>> def print_some_times(): - ... print(time.time()) - ... s.enter(10, 1, print_time) - ... s.enter(5, 2, print_time, argument=('positional',)) - ... # despite having higher priority, 'keyword' runs after 'positional' as enter() is relative - ... s.enter(5, 1, print_time, kwargs={'a': 'keyword'}) - ... s.enterabs(1_650_000_000, 10, print_time, argument=("first enterabs",)) - ... s.enterabs(1_650_000_000, 5, print_time, argument=("second enterabs",)) - ... s.run() - ... print(time.time()) - ... - >>> print_some_times() - 1652342830.3640375 - From print_time 1652342830.3642538 second enterabs - From print_time 1652342830.3643398 first enterabs - From print_time 1652342835.3694863 positional - From print_time 1652342835.3696074 keyword - From print_time 1652342840.369612 default - 1652342840.3697174 - - -.. _scheduler-objects: - -Scheduler Objects ------------------ - -:class:`scheduler` instances have the following methods and attributes: - - -.. method:: scheduler.enterabs(time, priority, action, argument=(), kwargs={}) - - Schedule a new event. The *time* argument should be a numeric type compatible - with the return value of the *timefunc* function passed to the constructor. - Events scheduled for the same *time* will be executed in the order of their - *priority*. A lower number represents a higher priority. - - Executing the event means executing ``action(*argument, **kwargs)``. - *argument* is a sequence holding the positional arguments for *action*. - *kwargs* is a dictionary holding the keyword arguments for *action*. - - Return value is an event which may be used for later cancellation of the event - (see :meth:`cancel`). - - .. versionchanged:: 3.3 - *argument* parameter is optional. - - .. versionchanged:: 3.3 - *kwargs* parameter was added. - - -.. method:: scheduler.enter(delay, priority, action, argument=(), kwargs={}) - - Schedule an event for *delay* more time units. Other than the relative time, the - other arguments, the effect and the return value are the same as those for - :meth:`enterabs`. - - .. versionchanged:: 3.3 - *argument* parameter is optional. - - .. versionchanged:: 3.3 - *kwargs* parameter was added. - -.. method:: scheduler.cancel(event) - - Remove the event from the queue. If *event* is not an event currently in the - queue, this method will raise a :exc:`ValueError`. - - -.. method:: scheduler.empty() - - Return ``True`` if the event queue is empty. - - -.. method:: scheduler.run(blocking=True) - - Run all scheduled events. This method will wait (using the *delayfunc* - function passed to the constructor) for the next event, then execute it and so - on until there are no more scheduled events. - - If *blocking* is false executes the scheduled events due to expire soonest - (if any) and then return the deadline of the next scheduled call in the - scheduler (if any). - - Either *action* or *delayfunc* can raise an exception. In either case, the - scheduler will maintain a consistent state and propagate the exception. If an - exception is raised by *action*, the event will not be attempted in future calls - to :meth:`run`. - - If a sequence of events takes longer to run than the time available before the - next event, the scheduler will simply fall behind. No events will be dropped; - the calling code is responsible for canceling events which are no longer - pertinent. - - .. versionchanged:: 3.3 - *blocking* parameter was added. - -.. attribute:: scheduler.queue - - Read-only attribute returning a list of upcoming events in the order they - will be run. Each event is shown as a :term:`named tuple` with the - following fields: time, priority, action, argument, kwargs. diff --git a/Doc/library/secrets.rst b/Doc/library/secrets.rst deleted file mode 100644 index 4405dfc0535973..00000000000000 --- a/Doc/library/secrets.rst +++ /dev/null @@ -1,204 +0,0 @@ -:mod:`secrets` --- Generate secure random numbers for managing secrets -====================================================================== - -.. module:: secrets - :synopsis: Generate secure random numbers for managing secrets. - -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano -.. versionadded:: 3.6 - -.. testsetup:: - - from secrets import * - __name__ = '' - -**Source code:** :source:`Lib/secrets.py` - -------------- - -The :mod:`secrets` module is used for generating cryptographically strong -random numbers suitable for managing data such as passwords, account -authentication, security tokens, and related secrets. - -In particular, :mod:`secrets` should be used in preference to the -default pseudo-random number generator in the :mod:`random` module, which -is designed for modelling and simulation, not security or cryptography. - -.. seealso:: - - :pep:`506` - - -Random numbers --------------- - -The :mod:`secrets` module provides access to the most secure source of -randomness that your operating system provides. - -.. class:: SystemRandom - - A class for generating random numbers using the highest-quality - sources provided by the operating system. See - :class:`random.SystemRandom` for additional details. - -.. function:: choice(sequence) - - Return a randomly chosen element from a non-empty sequence. - -.. function:: randbelow(n) - - Return a random int in the range [0, *n*). - -.. function:: randbits(k) - - Return an int with *k* random bits. - - -Generating tokens ------------------ - -The :mod:`secrets` module provides functions for generating secure -tokens, suitable for applications such as password resets, -hard-to-guess URLs, and similar. - -.. function:: token_bytes([nbytes=None]) - - Return a random byte string containing *nbytes* number of bytes. - If *nbytes* is ``None`` or not supplied, a reasonable default is - used. - - .. doctest:: - - >>> token_bytes(16) #doctest:+SKIP - b'\xebr\x17D*t\xae\xd4\xe3S\xb6\xe2\xebP1\x8b' - - -.. function:: token_hex([nbytes=None]) - - Return a random text string, in hexadecimal. The string has *nbytes* - random bytes, each byte converted to two hex digits. If *nbytes* is - ``None`` or not supplied, a reasonable default is used. - - .. doctest:: - - >>> token_hex(16) #doctest:+SKIP - 'f9bf78b9a18ce6d46a0cd2b0b86df9da' - -.. function:: token_urlsafe([nbytes=None]) - - Return a random URL-safe text string, containing *nbytes* random - bytes. The text is Base64 encoded, so on average each byte results - in approximately 1.3 characters. If *nbytes* is ``None`` or not - supplied, a reasonable default is used. - - .. doctest:: - - >>> token_urlsafe(16) #doctest:+SKIP - 'Drmhze6EPcv0fN_81Bj-nA' - - -How many bytes should tokens use? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -To be secure against -`brute-force attacks `_, -tokens need to have sufficient randomness. Unfortunately, what is -considered sufficient will necessarily increase as computers get more -powerful and able to make more guesses in a shorter period. As of 2015, -it is believed that 32 bytes (256 bits) of randomness is sufficient for -the typical use-case expected for the :mod:`secrets` module. - -For those who want to manage their own token length, you can explicitly -specify how much randomness is used for tokens by giving an :class:`int` -argument to the various ``token_*`` functions. That argument is taken -as the number of bytes of randomness to use. - -Otherwise, if no argument is provided, or if the argument is ``None``, -the ``token_*`` functions will use a reasonable default instead. - -.. note:: - - That default is subject to change at any time, including during - maintenance releases. - - -Other functions ---------------- - -.. function:: compare_digest(a, b) - - Return ``True`` if strings or - :term:`bytes-like objects ` - *a* and *b* are equal, otherwise ``False``, - using a "constant-time compare" to reduce the risk of - `timing attacks `_. - See :func:`hmac.compare_digest` for additional details. - - -Recipes and best practices --------------------------- - -This section shows recipes and best practices for using :mod:`secrets` -to manage a basic level of security. - -Generate an eight-character alphanumeric password: - -.. testcode:: - - import string - import secrets - alphabet = string.ascii_letters + string.digits - password = ''.join(secrets.choice(alphabet) for i in range(8)) - - -.. note:: - - Applications should not - `store passwords in a recoverable format `_, - whether plain text or encrypted. They should be salted and hashed - using a cryptographically strong one-way (irreversible) hash function. - - -Generate a ten-character alphanumeric password with at least one -lowercase character, at least one uppercase character, and at least -three digits: - -.. testcode:: - - import string - import secrets - alphabet = string.ascii_letters + string.digits - while True: - password = ''.join(secrets.choice(alphabet) for i in range(10)) - if (any(c.islower() for c in password) - and any(c.isupper() for c in password) - and sum(c.isdigit() for c in password) >= 3): - break - - -Generate an `XKCD-style passphrase `_: - -.. testcode:: - - import secrets - # On standard Linux systems, use a convenient dictionary file. - # Other platforms may need to provide their own word-list. - with open('/usr/share/dict/words') as f: - words = [word.strip() for word in f] - password = ' '.join(secrets.choice(words) for i in range(4)) - - -Generate a hard-to-guess temporary URL containing a security token -suitable for password recovery applications: - -.. testcode:: - - import secrets - url = 'https://example.com/reset=' + secrets.token_urlsafe() - - - -.. - # This modeline must appear within the last ten lines of the file. - kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/security_warnings.rst b/Doc/library/security_warnings.rst deleted file mode 100644 index 284f3658320623..00000000000000 --- a/Doc/library/security_warnings.rst +++ /dev/null @@ -1,40 +0,0 @@ -.. _security-warnings: - -.. index:: single: security considerations - -Security Considerations -======================= - -The following modules have specific security considerations: - -* :mod:`base64`: :ref:`base64 security considerations ` in - :rfc:`4648` -* :mod:`cgi`: :ref:`CGI security considerations ` -* :mod:`hashlib`: :ref:`all constructors take a "usedforsecurity" keyword-only - argument disabling known insecure and blocked algorithms - ` -* :mod:`http.server` is not suitable for production use, only implementing - basic security checks. See the :ref:`security considerations `. -* :mod:`logging`: :ref:`Logging configuration uses eval() - ` -* :mod:`multiprocessing`: :ref:`Connection.recv() uses pickle - ` -* :mod:`pickle`: :ref:`Restricting globals in pickle ` -* :mod:`random` shouldn't be used for security purposes, use :mod:`secrets` - instead -* :mod:`shelve`: :ref:`shelve is based on pickle and thus unsuitable for - dealing with untrusted sources ` -* :mod:`ssl`: :ref:`SSL/TLS security considerations ` -* :mod:`subprocess`: :ref:`Subprocess security considerations - ` -* :mod:`tempfile`: :ref:`mktemp is deprecated due to vulnerability to race - conditions ` -* :mod:`xml`: :ref:`XML vulnerabilities ` -* :mod:`zipfile`: :ref:`maliciously prepared .zip files can cause disk volume - exhaustion ` - -The :option:`-I` command line option can be used to run Python in isolated -mode. When it cannot be used, the :option:`-P` option or the -:envvar:`PYTHONSAFEPATH` environment variable can be used to not prepend a -potentially unsafe path to :data:`sys.path` such as the current directory, the -script's directory or an empty string. diff --git a/Doc/library/select.rst b/Doc/library/select.rst deleted file mode 100644 index c2941e628d9d78..00000000000000 --- a/Doc/library/select.rst +++ /dev/null @@ -1,650 +0,0 @@ -:mod:`select` --- Waiting for I/O completion -============================================ - -.. module:: select - :synopsis: Wait for I/O completion on multiple streams. - --------------- - -This module provides access to the :c:func:`!select` and :c:func:`!poll` functions -available in most operating systems, :c:func:`!devpoll` available on -Solaris and derivatives, :c:func:`!epoll` available on Linux 2.5+ and -:c:func:`!kqueue` available on most BSD. -Note that on Windows, it only works for sockets; on other operating systems, -it also works for other file types (in particular, on Unix, it works on pipes). -It cannot be used on regular files to determine whether a file has grown since -it was last read. - -.. note:: - - The :mod:`selectors` module allows high-level and efficient I/O - multiplexing, built upon the :mod:`select` module primitives. Users are - encouraged to use the :mod:`selectors` module instead, unless they want - precise control over the OS-level primitives used. - -.. include:: ../includes/wasm-notavail.rst - -The module defines the following: - - -.. exception:: error - - A deprecated alias of :exc:`OSError`. - - .. versionchanged:: 3.3 - Following :pep:`3151`, this class was made an alias of :exc:`OSError`. - - -.. function:: devpoll() - - (Only supported on Solaris and derivatives.) Returns a ``/dev/poll`` - polling object; see section :ref:`devpoll-objects` below for the - methods supported by devpoll objects. - - :c:func:`!devpoll` objects are linked to the number of file - descriptors allowed at the time of instantiation. If your program - reduces this value, :c:func:`!devpoll` will fail. If your program - increases this value, :c:func:`!devpoll` may return an - incomplete list of active file descriptors. - - The new file descriptor is :ref:`non-inheritable `. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - -.. function:: epoll(sizehint=-1, flags=0) - - (Only supported on Linux 2.5.44 and newer.) Return an edge polling object, - which can be used as Edge or Level Triggered interface for I/O - events. - - *sizehint* informs epoll about the expected number of events to be - registered. It must be positive, or ``-1`` to use the default. It is only - used on older systems where :c:func:`!epoll_create1` is not available; - otherwise it has no effect (though its value is still checked). - - *flags* is deprecated and completely ignored. However, when supplied, its - value must be ``0`` or ``select.EPOLL_CLOEXEC``, otherwise ``OSError`` is - raised. - - See the :ref:`epoll-objects` section below for the methods supported by - epolling objects. - - ``epoll`` objects support the context management protocol: when used in a - :keyword:`with` statement, the new file descriptor is automatically closed - at the end of the block. - - The new file descriptor is :ref:`non-inheritable `. - - .. versionchanged:: 3.3 - Added the *flags* parameter. - - .. versionchanged:: 3.4 - Support for the :keyword:`with` statement was added. - The new file descriptor is now non-inheritable. - - .. deprecated:: 3.4 - The *flags* parameter. ``select.EPOLL_CLOEXEC`` is used by default now. - Use :func:`os.set_inheritable` to make the file descriptor inheritable. - - -.. function:: poll() - - (Not supported by all operating systems.) Returns a polling object, which - supports registering and unregistering file descriptors, and then polling them - for I/O events; see section :ref:`poll-objects` below for the methods supported - by polling objects. - - -.. function:: kqueue() - - (Only supported on BSD.) Returns a kernel queue object; see section - :ref:`kqueue-objects` below for the methods supported by kqueue objects. - - The new file descriptor is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The new file descriptor is now non-inheritable. - - -.. function:: kevent(ident, filter=KQ_FILTER_READ, flags=KQ_EV_ADD, fflags=0, data=0, udata=0) - - (Only supported on BSD.) Returns a kernel event object; see section - :ref:`kevent-objects` below for the methods supported by kevent objects. - - -.. function:: select(rlist, wlist, xlist[, timeout]) - - This is a straightforward interface to the Unix :c:func:`!select` system call. - The first three arguments are iterables of 'waitable objects': either - integers representing file descriptors or objects with a parameterless method - named :meth:`~io.IOBase.fileno` returning such an integer: - - * *rlist*: wait until ready for reading - * *wlist*: wait until ready for writing - * *xlist*: wait for an "exceptional condition" (see the manual page for what - your system considers such a condition) - - Empty iterables are allowed, but acceptance of three empty iterables is - platform-dependent. (It is known to work on Unix but not on Windows.) The - optional *timeout* argument specifies a time-out as a floating point number - in seconds. When the *timeout* argument is omitted the function blocks until - at least one file descriptor is ready. A time-out value of zero specifies a - poll and never blocks. - - The return value is a triple of lists of objects that are ready: subsets of the - first three arguments. When the time-out is reached without a file descriptor - becoming ready, three empty lists are returned. - - .. index:: - single: socket() (in module socket) - single: popen() (in module os) - - Among the acceptable object types in the iterables are Python :term:`file - objects ` (e.g. ``sys.stdin``, or objects returned by - :func:`open` or :func:`os.popen`), socket objects returned by - :func:`socket.socket`. You may also define a :dfn:`wrapper` class yourself, - as long as it has an appropriate :meth:`~io.IOBase.fileno` method (that - really returns a file descriptor, not just a random integer). - - .. note:: - - .. index:: single: WinSock - - File objects on Windows are not acceptable, but sockets are. On Windows, - the underlying :c:func:`!select` function is provided by the WinSock - library, and does not handle file descriptors that don't originate from - WinSock. - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. attribute:: PIPE_BUF - - The minimum number of bytes which can be written without blocking to a pipe - when the pipe has been reported as ready for writing by :func:`~select.select`, - :func:`!poll` or another interface in this module. This doesn't apply - to other kind of file-like objects such as sockets. - - This value is guaranteed by POSIX to be at least 512. - - .. availability:: Unix - - .. versionadded:: 3.2 - - -.. _devpoll-objects: - -``/dev/poll`` Polling Objects ------------------------------ - -Solaris and derivatives have ``/dev/poll``. While :c:func:`!select` is -O(highest file descriptor) and :c:func:`!poll` is O(number of file -descriptors), ``/dev/poll`` is O(active file descriptors). - -``/dev/poll`` behaviour is very close to the standard :c:func:`!poll` -object. - - -.. method:: devpoll.close() - - Close the file descriptor of the polling object. - - .. versionadded:: 3.4 - - -.. attribute:: devpoll.closed - - ``True`` if the polling object is closed. - - .. versionadded:: 3.4 - - -.. method:: devpoll.fileno() - - Return the file descriptor number of the polling object. - - .. versionadded:: 3.4 - - -.. method:: devpoll.register(fd[, eventmask]) - - Register a file descriptor with the polling object. Future calls to the - :meth:`poll` method will then check whether the file descriptor has any - pending I/O events. *fd* can be either an integer, or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. File objects - implement :meth:`!fileno`, so they can also be used as the argument. - - *eventmask* is an optional bitmask describing the type of events you want to - check for. The constants are the same that with :c:func:`!poll` - object. The default value is a combination of the constants :const:`POLLIN`, - :const:`POLLPRI`, and :const:`POLLOUT`. - - .. warning:: - - Registering a file descriptor that's already registered is not an - error, but the result is undefined. The appropriate action is to - unregister or modify it first. This is an important difference - compared with :c:func:`!poll`. - - -.. method:: devpoll.modify(fd[, eventmask]) - - This method does an :meth:`unregister` followed by a - :meth:`register`. It is (a bit) more efficient that doing the same - explicitly. - - -.. method:: devpoll.unregister(fd) - - Remove a file descriptor being tracked by a polling object. Just like the - :meth:`register` method, *fd* can be an integer or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. - - Attempting to remove a file descriptor that was never registered is - safely ignored. - - -.. method:: devpoll.poll([timeout]) - - Polls the set of registered file descriptors, and returns a possibly empty list - containing ``(fd, event)`` 2-tuples for the descriptors that have events or - errors to report. *fd* is the file descriptor, and *event* is a bitmask with - bits set for the reported events for that descriptor --- :const:`POLLIN` for - waiting input, :const:`POLLOUT` to indicate that the descriptor can be written - to, and so forth. An empty list indicates that the call timed out and no file - descriptors had any events to report. If *timeout* is given, it specifies the - length of time in milliseconds which the system will wait for events before - returning. If *timeout* is omitted, -1, or :const:`None`, the call will - block until there is an event for this poll object. - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. _epoll-objects: - -Edge and Level Trigger Polling (epoll) Objects ----------------------------------------------- - - https://linux.die.net/man/4/epoll - - *eventmask* - - +-------------------------+-----------------------------------------------+ - | Constant | Meaning | - +=========================+===============================================+ - | :const:`EPOLLIN` | Available for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLOUT` | Available for write | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLPRI` | Urgent data for read | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLERR` | Error condition happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLHUP` | Hang up happened on the assoc. fd | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLET` | Set Edge Trigger behavior, the default is | - | | Level Trigger behavior | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLONESHOT` | Set one-shot behavior. After one event is | - | | pulled out, the fd is internally disabled | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLEXCLUSIVE` | Wake only one epoll object when the | - | | associated fd has an event. The default (if | - | | this flag is not set) is to wake all epoll | - | | objects polling on a fd. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDHUP` | Stream socket peer closed connection or shut | - | | down writing half of connection. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDNORM` | Equivalent to :const:`EPOLLIN` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLRDBAND` | Priority data band can be read. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRNORM` | Equivalent to :const:`EPOLLOUT` | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLWRBAND` | Priority data may be written. | - +-------------------------+-----------------------------------------------+ - | :const:`EPOLLMSG` | Ignored. | - +-------------------------+-----------------------------------------------+ - - .. versionadded:: 3.6 - :const:`EPOLLEXCLUSIVE` was added. It's only supported by Linux Kernel 4.5 - or later. - -.. method:: epoll.close() - - Close the control file descriptor of the epoll object. - - -.. attribute:: epoll.closed - - ``True`` if the epoll object is closed. - - -.. method:: epoll.fileno() - - Return the file descriptor number of the control fd. - - -.. method:: epoll.fromfd(fd) - - Create an epoll object from a given file descriptor. - - -.. method:: epoll.register(fd[, eventmask]) - - Register a fd descriptor with the epoll object. - - -.. method:: epoll.modify(fd, eventmask) - - Modify a registered file descriptor. - - -.. method:: epoll.unregister(fd) - - Remove a registered file descriptor from the epoll object. - - .. versionchanged:: 3.9 - The method no longer ignores the :data:`~errno.EBADF` error. - - -.. method:: epoll.poll(timeout=None, maxevents=-1) - - Wait for events. timeout in seconds (float) - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. _poll-objects: - -Polling Objects ---------------- - -The :c:func:`!poll` system call, supported on most Unix systems, provides better -scalability for network servers that service many, many clients at the same -time. :c:func:`!poll` scales better because the system call only requires listing -the file descriptors of interest, while :c:func:`!select` builds a bitmap, turns -on bits for the fds of interest, and then afterward the whole bitmap has to be -linearly scanned again. :c:func:`!select` is O(highest file descriptor), while -:c:func:`!poll` is O(number of file descriptors). - - -.. method:: poll.register(fd[, eventmask]) - - Register a file descriptor with the polling object. Future calls to the - :meth:`poll` method will then check whether the file descriptor has any - pending I/O events. *fd* can be either an integer, or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. File objects - implement :meth:`!fileno`, so they can also be used as the argument. - - *eventmask* is an optional bitmask describing the type of events you want to - check for, and can be a combination of the constants :const:`POLLIN`, - :const:`POLLPRI`, and :const:`POLLOUT`, described in the table below. If not - specified, the default value used will check for all 3 types of events. - - +-------------------+------------------------------------------+ - | Constant | Meaning | - +===================+==========================================+ - | :const:`POLLIN` | There is data to read | - +-------------------+------------------------------------------+ - | :const:`POLLPRI` | There is urgent data to read | - +-------------------+------------------------------------------+ - | :const:`POLLOUT` | Ready for output: writing will not block | - +-------------------+------------------------------------------+ - | :const:`POLLERR` | Error condition of some sort | - +-------------------+------------------------------------------+ - | :const:`POLLHUP` | Hung up | - +-------------------+------------------------------------------+ - | :const:`POLLRDHUP`| Stream socket peer closed connection, or | - | | shut down writing half of connection | - +-------------------+------------------------------------------+ - | :const:`POLLNVAL` | Invalid request: descriptor not open | - +-------------------+------------------------------------------+ - - Registering a file descriptor that's already registered is not an error, and has - the same effect as registering the descriptor exactly once. - - -.. method:: poll.modify(fd, eventmask) - - Modifies an already registered fd. This has the same effect as - ``register(fd, eventmask)``. Attempting to modify a file descriptor - that was never registered causes an :exc:`OSError` exception with errno - :const:`ENOENT` to be raised. - - -.. method:: poll.unregister(fd) - - Remove a file descriptor being tracked by a polling object. Just like the - :meth:`register` method, *fd* can be an integer or an object with a - :meth:`~io.IOBase.fileno` method that returns an integer. - - Attempting to remove a file descriptor that was never registered causes a - :exc:`KeyError` exception to be raised. - - -.. method:: poll.poll([timeout]) - - Polls the set of registered file descriptors, and returns a possibly empty list - containing ``(fd, event)`` 2-tuples for the descriptors that have events or - errors to report. *fd* is the file descriptor, and *event* is a bitmask with - bits set for the reported events for that descriptor --- :const:`POLLIN` for - waiting input, :const:`POLLOUT` to indicate that the descriptor can be written - to, and so forth. An empty list indicates that the call timed out and no file - descriptors had any events to report. If *timeout* is given, it specifies the - length of time in milliseconds which the system will wait for events before - returning. If *timeout* is omitted, negative, or :const:`None`, the call will - block until there is an event for this poll object. - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. _kqueue-objects: - -Kqueue Objects --------------- - -.. method:: kqueue.close() - - Close the control file descriptor of the kqueue object. - - -.. attribute:: kqueue.closed - - ``True`` if the kqueue object is closed. - - -.. method:: kqueue.fileno() - - Return the file descriptor number of the control fd. - - -.. method:: kqueue.fromfd(fd) - - Create a kqueue object from a given file descriptor. - - -.. method:: kqueue.control(changelist, max_events[, timeout]) -> eventlist - - Low level interface to kevent - - - changelist must be an iterable of kevent objects or ``None`` - - max_events must be 0 or a positive integer - - timeout in seconds (floats possible); the default is ``None``, - to wait forever - - .. versionchanged:: 3.5 - The function is now retried with a recomputed timeout when interrupted by - a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale), instead of raising - :exc:`InterruptedError`. - - -.. _kevent-objects: - -Kevent Objects --------------- - -https://man.freebsd.org/cgi/man.cgi?query=kqueue&sektion=2 - -.. attribute:: kevent.ident - - Value used to identify the event. The interpretation depends on the filter - but it's usually the file descriptor. In the constructor ident can either - be an int or an object with a :meth:`~io.IOBase.fileno` method. kevent - stores the integer internally. - -.. attribute:: kevent.filter - - Name of the kernel filter. - - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_FILTER_READ` | Takes a descriptor and returns whenever | - | | there is data available to read | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_WRITE` | Takes a descriptor and returns whenever | - | | there is data available to write | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_AIO` | AIO requests | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_VNODE` | Returns when one or more of the requested | - | | events watched in *fflag* occurs | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_PROC` | Watch for events on a process id | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_NETDEV` | Watch for events on a network device | - | | [not available on macOS] | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_SIGNAL` | Returns whenever the watched signal is | - | | delivered to the process | - +---------------------------+---------------------------------------------+ - | :const:`KQ_FILTER_TIMER` | Establishes an arbitrary timer | - +---------------------------+---------------------------------------------+ - -.. attribute:: kevent.flags - - Filter action. - - +---------------------------+---------------------------------------------+ - | Constant | Meaning | - +===========================+=============================================+ - | :const:`KQ_EV_ADD` | Adds or modifies an event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DELETE` | Removes an event from the queue | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ENABLE` | Permitscontrol() to returns the event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_DISABLE` | Disablesevent | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ONESHOT` | Removes event after first occurrence | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_CLEAR` | Reset the state after an event is retrieved | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_SYSFLAGS` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_FLAG1` | internal event | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_EOF` | Filter specific EOF condition | - +---------------------------+---------------------------------------------+ - | :const:`KQ_EV_ERROR` | See return values | - +---------------------------+---------------------------------------------+ - - -.. attribute:: kevent.fflags - - Filter specific flags. - - :const:`KQ_FILTER_READ` and :const:`KQ_FILTER_WRITE` filter flags: - - +----------------------------+--------------------------------------------+ - | Constant | Meaning | - +============================+============================================+ - | :const:`KQ_NOTE_LOWAT` | low water mark of a socket buffer | - +----------------------------+--------------------------------------------+ - - :const:`KQ_FILTER_VNODE` filter flags: - - +----------------------------+--------------------------------------------+ - | Constant | Meaning | - +============================+============================================+ - | :const:`KQ_NOTE_DELETE` | *unlink()* was called | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_WRITE` | a write occurred | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXTEND` | the file was extended | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_ATTRIB` | an attribute was changed | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINK` | the link count has changed | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_RENAME` | the file was renamed | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_REVOKE` | access to the file was revoked | - +----------------------------+--------------------------------------------+ - - :const:`KQ_FILTER_PROC` filter flags: - - +----------------------------+--------------------------------------------+ - | Constant | Meaning | - +============================+============================================+ - | :const:`KQ_NOTE_EXIT` | the process has exited | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_FORK` | the process has called *fork()* | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_EXEC` | the process has executed a new process | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PCTRLMASK` | internal filter flag | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_PDATAMASK` | internal filter flag | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACK` | follow a process across *fork()* | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_CHILD` | returned on the child process for | - | | *NOTE_TRACK* | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_TRACKERR` | unable to attach to a child | - +----------------------------+--------------------------------------------+ - - :const:`KQ_FILTER_NETDEV` filter flags (not available on macOS): - - +----------------------------+--------------------------------------------+ - | Constant | Meaning | - +============================+============================================+ - | :const:`KQ_NOTE_LINKUP` | link is up | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKDOWN` | link is down | - +----------------------------+--------------------------------------------+ - | :const:`KQ_NOTE_LINKINV` | link state is invalid | - +----------------------------+--------------------------------------------+ - - -.. attribute:: kevent.data - - Filter specific data. - - -.. attribute:: kevent.udata - - User defined value. diff --git a/Doc/library/selectors.rst b/Doc/library/selectors.rst deleted file mode 100644 index 76cbf91412f763..00000000000000 --- a/Doc/library/selectors.rst +++ /dev/null @@ -1,278 +0,0 @@ -:mod:`selectors` --- High-level I/O multiplexing -================================================ - -.. module:: selectors - :synopsis: High-level I/O multiplexing. - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/selectors.py` - --------------- - -Introduction ------------- - -This module allows high-level and efficient I/O multiplexing, built upon the -:mod:`select` module primitives. Users are encouraged to use this module -instead, unless they want precise control over the OS-level primitives used. - -It defines a :class:`BaseSelector` abstract base class, along with several -concrete implementations (:class:`KqueueSelector`, :class:`EpollSelector`...), -that can be used to wait for I/O readiness notification on multiple file -objects. In the following, "file object" refers to any object with a -:meth:`~io.IOBase.fileno` method, or a raw file descriptor. See :term:`file object`. - -:class:`DefaultSelector` is an alias to the most efficient implementation -available on the current platform: this should be the default choice for most -users. - -.. note:: - The type of file objects supported depends on the platform: on Windows, - sockets are supported, but not pipes, whereas on Unix, both are supported - (some other types may be supported as well, such as fifos or special file - devices). - -.. seealso:: - - :mod:`select` - Low-level I/O multiplexing module. - -.. include:: ../includes/wasm-notavail.rst - -Classes -------- - -Classes hierarchy:: - - BaseSelector - +-- SelectSelector - +-- PollSelector - +-- EpollSelector - +-- DevpollSelector - +-- KqueueSelector - - -In the following, *events* is a bitwise mask indicating which I/O events should -be waited for on a given file object. It can be a combination of the modules -constants below: - - +-----------------------+-----------------------------------------------+ - | Constant | Meaning | - +=======================+===============================================+ - | .. data:: EVENT_READ | Available for read | - +-----------------------+-----------------------------------------------+ - | .. data:: EVENT_WRITE | Available for write | - +-----------------------+-----------------------------------------------+ - - -.. class:: SelectorKey - - A :class:`SelectorKey` is a :class:`~collections.namedtuple` used to - associate a file object to its underlying file descriptor, selected event - mask and attached data. It is returned by several :class:`BaseSelector` - methods. - - .. attribute:: fileobj - - File object registered. - - .. attribute:: fd - - Underlying file descriptor. - - .. attribute:: events - - Events that must be waited for on this file object. - - .. attribute:: data - - Optional opaque data associated to this file object: for example, this - could be used to store a per-client session ID. - - -.. class:: BaseSelector - - A :class:`BaseSelector` is used to wait for I/O event readiness on multiple - file objects. It supports file stream registration, unregistration, and a - method to wait for I/O events on those streams, with an optional timeout. - It's an abstract base class, so cannot be instantiated. Use - :class:`DefaultSelector` instead, or one of :class:`SelectSelector`, - :class:`KqueueSelector` etc. if you want to specifically use an - implementation, and your platform supports it. - :class:`BaseSelector` and its concrete implementations support the - :term:`context manager` protocol. - - .. abstractmethod:: register(fileobj, events, data=None) - - Register a file object for selection, monitoring it for I/O events. - - *fileobj* is the file object to monitor. It may either be an integer - file descriptor or an object with a ``fileno()`` method. - *events* is a bitwise mask of events to monitor. - *data* is an opaque object. - - This returns a new :class:`SelectorKey` instance, or raises a - :exc:`ValueError` in case of invalid event mask or file descriptor, or - :exc:`KeyError` if the file object is already registered. - - .. abstractmethod:: unregister(fileobj) - - Unregister a file object from selection, removing it from monitoring. A - file object shall be unregistered prior to being closed. - - *fileobj* must be a file object previously registered. - - This returns the associated :class:`SelectorKey` instance, or raises a - :exc:`KeyError` if *fileobj* is not registered. It will raise - :exc:`ValueError` if *fileobj* is invalid (e.g. it has no ``fileno()`` - method or its ``fileno()`` method has an invalid return value). - - .. method:: modify(fileobj, events, data=None) - - Change a registered file object's monitored events or attached data. - - This is equivalent to ``BaseSelector.unregister(fileobj)`` followed - by ``BaseSelector.register(fileobj, events, data)``, except that it - can be implemented more efficiently. - - This returns a new :class:`SelectorKey` instance, or raises a - :exc:`ValueError` in case of invalid event mask or file descriptor, or - :exc:`KeyError` if the file object is not registered. - - .. abstractmethod:: select(timeout=None) - - Wait until some registered file objects become ready, or the timeout - expires. - - If ``timeout > 0``, this specifies the maximum wait time, in seconds. - If ``timeout <= 0``, the call won't block, and will report the currently - ready file objects. - If *timeout* is ``None``, the call will block until a monitored file object - becomes ready. - - This returns a list of ``(key, events)`` tuples, one for each ready file - object. - - *key* is the :class:`SelectorKey` instance corresponding to a ready file - object. - *events* is a bitmask of events ready on this file object. - - .. note:: - This method can return before any file object becomes ready or the - timeout has elapsed if the current process receives a signal: in this - case, an empty list will be returned. - - .. versionchanged:: 3.5 - The selector is now retried with a recomputed timeout when interrupted - by a signal if the signal handler did not raise an exception (see - :pep:`475` for the rationale), instead of returning an empty list - of events before the timeout. - - .. method:: close() - - Close the selector. - - This must be called to make sure that any underlying resource is freed. - The selector shall not be used once it has been closed. - - .. method:: get_key(fileobj) - - Return the key associated with a registered file object. - - This returns the :class:`SelectorKey` instance associated to this file - object, or raises :exc:`KeyError` if the file object is not registered. - - .. abstractmethod:: get_map() - - Return a mapping of file objects to selector keys. - - This returns a :class:`~collections.abc.Mapping` instance mapping - registered file objects to their associated :class:`SelectorKey` - instance. - - -.. class:: DefaultSelector() - - The default selector class, using the most efficient implementation - available on the current platform. This should be the default choice for - most users. - - -.. class:: SelectSelector() - - :func:`select.select`-based selector. - - -.. class:: PollSelector() - - :func:`select.poll`-based selector. - - -.. class:: EpollSelector() - - :func:`select.epoll`-based selector. - - .. method:: fileno() - - This returns the file descriptor used by the underlying - :func:`select.epoll` object. - -.. class:: DevpollSelector() - - :func:`select.devpoll`-based selector. - - .. method:: fileno() - - This returns the file descriptor used by the underlying - :func:`select.devpoll` object. - - .. versionadded:: 3.5 - -.. class:: KqueueSelector() - - :func:`select.kqueue`-based selector. - - .. method:: fileno() - - This returns the file descriptor used by the underlying - :func:`select.kqueue` object. - - -Examples --------- - -Here is a simple echo server implementation:: - - import selectors - import socket - - sel = selectors.DefaultSelector() - - def accept(sock, mask): - conn, addr = sock.accept() # Should be ready - print('accepted', conn, 'from', addr) - conn.setblocking(False) - sel.register(conn, selectors.EVENT_READ, read) - - def read(conn, mask): - data = conn.recv(1000) # Should be ready - if data: - print('echoing', repr(data), 'to', conn) - conn.send(data) # Hope it won't block - else: - print('closing', conn) - sel.unregister(conn) - conn.close() - - sock = socket.socket() - sock.bind(('localhost', 1234)) - sock.listen(100) - sock.setblocking(False) - sel.register(sock, selectors.EVENT_READ, accept) - - while True: - events = sel.select() - for key, mask in events: - callback = key.data - callback(key.fileobj, mask) diff --git a/Doc/library/shelve.rst b/Doc/library/shelve.rst deleted file mode 100644 index 219219af6fd87f..00000000000000 --- a/Doc/library/shelve.rst +++ /dev/null @@ -1,219 +0,0 @@ -:mod:`shelve` --- Python object persistence -=========================================== - -.. module:: shelve - :synopsis: Python object persistence. - -**Source code:** :source:`Lib/shelve.py` - -.. index:: pair: module; pickle - --------------- - -A "shelf" is a persistent, dictionary-like object. The difference with "dbm" -databases is that the values (not the keys!) in a shelf can be essentially -arbitrary Python objects --- anything that the :mod:`pickle` module can handle. -This includes most class instances, recursive data types, and objects containing -lots of shared sub-objects. The keys are ordinary strings. - - -.. function:: open(filename, flag='c', protocol=None, writeback=False) - - Open a persistent dictionary. The filename specified is the base filename for - the underlying database. As a side-effect, an extension may be added to the - filename and more than one file may be created. By default, the underlying - database file is opened for reading and writing. The optional *flag* parameter - has the same interpretation as the *flag* parameter of :func:`dbm.open`. - - By default, pickles created with :const:`pickle.DEFAULT_PROTOCOL` are used - to serialize values. The version of the pickle protocol can be specified - with the *protocol* parameter. - - Because of Python semantics, a shelf cannot know when a mutable - persistent-dictionary entry is modified. By default modified objects are - written *only* when assigned to the shelf (see :ref:`shelve-example`). If the - optional *writeback* parameter is set to ``True``, all entries accessed are also - cached in memory, and written back on :meth:`~Shelf.sync` and - :meth:`~Shelf.close`; this can make it handier to mutate mutable entries in - the persistent dictionary, but, if many entries are accessed, it can consume - vast amounts of memory for the cache, and it can make the close operation - very slow since all accessed entries are written back (there is no way to - determine which accessed entries are mutable, nor which ones were actually - mutated). - - .. versionchanged:: 3.10 - :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle - protocol. - - .. versionchanged:: 3.11 - Accepts :term:`path-like object` for filename. - - .. note:: - - Do not rely on the shelf being closed automatically; always call - :meth:`~Shelf.close` explicitly when you don't need it any more, or - use :func:`shelve.open` as a context manager:: - - with shelve.open('spam') as db: - db['eggs'] = 'eggs' - -.. _shelve-security: - -.. warning:: - - Because the :mod:`shelve` module is backed by :mod:`pickle`, it is insecure - to load a shelf from an untrusted source. Like with pickle, loading a shelf - can execute arbitrary code. - -Shelf objects support most of methods and operations supported by dictionaries -(except copying, constructors and operators ``|`` and ``|=``). This eases the -transition from dictionary based scripts to those requiring persistent storage. - -Two additional methods are supported: - -.. method:: Shelf.sync() - - Write back all entries in the cache if the shelf was opened with *writeback* - set to :const:`True`. Also empty the cache and synchronize the persistent - dictionary on disk, if feasible. This is called automatically when the shelf - is closed with :meth:`close`. - -.. method:: Shelf.close() - - Synchronize and close the persistent *dict* object. Operations on a closed - shelf will fail with a :exc:`ValueError`. - - -.. seealso:: - - `Persistent dictionary recipe `_ - with widely supported storage formats and having the speed of native - dictionaries. - - -Restrictions ------------- - -.. index:: - pair: module; dbm.ndbm - pair: module; dbm.gnu - -* The choice of which database package will be used (such as :mod:`dbm.ndbm` or - :mod:`dbm.gnu`) depends on which interface is available. Therefore it is not - safe to open the database directly using :mod:`dbm`. The database is also - (unfortunately) subject to the limitations of :mod:`dbm`, if it is used --- - this means that (the pickled representation of) the objects stored in the - database should be fairly small, and in rare cases key collisions may cause - the database to refuse updates. - -* The :mod:`shelve` module does not support *concurrent* read/write access to - shelved objects. (Multiple simultaneous read accesses are safe.) When a - program has a shelf open for writing, no other program should have it open for - reading or writing. Unix file locking can be used to solve this, but this - differs across Unix versions and requires knowledge about the database - implementation used. - - -.. class:: Shelf(dict, protocol=None, writeback=False, keyencoding='utf-8') - - A subclass of :class:`collections.abc.MutableMapping` which stores pickled - values in the *dict* object. - - By default, pickles created with :const:`pickle.DEFAULT_PROTOCOL` are used - to serialize values. The version of the pickle protocol can be specified - with the *protocol* parameter. See the :mod:`pickle` documentation for a - discussion of the pickle protocols. - - If the *writeback* parameter is ``True``, the object will hold a cache of all - entries accessed and write them back to the *dict* at sync and close times. - This allows natural operations on mutable entries, but can consume much more - memory and make sync and close take a long time. - - The *keyencoding* parameter is the encoding used to encode keys before they - are used with the underlying dict. - - A :class:`Shelf` object can also be used as a context manager, in which - case it will be automatically closed when the :keyword:`with` block ends. - - .. versionchanged:: 3.2 - Added the *keyencoding* parameter; previously, keys were always encoded in - UTF-8. - - .. versionchanged:: 3.4 - Added context manager support. - - .. versionchanged:: 3.10 - :const:`pickle.DEFAULT_PROTOCOL` is now used as the default pickle - protocol. - - -.. class:: BsdDbShelf(dict, protocol=None, writeback=False, keyencoding='utf-8') - - A subclass of :class:`Shelf` which exposes :meth:`first`, :meth:`!next`, - :meth:`previous`, :meth:`last` and :meth:`set_location` which are available - in the third-party :mod:`bsddb` module from `pybsddb - `_ but not in other database - modules. The *dict* object passed to the constructor must support those - methods. This is generally accomplished by calling one of - :func:`bsddb.hashopen`, :func:`bsddb.btopen` or :func:`bsddb.rnopen`. The - optional *protocol*, *writeback*, and *keyencoding* parameters have the same - interpretation as for the :class:`Shelf` class. - - -.. class:: DbfilenameShelf(filename, flag='c', protocol=None, writeback=False) - - A subclass of :class:`Shelf` which accepts a *filename* instead of a dict-like - object. The underlying file will be opened using :func:`dbm.open`. By - default, the file will be created and opened for both read and write. The - optional *flag* parameter has the same interpretation as for the :func:`.open` - function. The optional *protocol* and *writeback* parameters have the same - interpretation as for the :class:`Shelf` class. - - -.. _shelve-example: - -Example -------- - -To summarize the interface (``key`` is a string, ``data`` is an arbitrary -object):: - - import shelve - - d = shelve.open(filename) # open -- file may get suffix added by low-level - # library - - d[key] = data # store data at key (overwrites old data if - # using an existing key) - data = d[key] # retrieve a COPY of data at key (raise KeyError - # if no such key) - del d[key] # delete data stored at key (raises KeyError - # if no such key) - - flag = key in d # true if the key exists - klist = list(d.keys()) # a list of all existing keys (slow!) - - # as d was opened WITHOUT writeback=True, beware: - d['xx'] = [0, 1, 2] # this works as expected, but... - d['xx'].append(3) # *this doesn't!* -- d['xx'] is STILL [0, 1, 2]! - - # having opened d without writeback=True, you need to code carefully: - temp = d['xx'] # extracts the copy - temp.append(5) # mutates the copy - d['xx'] = temp # stores the copy right back, to persist it - - # or, d=shelve.open(filename,writeback=True) would let you just code - # d['xx'].append(5) and have it work as expected, BUT it would also - # consume more memory and make the d.close() operation slower. - - d.close() # close it - - -.. seealso:: - - Module :mod:`dbm` - Generic interface to ``dbm``-style databases. - - Module :mod:`pickle` - Object serialization used by :mod:`shelve`. - diff --git a/Doc/library/shlex.rst b/Doc/library/shlex.rst deleted file mode 100644 index aab6a543792096..00000000000000 --- a/Doc/library/shlex.rst +++ /dev/null @@ -1,463 +0,0 @@ -:mod:`shlex` --- Simple lexical analysis -======================================== - -.. module:: shlex - :synopsis: Simple lexical analysis for Unix shell-like languages. - -.. moduleauthor:: Eric S. Raymond -.. moduleauthor:: Gustavo Niemeyer -.. sectionauthor:: Eric S. Raymond -.. sectionauthor:: Gustavo Niemeyer - -**Source code:** :source:`Lib/shlex.py` - --------------- - -The :class:`~shlex.shlex` class makes it easy to write lexical analyzers for -simple syntaxes resembling that of the Unix shell. This will often be useful -for writing minilanguages, (for example, in run control files for Python -applications) or for parsing quoted strings. - -The :mod:`shlex` module defines the following functions: - - -.. function:: split(s, comments=False, posix=True) - - Split the string *s* using shell-like syntax. If *comments* is :const:`False` - (the default), the parsing of comments in the given string will be disabled - (setting the :attr:`~shlex.commenters` attribute of the - :class:`~shlex.shlex` instance to the empty string). This function operates - in POSIX mode by default, but uses non-POSIX mode if the *posix* argument is - false. - - .. note:: - - Since the :func:`split` function instantiates a :class:`~shlex.shlex` - instance, passing ``None`` for *s* will read the string to split from - standard input. - - .. deprecated:: 3.9 - Passing ``None`` for *s* will raise an exception in future Python - versions. - -.. function:: join(split_command) - - Concatenate the tokens of the list *split_command* and return a string. - This function is the inverse of :func:`split`. - - >>> from shlex import join - >>> print(join(['echo', '-n', 'Multiple words'])) - echo -n 'Multiple words' - - The returned value is shell-escaped to protect against injection - vulnerabilities (see :func:`quote`). - - .. versionadded:: 3.8 - - -.. function:: quote(s) - - Return a shell-escaped version of the string *s*. The returned value is a - string that can safely be used as one token in a shell command line, for - cases where you cannot use a list. - - .. _shlex-quote-warning: - - .. warning:: - - The ``shlex`` module is **only designed for Unix shells**. - - The :func:`quote` function is not guaranteed to be correct on non-POSIX - compliant shells or shells from other operating systems such as Windows. - Executing commands quoted by this module on such shells can open up the - possibility of a command injection vulnerability. - - Consider using functions that pass command arguments with lists such as - :func:`subprocess.run` with ``shell=False``. - - This idiom would be unsafe: - - >>> filename = 'somefile; rm -rf ~' - >>> command = 'ls -l {}'.format(filename) - >>> print(command) # executed by a shell: boom! - ls -l somefile; rm -rf ~ - - :func:`quote` lets you plug the security hole: - - >>> from shlex import quote - >>> command = 'ls -l {}'.format(quote(filename)) - >>> print(command) - ls -l 'somefile; rm -rf ~' - >>> remote_command = 'ssh home {}'.format(quote(command)) - >>> print(remote_command) - ssh home 'ls -l '"'"'somefile; rm -rf ~'"'"'' - - The quoting is compatible with UNIX shells and with :func:`split`: - - >>> from shlex import split - >>> remote_command = split(remote_command) - >>> remote_command - ['ssh', 'home', "ls -l 'somefile; rm -rf ~'"] - >>> command = split(remote_command[-1]) - >>> command - ['ls', '-l', 'somefile; rm -rf ~'] - - .. versionadded:: 3.3 - -The :mod:`shlex` module defines the following class: - - -.. class:: shlex(instream=None, infile=None, posix=False, punctuation_chars=False) - - A :class:`~shlex.shlex` instance or subclass instance is a lexical analyzer - object. The initialization argument, if present, specifies where to read - characters from. It must be a file-/stream-like object with - :meth:`~io.TextIOBase.read` and :meth:`~io.TextIOBase.readline` methods, or - a string. If no argument is given, input will be taken from ``sys.stdin``. - The second optional argument is a filename string, which sets the initial - value of the :attr:`~shlex.infile` attribute. If the *instream* - argument is omitted or equal to ``sys.stdin``, this second argument - defaults to "stdin". The *posix* argument defines the operational mode: - when *posix* is not true (default), the :class:`~shlex.shlex` instance will - operate in compatibility mode. When operating in POSIX mode, - :class:`~shlex.shlex` will try to be as close as possible to the POSIX shell - parsing rules. The *punctuation_chars* argument provides a way to make the - behaviour even closer to how real shells parse. This can take a number of - values: the default value, ``False``, preserves the behaviour seen under - Python 3.5 and earlier. If set to ``True``, then parsing of the characters - ``();<>|&`` is changed: any run of these characters (considered punctuation - characters) is returned as a single token. If set to a non-empty string of - characters, those characters will be used as the punctuation characters. Any - characters in the :attr:`wordchars` attribute that appear in - *punctuation_chars* will be removed from :attr:`wordchars`. See - :ref:`improved-shell-compatibility` for more information. *punctuation_chars* - can be set only upon :class:`~shlex.shlex` instance creation and can't be - modified later. - - .. versionchanged:: 3.6 - The *punctuation_chars* parameter was added. - -.. seealso:: - - Module :mod:`configparser` - Parser for configuration files similar to the Windows :file:`.ini` files. - - -.. _shlex-objects: - -shlex Objects -------------- - -A :class:`~shlex.shlex` instance has the following methods: - - -.. method:: shlex.get_token() - - Return a token. If tokens have been stacked using :meth:`push_token`, pop a - token off the stack. Otherwise, read one from the input stream. If reading - encounters an immediate end-of-file, :attr:`eof` is returned (the empty - string (``''``) in non-POSIX mode, and ``None`` in POSIX mode). - - -.. method:: shlex.push_token(str) - - Push the argument onto the token stack. - - -.. method:: shlex.read_token() - - Read a raw token. Ignore the pushback stack, and do not interpret source - requests. (This is not ordinarily a useful entry point, and is documented here - only for the sake of completeness.) - - -.. method:: shlex.sourcehook(filename) - - When :class:`~shlex.shlex` detects a source request (see :attr:`source` - below) this method is given the following token as argument, and expected - to return a tuple consisting of a filename and an open file-like object. - - Normally, this method first strips any quotes off the argument. If the result - is an absolute pathname, or there was no previous source request in effect, or - the previous source was a stream (such as ``sys.stdin``), the result is left - alone. Otherwise, if the result is a relative pathname, the directory part of - the name of the file immediately before it on the source inclusion stack is - prepended (this behavior is like the way the C preprocessor handles ``#include - "file.h"``). - - The result of the manipulations is treated as a filename, and returned as the - first component of the tuple, with :func:`open` called on it to yield the second - component. (Note: this is the reverse of the order of arguments in instance - initialization!) - - This hook is exposed so that you can use it to implement directory search paths, - addition of file extensions, and other namespace hacks. There is no - corresponding 'close' hook, but a shlex instance will call the - :meth:`~io.IOBase.close` method of the sourced input stream when it returns - EOF. - - For more explicit control of source stacking, use the :meth:`push_source` and - :meth:`pop_source` methods. - - -.. method:: shlex.push_source(newstream, newfile=None) - - Push an input source stream onto the input stack. If the filename argument is - specified it will later be available for use in error messages. This is the - same method used internally by the :meth:`sourcehook` method. - - -.. method:: shlex.pop_source() - - Pop the last-pushed input source from the input stack. This is the same method - used internally when the lexer reaches EOF on a stacked input stream. - - -.. method:: shlex.error_leader(infile=None, lineno=None) - - This method generates an error message leader in the format of a Unix C compiler - error label; the format is ``'"%s", line %d: '``, where the ``%s`` is replaced - with the name of the current source file and the ``%d`` with the current input - line number (the optional arguments can be used to override these). - - This convenience is provided to encourage :mod:`shlex` users to generate error - messages in the standard, parseable format understood by Emacs and other Unix - tools. - -Instances of :class:`~shlex.shlex` subclasses have some public instance -variables which either control lexical analysis or can be used for debugging: - - -.. attribute:: shlex.commenters - - The string of characters that are recognized as comment beginners. All - characters from the comment beginner to end of line are ignored. Includes just - ``'#'`` by default. - - -.. attribute:: shlex.wordchars - - The string of characters that will accumulate into multi-character tokens. By - default, includes all ASCII alphanumerics and underscore. In POSIX mode, the - accented characters in the Latin-1 set are also included. If - :attr:`punctuation_chars` is not empty, the characters ``~-./*?=``, which can - appear in filename specifications and command line parameters, will also be - included in this attribute, and any characters which appear in - ``punctuation_chars`` will be removed from ``wordchars`` if they are present - there. If :attr:`whitespace_split` is set to ``True``, this will have no - effect. - - -.. attribute:: shlex.whitespace - - Characters that will be considered whitespace and skipped. Whitespace bounds - tokens. By default, includes space, tab, linefeed and carriage-return. - - -.. attribute:: shlex.escape - - Characters that will be considered as escape. This will be only used in POSIX - mode, and includes just ``'\'`` by default. - - -.. attribute:: shlex.quotes - - Characters that will be considered string quotes. The token accumulates until - the same quote is encountered again (thus, different quote types protect each - other as in the shell.) By default, includes ASCII single and double quotes. - - -.. attribute:: shlex.escapedquotes - - Characters in :attr:`quotes` that will interpret escape characters defined in - :attr:`escape`. This is only used in POSIX mode, and includes just ``'"'`` by - default. - - -.. attribute:: shlex.whitespace_split - - If ``True``, tokens will only be split in whitespaces. This is useful, for - example, for parsing command lines with :class:`~shlex.shlex`, getting - tokens in a similar way to shell arguments. When used in combination with - :attr:`punctuation_chars`, tokens will be split on whitespace in addition to - those characters. - - .. versionchanged:: 3.8 - The :attr:`punctuation_chars` attribute was made compatible with the - :attr:`whitespace_split` attribute. - - -.. attribute:: shlex.infile - - The name of the current input file, as initially set at class instantiation time - or stacked by later source requests. It may be useful to examine this when - constructing error messages. - - -.. attribute:: shlex.instream - - The input stream from which this :class:`~shlex.shlex` instance is reading - characters. - - -.. attribute:: shlex.source - - This attribute is ``None`` by default. If you assign a string to it, that - string will be recognized as a lexical-level inclusion request similar to the - ``source`` keyword in various shells. That is, the immediately following token - will be opened as a filename and input will be taken from that stream until - EOF, at which point the :meth:`~io.IOBase.close` method of that stream will be - called and the input source will again become the original input stream. Source - requests may be stacked any number of levels deep. - - -.. attribute:: shlex.debug - - If this attribute is numeric and ``1`` or more, a :class:`~shlex.shlex` - instance will print verbose progress output on its behavior. If you need - to use this, you can read the module source code to learn the details. - - -.. attribute:: shlex.lineno - - Source line number (count of newlines seen so far plus one). - - -.. attribute:: shlex.token - - The token buffer. It may be useful to examine this when catching exceptions. - - -.. attribute:: shlex.eof - - Token used to determine end of file. This will be set to the empty string - (``''``), in non-POSIX mode, and to ``None`` in POSIX mode. - - -.. attribute:: shlex.punctuation_chars - - A read-only property. Characters that will be considered punctuation. Runs of - punctuation characters will be returned as a single token. However, note that no - semantic validity checking will be performed: for example, '>>>' could be - returned as a token, even though it may not be recognised as such by shells. - - .. versionadded:: 3.6 - - -.. _shlex-parsing-rules: - -Parsing Rules -------------- - -When operating in non-POSIX mode, :class:`~shlex.shlex` will try to obey to the -following rules. - -* Quote characters are not recognized within words (``Do"Not"Separate`` is - parsed as the single word ``Do"Not"Separate``); - -* Escape characters are not recognized; - -* Enclosing characters in quotes preserve the literal value of all characters - within the quotes; - -* Closing quotes separate words (``"Do"Separate`` is parsed as ``"Do"`` and - ``Separate``); - -* If :attr:`~shlex.whitespace_split` is ``False``, any character not - declared to be a word character, whitespace, or a quote will be returned as - a single-character token. If it is ``True``, :class:`~shlex.shlex` will only - split words in whitespaces; - -* EOF is signaled with an empty string (``''``); - -* It's not possible to parse empty strings, even if quoted. - -When operating in POSIX mode, :class:`~shlex.shlex` will try to obey to the -following parsing rules. - -* Quotes are stripped out, and do not separate words (``"Do"Not"Separate"`` is - parsed as the single word ``DoNotSeparate``); - -* Non-quoted escape characters (e.g. ``'\'``) preserve the literal value of the - next character that follows; - -* Enclosing characters in quotes which are not part of - :attr:`~shlex.escapedquotes` (e.g. ``"'"``) preserve the literal value - of all characters within the quotes; - -* Enclosing characters in quotes which are part of - :attr:`~shlex.escapedquotes` (e.g. ``'"'``) preserves the literal value - of all characters within the quotes, with the exception of the characters - mentioned in :attr:`~shlex.escape`. The escape characters retain its - special meaning only when followed by the quote in use, or the escape - character itself. Otherwise the escape character will be considered a - normal character. - -* EOF is signaled with a :const:`None` value; - -* Quoted empty strings (``''``) are allowed. - -.. _improved-shell-compatibility: - -Improved Compatibility with Shells ----------------------------------- - -.. versionadded:: 3.6 - -The :class:`shlex` class provides compatibility with the parsing performed by -common Unix shells like ``bash``, ``dash``, and ``sh``. To take advantage of -this compatibility, specify the ``punctuation_chars`` argument in the -constructor. This defaults to ``False``, which preserves pre-3.6 behaviour. -However, if it is set to ``True``, then parsing of the characters ``();<>|&`` -is changed: any run of these characters is returned as a single token. While -this is short of a full parser for shells (which would be out of scope for the -standard library, given the multiplicity of shells out there), it does allow -you to perform processing of command lines more easily than you could -otherwise. To illustrate, you can see the difference in the following snippet: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> import shlex - >>> text = "a && b; c && d || e; f >'abc'; (def \"ghi\")" - >>> s = shlex.shlex(text, posix=True) - >>> s.whitespace_split = True - >>> list(s) - ['a', '&&', 'b;', 'c', '&&', 'd', '||', 'e;', 'f', '>abc;', '(def', 'ghi)'] - >>> s = shlex.shlex(text, posix=True, punctuation_chars=True) - >>> s.whitespace_split = True - >>> list(s) - ['a', '&&', 'b', ';', 'c', '&&', 'd', '||', 'e', ';', 'f', '>', 'abc', ';', - '(', 'def', 'ghi', ')'] - -Of course, tokens will be returned which are not valid for shells, and you'll -need to implement your own error checks on the returned tokens. - -Instead of passing ``True`` as the value for the punctuation_chars parameter, -you can pass a string with specific characters, which will be used to determine -which characters constitute punctuation. For example:: - - >>> import shlex - >>> s = shlex.shlex("a && b || c", punctuation_chars="|") - >>> list(s) - ['a', '&', '&', 'b', '||', 'c'] - -.. note:: When ``punctuation_chars`` is specified, the :attr:`~shlex.wordchars` - attribute is augmented with the characters ``~-./*?=``. That is because these - characters can appear in file names (including wildcards) and command-line - arguments (e.g. ``--color=auto``). Hence:: - - >>> import shlex - >>> s = shlex.shlex('~/a && b-c --color=auto || d *.py?', - ... punctuation_chars=True) - >>> list(s) - ['~/a', '&&', 'b-c', '--color=auto', '||', 'd', '*.py?'] - - However, to match the shell as closely as possible, it is recommended to - always use ``posix`` and :attr:`~shlex.whitespace_split` when using - :attr:`~shlex.punctuation_chars`, which will negate - :attr:`~shlex.wordchars` entirely. - -For best effect, ``punctuation_chars`` should be set in conjunction with -``posix=True``. (Note that ``posix=False`` is the default for -:class:`~shlex.shlex`.) diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst deleted file mode 100644 index 4ba2856fce2654..00000000000000 --- a/Doc/library/shutil.rst +++ /dev/null @@ -1,825 +0,0 @@ -:mod:`shutil` --- High-level file operations -============================================ - -.. module:: shutil - :synopsis: High-level file operations, including copying. - -.. sectionauthor:: Fred L. Drake, Jr. -.. partly based on the docstrings - -**Source code:** :source:`Lib/shutil.py` - -.. index:: - single: file; copying - single: copying files - --------------- - -The :mod:`shutil` module offers a number of high-level operations on files and -collections of files. In particular, functions are provided which support file -copying and removal. For operations on individual files, see also the -:mod:`os` module. - -.. warning:: - - Even the higher-level file copying functions (:func:`shutil.copy`, - :func:`shutil.copy2`) cannot copy all file metadata. - - On POSIX platforms, this means that file owner and group are lost as well - as ACLs. On Mac OS, the resource fork and other metadata are not used. - This means that resources will be lost and file type and creator codes will - not be correct. On Windows, file owners, ACLs and alternate data streams - are not copied. - - -.. _file-operations: - -Directory and files operations ------------------------------- - -.. function:: copyfileobj(fsrc, fdst[, length]) - - Copy the contents of the file-like object *fsrc* to the file-like object *fdst*. - The integer *length*, if given, is the buffer size. In particular, a negative - *length* value means to copy the data without looping over the source data in - chunks; by default the data is read in chunks to avoid uncontrolled memory - consumption. Note that if the current file position of the *fsrc* object is not - 0, only the contents from the current file position to the end of the file will - be copied. - - -.. function:: copyfile(src, dst, *, follow_symlinks=True) - - Copy the contents (no metadata) of the file named *src* to a file named - *dst* and return *dst* in the most efficient way possible. - *src* and *dst* are path-like objects or path names given as strings. - - *dst* must be the complete target file name; look at :func:`~shutil.copy` - for a copy that accepts a target directory path. If *src* and *dst* - specify the same file, :exc:`SameFileError` is raised. - - The destination location must be writable; otherwise, an :exc:`OSError` - exception will be raised. If *dst* already exists, it will be replaced. - Special files such as character or block devices and pipes cannot be - copied with this function. - - If *follow_symlinks* is false and *src* is a symbolic link, - a new symbolic link will be created instead of copying the - file *src* points to. - - .. audit-event:: shutil.copyfile src,dst shutil.copyfile - - .. versionchanged:: 3.3 - :exc:`IOError` used to be raised instead of :exc:`OSError`. - Added *follow_symlinks* argument. - Now returns *dst*. - - .. versionchanged:: 3.4 - Raise :exc:`SameFileError` instead of :exc:`Error`. Since the former is - a subclass of the latter, this change is backward compatible. - - .. versionchanged:: 3.8 - Platform-specific fast-copy syscalls may be used internally in order to - copy the file more efficiently. See - :ref:`shutil-platform-dependent-efficient-copy-operations` section. - -.. exception:: SameFileError - - This exception is raised if source and destination in :func:`copyfile` - are the same file. - - .. versionadded:: 3.4 - - -.. function:: copymode(src, dst, *, follow_symlinks=True) - - Copy the permission bits from *src* to *dst*. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path names - given as strings. - If *follow_symlinks* is false, and both *src* and *dst* are symbolic links, - :func:`copymode` will attempt to modify the mode of *dst* itself (rather - than the file it points to). This functionality is not available on every - platform; please see :func:`copystat` for more information. If - :func:`copymode` cannot modify symbolic links on the local platform, and it - is asked to do so, it will do nothing and return. - - .. audit-event:: shutil.copymode src,dst shutil.copymode - - .. versionchanged:: 3.3 - Added *follow_symlinks* argument. - -.. function:: copystat(src, dst, *, follow_symlinks=True) - - Copy the permission bits, last access time, last modification time, and - flags from *src* to *dst*. On Linux, :func:`copystat` also copies the - "extended attributes" where possible. The file contents, owner, and - group are unaffected. *src* and *dst* are path-like objects or path - names given as strings. - - If *follow_symlinks* is false, and *src* and *dst* both - refer to symbolic links, :func:`copystat` will operate on - the symbolic links themselves rather than the files the - symbolic links refer to—reading the information from the - *src* symbolic link, and writing the information to the - *dst* symbolic link. - - .. note:: - - Not all platforms provide the ability to examine and - modify symbolic links. Python itself can tell you what - functionality is locally available. - - * If ``os.chmod in os.supports_follow_symlinks`` is - ``True``, :func:`copystat` can modify the permission - bits of a symbolic link. - - * If ``os.utime in os.supports_follow_symlinks`` is - ``True``, :func:`copystat` can modify the last access - and modification times of a symbolic link. - - * If ``os.chflags in os.supports_follow_symlinks`` is - ``True``, :func:`copystat` can modify the flags of - a symbolic link. (``os.chflags`` is not available on - all platforms.) - - On platforms where some or all of this functionality - is unavailable, when asked to modify a symbolic link, - :func:`copystat` will copy everything it can. - :func:`copystat` never returns failure. - - Please see :data:`os.supports_follow_symlinks` - for more information. - - .. audit-event:: shutil.copystat src,dst shutil.copystat - - .. versionchanged:: 3.3 - Added *follow_symlinks* argument and support for Linux extended attributes. - -.. function:: copy(src, dst, *, follow_symlinks=True) - - Copies the file *src* to the file or directory *dst*. *src* and *dst* - should be :term:`path-like objects ` or strings. If - *dst* specifies a directory, the file will be copied into *dst* using the - base filename from *src*. If *dst* specifies a file that already exists, - it will be replaced. Returns the path to the newly created file. - - If *follow_symlinks* is false, and *src* is a symbolic link, - *dst* will be created as a symbolic link. If *follow_symlinks* - is true and *src* is a symbolic link, *dst* will be a copy of - the file *src* refers to. - - :func:`~shutil.copy` copies the file data and the file's permission - mode (see :func:`os.chmod`). Other metadata, like the - file's creation and modification times, is not preserved. - To preserve all file metadata from the original, use - :func:`~shutil.copy2` instead. - - .. audit-event:: shutil.copyfile src,dst shutil.copy - - .. audit-event:: shutil.copymode src,dst shutil.copy - - .. versionchanged:: 3.3 - Added *follow_symlinks* argument. - Now returns path to the newly created file. - - .. versionchanged:: 3.8 - Platform-specific fast-copy syscalls may be used internally in order to - copy the file more efficiently. See - :ref:`shutil-platform-dependent-efficient-copy-operations` section. - -.. function:: copy2(src, dst, *, follow_symlinks=True) - - Identical to :func:`~shutil.copy` except that :func:`copy2` - also attempts to preserve file metadata. - - When *follow_symlinks* is false, and *src* is a symbolic - link, :func:`copy2` attempts to copy all metadata from the - *src* symbolic link to the newly created *dst* symbolic link. - However, this functionality is not available on all platforms. - On platforms where some or all of this functionality is - unavailable, :func:`copy2` will preserve all the metadata - it can; :func:`copy2` never raises an exception because it - cannot preserve file metadata. - - :func:`copy2` uses :func:`copystat` to copy the file metadata. - Please see :func:`copystat` for more information - about platform support for modifying symbolic link metadata. - - .. audit-event:: shutil.copyfile src,dst shutil.copy2 - - .. audit-event:: shutil.copystat src,dst shutil.copy2 - - .. versionchanged:: 3.3 - Added *follow_symlinks* argument, try to copy extended - file system attributes too (currently Linux only). - Now returns path to the newly created file. - - .. versionchanged:: 3.8 - Platform-specific fast-copy syscalls may be used internally in order to - copy the file more efficiently. See - :ref:`shutil-platform-dependent-efficient-copy-operations` section. - -.. function:: ignore_patterns(*patterns) - - This factory function creates a function that can be used as a callable for - :func:`copytree`\'s *ignore* argument, ignoring files and directories that - match one of the glob-style *patterns* provided. See the example below. - - -.. function:: copytree(src, dst, symlinks=False, ignore=None, \ - copy_function=copy2, ignore_dangling_symlinks=False, \ - dirs_exist_ok=False) - - Recursively copy an entire directory tree rooted at *src* to a directory - named *dst* and return the destination directory. All intermediate - directories needed to contain *dst* will also be created by default. - - Permissions and times of directories are copied with :func:`copystat`, - individual files are copied using :func:`~shutil.copy2`. - - If *symlinks* is true, symbolic links in the source tree are represented as - symbolic links in the new tree and the metadata of the original links will - be copied as far as the platform allows; if false or omitted, the contents - and metadata of the linked files are copied to the new tree. - - When *symlinks* is false, if the file pointed by the symlink doesn't - exist, an exception will be added in the list of errors raised in - an :exc:`Error` exception at the end of the copy process. - You can set the optional *ignore_dangling_symlinks* flag to true if you - want to silence this exception. Notice that this option has no effect - on platforms that don't support :func:`os.symlink`. - - If *ignore* is given, it must be a callable that will receive as its - arguments the directory being visited by :func:`copytree`, and a list of its - contents, as returned by :func:`os.listdir`. Since :func:`copytree` is - called recursively, the *ignore* callable will be called once for each - directory that is copied. The callable must return a sequence of directory - and file names relative to the current directory (i.e. a subset of the items - in its second argument); these names will then be ignored in the copy - process. :func:`ignore_patterns` can be used to create such a callable that - ignores names based on glob-style patterns. - - If exception(s) occur, an :exc:`Error` is raised with a list of reasons. - - If *copy_function* is given, it must be a callable that will be used to copy - each file. It will be called with the source path and the destination path - as arguments. By default, :func:`~shutil.copy2` is used, but any function - that supports the same signature (like :func:`~shutil.copy`) can be used. - - If *dirs_exist_ok* is false (the default) and *dst* already exists, a - :exc:`FileExistsError` is raised. If *dirs_exist_ok* is true, the copying - operation will continue if it encounters existing directories, and files - within the *dst* tree will be overwritten by corresponding files from the - *src* tree. - - .. audit-event:: shutil.copytree src,dst shutil.copytree - - .. versionchanged:: 3.3 - Copy metadata when *symlinks* is false. - Now returns *dst*. - - .. versionchanged:: 3.2 - Added the *copy_function* argument to be able to provide a custom copy - function. - Added the *ignore_dangling_symlinks* argument to silence dangling symlinks - errors when *symlinks* is false. - - .. versionchanged:: 3.8 - Platform-specific fast-copy syscalls may be used internally in order to - copy the file more efficiently. See - :ref:`shutil-platform-dependent-efficient-copy-operations` section. - - .. versionadded:: 3.8 - The *dirs_exist_ok* parameter. - -.. function:: rmtree(path, ignore_errors=False, onerror=None, *, dir_fd=None) - - .. index:: single: directory; deleting - - Delete an entire directory tree; *path* must point to a directory (but not a - symbolic link to a directory). If *ignore_errors* is true, errors resulting - from failed removals will be ignored; if false or omitted, such errors are - handled by calling a handler specified by *onerror* or, if that is omitted, - they raise an exception. - - This function can support :ref:`paths relative to directory descriptors - `. - - .. note:: - - On platforms that support the necessary fd-based functions a symlink - attack resistant version of :func:`rmtree` is used by default. On other - platforms, the :func:`rmtree` implementation is susceptible to a symlink - attack: given proper timing and circumstances, attackers can manipulate - symlinks on the filesystem to delete files they wouldn't be able to access - otherwise. Applications can use the :data:`rmtree.avoids_symlink_attacks` - function attribute to determine which case applies. - - If *onerror* is provided, it must be a callable that accepts three - parameters: *function*, *path*, and *excinfo*. - - The first parameter, *function*, is the function which raised the exception; - it depends on the platform and implementation. The second parameter, - *path*, will be the path name passed to *function*. The third parameter, - *excinfo*, will be the exception information returned by - :func:`sys.exc_info`. Exceptions raised by *onerror* will not be caught. - - .. audit-event:: shutil.rmtree path,dir_fd shutil.rmtree - - .. versionchanged:: 3.3 - Added a symlink attack resistant version that is used automatically - if platform supports fd-based functions. - - .. versionchanged:: 3.8 - On Windows, will no longer delete the contents of a directory junction - before removing the junction. - - .. versionchanged:: 3.11 - The *dir_fd* parameter. - - .. attribute:: rmtree.avoids_symlink_attacks - - Indicates whether the current platform and implementation provides a - symlink attack resistant version of :func:`rmtree`. Currently this is - only true for platforms supporting fd-based directory access functions. - - .. versionadded:: 3.3 - - -.. function:: move(src, dst, copy_function=copy2) - - Recursively move a file or directory (*src*) to another location (*dst*) - and return the destination. - - If the destination is an existing directory, then *src* is moved inside that - directory. If the destination already exists but is not a directory, it may - be overwritten depending on :func:`os.rename` semantics. - - If the destination is on the current filesystem, then :func:`os.rename` is - used. Otherwise, *src* is copied to *dst* using *copy_function* and then - removed. In case of symlinks, a new symlink pointing to the target of *src* - will be created in or as *dst* and *src* will be removed. - - If *copy_function* is given, it must be a callable that takes two arguments - *src* and *dst*, and will be used to copy *src* to *dst* if - :func:`os.rename` cannot be used. If the source is a directory, - :func:`copytree` is called, passing it the *copy_function*. The - default *copy_function* is :func:`copy2`. Using :func:`~shutil.copy` as the - *copy_function* allows the move to succeed when it is not possible to also - copy the metadata, at the expense of not copying any of the metadata. - - .. audit-event:: shutil.move src,dst shutil.move - - .. versionchanged:: 3.3 - Added explicit symlink handling for foreign filesystems, thus adapting - it to the behavior of GNU's :program:`mv`. - Now returns *dst*. - - .. versionchanged:: 3.5 - Added the *copy_function* keyword argument. - - .. versionchanged:: 3.8 - Platform-specific fast-copy syscalls may be used internally in order to - copy the file more efficiently. See - :ref:`shutil-platform-dependent-efficient-copy-operations` section. - - .. versionchanged:: 3.9 - Accepts a :term:`path-like object` for both *src* and *dst*. - -.. function:: disk_usage(path) - - Return disk usage statistics about the given path as a :term:`named tuple` - with the attributes *total*, *used* and *free*, which are the amount of - total, used and free space, in bytes. *path* may be a file or a - directory. - - .. note:: - - On Unix filesystems, *path* must point to a path within a **mounted** - filesystem partition. On those platforms, CPython doesn't attempt to - retrieve disk usage information from non-mounted filesystems. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.8 - On Windows, *path* can now be a file or directory. - - .. availability:: Unix, Windows. - -.. function:: chown(path, user=None, group=None) - - Change owner *user* and/or *group* of the given *path*. - - *user* can be a system user name or a uid; the same applies to *group*. At - least one argument is required. - - See also :func:`os.chown`, the underlying function. - - .. audit-event:: shutil.chown path,user,group shutil.chown - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: which(cmd, mode=os.F_OK | os.X_OK, path=None) - - Return the path to an executable which would be run if the given *cmd* was - called. If no *cmd* would be called, return ``None``. - - *mode* is a permission mask passed to :func:`os.access`, by default - determining if the file exists and executable. - - When no *path* is specified, the results of :func:`os.environ` are used, - returning either the "PATH" value or a fallback of :data:`os.defpath`. - - On Windows, the current directory is always prepended to the *path* whether - or not you use the default or provide your own, which is the behavior the - command shell uses when finding executables. Additionally, when finding the - *cmd* in the *path*, the ``PATHEXT`` environment variable is checked. For - example, if you call ``shutil.which("python")``, :func:`which` will search - ``PATHEXT`` to know that it should look for ``python.exe`` within the *path* - directories. For example, on Windows:: - - >>> shutil.which("python") - 'C:\\Python33\\python.EXE' - - .. versionadded:: 3.3 - - .. versionchanged:: 3.8 - The :class:`bytes` type is now accepted. If *cmd* type is - :class:`bytes`, the result type is also :class:`bytes`. - -.. exception:: Error - - This exception collects exceptions that are raised during a multi-file - operation. For :func:`copytree`, the exception argument is a list of 3-tuples - (*srcname*, *dstname*, *exception*). - -.. _shutil-platform-dependent-efficient-copy-operations: - -Platform-dependent efficient copy operations -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Starting from Python 3.8, all functions involving a file copy -(:func:`copyfile`, :func:`~shutil.copy`, :func:`copy2`, -:func:`copytree`, and :func:`move`) may use -platform-specific "fast-copy" syscalls in order to copy the file more -efficiently (see :issue:`33671`). -"fast-copy" means that the copying operation occurs within the kernel, avoiding -the use of userspace buffers in Python as in "``outfd.write(infd.read())``". - -On macOS `fcopyfile`_ is used to copy the file content (not metadata). - -On Linux :func:`os.sendfile` is used. - -On Windows :func:`shutil.copyfile` uses a bigger default buffer size (1 MiB -instead of 64 KiB) and a :func:`memoryview`-based variant of -:func:`shutil.copyfileobj` is used. - -If the fast-copy operation fails and no data was written in the destination -file then shutil will silently fallback on using less efficient -:func:`copyfileobj` function internally. - -.. versionchanged:: 3.8 - -.. _shutil-copytree-example: - -copytree example -~~~~~~~~~~~~~~~~ - -An example that uses the :func:`ignore_patterns` helper:: - - from shutil import copytree, ignore_patterns - - copytree(source, destination, ignore=ignore_patterns('*.pyc', 'tmp*')) - -This will copy everything except ``.pyc`` files and files or directories whose -name starts with ``tmp``. - -Another example that uses the *ignore* argument to add a logging call:: - - from shutil import copytree - import logging - - def _logpath(path, names): - logging.info('Working in %s', path) - return [] # nothing will be ignored - - copytree(source, destination, ignore=_logpath) - - -.. _shutil-rmtree-example: - -rmtree example -~~~~~~~~~~~~~~ - -This example shows how to remove a directory tree on Windows where some -of the files have their read-only bit set. It uses the onerror callback -to clear the readonly bit and reattempt the remove. Any subsequent failure -will propagate. :: - - import os, stat - import shutil - - def remove_readonly(func, path, _): - "Clear the readonly bit and reattempt the removal" - os.chmod(path, stat.S_IWRITE) - func(path) - - shutil.rmtree(directory, onerror=remove_readonly) - -.. _archiving-operations: - -Archiving operations --------------------- - -.. versionadded:: 3.2 - -.. versionchanged:: 3.5 - Added support for the *xztar* format. - - -High-level utilities to create and read compressed and archived files are also -provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. - -.. function:: make_archive(base_name, format, [root_dir, [base_dir, [verbose, [dry_run, [owner, [group, [logger]]]]]]]) - - Create an archive file (such as zip or tar) and return its name. - - *base_name* is the name of the file to create, including the path, minus - any format-specific extension. *format* is the archive format: one of - "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the - :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is - available), or "xztar" (if the :mod:`lzma` module is available). - - *root_dir* is a directory that will be the root directory of the - archive, all paths in the archive will be relative to it; for example, - we typically chdir into *root_dir* before creating the archive. - - *base_dir* is the directory where we start archiving from; - i.e. *base_dir* will be the common prefix of all files and - directories in the archive. *base_dir* must be given relative - to *root_dir*. See :ref:`shutil-archiving-example-with-basedir` for how to - use *base_dir* and *root_dir* together. - - *root_dir* and *base_dir* both default to the current directory. - - If *dry_run* is true, no archive is created, but the operations that would be - executed are logged to *logger*. - - *owner* and *group* are used when creating a tar archive. By default, - uses the current owner and group. - - *logger* must be an object compatible with :pep:`282`, usually an instance of - :class:`logging.Logger`. - - The *verbose* argument is unused and deprecated. - - .. audit-event:: shutil.make_archive base_name,format,root_dir,base_dir shutil.make_archive - - .. note:: - - This function is not thread-safe when custom archivers registered - with :func:`register_archive_format` are used. In this case it - temporarily changes the current working directory of the process - to perform archiving. - - .. versionchanged:: 3.8 - The modern pax (POSIX.1-2001) format is now used instead of - the legacy GNU format for archives created with ``format="tar"``. - - .. versionchanged:: 3.10.6 - This function is now made thread-safe during creation of standard - ``.zip`` and tar archives. - -.. function:: get_archive_formats() - - Return a list of supported formats for archiving. - Each element of the returned sequence is a tuple ``(name, description)``. - - By default :mod:`shutil` provides these formats: - - - *zip*: ZIP file (if the :mod:`zlib` module is available). - - *tar*: Uncompressed tar file. Uses POSIX.1-2001 pax format for new archives. - - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). - - You can register new formats or provide your own archiver for any existing - formats, by using :func:`register_archive_format`. - - -.. function:: register_archive_format(name, function, [extra_args, [description]]) - - Register an archiver for the format *name*. - - *function* is the callable that will be used to unpack archives. The callable - will receive the *base_name* of the file to create, followed by the - *base_dir* (which defaults to :data:`os.curdir`) to start archiving from. - Further arguments are passed as keyword arguments: *owner*, *group*, - *dry_run* and *logger* (as passed in :func:`make_archive`). - - If given, *extra_args* is a sequence of ``(name, value)`` pairs that will be - used as extra keywords arguments when the archiver callable is used. - - *description* is used by :func:`get_archive_formats` which returns the - list of archivers. Defaults to an empty string. - - -.. function:: unregister_archive_format(name) - - Remove the archive format *name* from the list of supported formats. - - -.. function:: unpack_archive(filename[, extract_dir[, format[, filter]]]) - - Unpack an archive. *filename* is the full path of the archive. - - *extract_dir* is the name of the target directory where the archive is - unpacked. If not provided, the current working directory is used. - - *format* is the archive format: one of "zip", "tar", "gztar", "bztar", or - "xztar". Or any other format registered with - :func:`register_unpack_format`. If not provided, :func:`unpack_archive` - will use the archive file name extension and see if an unpacker was - registered for that extension. In case none is found, - a :exc:`ValueError` is raised. - - The keyword-only *filter* argument, which was added in Python 3.11.4, - is passed to the underlying unpacking function. - For zip files, *filter* is not accepted. - For tar files, it is recommended to set it to ``'data'``, - unless using features specific to tar and UNIX-like filesystems. - (See :ref:`tarfile-extraction-filter` for details.) - The ``'data'`` filter will become the default for tar files - in Python 3.14. - - .. audit-event:: shutil.unpack_archive filename,extract_dir,format shutil.unpack_archive - - .. warning:: - - Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of the path specified in - the *extract_dir* argument, e.g. members that have absolute filenames - starting with "/" or filenames with two dots "..". - - .. versionchanged:: 3.7 - Accepts a :term:`path-like object` for *filename* and *extract_dir*. - - .. versionchanged:: 3.11.4 - Added the *filter* argument. - -.. function:: register_unpack_format(name, extensions, function[, extra_args[, description]]) - - Registers an unpack format. *name* is the name of the format and - *extensions* is a list of extensions corresponding to the format, like - ``.zip`` for Zip files. - - *function* is the callable that will be used to unpack archives. The - callable will receive: - - - the path of the archive, as a positional argument; - - the directory the archive must be extracted to, as a positional argument; - - possibly a *filter* keyword argument, if it was given to - :func:`unpack_archive`; - - additional keyword arguments, specified by *extra_args* as a sequence - of ``(name, value)`` tuples. - - *description* can be provided to describe the format, and will be returned - by the :func:`get_unpack_formats` function. - - -.. function:: unregister_unpack_format(name) - - Unregister an unpack format. *name* is the name of the format. - - -.. function:: get_unpack_formats() - - Return a list of all registered formats for unpacking. - Each element of the returned sequence is a tuple - ``(name, extensions, description)``. - - By default :mod:`shutil` provides these formats: - - - *zip*: ZIP file (unpacking compressed files works only if the corresponding - module is available). - - *tar*: uncompressed tar file. - - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). - - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). - - You can register new formats or provide your own unpacker for any existing - formats, by using :func:`register_unpack_format`. - - -.. _shutil-archiving-example: - -Archiving example -~~~~~~~~~~~~~~~~~ - -In this example, we create a gzip'ed tar-file archive containing all files -found in the :file:`.ssh` directory of the user:: - - >>> from shutil import make_archive - >>> import os - >>> archive_name = os.path.expanduser(os.path.join('~', 'myarchive')) - >>> root_dir = os.path.expanduser(os.path.join('~', '.ssh')) - >>> make_archive(archive_name, 'gztar', root_dir) - '/Users/tarek/myarchive.tar.gz' - -The resulting archive contains: - -.. code-block:: shell-session - - $ tar -tzvf /Users/tarek/myarchive.tar.gz - drwx------ tarek/staff 0 2010-02-01 16:23:40 ./ - -rw-r--r-- tarek/staff 609 2008-06-09 13:26:54 ./authorized_keys - -rwxr-xr-x tarek/staff 65 2008-06-09 13:26:54 ./config - -rwx------ tarek/staff 668 2008-06-09 13:26:54 ./id_dsa - -rwxr-xr-x tarek/staff 609 2008-06-09 13:26:54 ./id_dsa.pub - -rw------- tarek/staff 1675 2008-06-09 13:26:54 ./id_rsa - -rw-r--r-- tarek/staff 397 2008-06-09 13:26:54 ./id_rsa.pub - -rw-r--r-- tarek/staff 37192 2010-02-06 18:23:10 ./known_hosts - - -.. _shutil-archiving-example-with-basedir: - -Archiving example with *base_dir* -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In this example, similar to the `one above `_, -we show how to use :func:`make_archive`, but this time with the usage of -*base_dir*. We now have the following directory structure: - -.. code-block:: shell-session - - $ tree tmp - tmp - └── root - └── structure - ├── content - └── please_add.txt - └── do_not_add.txt - -In the final archive, :file:`please_add.txt` should be included, but -:file:`do_not_add.txt` should not. Therefore we use the following:: - - >>> from shutil import make_archive - >>> import os - >>> archive_name = os.path.expanduser(os.path.join('~', 'myarchive')) - >>> make_archive( - ... archive_name, - ... 'tar', - ... root_dir='tmp/root', - ... base_dir='structure/content', - ... ) - '/Users/tarek/my_archive.tar' - -Listing the files in the resulting archive gives us: - -.. code-block:: shell-session - - $ python -m tarfile -l /Users/tarek/myarchive.tar - structure/content/ - structure/content/please_add.txt - - -Querying the size of the output terminal ----------------------------------------- - -.. function:: get_terminal_size(fallback=(columns, lines)) - - Get the size of the terminal window. - - For each of the two dimensions, the environment variable, ``COLUMNS`` - and ``LINES`` respectively, is checked. If the variable is defined and - the value is a positive integer, it is used. - - When ``COLUMNS`` or ``LINES`` is not defined, which is the common case, - the terminal connected to :data:`sys.__stdout__` is queried - by invoking :func:`os.get_terminal_size`. - - If the terminal size cannot be successfully queried, either because - the system doesn't support querying, or because we are not - connected to a terminal, the value given in ``fallback`` parameter - is used. ``fallback`` defaults to ``(80, 24)`` which is the default - size used by many terminal emulators. - - The value returned is a named tuple of type :class:`os.terminal_size`. - - See also: The Single UNIX Specification, Version 2, - `Other Environment Variables`_. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.11 - The ``fallback`` values are also used if :func:`os.get_terminal_size` - returns zeroes. - -.. _`fcopyfile`: - http://www.manpagez.com/man/3/copyfile/ - -.. _`Other Environment Variables`: - https://pubs.opengroup.org/onlinepubs/7908799/xbd/envvar.html#tag_002_003 diff --git a/Doc/library/signal.rst b/Doc/library/signal.rst deleted file mode 100644 index 7ee5ece8859825..00000000000000 --- a/Doc/library/signal.rst +++ /dev/null @@ -1,805 +0,0 @@ -:mod:`signal` --- Set handlers for asynchronous events -====================================================== - -.. module:: signal - :synopsis: Set handlers for asynchronous events. - -**Source code:** :source:`Lib/signal.py` - --------------- - -This module provides mechanisms to use signal handlers in Python. - - -General rules -------------- - -The :func:`signal.signal` function allows defining custom handlers to be -executed when a signal is received. A small number of default handlers are -installed: :const:`SIGPIPE` is ignored (so write errors on pipes and sockets -can be reported as ordinary Python exceptions) and :const:`SIGINT` is -translated into a :exc:`KeyboardInterrupt` exception if the parent process -has not changed it. - -A handler for a particular signal, once set, remains installed until it is -explicitly reset (Python emulates the BSD style interface regardless of the -underlying implementation), with the exception of the handler for -:const:`SIGCHLD`, which follows the underlying implementation. - -On WebAssembly platforms ``wasm32-emscripten`` and ``wasm32-wasi``, signals -are emulated and therefore behave differently. Several functions and signals -are not available on these platforms. - -Execution of Python signal handlers -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A Python signal handler does not get executed inside the low-level (C) signal -handler. Instead, the low-level signal handler sets a flag which tells the -:term:`virtual machine` to execute the corresponding Python signal handler -at a later point(for example at the next :term:`bytecode` instruction). -This has consequences: - -* It makes little sense to catch synchronous errors like :const:`SIGFPE` or - :const:`SIGSEGV` that are caused by an invalid operation in C code. Python - will return from the signal handler to the C code, which is likely to raise - the same signal again, causing Python to apparently hang. From Python 3.3 - onwards, you can use the :mod:`faulthandler` module to report on synchronous - errors. - -* A long-running calculation implemented purely in C (such as regular - expression matching on a large body of text) may run uninterrupted for an - arbitrary amount of time, regardless of any signals received. The Python - signal handlers will be called when the calculation finishes. - -* If the handler raises an exception, it will be raised "out of thin air" in - the main thread. See the :ref:`note below ` for a - discussion. - -.. _signals-and-threads: - - -Signals and threads -^^^^^^^^^^^^^^^^^^^ - -Python signal handlers are always executed in the main Python thread of the main interpreter, -even if the signal was received in another thread. This means that signals -can't be used as a means of inter-thread communication. You can use -the synchronization primitives from the :mod:`threading` module instead. - -Besides, only the main thread of the main interpreter is allowed to set a new signal handler. - - -Module contents ---------------- - -.. versionchanged:: 3.5 - signal (SIG*), handler (:const:`SIG_DFL`, :const:`SIG_IGN`) and sigmask - (:const:`SIG_BLOCK`, :const:`SIG_UNBLOCK`, :const:`SIG_SETMASK`) - related constants listed below were turned into - :class:`enums ` (:class:`Signals`, :class:`Handlers` and :class:`Sigmasks` respectively). - :func:`getsignal`, :func:`pthread_sigmask`, :func:`sigpending` and - :func:`sigwait` functions return human-readable - :class:`enums ` as :class:`Signals` objects. - - -The signal module defines three enums: - -.. class:: Signals - - :class:`enum.IntEnum` collection of SIG* constants and the CTRL_* constants. - - .. versionadded:: 3.5 - -.. class:: Handlers - - :class:`enum.IntEnum` collection the constants :const:`SIG_DFL` and :const:`SIG_IGN`. - - .. versionadded:: 3.5 - -.. class:: Sigmasks - - :class:`enum.IntEnum` collection the constants :const:`SIG_BLOCK`, :const:`SIG_UNBLOCK` and :const:`SIG_SETMASK`. - - .. availability:: Unix. - - See the man page :manpage:`sigprocmask(2)` and - :manpage:`pthread_sigmask(3)` for further information. - - .. versionadded:: 3.5 - - -The variables defined in the :mod:`signal` module are: - - -.. data:: SIG_DFL - - This is one of two standard signal handling options; it will simply perform - the default function for the signal. For example, on most systems the - default action for :const:`SIGQUIT` is to dump core and exit, while the - default action for :const:`SIGCHLD` is to simply ignore it. - - -.. data:: SIG_IGN - - This is another standard signal handler, which will simply ignore the given - signal. - - -.. data:: SIGABRT - - Abort signal from :manpage:`abort(3)`. - -.. data:: SIGALRM - - Timer signal from :manpage:`alarm(2)`. - - .. availability:: Unix. - -.. data:: SIGBREAK - - Interrupt from keyboard (CTRL + BREAK). - - .. availability:: Windows. - -.. data:: SIGBUS - - Bus error (bad memory access). - - .. availability:: Unix. - -.. data:: SIGCHLD - - Child process stopped or terminated. - - .. availability:: Unix. - -.. data:: SIGCLD - - Alias to :data:`SIGCHLD`. - -.. data:: SIGCONT - - Continue the process if it is currently stopped - - .. availability:: Unix. - -.. data:: SIGFPE - - Floating-point exception. For example, division by zero. - - .. seealso:: - :exc:`ZeroDivisionError` is raised when the second argument of a division - or modulo operation is zero. - -.. data:: SIGHUP - - Hangup detected on controlling terminal or death of controlling process. - - .. availability:: Unix. - -.. data:: SIGILL - - Illegal instruction. - -.. data:: SIGINT - - Interrupt from keyboard (CTRL + C). - - Default action is to raise :exc:`KeyboardInterrupt`. - -.. data:: SIGKILL - - Kill signal. - - It cannot be caught, blocked, or ignored. - - .. availability:: Unix. - -.. data:: SIGPIPE - - Broken pipe: write to pipe with no readers. - - Default action is to ignore the signal. - - .. availability:: Unix. - -.. data:: SIGSEGV - - Segmentation fault: invalid memory reference. - -.. data:: SIGSTKFLT - - Stack fault on coprocessor. The Linux kernel does not raise this signal: it - can only be raised in user space. - - .. availability:: Linux. - - On architectures where the signal is available. See - the man page :manpage:`signal(7)` for further information. - - .. versionadded:: 3.11 - -.. data:: SIGTERM - - Termination signal. - -.. data:: SIGUSR1 - - User-defined signal 1. - - .. availability:: Unix. - -.. data:: SIGUSR2 - - User-defined signal 2. - - .. availability:: Unix. - -.. data:: SIGWINCH - - Window resize signal. - - .. availability:: Unix. - -.. data:: SIG* - - All the signal numbers are defined symbolically. For example, the hangup signal - is defined as :const:`signal.SIGHUP`; the variable names are identical to the - names used in C programs, as found in ````. The Unix man page for - ':c:func:`signal`' lists the existing signals (on some systems this is - :manpage:`signal(2)`, on others the list is in :manpage:`signal(7)`). Note that - not all systems define the same set of signal names; only those names defined by - the system are defined by this module. - - -.. data:: CTRL_C_EVENT - - The signal corresponding to the :kbd:`Ctrl+C` keystroke event. This signal can - only be used with :func:`os.kill`. - - .. availability:: Windows. - - .. versionadded:: 3.2 - - -.. data:: CTRL_BREAK_EVENT - - The signal corresponding to the :kbd:`Ctrl+Break` keystroke event. This signal can - only be used with :func:`os.kill`. - - .. availability:: Windows. - - .. versionadded:: 3.2 - - -.. data:: NSIG - - One more than the number of the highest signal number. - Use :func:`valid_signals` to get valid signal numbers. - - -.. data:: ITIMER_REAL - - Decrements interval timer in real time, and delivers :const:`SIGALRM` upon - expiration. - - -.. data:: ITIMER_VIRTUAL - - Decrements interval timer only when the process is executing, and delivers - SIGVTALRM upon expiration. - - -.. data:: ITIMER_PROF - - Decrements interval timer both when the process executes and when the - system is executing on behalf of the process. Coupled with ITIMER_VIRTUAL, - this timer is usually used to profile the time spent by the application - in user and kernel space. SIGPROF is delivered upon expiration. - - -.. data:: SIG_BLOCK - - A possible value for the *how* parameter to :func:`pthread_sigmask` - indicating that signals are to be blocked. - - .. versionadded:: 3.3 - -.. data:: SIG_UNBLOCK - - A possible value for the *how* parameter to :func:`pthread_sigmask` - indicating that signals are to be unblocked. - - .. versionadded:: 3.3 - -.. data:: SIG_SETMASK - - A possible value for the *how* parameter to :func:`pthread_sigmask` - indicating that the signal mask is to be replaced. - - .. versionadded:: 3.3 - - -The :mod:`signal` module defines one exception: - -.. exception:: ItimerError - - Raised to signal an error from the underlying :func:`setitimer` or - :func:`getitimer` implementation. Expect this error if an invalid - interval timer or a negative time is passed to :func:`setitimer`. - This error is a subtype of :exc:`OSError`. - - .. versionadded:: 3.3 - This error used to be a subtype of :exc:`IOError`, which is now an - alias of :exc:`OSError`. - - -The :mod:`signal` module defines the following functions: - - -.. function:: alarm(time) - - If *time* is non-zero, this function requests that a :const:`SIGALRM` signal be - sent to the process in *time* seconds. Any previously scheduled alarm is - canceled (only one alarm can be scheduled at any time). The returned value is - then the number of seconds before any previously set alarm was to have been - delivered. If *time* is zero, no alarm is scheduled, and any scheduled alarm is - canceled. If the return value is zero, no alarm is currently scheduled. - - .. availability:: Unix. - - See the man page :manpage:`alarm(2)` for further information. - - -.. function:: getsignal(signalnum) - - Return the current signal handler for the signal *signalnum*. The returned value - may be a callable Python object, or one of the special values - :const:`signal.SIG_IGN`, :const:`signal.SIG_DFL` or :const:`None`. Here, - :const:`signal.SIG_IGN` means that the signal was previously ignored, - :const:`signal.SIG_DFL` means that the default way of handling the signal was - previously in use, and ``None`` means that the previous signal handler was not - installed from Python. - - -.. function:: strsignal(signalnum) - - Returns the description of signal *signalnum*, such as "Interrupt" - for :const:`SIGINT`. Returns :const:`None` if *signalnum* has no - description. Raises :exc:`ValueError` if *signalnum* is invalid. - - .. versionadded:: 3.8 - - -.. function:: valid_signals() - - Return the set of valid signal numbers on this platform. This can be - less than ``range(1, NSIG)`` if some signals are reserved by the system - for internal use. - - .. versionadded:: 3.8 - - -.. function:: pause() - - Cause the process to sleep until a signal is received; the appropriate handler - will then be called. Returns nothing. - - .. availability:: Unix. - - See the man page :manpage:`signal(2)` for further information. - - See also :func:`sigwait`, :func:`sigwaitinfo`, :func:`sigtimedwait` and - :func:`sigpending`. - - -.. function:: raise_signal(signum) - - Sends a signal to the calling process. Returns nothing. - - .. versionadded:: 3.8 - - -.. function:: pidfd_send_signal(pidfd, sig, siginfo=None, flags=0) - - Send signal *sig* to the process referred to by file descriptor *pidfd*. - Python does not currently support the *siginfo* parameter; it must be - ``None``. The *flags* argument is provided for future extensions; no flag - values are currently defined. - - See the :manpage:`pidfd_send_signal(2)` man page for more information. - - .. availability:: Linux >= 5.1 - .. versionadded:: 3.9 - - -.. function:: pthread_kill(thread_id, signalnum) - - Send the signal *signalnum* to the thread *thread_id*, another thread in the - same process as the caller. The target thread can be executing any code - (Python or not). However, if the target thread is executing the Python - interpreter, the Python signal handlers will be :ref:`executed by the main - thread of the main interpreter `. Therefore, the only point of sending a - signal to a particular Python thread would be to force a running system call - to fail with :exc:`InterruptedError`. - - Use :func:`threading.get_ident()` or the :attr:`~threading.Thread.ident` - attribute of :class:`threading.Thread` objects to get a suitable value - for *thread_id*. - - If *signalnum* is 0, then no signal is sent, but error checking is still - performed; this can be used to check if the target thread is still running. - - .. audit-event:: signal.pthread_kill thread_id,signalnum signal.pthread_kill - - .. availability:: Unix. - - See the man page :manpage:`pthread_kill(3)` for further information. - - See also :func:`os.kill`. - - .. versionadded:: 3.3 - - -.. function:: pthread_sigmask(how, mask) - - Fetch and/or change the signal mask of the calling thread. The signal mask - is the set of signals whose delivery is currently blocked for the caller. - Return the old signal mask as a set of signals. - - The behavior of the call is dependent on the value of *how*, as follows. - - * :data:`SIG_BLOCK`: The set of blocked signals is the union of the current - set and the *mask* argument. - * :data:`SIG_UNBLOCK`: The signals in *mask* are removed from the current - set of blocked signals. It is permissible to attempt to unblock a - signal which is not blocked. - * :data:`SIG_SETMASK`: The set of blocked signals is set to the *mask* - argument. - - *mask* is a set of signal numbers (e.g. {:const:`signal.SIGINT`, - :const:`signal.SIGTERM`}). Use :func:`~signal.valid_signals` for a full - mask including all signals. - - For example, ``signal.pthread_sigmask(signal.SIG_BLOCK, [])`` reads the - signal mask of the calling thread. - - :data:`SIGKILL` and :data:`SIGSTOP` cannot be blocked. - - .. availability:: Unix. - - See the man page :manpage:`sigprocmask(2)` and - :manpage:`pthread_sigmask(3)` for further information. - - See also :func:`pause`, :func:`sigpending` and :func:`sigwait`. - - .. versionadded:: 3.3 - - -.. function:: setitimer(which, seconds, interval=0.0) - - Sets given interval timer (one of :const:`signal.ITIMER_REAL`, - :const:`signal.ITIMER_VIRTUAL` or :const:`signal.ITIMER_PROF`) specified - by *which* to fire after *seconds* (float is accepted, different from - :func:`alarm`) and after that every *interval* seconds (if *interval* - is non-zero). The interval timer specified by *which* can be cleared by - setting *seconds* to zero. - - When an interval timer fires, a signal is sent to the process. - The signal sent is dependent on the timer being used; - :const:`signal.ITIMER_REAL` will deliver :const:`SIGALRM`, - :const:`signal.ITIMER_VIRTUAL` sends :const:`SIGVTALRM`, - and :const:`signal.ITIMER_PROF` will deliver :const:`SIGPROF`. - - The old values are returned as a tuple: (delay, interval). - - Attempting to pass an invalid interval timer will cause an - :exc:`ItimerError`. - - .. availability:: Unix. - - -.. function:: getitimer(which) - - Returns current value of a given interval timer specified by *which*. - - .. availability:: Unix. - - -.. function:: set_wakeup_fd(fd, *, warn_on_full_buffer=True) - - Set the wakeup file descriptor to *fd*. When a signal is received, the - signal number is written as a single byte into the fd. This can be used by - a library to wakeup a poll or select call, allowing the signal to be fully - processed. - - The old wakeup fd is returned (or -1 if file descriptor wakeup was not - enabled). If *fd* is -1, file descriptor wakeup is disabled. - If not -1, *fd* must be non-blocking. It is up to the library to remove - any bytes from *fd* before calling poll or select again. - - When threads are enabled, this function can only be called - from :ref:`the main thread of the main interpreter `; - attempting to call it from other threads will cause a :exc:`ValueError` - exception to be raised. - - There are two common ways to use this function. In both approaches, - you use the fd to wake up when a signal arrives, but then they - differ in how they determine *which* signal or signals have - arrived. - - In the first approach, we read the data out of the fd's buffer, and - the byte values give you the signal numbers. This is simple, but in - rare cases it can run into a problem: generally the fd will have a - limited amount of buffer space, and if too many signals arrive too - quickly, then the buffer may become full, and some signals may be - lost. If you use this approach, then you should set - ``warn_on_full_buffer=True``, which will at least cause a warning - to be printed to stderr when signals are lost. - - In the second approach, we use the wakeup fd *only* for wakeups, - and ignore the actual byte values. In this case, all we care about - is whether the fd's buffer is empty or non-empty; a full buffer - doesn't indicate a problem at all. If you use this approach, then - you should set ``warn_on_full_buffer=False``, so that your users - are not confused by spurious warning messages. - - .. versionchanged:: 3.5 - On Windows, the function now also supports socket handles. - - .. versionchanged:: 3.7 - Added ``warn_on_full_buffer`` parameter. - -.. function:: siginterrupt(signalnum, flag) - - Change system call restart behaviour: if *flag* is :const:`False`, system - calls will be restarted when interrupted by signal *signalnum*, otherwise - system calls will be interrupted. Returns nothing. - - .. availability:: Unix. - - See the man page :manpage:`siginterrupt(3)` for further information. - - Note that installing a signal handler with :func:`signal` will reset the - restart behaviour to interruptible by implicitly calling - :c:func:`!siginterrupt` with a true *flag* value for the given signal. - - -.. function:: signal(signalnum, handler) - - Set the handler for signal *signalnum* to the function *handler*. *handler* can - be a callable Python object taking two arguments (see below), or one of the - special values :const:`signal.SIG_IGN` or :const:`signal.SIG_DFL`. The previous - signal handler will be returned (see the description of :func:`getsignal` - above). (See the Unix man page :manpage:`signal(2)` for further information.) - - When threads are enabled, this function can only be called - from :ref:`the main thread of the main interpreter `; - attempting to call it from other threads will cause a :exc:`ValueError` - exception to be raised. - - The *handler* is called with two arguments: the signal number and the current - stack frame (``None`` or a frame object; for a description of frame objects, - see the :ref:`description in the type hierarchy ` or see the - attribute descriptions in the :mod:`inspect` module). - - On Windows, :func:`signal` can only be called with :const:`SIGABRT`, - :const:`SIGFPE`, :const:`SIGILL`, :const:`SIGINT`, :const:`SIGSEGV`, - :const:`SIGTERM`, or :const:`SIGBREAK`. - A :exc:`ValueError` will be raised in any other case. - Note that not all systems define the same set of signal names; an - :exc:`AttributeError` will be raised if a signal name is not defined as - ``SIG*`` module level constant. - - -.. function:: sigpending() - - Examine the set of signals that are pending for delivery to the calling - thread (i.e., the signals which have been raised while blocked). Return the - set of the pending signals. - - .. availability:: Unix. - - See the man page :manpage:`sigpending(2)` for further information. - - See also :func:`pause`, :func:`pthread_sigmask` and :func:`sigwait`. - - .. versionadded:: 3.3 - - -.. function:: sigwait(sigset) - - Suspend execution of the calling thread until the delivery of one of the - signals specified in the signal set *sigset*. The function accepts the signal - (removes it from the pending list of signals), and returns the signal number. - - .. availability:: Unix. - - See the man page :manpage:`sigwait(3)` for further information. - - See also :func:`pause`, :func:`pthread_sigmask`, :func:`sigpending`, - :func:`sigwaitinfo` and :func:`sigtimedwait`. - - .. versionadded:: 3.3 - - -.. function:: sigwaitinfo(sigset) - - Suspend execution of the calling thread until the delivery of one of the - signals specified in the signal set *sigset*. The function accepts the - signal and removes it from the pending list of signals. If one of the - signals in *sigset* is already pending for the calling thread, the function - will return immediately with information about that signal. The signal - handler is not called for the delivered signal. The function raises an - :exc:`InterruptedError` if it is interrupted by a signal that is not in - *sigset*. - - The return value is an object representing the data contained in the - :c:type:`siginfo_t` structure, namely: :attr:`si_signo`, :attr:`si_code`, - :attr:`si_errno`, :attr:`si_pid`, :attr:`si_uid`, :attr:`si_status`, - :attr:`si_band`. - - .. availability:: Unix. - - See the man page :manpage:`sigwaitinfo(2)` for further information. - - See also :func:`pause`, :func:`sigwait` and :func:`sigtimedwait`. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - The function is now retried if interrupted by a signal not in *sigset* - and the signal handler does not raise an exception (see :pep:`475` for - the rationale). - - -.. function:: sigtimedwait(sigset, timeout) - - Like :func:`sigwaitinfo`, but takes an additional *timeout* argument - specifying a timeout. If *timeout* is specified as ``0``, a poll is - performed. Returns :const:`None` if a timeout occurs. - - .. availability:: Unix. - - See the man page :manpage:`sigtimedwait(2)` for further information. - - See also :func:`pause`, :func:`sigwait` and :func:`sigwaitinfo`. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - The function is now retried with the recomputed *timeout* if interrupted - by a signal not in *sigset* and the signal handler does not raise an - exception (see :pep:`475` for the rationale). - - -.. _signal-example: - -Examples --------- - -Here is a minimal example program. It uses the :func:`alarm` function to limit -the time spent waiting to open a file; this is useful if the file is for a -serial device that may not be turned on, which would normally cause the -:func:`os.open` to hang indefinitely. The solution is to set a 5-second alarm -before opening the file; if the operation takes too long, the alarm signal will -be sent, and the handler raises an exception. :: - - import signal, os - - def handler(signum, frame): - signame = signal.Signals(signum).name - print(f'Signal handler called with signal {signame} ({signum})') - raise OSError("Couldn't open device!") - - # Set the signal handler and a 5-second alarm - signal.signal(signal.SIGALRM, handler) - signal.alarm(5) - - # This open() may hang indefinitely - fd = os.open('/dev/ttyS0', os.O_RDWR) - - signal.alarm(0) # Disable the alarm - -Note on SIGPIPE ---------------- - -Piping output of your program to tools like :manpage:`head(1)` will -cause a :const:`SIGPIPE` signal to be sent to your process when the receiver -of its standard output closes early. This results in an exception -like :code:`BrokenPipeError: [Errno 32] Broken pipe`. To handle this -case, wrap your entry point to catch this exception as follows:: - - import os - import sys - - def main(): - try: - # simulate large output (your code replaces this loop) - for x in range(10000): - print("y") - # flush output here to force SIGPIPE to be triggered - # while inside this try block. - sys.stdout.flush() - except BrokenPipeError: - # Python flushes standard streams on exit; redirect remaining output - # to devnull to avoid another BrokenPipeError at shutdown - devnull = os.open(os.devnull, os.O_WRONLY) - os.dup2(devnull, sys.stdout.fileno()) - sys.exit(1) # Python exits with error code 1 on EPIPE - - if __name__ == '__main__': - main() - -Do not set :const:`SIGPIPE`'s disposition to :const:`SIG_DFL` in -order to avoid :exc:`BrokenPipeError`. Doing that would cause -your program to exit unexpectedly whenever any socket -connection is interrupted while your program is still writing to -it. - -.. _handlers-and-exceptions: - -Note on Signal Handlers and Exceptions --------------------------------------- - -If a signal handler raises an exception, the exception will be propagated to -the main thread and may be raised after any :term:`bytecode` instruction. Most -notably, a :exc:`KeyboardInterrupt` may appear at any point during execution. -Most Python code, including the standard library, cannot be made robust against -this, and so a :exc:`KeyboardInterrupt` (or any other exception resulting from -a signal handler) may on rare occasions put the program in an unexpected state. - -To illustrate this issue, consider the following code:: - - class SpamContext: - def __init__(self): - self.lock = threading.Lock() - - def __enter__(self): - # If KeyboardInterrupt occurs here, everything is fine - self.lock.acquire() - # If KeyboardInterrupt occurs here, __exit__ will not be called - ... - # KeyboardInterrupt could occur just before the function returns - - def __exit__(self, exc_type, exc_val, exc_tb): - ... - self.lock.release() - -For many programs, especially those that merely want to exit on -:exc:`KeyboardInterrupt`, this is not a problem, but applications that are -complex or require high reliability should avoid raising exceptions from signal -handlers. They should also avoid catching :exc:`KeyboardInterrupt` as a means -of gracefully shutting down. Instead, they should install their own -:const:`SIGINT` handler. Below is an example of an HTTP server that avoids -:exc:`KeyboardInterrupt`:: - - import signal - import socket - from selectors import DefaultSelector, EVENT_READ - from http.server import HTTPServer, SimpleHTTPRequestHandler - - interrupt_read, interrupt_write = socket.socketpair() - - def handler(signum, frame): - print('Signal handler called with signal', signum) - interrupt_write.send(b'\0') - signal.signal(signal.SIGINT, handler) - - def serve_forever(httpd): - sel = DefaultSelector() - sel.register(interrupt_read, EVENT_READ) - sel.register(httpd, EVENT_READ) - - while True: - for key, _ in sel.select(): - if key.fileobj == interrupt_read: - interrupt_read.recv(1) - return - if key.fileobj == httpd: - httpd.handle_request() - - print("Serving on port 8000") - httpd = HTTPServer(('', 8000), SimpleHTTPRequestHandler) - serve_forever(httpd) - print("Shutdown...") diff --git a/Doc/library/site.rst b/Doc/library/site.rst deleted file mode 100644 index ab4fa765bf63d5..00000000000000 --- a/Doc/library/site.rst +++ /dev/null @@ -1,289 +0,0 @@ -:mod:`site` --- Site-specific configuration hook -================================================ - -.. module:: site - :synopsis: Module responsible for site-specific configuration. - -**Source code:** :source:`Lib/site.py` - --------------- - -.. highlight:: none - -**This module is automatically imported during initialization.** The automatic -import can be suppressed using the interpreter's :option:`-S` option. - -.. index:: triple: module; search; path - -Importing this module will append site-specific paths to the module search path -and add a few builtins, unless :option:`-S` was used. In that case, this module -can be safely imported with no automatic modifications to the module search path -or additions to the builtins. To explicitly trigger the usual site-specific -additions, call the :func:`main` function. - -.. versionchanged:: 3.3 - Importing the module used to trigger paths manipulation even when using - :option:`-S`. - -.. index:: - pair: site-packages; directory - -It starts by constructing up to four directories from a head and a tail part. -For the head part, it uses ``sys.prefix`` and ``sys.exec_prefix``; empty heads -are skipped. For the tail part, it uses the empty string and then -:file:`lib/site-packages` (on Windows) or -:file:`lib/python{X.Y}/site-packages` (on Unix and macOS). For each -of the distinct head-tail combinations, it sees if it refers to an existing -directory, and if so, adds it to ``sys.path`` and also inspects the newly -added path for configuration files. - -.. versionchanged:: 3.5 - Support for the "site-python" directory has been removed. - -If a file named "pyvenv.cfg" exists one directory above sys.executable, -sys.prefix and sys.exec_prefix are set to that directory and -it is also checked for site-packages (sys.base_prefix and -sys.base_exec_prefix will always be the "real" prefixes of the Python -installation). If "pyvenv.cfg" (a bootstrap configuration file) contains -the key "include-system-site-packages" set to anything other than "true" -(case-insensitive), the system-level prefixes will not be -searched for site-packages; otherwise they will. - -.. index:: - single: # (hash); comment - pair: statement; import - -A path configuration file is a file whose name has the form :file:`{name}.pth` -and exists in one of the four directories mentioned above; its contents are -additional items (one per line) to be added to ``sys.path``. Non-existing items -are never added to ``sys.path``, and no check is made that the item refers to a -directory rather than a file. No item is added to ``sys.path`` more than -once. Blank lines and lines beginning with ``#`` are skipped. Lines starting -with ``import`` (followed by space or tab) are executed. - -.. note:: - - An executable line in a :file:`.pth` file is run at every Python startup, - regardless of whether a particular module is actually going to be used. - Its impact should thus be kept to a minimum. - The primary intended purpose of executable lines is to make the - corresponding module(s) importable - (load 3rd-party import hooks, adjust :envvar:`PATH` etc). - Any other initialization is supposed to be done upon a module's - actual import, if and when it happens. - Limiting a code chunk to a single line is a deliberate measure - to discourage putting anything more complex here. - -.. index:: - single: package - triple: path; configuration; file - -For example, suppose ``sys.prefix`` and ``sys.exec_prefix`` are set to -:file:`/usr/local`. The Python X.Y library is then installed in -:file:`/usr/local/lib/python{X.Y}`. Suppose this has -a subdirectory :file:`/usr/local/lib/python{X.Y}/site-packages` with three -subsubdirectories, :file:`foo`, :file:`bar` and :file:`spam`, and two path -configuration files, :file:`foo.pth` and :file:`bar.pth`. Assume -:file:`foo.pth` contains the following:: - - # foo package configuration - - foo - bar - bletch - -and :file:`bar.pth` contains:: - - # bar package configuration - - bar - -Then the following version-specific directories are added to -``sys.path``, in this order:: - - /usr/local/lib/pythonX.Y/site-packages/bar - /usr/local/lib/pythonX.Y/site-packages/foo - -Note that :file:`bletch` is omitted because it doesn't exist; the :file:`bar` -directory precedes the :file:`foo` directory because :file:`bar.pth` comes -alphabetically before :file:`foo.pth`; and :file:`spam` is omitted because it is -not mentioned in either path configuration file. - -:mod:`sitecustomize` --------------------- - -.. module:: sitecustomize - -After these path manipulations, an attempt is made to import a module named -:mod:`sitecustomize`, which can perform arbitrary site-specific customizations. -It is typically created by a system administrator in the site-packages -directory. If this import fails with an :exc:`ImportError` or its subclass -exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'sitecustomize'``, -it is silently ignored. If Python is started without output streams available, as -with :file:`pythonw.exe` on Windows (which is used by default to start IDLE), -attempted output from :mod:`sitecustomize` is ignored. Any other exception -causes a silent and perhaps mysterious failure of the process. - -:mod:`usercustomize` --------------------- - -.. module:: usercustomize - -After this, an attempt is made to import a module named :mod:`usercustomize`, -which can perform arbitrary user-specific customizations, if -:data:`~site.ENABLE_USER_SITE` is true. This file is intended to be created in the -user site-packages directory (see below), which is part of ``sys.path`` unless -disabled by :option:`-s`. If this import fails with an :exc:`ImportError` or -its subclass exception, and the exception's :attr:`~ImportError.name` -attribute equals to ``'usercustomize'``, it is silently ignored. - -Note that for some non-Unix systems, ``sys.prefix`` and ``sys.exec_prefix`` are -empty, and the path manipulations are skipped; however the import of -:mod:`sitecustomize` and :mod:`usercustomize` is still attempted. - -.. currentmodule:: site - -.. _rlcompleter-config: - -Readline configuration ----------------------- - -On systems that support :mod:`readline`, this module will also import and -configure the :mod:`rlcompleter` module, if Python is started in -:ref:`interactive mode ` and without the :option:`-S` option. -The default behavior is enable tab-completion and to use -:file:`~/.python_history` as the history save file. To disable it, delete (or -override) the :data:`sys.__interactivehook__` attribute in your -:mod:`sitecustomize` or :mod:`usercustomize` module or your -:envvar:`PYTHONSTARTUP` file. - -.. versionchanged:: 3.4 - Activation of rlcompleter and history was made automatic. - - -Module contents ---------------- - -.. data:: PREFIXES - - A list of prefixes for site-packages directories. - - -.. data:: ENABLE_USER_SITE - - Flag showing the status of the user site-packages directory. ``True`` means - that it is enabled and was added to ``sys.path``. ``False`` means that it - was disabled by user request (with :option:`-s` or - :envvar:`PYTHONNOUSERSITE`). ``None`` means it was disabled for security - reasons (mismatch between user or group id and effective id) or by an - administrator. - - -.. data:: USER_SITE - - Path to the user site-packages for the running Python. Can be ``None`` if - :func:`getusersitepackages` hasn't been called yet. Default value is - :file:`~/.local/lib/python{X.Y}/site-packages` for UNIX and non-framework - macOS builds, :file:`~/Library/Python/{X.Y}/lib/python/site-packages` for macOS - framework builds, and :file:`{%APPDATA%}\\Python\\Python{XY}\\site-packages` - on Windows. This directory is a site directory, which means that - :file:`.pth` files in it will be processed. - - -.. data:: USER_BASE - - Path to the base directory for the user site-packages. Can be ``None`` if - :func:`getuserbase` hasn't been called yet. Default value is - :file:`~/.local` for UNIX and macOS non-framework builds, - :file:`~/Library/Python/{X.Y}` for macOS framework builds, and - :file:`{%APPDATA%}\\Python` for Windows. This value is used by Distutils to - compute the installation directories for scripts, data files, Python modules, - etc. for the :ref:`user installation scheme `. - See also :envvar:`PYTHONUSERBASE`. - - -.. function:: main() - - Adds all the standard site-specific directories to the module search - path. This function is called automatically when this module is imported, - unless the Python interpreter was started with the :option:`-S` flag. - - .. versionchanged:: 3.3 - This function used to be called unconditionally. - - -.. function:: addsitedir(sitedir, known_paths=None) - - Add a directory to sys.path and process its :file:`.pth` files. Typically - used in :mod:`sitecustomize` or :mod:`usercustomize` (see above). - - -.. function:: getsitepackages() - - Return a list containing all global site-packages directories. - - .. versionadded:: 3.2 - - -.. function:: getuserbase() - - Return the path of the user base directory, :data:`USER_BASE`. If it is not - initialized yet, this function will also set it, respecting - :envvar:`PYTHONUSERBASE`. - - .. versionadded:: 3.2 - - -.. function:: getusersitepackages() - - Return the path of the user-specific site-packages directory, - :data:`USER_SITE`. If it is not initialized yet, this function will also set - it, respecting :data:`USER_BASE`. To determine if the user-specific - site-packages was added to ``sys.path`` :data:`ENABLE_USER_SITE` should be - used. - - .. versionadded:: 3.2 - - -.. _site-commandline: - -Command Line Interface ----------------------- - -.. program:: site - -The :mod:`site` module also provides a way to get the user directories from the -command line: - -.. code-block:: shell-session - - $ python3 -m site --user-site - /home/user/.local/lib/python3.3/site-packages - -If it is called without arguments, it will print the contents of -:data:`sys.path` on the standard output, followed by the value of -:data:`USER_BASE` and whether the directory exists, then the same thing for -:data:`USER_SITE`, and finally the value of :data:`ENABLE_USER_SITE`. - -.. option:: --user-base - - Print the path to the user base directory. - -.. option:: --user-site - - Print the path to the user site-packages directory. - -If both options are given, user base and user site will be printed (always in -this order), separated by :data:`os.pathsep`. - -If any option is given, the script will exit with one of these values: ``0`` if -the user site-packages directory is enabled, ``1`` if it was disabled by the -user, ``2`` if it is disabled for security reasons or by an administrator, and a -value greater than 2 if there is an error. - -.. seealso:: - - * :pep:`370` -- Per user site-packages directory - * :ref:`sys-path-init` -- The initialization of :data:`sys.path`. - diff --git a/Doc/library/smtpd.rst b/Doc/library/smtpd.rst deleted file mode 100644 index ac0c9aeb236091..00000000000000 --- a/Doc/library/smtpd.rst +++ /dev/null @@ -1,268 +0,0 @@ -:mod:`smtpd` --- SMTP Server -============================ - -.. module:: smtpd - :synopsis: A SMTP server implementation in Python. - :deprecated: - -.. moduleauthor:: Barry Warsaw -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/smtpd.py` - --------------- - -This module offers several classes to implement SMTP (email) servers. - -.. deprecated-removed:: 3.6 3.12 - The :mod:`smtpd` module is deprecated - (see :pep:`PEP 594 <594#smtpd>` for details). - The `aiosmtpd `_ package is a recommended - replacement for this module. It is based on :mod:`asyncio` and provides a - more straightforward API. - -Several server implementations are present; one is a generic -do-nothing implementation, which can be overridden, while the other two offer -specific mail-sending strategies. - -Additionally the SMTPChannel may be extended to implement very specific -interaction behaviour with SMTP clients. - -The code supports :RFC:`5321`, plus the :rfc:`1870` SIZE and :rfc:`6531` -SMTPUTF8 extensions. - -.. include:: ../includes/wasm-notavail.rst - -SMTPServer Objects ------------------- - - -.. class:: SMTPServer(localaddr, remoteaddr, data_size_limit=33554432,\ - map=None, enable_SMTPUTF8=False, decode_data=False) - - Create a new :class:`SMTPServer` object, which binds to local address - *localaddr*. It will treat *remoteaddr* as an upstream SMTP relayer. Both - *localaddr* and *remoteaddr* should be a :ref:`(host, port) ` - tuple. The object inherits from :class:`asyncore.dispatcher`, and so will - insert itself into :mod:`asyncore`'s event loop on instantiation. - - *data_size_limit* specifies the maximum number of bytes that will be - accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no - limit. - - *map* is the socket map to use for connections (an initially empty - dictionary is a suitable value). If not specified the :mod:`asyncore` - global socket map is used. - - *enable_SMTPUTF8* determines whether the ``SMTPUTF8`` extension (as defined - in :RFC:`6531`) should be enabled. The default is ``False``. - When ``True``, ``SMTPUTF8`` is accepted as a parameter to the ``MAIL`` - command and when present is passed to :meth:`process_message` in the - ``kwargs['mail_options']`` list. *decode_data* and *enable_SMTPUTF8* - cannot be set to ``True`` at the same time. - - *decode_data* specifies whether the data portion of the SMTP transaction - should be decoded using UTF-8. When *decode_data* is ``False`` (the - default), the server advertises the ``8BITMIME`` - extension (:rfc:`6152`), accepts the ``BODY=8BITMIME`` parameter to - the ``MAIL`` command, and when present passes it to :meth:`process_message` - in the ``kwargs['mail_options']`` list. *decode_data* and *enable_SMTPUTF8* - cannot be set to ``True`` at the same time. - - .. method:: process_message(peer, mailfrom, rcpttos, data, **kwargs) - - Raise a :exc:`NotImplementedError` exception. Override this in subclasses to - do something useful with this message. Whatever was passed in the - constructor as *remoteaddr* will be available as the :attr:`_remoteaddr` - attribute. *peer* is the remote host's address, *mailfrom* is the envelope - originator, *rcpttos* are the envelope recipients and *data* is a string - containing the contents of the e-mail (which should be in :rfc:`5321` - format). - - If the *decode_data* constructor keyword is set to ``True``, the *data* - argument will be a unicode string. If it is set to ``False``, it - will be a bytes object. - - *kwargs* is a dictionary containing additional information. It is empty - if ``decode_data=True`` was given as an init argument, otherwise - it contains the following keys: - - *mail_options*: - a list of all received parameters to the ``MAIL`` - command (the elements are uppercase strings; example: - ``['BODY=8BITMIME', 'SMTPUTF8']``). - - *rcpt_options*: - same as *mail_options* but for the ``RCPT`` command. - Currently no ``RCPT TO`` options are supported, so for now - this will always be an empty list. - - Implementations of ``process_message`` should use the ``**kwargs`` - signature to accept arbitrary keyword arguments, since future feature - enhancements may add keys to the kwargs dictionary. - - Return ``None`` to request a normal ``250 Ok`` response; otherwise - return the desired response string in :RFC:`5321` format. - - .. attribute:: channel_class - - Override this in subclasses to use a custom :class:`SMTPChannel` for - managing SMTP clients. - - .. versionadded:: 3.4 - The *map* constructor argument. - - .. versionchanged:: 3.5 - *localaddr* and *remoteaddr* may now contain IPv6 addresses. - - .. versionadded:: 3.5 - The *decode_data* and *enable_SMTPUTF8* constructor parameters, and the - *kwargs* parameter to :meth:`process_message` when *decode_data* is - ``False``. - - .. versionchanged:: 3.6 - *decode_data* is now ``False`` by default. - - -DebuggingServer Objects ------------------------ - - -.. class:: DebuggingServer(localaddr, remoteaddr) - - Create a new debugging server. Arguments are as per :class:`SMTPServer`. - Messages will be discarded, and printed on stdout. - - -PureProxy Objects ------------------ - - -.. class:: PureProxy(localaddr, remoteaddr) - - Create a new pure proxy server. Arguments are as per :class:`SMTPServer`. - Everything will be relayed to *remoteaddr*. Note that running this has a good - chance to make you into an open relay, so please be careful. - - -SMTPChannel Objects -------------------- - -.. class:: SMTPChannel(server, conn, addr, data_size_limit=33554432,\ - map=None, enable_SMTPUTF8=False, decode_data=False) - - Create a new :class:`SMTPChannel` object which manages the communication - between the server and a single SMTP client. - - *conn* and *addr* are as per the instance variables described below. - - *data_size_limit* specifies the maximum number of bytes that will be - accepted in a ``DATA`` command. A value of ``None`` or ``0`` means no - limit. - - *enable_SMTPUTF8* determines whether the ``SMTPUTF8`` extension (as defined - in :RFC:`6531`) should be enabled. The default is ``False``. - *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same - time. - - A dictionary can be specified in *map* to avoid using a global socket map. - - *decode_data* specifies whether the data portion of the SMTP transaction - should be decoded using UTF-8. The default is ``False``. - *decode_data* and *enable_SMTPUTF8* cannot be set to ``True`` at the same - time. - - To use a custom SMTPChannel implementation you need to override the - :attr:`SMTPServer.channel_class` of your :class:`SMTPServer`. - - .. versionchanged:: 3.5 - The *decode_data* and *enable_SMTPUTF8* parameters were added. - - .. versionchanged:: 3.6 - *decode_data* is now ``False`` by default. - - The :class:`SMTPChannel` has the following instance variables: - - .. attribute:: smtp_server - - Holds the :class:`SMTPServer` that spawned this channel. - - .. attribute:: conn - - Holds the socket object connecting to the client. - - .. attribute:: addr - - Holds the address of the client, the second value returned by - :func:`socket.accept ` - - .. attribute:: received_lines - - Holds a list of the line strings (decoded using UTF-8) received from - the client. The lines have their ``"\r\n"`` line ending translated to - ``"\n"``. - - .. attribute:: smtp_state - - Holds the current state of the channel. This will be either - :attr:`COMMAND` initially and then :attr:`DATA` after the client sends - a "DATA" line. - - .. attribute:: seen_greeting - - Holds a string containing the greeting sent by the client in its "HELO". - - .. attribute:: mailfrom - - Holds a string containing the address identified in the "MAIL FROM:" line - from the client. - - .. attribute:: rcpttos - - Holds a list of strings containing the addresses identified in the - "RCPT TO:" lines from the client. - - .. attribute:: received_data - - Holds a string containing all of the data sent by the client during the - DATA state, up to but not including the terminating ``"\r\n.\r\n"``. - - .. attribute:: fqdn - - Holds the fully qualified domain name of the server as returned by - :func:`socket.getfqdn`. - - .. attribute:: peer - - Holds the name of the client peer as returned by ``conn.getpeername()`` - where ``conn`` is :attr:`conn`. - - The :class:`SMTPChannel` operates by invoking methods named ``smtp_`` - upon reception of a command line from the client. Built into the base - :class:`SMTPChannel` class are methods for handling the following commands - (and responding to them appropriately): - - ======== =================================================================== - Command Action taken - ======== =================================================================== - HELO Accepts the greeting from the client and stores it in - :attr:`seen_greeting`. Sets server to base command mode. - EHLO Accepts the greeting from the client and stores it in - :attr:`seen_greeting`. Sets server to extended command mode. - NOOP Takes no action. - QUIT Closes the connection cleanly. - MAIL Accepts the "MAIL FROM:" syntax and stores the supplied address as - :attr:`mailfrom`. In extended command mode, accepts the - :rfc:`1870` SIZE attribute and responds appropriately based on the - value of *data_size_limit*. - RCPT Accepts the "RCPT TO:" syntax and stores the supplied addresses in - the :attr:`rcpttos` list. - RSET Resets the :attr:`mailfrom`, :attr:`rcpttos`, and - :attr:`received_data`, but not the greeting. - DATA Sets the internal state to :attr:`DATA` and stores remaining lines - from the client in :attr:`received_data` until the terminator - ``"\r\n.\r\n"`` is received. - HELP Returns minimal information on command syntax - VRFY Returns code 252 (the server doesn't know if the address is valid) - EXPN Reports that the command is not implemented. - ======== =================================================================== diff --git a/Doc/library/smtplib.rst b/Doc/library/smtplib.rst deleted file mode 100644 index 67cd5b000afb70..00000000000000 --- a/Doc/library/smtplib.rst +++ /dev/null @@ -1,606 +0,0 @@ -:mod:`smtplib` --- SMTP protocol client -======================================= - -.. module:: smtplib - :synopsis: SMTP protocol client (requires sockets). - -.. sectionauthor:: Eric S. Raymond - -**Source code:** :source:`Lib/smtplib.py` - -.. index:: - pair: SMTP; protocol - single: Simple Mail Transfer Protocol - --------------- - -The :mod:`smtplib` module defines an SMTP client session object that can be used -to send mail to any internet machine with an SMTP or ESMTP listener daemon. For -details of SMTP and ESMTP operation, consult :rfc:`821` (Simple Mail Transfer -Protocol) and :rfc:`1869` (SMTP Service Extensions). - -.. include:: ../includes/wasm-notavail.rst - -.. class:: SMTP(host='', port=0, local_hostname=None[, timeout], source_address=None) - - An :class:`SMTP` instance encapsulates an SMTP connection. It has methods - that support a full repertoire of SMTP and ESMTP operations. If the optional - *host* and *port* parameters are given, the SMTP :meth:`connect` method is - called with those parameters during initialization. If specified, - *local_hostname* is used as the FQDN of the local host in the HELO/EHLO - command. Otherwise, the local hostname is found using - :func:`socket.getfqdn`. If the :meth:`connect` call returns anything other - than a success code, an :exc:`SMTPConnectError` is raised. The optional - *timeout* parameter specifies a timeout in seconds for blocking operations - like the connection attempt (if not specified, the global default timeout - setting will be used). If the timeout expires, :exc:`TimeoutError` is - raised. The optional *source_address* parameter allows binding - to some specific source address in a machine with multiple network - interfaces, and/or to some specific source TCP port. It takes a 2-tuple - ``(host, port)``, for the socket to bind to as its source address before - connecting. If omitted (or if *host* or *port* are ``''`` and/or ``0`` - respectively) the OS default behavior will be used. - - For normal use, you should only require the initialization/connect, - :meth:`sendmail`, and :meth:`SMTP.quit` methods. - An example is included below. - - The :class:`SMTP` class supports the :keyword:`with` statement. When used - like this, the SMTP ``QUIT`` command is issued automatically when the - :keyword:`!with` statement exits. E.g.:: - - >>> from smtplib import SMTP - >>> with SMTP("domain.org") as smtp: - ... smtp.noop() - ... - (250, b'Ok') - >>> - - .. audit-event:: smtplib.send self,data smtplib.SMTP - - All commands will raise an :ref:`auditing event ` - ``smtplib.SMTP.send`` with arguments ``self`` and ``data``, - where ``data`` is the bytes about to be sent to the remote host. - - .. versionchanged:: 3.3 - Support for the :keyword:`with` statement was added. - - .. versionchanged:: 3.3 - source_address argument was added. - - .. versionadded:: 3.5 - The SMTPUTF8 extension (:rfc:`6531`) is now supported. - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket - -.. class:: SMTP_SSL(host='', port=0, local_hostname=None, keyfile=None, \ - certfile=None [, timeout], context=None, \ - source_address=None) - - An :class:`SMTP_SSL` instance behaves exactly the same as instances of - :class:`SMTP`. :class:`SMTP_SSL` should be used for situations where SSL is - required from the beginning of the connection and using :meth:`starttls` is - not appropriate. If *host* is not specified, the local host is used. If - *port* is zero, the standard SMTP-over-SSL port (465) is used. The optional - arguments *local_hostname*, *timeout* and *source_address* have the same - meaning as they do in the :class:`SMTP` class. *context*, also optional, - can contain a :class:`~ssl.SSLContext` and allows configuring various - aspects of the secure connection. Please read :ref:`ssl-security` for - best practices. - - *keyfile* and *certfile* are a legacy alternative to *context*, and can - point to a PEM formatted private key and certificate chain file for the - SSL connection. - - .. versionchanged:: 3.3 - *context* was added. - - .. versionchanged:: 3.3 - source_address argument was added. - - .. versionchanged:: 3.4 - The class now supports hostname check with - :attr:`ssl.SSLContext.check_hostname` and *Server Name Indication* (see - :const:`ssl.HAS_SNI`). - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - .. versionchanged:: 3.9 - If the *timeout* parameter is set to be zero, it will raise a - :class:`ValueError` to prevent the creation of a non-blocking socket - -.. class:: LMTP(host='', port=LMTP_PORT, local_hostname=None, \ - source_address=None[, timeout]) - - The LMTP protocol, which is very similar to ESMTP, is heavily based on the - standard SMTP client. It's common to use Unix sockets for LMTP, so our - :meth:`connect` method must support that as well as a regular host:port - server. The optional arguments local_hostname and source_address have the - same meaning as they do in the :class:`SMTP` class. To specify a Unix - socket, you must use an absolute path for *host*, starting with a '/'. - - Authentication is supported, using the regular SMTP mechanism. When using a - Unix socket, LMTP generally don't support or require any authentication, but - your mileage might vary. - - .. versionchanged:: 3.9 - The optional *timeout* parameter was added. - - -A nice selection of exceptions is defined as well: - - -.. exception:: SMTPException - - Subclass of :exc:`OSError` that is the base exception class for all - the other exceptions provided by this module. - - .. versionchanged:: 3.4 - SMTPException became subclass of :exc:`OSError` - - -.. exception:: SMTPServerDisconnected - - This exception is raised when the server unexpectedly disconnects, or when an - attempt is made to use the :class:`SMTP` instance before connecting it to a - server. - - -.. exception:: SMTPResponseException - - Base class for all exceptions that include an SMTP error code. These exceptions - are generated in some instances when the SMTP server returns an error code. The - error code is stored in the :attr:`smtp_code` attribute of the error, and the - :attr:`smtp_error` attribute is set to the error message. - - -.. exception:: SMTPSenderRefused - - Sender address refused. In addition to the attributes set by on all - :exc:`SMTPResponseException` exceptions, this sets 'sender' to the string that - the SMTP server refused. - - -.. exception:: SMTPRecipientsRefused - - All recipient addresses refused. The errors for each recipient are accessible - through the attribute :attr:`recipients`, which is a dictionary of exactly the - same sort as :meth:`SMTP.sendmail` returns. - - -.. exception:: SMTPDataError - - The SMTP server refused to accept the message data. - - -.. exception:: SMTPConnectError - - Error occurred during establishment of a connection with the server. - - -.. exception:: SMTPHeloError - - The server refused our ``HELO`` message. - - -.. exception:: SMTPNotSupportedError - - The command or option attempted is not supported by the server. - - .. versionadded:: 3.5 - - -.. exception:: SMTPAuthenticationError - - SMTP authentication went wrong. Most probably the server didn't accept the - username/password combination provided. - - -.. seealso:: - - :rfc:`821` - Simple Mail Transfer Protocol - Protocol definition for SMTP. This document covers the model, operating - procedure, and protocol details for SMTP. - - :rfc:`1869` - SMTP Service Extensions - Definition of the ESMTP extensions for SMTP. This describes a framework for - extending SMTP with new commands, supporting dynamic discovery of the commands - provided by the server, and defines a few additional commands. - - -.. _smtp-objects: - -SMTP Objects ------------- - -An :class:`SMTP` instance has the following methods: - - -.. method:: SMTP.set_debuglevel(level) - - Set the debug output level. A value of 1 or ``True`` for *level* results in - debug messages for connection and for all messages sent to and received from - the server. A value of 2 for *level* results in these messages being - timestamped. - - .. versionchanged:: 3.5 Added debuglevel 2. - - -.. method:: SMTP.docmd(cmd, args='') - - Send a command *cmd* to the server. The optional argument *args* is simply - concatenated to the command, separated by a space. - - This returns a 2-tuple composed of a numeric response code and the actual - response line (multiline responses are joined into one long line.) - - In normal operation it should not be necessary to call this method explicitly. - It is used to implement other methods and may be useful for testing private - extensions. - - If the connection to the server is lost while waiting for the reply, - :exc:`SMTPServerDisconnected` will be raised. - - -.. method:: SMTP.connect(host='localhost', port=0) - - Connect to a host on a given port. The defaults are to connect to the local - host at the standard SMTP port (25). If the hostname ends with a colon (``':'``) - followed by a number, that suffix will be stripped off and the number - interpreted as the port number to use. This method is automatically invoked by - the constructor if a host is specified during instantiation. Returns a - 2-tuple of the response code and message sent by the server in its - connection response. - - .. audit-event:: smtplib.connect self,host,port smtplib.SMTP.connect - - -.. method:: SMTP.helo(name='') - - Identify yourself to the SMTP server using ``HELO``. The hostname argument - defaults to the fully qualified domain name of the local host. - The message returned by the server is stored as the :attr:`helo_resp` attribute - of the object. - - In normal operation it should not be necessary to call this method explicitly. - It will be implicitly called by the :meth:`sendmail` when necessary. - - -.. method:: SMTP.ehlo(name='') - - Identify yourself to an ESMTP server using ``EHLO``. The hostname argument - defaults to the fully qualified domain name of the local host. Examine the - response for ESMTP option and store them for use by :meth:`has_extn`. - Also sets several informational attributes: the message returned by - the server is stored as the :attr:`ehlo_resp` attribute, :attr:`does_esmtp` - is set to ``True`` or ``False`` depending on whether the server supports - ESMTP, and :attr:`esmtp_features` will be a dictionary containing the names - of the SMTP service extensions this server supports, and their parameters - (if any). - - Unless you wish to use :meth:`has_extn` before sending mail, it should not be - necessary to call this method explicitly. It will be implicitly called by - :meth:`sendmail` when necessary. - -.. method:: SMTP.ehlo_or_helo_if_needed() - - This method calls :meth:`ehlo` and/or :meth:`helo` if there has been no - previous ``EHLO`` or ``HELO`` command this session. It tries ESMTP ``EHLO`` - first. - - :exc:`SMTPHeloError` - The server didn't reply properly to the ``HELO`` greeting. - -.. method:: SMTP.has_extn(name) - - Return :const:`True` if *name* is in the set of SMTP service extensions returned - by the server, :const:`False` otherwise. Case is ignored. - - -.. method:: SMTP.verify(address) - - Check the validity of an address on this server using SMTP ``VRFY``. Returns a - tuple consisting of code 250 and a full :rfc:`822` address (including human - name) if the user address is valid. Otherwise returns an SMTP error code of 400 - or greater and an error string. - - .. note:: - - Many sites disable SMTP ``VRFY`` in order to foil spammers. - - -.. method:: SMTP.login(user, password, *, initial_response_ok=True) - - Log in on an SMTP server that requires authentication. The arguments are the - username and the password to authenticate with. If there has been no previous - ``EHLO`` or ``HELO`` command this session, this method tries ESMTP ``EHLO`` - first. This method will return normally if the authentication was successful, or - may raise the following exceptions: - - :exc:`SMTPHeloError` - The server didn't reply properly to the ``HELO`` greeting. - - :exc:`SMTPAuthenticationError` - The server didn't accept the username/password combination. - - :exc:`SMTPNotSupportedError` - The ``AUTH`` command is not supported by the server. - - :exc:`SMTPException` - No suitable authentication method was found. - - Each of the authentication methods supported by :mod:`smtplib` are tried in - turn if they are advertised as supported by the server. See :meth:`auth` - for a list of supported authentication methods. *initial_response_ok* is - passed through to :meth:`auth`. - - Optional keyword argument *initial_response_ok* specifies whether, for - authentication methods that support it, an "initial response" as specified - in :rfc:`4954` can be sent along with the ``AUTH`` command, rather than - requiring a challenge/response. - - .. versionchanged:: 3.5 - :exc:`SMTPNotSupportedError` may be raised, and the - *initial_response_ok* parameter was added. - - -.. method:: SMTP.auth(mechanism, authobject, *, initial_response_ok=True) - - Issue an ``SMTP`` ``AUTH`` command for the specified authentication - *mechanism*, and handle the challenge response via *authobject*. - - *mechanism* specifies which authentication mechanism is to - be used as argument to the ``AUTH`` command; the valid values are - those listed in the ``auth`` element of :attr:`esmtp_features`. - - *authobject* must be a callable object taking an optional single argument: - - data = authobject(challenge=None) - - If optional keyword argument *initial_response_ok* is true, - ``authobject()`` will be called first with no argument. It can return the - :rfc:`4954` "initial response" ASCII ``str`` which will be encoded and sent with - the ``AUTH`` command as below. If the ``authobject()`` does not support an - initial response (e.g. because it requires a challenge), it should return - ``None`` when called with ``challenge=None``. If *initial_response_ok* is - false, then ``authobject()`` will not be called first with ``None``. - - If the initial response check returns ``None``, or if *initial_response_ok* is - false, ``authobject()`` will be called to process the server's challenge - response; the *challenge* argument it is passed will be a ``bytes``. It - should return ASCII ``str`` *data* that will be base64 encoded and sent to the - server. - - The ``SMTP`` class provides ``authobjects`` for the ``CRAM-MD5``, ``PLAIN``, - and ``LOGIN`` mechanisms; they are named ``SMTP.auth_cram_md5``, - ``SMTP.auth_plain``, and ``SMTP.auth_login`` respectively. They all require - that the ``user`` and ``password`` properties of the ``SMTP`` instance are - set to appropriate values. - - User code does not normally need to call ``auth`` directly, but can instead - call the :meth:`login` method, which will try each of the above mechanisms - in turn, in the order listed. ``auth`` is exposed to facilitate the - implementation of authentication methods not (or not yet) supported - directly by :mod:`smtplib`. - - .. versionadded:: 3.5 - - -.. method:: SMTP.starttls(keyfile=None, certfile=None, context=None) - - Put the SMTP connection in TLS (Transport Layer Security) mode. All SMTP - commands that follow will be encrypted. You should then call :meth:`ehlo` - again. - - If *keyfile* and *certfile* are provided, they are used to create an - :class:`ssl.SSLContext`. - - Optional *context* parameter is an :class:`ssl.SSLContext` object; This is - an alternative to using a keyfile and a certfile and if specified both - *keyfile* and *certfile* should be ``None``. - - If there has been no previous ``EHLO`` or ``HELO`` command this session, - this method tries ESMTP ``EHLO`` first. - - .. deprecated:: 3.6 - - *keyfile* and *certfile* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - :exc:`SMTPHeloError` - The server didn't reply properly to the ``HELO`` greeting. - - :exc:`SMTPNotSupportedError` - The server does not support the STARTTLS extension. - - :exc:`RuntimeError` - SSL/TLS support is not available to your Python interpreter. - - .. versionchanged:: 3.3 - *context* was added. - - .. versionchanged:: 3.4 - The method now supports hostname check with - :attr:`SSLContext.check_hostname` and *Server Name Indicator* (see - :const:`~ssl.HAS_SNI`). - - .. versionchanged:: 3.5 - The error raised for lack of STARTTLS support is now the - :exc:`SMTPNotSupportedError` subclass instead of the base - :exc:`SMTPException`. - - -.. method:: SMTP.sendmail(from_addr, to_addrs, msg, mail_options=(), rcpt_options=()) - - Send mail. The required arguments are an :rfc:`822` from-address string, a list - of :rfc:`822` to-address strings (a bare string will be treated as a list with 1 - address), and a message string. The caller may pass a list of ESMTP options - (such as ``8bitmime``) to be used in ``MAIL FROM`` commands as *mail_options*. - ESMTP options (such as ``DSN`` commands) that should be used with all ``RCPT`` - commands can be passed as *rcpt_options*. (If you need to use different ESMTP - options to different recipients you have to use the low-level methods such as - :meth:`mail`, :meth:`rcpt` and :meth:`data` to send the message.) - - .. note:: - - The *from_addr* and *to_addrs* parameters are used to construct the message - envelope used by the transport agents. ``sendmail`` does not modify the - message headers in any way. - - *msg* may be a string containing characters in the ASCII range, or a byte - string. A string is encoded to bytes using the ascii codec, and lone ``\r`` - and ``\n`` characters are converted to ``\r\n`` characters. A byte string is - not modified. - - If there has been no previous ``EHLO`` or ``HELO`` command this session, this - method tries ESMTP ``EHLO`` first. If the server does ESMTP, message size and - each of the specified options will be passed to it (if the option is in the - feature set the server advertises). If ``EHLO`` fails, ``HELO`` will be tried - and ESMTP options suppressed. - - This method will return normally if the mail is accepted for at least one - recipient. Otherwise it will raise an exception. That is, if this method does - not raise an exception, then someone should get your mail. If this method does - not raise an exception, it returns a dictionary, with one entry for each - recipient that was refused. Each entry contains a tuple of the SMTP error code - and the accompanying error message sent by the server. - - If ``SMTPUTF8`` is included in *mail_options*, and the server supports it, - *from_addr* and *to_addrs* may contain non-ASCII characters. - - This method may raise the following exceptions: - - :exc:`SMTPRecipientsRefused` - All recipients were refused. Nobody got the mail. The :attr:`recipients` - attribute of the exception object is a dictionary with information about the - refused recipients (like the one returned when at least one recipient was - accepted). - - :exc:`SMTPHeloError` - The server didn't reply properly to the ``HELO`` greeting. - - :exc:`SMTPSenderRefused` - The server didn't accept the *from_addr*. - - :exc:`SMTPDataError` - The server replied with an unexpected error code (other than a refusal of a - recipient). - - :exc:`SMTPNotSupportedError` - ``SMTPUTF8`` was given in the *mail_options* but is not supported by the - server. - - Unless otherwise noted, the connection will be open even after an exception is - raised. - - .. versionchanged:: 3.2 - *msg* may be a byte string. - - .. versionchanged:: 3.5 - ``SMTPUTF8`` support added, and :exc:`SMTPNotSupportedError` may be - raised if ``SMTPUTF8`` is specified but the server does not support it. - - -.. method:: SMTP.send_message(msg, from_addr=None, to_addrs=None, \ - mail_options=(), rcpt_options=()) - - This is a convenience method for calling :meth:`sendmail` with the message - represented by an :class:`email.message.Message` object. The arguments have - the same meaning as for :meth:`sendmail`, except that *msg* is a ``Message`` - object. - - If *from_addr* is ``None`` or *to_addrs* is ``None``, ``send_message`` fills - those arguments with addresses extracted from the headers of *msg* as - specified in :rfc:`5322`\: *from_addr* is set to the :mailheader:`Sender` - field if it is present, and otherwise to the :mailheader:`From` field. - *to_addrs* combines the values (if any) of the :mailheader:`To`, - :mailheader:`Cc`, and :mailheader:`Bcc` fields from *msg*. If exactly one - set of :mailheader:`Resent-*` headers appear in the message, the regular - headers are ignored and the :mailheader:`Resent-*` headers are used instead. - If the message contains more than one set of :mailheader:`Resent-*` headers, - a :exc:`ValueError` is raised, since there is no way to unambiguously detect - the most recent set of :mailheader:`Resent-` headers. - - ``send_message`` serializes *msg* using - :class:`~email.generator.BytesGenerator` with ``\r\n`` as the *linesep*, and - calls :meth:`sendmail` to transmit the resulting message. Regardless of the - values of *from_addr* and *to_addrs*, ``send_message`` does not transmit any - :mailheader:`Bcc` or :mailheader:`Resent-Bcc` headers that may appear - in *msg*. If any of the addresses in *from_addr* and *to_addrs* contain - non-ASCII characters and the server does not advertise ``SMTPUTF8`` support, - an :exc:`SMTPNotSupported` error is raised. Otherwise the ``Message`` is - serialized with a clone of its :mod:`~email.policy` with the - :attr:`~email.policy.EmailPolicy.utf8` attribute set to ``True``, and - ``SMTPUTF8`` and ``BODY=8BITMIME`` are added to *mail_options*. - - .. versionadded:: 3.2 - - .. versionadded:: 3.5 - Support for internationalized addresses (``SMTPUTF8``). - - -.. method:: SMTP.quit() - - Terminate the SMTP session and close the connection. Return the result of - the SMTP ``QUIT`` command. - - -Low-level methods corresponding to the standard SMTP/ESMTP commands ``HELP``, -``RSET``, ``NOOP``, ``MAIL``, ``RCPT``, and ``DATA`` are also supported. -Normally these do not need to be called directly, so they are not documented -here. For details, consult the module code. - - -.. _smtp-example: - -SMTP Example ------------- - -This example prompts the user for addresses needed in the message envelope ('To' -and 'From' addresses), and the message to be delivered. Note that the headers -to be included with the message must be included in the message as entered; this -example doesn't do any processing of the :rfc:`822` headers. In particular, the -'To' and 'From' addresses must be included in the message headers explicitly. :: - - import smtplib - - def prompt(prompt): - return input(prompt).strip() - - fromaddr = prompt("From: ") - toaddrs = prompt("To: ").split() - print("Enter message, end with ^D (Unix) or ^Z (Windows):") - - # Add the From: and To: headers at the start! - msg = ("From: %s\r\nTo: %s\r\n\r\n" - % (fromaddr, ", ".join(toaddrs))) - while True: - try: - line = input() - except EOFError: - break - if not line: - break - msg = msg + line - - print("Message length is", len(msg)) - - server = smtplib.SMTP('localhost') - server.set_debuglevel(1) - server.sendmail(fromaddr, toaddrs, msg) - server.quit() - -.. note:: - - In general, you will want to use the :mod:`email` package's features to - construct an email message, which you can then send - via :meth:`~smtplib.SMTP.send_message`; see :ref:`email-examples`. diff --git a/Doc/library/sndhdr.rst b/Doc/library/sndhdr.rst deleted file mode 100644 index fa9323e18dc348..00000000000000 --- a/Doc/library/sndhdr.rst +++ /dev/null @@ -1,104 +0,0 @@ -:mod:`sndhdr` --- Determine type of sound file -============================================== - -.. module:: sndhdr - :synopsis: Determine type of a sound file. - :deprecated: - -.. sectionauthor:: Fred L. Drake, Jr. -.. Based on comments in the module source file. - -**Source code:** :source:`Lib/sndhdr.py` - -.. index:: - single: A-LAW - single: u-LAW - -.. deprecated-removed:: 3.11 3.13 - The :mod:`sndhdr` module is deprecated - (see :pep:`PEP 594 <594#sndhdr>` for details and alternatives). - --------------- - -The :mod:`sndhdr` provides utility functions which attempt to determine the type -of sound data which is in a file. When these functions are able to determine -what type of sound data is stored in a file, they return a -:func:`~collections.namedtuple`, containing five attributes: (``filetype``, -``framerate``, ``nchannels``, ``nframes``, ``sampwidth``). The value for *type* -indicates the data type and will be one of the strings ``'aifc'``, ``'aiff'``, -``'au'``, ``'hcom'``, ``'sndr'``, ``'sndt'``, ``'voc'``, ``'wav'``, ``'8svx'``, -``'sb'``, ``'ub'``, or ``'ul'``. The *sampling_rate* will be either the actual -value or ``0`` if unknown or difficult to decode. Similarly, *channels* will be -either the number of channels or ``0`` if it cannot be determined or if the -value is difficult to decode. The value for *frames* will be either the number -of frames or ``-1``. The last item in the tuple, *bits_per_sample*, will either -be the sample size in bits or ``'A'`` for A-LAW or ``'U'`` for u-LAW. - - -.. function:: what(filename) - - Determines the type of sound data stored in the file *filename* using - :func:`whathdr`. If it succeeds, returns a namedtuple as described above, otherwise - ``None`` is returned. - - .. versionchanged:: 3.5 - Result changed from a tuple to a namedtuple. - - -.. function:: whathdr(filename) - - Determines the type of sound data stored in a file based on the file header. - The name of the file is given by *filename*. This function returns a namedtuple as - described above on success, or ``None``. - - .. versionchanged:: 3.5 - Result changed from a tuple to a namedtuple. - -The following sound header types are recognized, as listed below with the return value -from :func:`whathdr`: and :func:`what`: - -+------------+------------------------------------+ -| Value | Sound header format | -+============+====================================+ -| ``'aifc'`` | Compressed Audio Interchange Files | -+------------+------------------------------------+ -| ``'aiff'`` | Audio Interchange Files | -+------------+------------------------------------+ -| ``'au'`` | Au Files | -+------------+------------------------------------+ -| ``'hcom'`` | HCOM Files | -+------------+------------------------------------+ -| ``'sndt'`` | Sndtool Sound Files | -+------------+------------------------------------+ -| ``'voc'`` | Creative Labs Audio Files | -+------------+------------------------------------+ -| ``'wav'`` | Waveform Audio File Format Files | -+------------+------------------------------------+ -| ``'8svx'`` | 8-Bit Sampled Voice Files | -+------------+------------------------------------+ -| ``'sb'`` | Signed Byte Audio Data Files | -+------------+------------------------------------+ -| ``'ub'`` | UB Files | -+------------+------------------------------------+ -| ``'ul'`` | uLAW Audio Files | -+------------+------------------------------------+ - -.. data:: tests - - A list of functions performing the individual tests. Each function takes two - arguments: the byte-stream and an open file-like object. When :func:`what` is - called with a byte-stream, the file-like object will be ``None``. - - The test function should return a string describing the image type if the test - succeeded, or ``None`` if it failed. - -Example: - -.. code-block:: pycon - - >>> import sndhdr - >>> imghdr.what('bass.wav') - 'wav' - >>> imghdr.whathdr('bass.wav') - 'wav' - diff --git a/Doc/library/socket.rst b/Doc/library/socket.rst deleted file mode 100644 index b5c1edaf5ad8e0..00000000000000 --- a/Doc/library/socket.rst +++ /dev/null @@ -1,2187 +0,0 @@ -:mod:`socket` --- Low-level networking interface -================================================ - -.. module:: socket - :synopsis: Low-level networking interface. - -**Source code:** :source:`Lib/socket.py` - --------------- - -This module provides access to the BSD *socket* interface. It is available on -all modern Unix systems, Windows, MacOS, and probably additional platforms. - -.. note:: - - Some behavior may be platform dependent, since calls are made to the operating - system socket APIs. - - -.. include:: ../includes/wasm-notavail.rst - -.. index:: pair: object; socket - -The Python interface is a straightforward transliteration of the Unix system -call and library interface for sockets to Python's object-oriented style: the -:func:`.socket` function returns a :dfn:`socket object` whose methods implement -the various socket system calls. Parameter types are somewhat higher-level than -in the C interface: as with :meth:`read` and :meth:`write` operations on Python -files, buffer allocation on receive operations is automatic, and buffer length -is implicit on send operations. - - -.. seealso:: - - Module :mod:`socketserver` - Classes that simplify writing network servers. - - Module :mod:`ssl` - A TLS/SSL wrapper for socket objects. - - -Socket families ---------------- - -Depending on the system and the build options, various socket families -are supported by this module. - -The address format required by a particular socket object is automatically -selected based on the address family specified when the socket object was -created. Socket addresses are represented as follows: - -- The address of an :const:`AF_UNIX` socket bound to a file system node - is represented as a string, using the file system encoding and the - ``'surrogateescape'`` error handler (see :pep:`383`). An address in - Linux's abstract namespace is returned as a :term:`bytes-like object` with - an initial null byte; note that sockets in this namespace can - communicate with normal file system sockets, so programs intended to - run on Linux may need to deal with both types of address. A string or - bytes-like object can be used for either type of address when - passing it as an argument. - - .. versionchanged:: 3.3 - Previously, :const:`AF_UNIX` socket paths were assumed to use UTF-8 - encoding. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - -.. _host_port: - -- A pair ``(host, port)`` is used for the :const:`AF_INET` address family, - where *host* is a string representing either a hostname in internet domain - notation like ``'daring.cwi.nl'`` or an IPv4 address like ``'100.50.200.5'``, - and *port* is an integer. - - - For IPv4 addresses, two special forms are accepted instead of a host - address: ``''`` represents :const:`INADDR_ANY`, which is used to bind to all - interfaces, and the string ``''`` represents - :const:`INADDR_BROADCAST`. This behavior is not compatible with IPv6, - therefore, you may want to avoid these if you intend to support IPv6 with your - Python programs. - -- For :const:`AF_INET6` address family, a four-tuple ``(host, port, flowinfo, - scope_id)`` is used, where *flowinfo* and *scope_id* represent the ``sin6_flowinfo`` - and ``sin6_scope_id`` members in :const:`struct sockaddr_in6` in C. For - :mod:`socket` module methods, *flowinfo* and *scope_id* can be omitted just for - backward compatibility. Note, however, omission of *scope_id* can cause problems - in manipulating scoped IPv6 addresses. - - .. versionchanged:: 3.7 - For multicast addresses (with *scope_id* meaningful) *address* may not contain - ``%scope_id`` (or ``zone id``) part. This information is superfluous and may - be safely omitted (recommended). - -- :const:`AF_NETLINK` sockets are represented as pairs ``(pid, groups)``. - -- Linux-only support for TIPC is available using the :const:`AF_TIPC` - address family. TIPC is an open, non-IP based networked protocol designed - for use in clustered computer environments. Addresses are represented by a - tuple, and the fields depend on the address type. The general tuple form is - ``(addr_type, v1, v2, v3 [, scope])``, where: - - - *addr_type* is one of :const:`TIPC_ADDR_NAMESEQ`, :const:`TIPC_ADDR_NAME`, - or :const:`TIPC_ADDR_ID`. - - *scope* is one of :const:`TIPC_ZONE_SCOPE`, :const:`TIPC_CLUSTER_SCOPE`, and - :const:`TIPC_NODE_SCOPE`. - - If *addr_type* is :const:`TIPC_ADDR_NAME`, then *v1* is the server type, *v2* is - the port identifier, and *v3* should be 0. - - If *addr_type* is :const:`TIPC_ADDR_NAMESEQ`, then *v1* is the server type, *v2* - is the lower port number, and *v3* is the upper port number. - - If *addr_type* is :const:`TIPC_ADDR_ID`, then *v1* is the node, *v2* is the - reference, and *v3* should be set to 0. - -- A tuple ``(interface, )`` is used for the :const:`AF_CAN` address family, - where *interface* is a string representing a network interface name like - ``'can0'``. The network interface name ``''`` can be used to receive packets - from all network interfaces of this family. - - - :const:`CAN_ISOTP` protocol require a tuple ``(interface, rx_addr, tx_addr)`` - where both additional parameters are unsigned long integer that represent a - CAN identifier (standard or extended). - - :const:`CAN_J1939` protocol require a tuple ``(interface, name, pgn, addr)`` - where additional parameters are 64-bit unsigned integer representing the - ECU name, a 32-bit unsigned integer representing the Parameter Group Number - (PGN), and an 8-bit integer representing the address. - -- A string or a tuple ``(id, unit)`` is used for the :const:`SYSPROTO_CONTROL` - protocol of the :const:`PF_SYSTEM` family. The string is the name of a - kernel control using a dynamically assigned ID. The tuple can be used if ID - and unit number of the kernel control are known or if a registered ID is - used. - - .. versionadded:: 3.3 - -- :const:`AF_BLUETOOTH` supports the following protocols and address - formats: - - - :const:`BTPROTO_L2CAP` accepts ``(bdaddr, psm)`` where ``bdaddr`` is - the Bluetooth address as a string and ``psm`` is an integer. - - - :const:`BTPROTO_RFCOMM` accepts ``(bdaddr, channel)`` where ``bdaddr`` - is the Bluetooth address as a string and ``channel`` is an integer. - - - :const:`BTPROTO_HCI` accepts ``(device_id,)`` where ``device_id`` is - either an integer or a string with the Bluetooth address of the - interface. (This depends on your OS; NetBSD and DragonFlyBSD expect - a Bluetooth address while everything else expects an integer.) - - .. versionchanged:: 3.2 - NetBSD and DragonFlyBSD support added. - - - :const:`BTPROTO_SCO` accepts ``bdaddr`` where ``bdaddr`` is a - :class:`bytes` object containing the Bluetooth address in a - string format. (ex. ``b'12:23:34:45:56:67'``) This protocol is not - supported under FreeBSD. - -- :const:`AF_ALG` is a Linux-only socket based interface to Kernel - cryptography. An algorithm socket is configured with a tuple of two to four - elements ``(type, name [, feat [, mask]])``, where: - - - *type* is the algorithm type as string, e.g. ``aead``, ``hash``, - ``skcipher`` or ``rng``. - - - *name* is the algorithm name and operation mode as string, e.g. - ``sha256``, ``hmac(sha256)``, ``cbc(aes)`` or ``drbg_nopr_ctr_aes256``. - - - *feat* and *mask* are unsigned 32bit integers. - - .. availability:: Linux >= 2.6.38. - - Some algorithm types require more recent Kernels. - - .. versionadded:: 3.6 - -- :const:`AF_VSOCK` allows communication between virtual machines and - their hosts. The sockets are represented as a ``(CID, port)`` tuple - where the context ID or CID and port are integers. - - .. availability:: Linux >= 3.9 - - See :manpage:`vsock(7)` - - .. versionadded:: 3.7 - -- :const:`AF_PACKET` is a low-level interface directly to network devices. - The packets are represented by the tuple - ``(ifname, proto[, pkttype[, hatype[, addr]]])`` where: - - - *ifname* - String specifying the device name. - - *proto* - An in network-byte-order integer specifying the Ethernet - protocol number. - - *pkttype* - Optional integer specifying the packet type: - - - ``PACKET_HOST`` (the default) - Packet addressed to the local host. - - ``PACKET_BROADCAST`` - Physical-layer broadcast packet. - - ``PACKET_MULTICAST`` - Packet sent to a physical-layer multicast address. - - ``PACKET_OTHERHOST`` - Packet to some other host that has been caught by - a device driver in promiscuous mode. - - ``PACKET_OUTGOING`` - Packet originating from the local host that is - looped back to a packet socket. - - *hatype* - Optional integer specifying the ARP hardware address type. - - *addr* - Optional bytes-like object specifying the hardware physical - address, whose interpretation depends on the device. - - .. availability:: Linux >= 2.2. - -- :const:`AF_QIPCRTR` is a Linux-only socket based interface for communicating - with services running on co-processors in Qualcomm platforms. The address - family is represented as a ``(node, port)`` tuple where the *node* and *port* - are non-negative integers. - - .. availability:: Linux >= 4.7. - - .. versionadded:: 3.8 - -- :const:`IPPROTO_UDPLITE` is a variant of UDP which allows you to specify - what portion of a packet is covered with the checksum. It adds two socket - options that you can change. - ``self.setsockopt(IPPROTO_UDPLITE, UDPLITE_SEND_CSCOV, length)`` will - change what portion of outgoing packets are covered by the checksum and - ``self.setsockopt(IPPROTO_UDPLITE, UDPLITE_RECV_CSCOV, length)`` will - filter out packets which cover too little of their data. In both cases - ``length`` should be in ``range(8, 2**16, 8)``. - - Such a socket should be constructed with - ``socket(AF_INET, SOCK_DGRAM, IPPROTO_UDPLITE)`` for IPv4 or - ``socket(AF_INET6, SOCK_DGRAM, IPPROTO_UDPLITE)`` for IPv6. - - .. availability:: Linux >= 2.6.20, FreeBSD >= 10.1 - - .. versionadded:: 3.9 - -If you use a hostname in the *host* portion of IPv4/v6 socket address, the -program may show a nondeterministic behavior, as Python uses the first address -returned from the DNS resolution. The socket address will be resolved -differently into an actual IPv4/v6 address, depending on the results from DNS -resolution and/or the host configuration. For deterministic behavior use a -numeric address in *host* portion. - -All errors raise exceptions. The normal exceptions for invalid argument types -and out-of-memory conditions can be raised. Errors -related to socket or address semantics raise :exc:`OSError` or one of its -subclasses. - -Non-blocking mode is supported through :meth:`~socket.setblocking`. A -generalization of this based on timeouts is supported through -:meth:`~socket.settimeout`. - - -Module contents ---------------- - -The module :mod:`socket` exports the following elements. - - -Exceptions -^^^^^^^^^^ - -.. exception:: error - - A deprecated alias of :exc:`OSError`. - - .. versionchanged:: 3.3 - Following :pep:`3151`, this class was made an alias of :exc:`OSError`. - - -.. exception:: herror - - A subclass of :exc:`OSError`, this exception is raised for - address-related errors, i.e. for functions that use *h_errno* in the POSIX - C API, including :func:`gethostbyname_ex` and :func:`gethostbyaddr`. - The accompanying value is a pair ``(h_errno, string)`` representing an - error returned by a library call. *h_errno* is a numeric value, while - *string* represents the description of *h_errno*, as returned by the - :c:func:`hstrerror` C function. - - .. versionchanged:: 3.3 - This class was made a subclass of :exc:`OSError`. - -.. exception:: gaierror - - A subclass of :exc:`OSError`, this exception is raised for - address-related errors by :func:`getaddrinfo` and :func:`getnameinfo`. - The accompanying value is a pair ``(error, string)`` representing an error - returned by a library call. *string* represents the description of - *error*, as returned by the :c:func:`gai_strerror` C function. The - numeric *error* value will match one of the :const:`EAI_\*` constants - defined in this module. - - .. versionchanged:: 3.3 - This class was made a subclass of :exc:`OSError`. - -.. exception:: timeout - - A deprecated alias of :exc:`TimeoutError`. - - A subclass of :exc:`OSError`, this exception is raised when a timeout - occurs on a socket which has had timeouts enabled via a prior call to - :meth:`~socket.settimeout` (or implicitly through - :func:`~socket.setdefaulttimeout`). The accompanying value is a string - whose value is currently always "timed out". - - .. versionchanged:: 3.3 - This class was made a subclass of :exc:`OSError`. - - .. versionchanged:: 3.10 - This class was made an alias of :exc:`TimeoutError`. - - -Constants -^^^^^^^^^ - - The AF_* and SOCK_* constants are now :class:`AddressFamily` and - :class:`SocketKind` :class:`.IntEnum` collections. - - .. versionadded:: 3.4 - -.. data:: AF_UNIX - AF_INET - AF_INET6 - - These constants represent the address (and protocol) families, used for the - first argument to :func:`.socket`. If the :const:`AF_UNIX` constant is not - defined then this protocol is unsupported. More constants may be available - depending on the system. - -.. data:: AF_UNSPEC - - :const:`AF_UNSPEC` means that - :func:`getaddrinfo` should return socket addresses for any - address family (either IPv4, IPv6, or any other) that can be used. - -.. data:: SOCK_STREAM - SOCK_DGRAM - SOCK_RAW - SOCK_RDM - SOCK_SEQPACKET - - These constants represent the socket types, used for the second argument to - :func:`.socket`. More constants may be available depending on the system. - (Only :const:`SOCK_STREAM` and :const:`SOCK_DGRAM` appear to be generally - useful.) - -.. data:: SOCK_CLOEXEC - SOCK_NONBLOCK - - These two constants, if defined, can be combined with the socket types and - allow you to set some flags atomically (thus avoiding possible race - conditions and the need for separate calls). - - .. seealso:: - - `Secure File Descriptor Handling `_ - for a more thorough explanation. - - .. availability:: Linux >= 2.6.27. - - .. versionadded:: 3.2 - -.. _socket-unix-constants: - -.. data:: SO_* - SOMAXCONN - MSG_* - SOL_* - SCM_* - IPPROTO_* - IPPORT_* - INADDR_* - IP_* - IPV6_* - EAI_* - AI_* - NI_* - TCP_* - - Many constants of these forms, documented in the Unix documentation on sockets - and/or the IP protocol, are also defined in the socket module. They are - generally used in arguments to the :meth:`setsockopt` and :meth:`getsockopt` - methods of socket objects. In most cases, only those symbols that are defined - in the Unix header files are defined; for a few symbols, default values are - provided. - - .. versionchanged:: 3.6 - ``SO_DOMAIN``, ``SO_PROTOCOL``, ``SO_PEERSEC``, ``SO_PASSSEC``, - ``TCP_USER_TIMEOUT``, ``TCP_CONGESTION`` were added. - - .. versionchanged:: 3.6.5 - On Windows, ``TCP_FASTOPEN``, ``TCP_KEEPCNT`` appear if run-time Windows - supports. - - .. versionchanged:: 3.7 - ``TCP_NOTSENT_LOWAT`` was added. - - On Windows, ``TCP_KEEPIDLE``, ``TCP_KEEPINTVL`` appear if run-time Windows - supports. - - .. versionchanged:: 3.10 - ``IP_RECVTOS`` was added. - Added ``TCP_KEEPALIVE``. On MacOS this constant can be used in the same - way that ``TCP_KEEPIDLE`` is used on Linux. - - .. versionchanged:: 3.11 - Added ``TCP_CONNECTION_INFO``. On MacOS this constant can be used in the - same way that ``TCP_INFO`` is used on Linux and BSD. - -.. data:: AF_CAN - PF_CAN - SOL_CAN_* - CAN_* - - Many constants of these forms, documented in the Linux documentation, are - also defined in the socket module. - - .. availability:: Linux >= 2.6.25, NetBSD >= 8. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.11 - NetBSD support was added. - -.. data:: CAN_BCM - CAN_BCM_* - - CAN_BCM, in the CAN protocol family, is the broadcast manager (BCM) protocol. - Broadcast manager constants, documented in the Linux documentation, are also - defined in the socket module. - - .. availability:: Linux >= 2.6.25. - - .. note:: - The :data:`CAN_BCM_CAN_FD_FRAME` flag is only available on Linux >= 4.8. - - .. versionadded:: 3.4 - -.. data:: CAN_RAW_FD_FRAMES - - Enables CAN FD support in a CAN_RAW socket. This is disabled by default. - This allows your application to send both CAN and CAN FD frames; however, - you must accept both CAN and CAN FD frames when reading from the socket. - - This constant is documented in the Linux documentation. - - .. availability:: Linux >= 3.6. - - .. versionadded:: 3.5 - -.. data:: CAN_RAW_JOIN_FILTERS - - Joins the applied CAN filters such that only CAN frames that match all - given CAN filters are passed to user space. - - This constant is documented in the Linux documentation. - - .. availability:: Linux >= 4.1. - - .. versionadded:: 3.9 - -.. data:: CAN_ISOTP - - CAN_ISOTP, in the CAN protocol family, is the ISO-TP (ISO 15765-2) protocol. - ISO-TP constants, documented in the Linux documentation. - - .. availability:: Linux >= 2.6.25. - - .. versionadded:: 3.7 - -.. data:: CAN_J1939 - - CAN_J1939, in the CAN protocol family, is the SAE J1939 protocol. - J1939 constants, documented in the Linux documentation. - - .. availability:: Linux >= 5.4. - - .. versionadded:: 3.9 - - -.. data:: AF_PACKET - PF_PACKET - PACKET_* - - Many constants of these forms, documented in the Linux documentation, are - also defined in the socket module. - - .. availability:: Linux >= 2.2. - - -.. data:: AF_RDS - PF_RDS - SOL_RDS - RDS_* - - Many constants of these forms, documented in the Linux documentation, are - also defined in the socket module. - - .. availability:: Linux >= 2.6.30. - - .. versionadded:: 3.3 - - -.. data:: SIO_RCVALL - SIO_KEEPALIVE_VALS - SIO_LOOPBACK_FAST_PATH - RCVALL_* - - Constants for Windows' WSAIoctl(). The constants are used as arguments to the - :meth:`~socket.socket.ioctl` method of socket objects. - - .. versionchanged:: 3.6 - ``SIO_LOOPBACK_FAST_PATH`` was added. - - -.. data:: TIPC_* - - TIPC related constants, matching the ones exported by the C socket API. See - the TIPC documentation for more information. - -.. data:: AF_ALG - SOL_ALG - ALG_* - - Constants for Linux Kernel cryptography. - - .. availability:: Linux >= 2.6.38. - - .. versionadded:: 3.6 - - -.. data:: AF_VSOCK - IOCTL_VM_SOCKETS_GET_LOCAL_CID - VMADDR* - SO_VM* - - Constants for Linux host/guest communication. - - .. availability:: Linux >= 4.8. - - .. versionadded:: 3.7 - -.. data:: AF_LINK - - .. availability:: BSD, macOS. - - .. versionadded:: 3.4 - -.. data:: has_ipv6 - - This constant contains a boolean value which indicates if IPv6 is supported on - this platform. - -.. data:: BDADDR_ANY - BDADDR_LOCAL - - These are string constants containing Bluetooth addresses with special - meanings. For example, :const:`BDADDR_ANY` can be used to indicate - any address when specifying the binding socket with - :const:`BTPROTO_RFCOMM`. - -.. data:: HCI_FILTER - HCI_TIME_STAMP - HCI_DATA_DIR - - For use with :const:`BTPROTO_HCI`. :const:`HCI_FILTER` is not - available for NetBSD or DragonFlyBSD. :const:`HCI_TIME_STAMP` and - :const:`HCI_DATA_DIR` are not available for FreeBSD, NetBSD, or - DragonFlyBSD. - -.. data:: AF_QIPCRTR - - Constant for Qualcomm's IPC router protocol, used to communicate with - service providing remote processors. - - .. availability:: Linux >= 4.7. - -.. data:: SCM_CREDS2 - LOCAL_CREDS - LOCAL_CREDS_PERSISTENT - - LOCAL_CREDS and LOCAL_CREDS_PERSISTENT can be used - with SOCK_DGRAM, SOCK_STREAM sockets, equivalent to - Linux/DragonFlyBSD SO_PASSCRED, while LOCAL_CREDS - sends the credentials at first read, LOCAL_CREDS_PERSISTENT - sends for each read, SCM_CREDS2 must be then used for - the latter for the message type. - - .. versionadded:: 3.11 - - .. availability:: FreeBSD. - -.. data:: SO_INCOMING_CPU - - Constant to optimize CPU locality, to be used in conjunction with - :data:`SO_REUSEPORT`. - - .. versionadded:: 3.11 - - .. availability:: Linux >= 3.9 - -Functions -^^^^^^^^^ - -Creating sockets -'''''''''''''''' - -The following functions all create :ref:`socket objects `. - - -.. class:: socket(family=AF_INET, type=SOCK_STREAM, proto=0, fileno=None) - - Create a new socket using the given address family, socket type and protocol - number. The address family should be :const:`AF_INET` (the default), - :const:`AF_INET6`, :const:`AF_UNIX`, :const:`AF_CAN`, :const:`AF_PACKET`, - or :const:`AF_RDS`. The socket type should be :const:`SOCK_STREAM` (the - default), :const:`SOCK_DGRAM`, :const:`SOCK_RAW` or perhaps one of the other - ``SOCK_`` constants. The protocol number is usually zero and may be omitted - or in the case where the address family is :const:`AF_CAN` the protocol - should be one of :const:`CAN_RAW`, :const:`CAN_BCM`, :const:`CAN_ISOTP` or - :const:`CAN_J1939`. - - If *fileno* is specified, the values for *family*, *type*, and *proto* are - auto-detected from the specified file descriptor. Auto-detection can be - overruled by calling the function with explicit *family*, *type*, or *proto* - arguments. This only affects how Python represents e.g. the return value - of :meth:`socket.getpeername` but not the actual OS resource. Unlike - :func:`socket.fromfd`, *fileno* will return the same socket and not a - duplicate. This may help close a detached socket using - :meth:`socket.close()`. - - The newly created socket is :ref:`non-inheritable `. - - .. audit-event:: socket.__new__ self,family,type,protocol socket.socket - - .. versionchanged:: 3.3 - The AF_CAN family was added. - The AF_RDS family was added. - - .. versionchanged:: 3.4 - The CAN_BCM protocol was added. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - .. versionchanged:: 3.7 - The CAN_ISOTP protocol was added. - - .. versionchanged:: 3.7 - When :const:`SOCK_NONBLOCK` or :const:`SOCK_CLOEXEC` - bit flags are applied to *type* they are cleared, and - :attr:`socket.type` will not reflect them. They are still passed - to the underlying system ``socket()`` call. Therefore, - - :: - - sock = socket.socket( - socket.AF_INET, - socket.SOCK_STREAM | socket.SOCK_NONBLOCK) - - will still create a non-blocking socket on OSes that support - ``SOCK_NONBLOCK``, but ``sock.type`` will be set to - ``socket.SOCK_STREAM``. - - .. versionchanged:: 3.9 - The CAN_J1939 protocol was added. - - .. versionchanged:: 3.10 - The IPPROTO_MPTCP protocol was added. - -.. function:: socketpair([family[, type[, proto]]]) - - Build a pair of connected socket objects using the given address family, socket - type, and protocol number. Address family, socket type, and protocol number are - as for the :func:`.socket` function above. The default family is :const:`AF_UNIX` - if defined on the platform; otherwise, the default is :const:`AF_INET`. - - The newly created sockets are :ref:`non-inheritable `. - - .. versionchanged:: 3.2 - The returned socket objects now support the whole socket API, rather - than a subset. - - .. versionchanged:: 3.4 - The returned sockets are now non-inheritable. - - .. versionchanged:: 3.5 - Windows support added. - - -.. function:: create_connection(address, timeout=GLOBAL_DEFAULT, source_address=None, *, all_errors=False) - - Connect to a TCP service listening on the internet *address* (a 2-tuple - ``(host, port)``), and return the socket object. This is a higher-level - function than :meth:`socket.connect`: if *host* is a non-numeric hostname, - it will try to resolve it for both :data:`AF_INET` and :data:`AF_INET6`, - and then try to connect to all possible addresses in turn until a - connection succeeds. This makes it easy to write clients that are - compatible to both IPv4 and IPv6. - - Passing the optional *timeout* parameter will set the timeout on the - socket instance before attempting to connect. If no *timeout* is - supplied, the global default timeout setting returned by - :func:`getdefaulttimeout` is used. - - If supplied, *source_address* must be a 2-tuple ``(host, port)`` for the - socket to bind to as its source address before connecting. If host or port - are '' or 0 respectively the OS default behavior will be used. - - When a connection cannot be created, an exception is raised. By default, - it is the exception from the last address in the list. If *all_errors* - is ``True``, it is an :exc:`ExceptionGroup` containing the errors of all - attempts. - - .. versionchanged:: 3.2 - *source_address* was added. - - .. versionchanged:: 3.11 - *all_errors* was added. - - -.. function:: create_server(address, *, family=AF_INET, backlog=None, reuse_port=False, dualstack_ipv6=False) - - Convenience function which creates a TCP socket bound to *address* (a 2-tuple - ``(host, port)``) and return the socket object. - - *family* should be either :data:`AF_INET` or :data:`AF_INET6`. - *backlog* is the queue size passed to :meth:`socket.listen`; if not specified - , a default reasonable value is chosen. - *reuse_port* dictates whether to set the :data:`SO_REUSEPORT` socket option. - - If *dualstack_ipv6* is true and the platform supports it the socket will - be able to accept both IPv4 and IPv6 connections, else it will raise - :exc:`ValueError`. Most POSIX platforms and Windows are supposed to support - this functionality. - When this functionality is enabled the address returned by - :meth:`socket.getpeername` when an IPv4 connection occurs will be an IPv6 - address represented as an IPv4-mapped IPv6 address. - If *dualstack_ipv6* is false it will explicitly disable this functionality - on platforms that enable it by default (e.g. Linux). - This parameter can be used in conjunction with :func:`has_dualstack_ipv6`: - - :: - - import socket - - addr = ("", 8080) # all interfaces, port 8080 - if socket.has_dualstack_ipv6(): - s = socket.create_server(addr, family=socket.AF_INET6, dualstack_ipv6=True) - else: - s = socket.create_server(addr) - - .. note:: - On POSIX platforms the :data:`SO_REUSEADDR` socket option is set in order to - immediately reuse previous sockets which were bound on the same *address* - and remained in TIME_WAIT state. - - .. versionadded:: 3.8 - -.. function:: has_dualstack_ipv6() - - Return ``True`` if the platform supports creating a TCP socket which can - handle both IPv4 and IPv6 connections. - - .. versionadded:: 3.8 - -.. function:: fromfd(fd, family, type, proto=0) - - Duplicate the file descriptor *fd* (an integer as returned by a file object's - :meth:`~io.IOBase.fileno` method) and build a socket object from the result. Address - family, socket type and protocol number are as for the :func:`.socket` function - above. The file descriptor should refer to a socket, but this is not checked --- - subsequent operations on the object may fail if the file descriptor is invalid. - This function is rarely needed, but can be used to get or set socket options on - a socket passed to a program as standard input or output (such as a server - started by the Unix inet daemon). The socket is assumed to be in blocking mode. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The returned socket is now non-inheritable. - - -.. function:: fromshare(data) - - Instantiate a socket from data obtained from the :meth:`socket.share` - method. The socket is assumed to be in blocking mode. - - .. availability:: Windows. - - .. versionadded:: 3.3 - - -.. data:: SocketType - - This is a Python type object that represents the socket object type. It is the - same as ``type(socket(...))``. - - -Other functions -''''''''''''''' - -The :mod:`socket` module also offers various network-related services: - - -.. function:: close(fd) - - Close a socket file descriptor. This is like :func:`os.close`, but for - sockets. On some platforms (most noticeable Windows) :func:`os.close` - does not work for socket file descriptors. - - .. versionadded:: 3.7 - -.. function:: getaddrinfo(host, port, family=0, type=0, proto=0, flags=0) - - Translate the *host*/*port* argument into a sequence of 5-tuples that contain - all the necessary arguments for creating a socket connected to that service. - *host* is a domain name, a string representation of an IPv4/v6 address - or ``None``. *port* is a string service name such as ``'http'``, a numeric - port number or ``None``. By passing ``None`` as the value of *host* - and *port*, you can pass ``NULL`` to the underlying C API. - - The *family*, *type* and *proto* arguments can be optionally specified - in order to narrow the list of addresses returned. Passing zero as a - value for each of these arguments selects the full range of results. - The *flags* argument can be one or several of the ``AI_*`` constants, - and will influence how results are computed and returned. - For example, :const:`AI_NUMERICHOST` will disable domain name resolution - and will raise an error if *host* is a domain name. - - The function returns a list of 5-tuples with the following structure: - - ``(family, type, proto, canonname, sockaddr)`` - - In these tuples, *family*, *type*, *proto* are all integers and are - meant to be passed to the :func:`.socket` function. *canonname* will be - a string representing the canonical name of the *host* if - :const:`AI_CANONNAME` is part of the *flags* argument; else *canonname* - will be empty. *sockaddr* is a tuple describing a socket address, whose - format depends on the returned *family* (a ``(address, port)`` 2-tuple for - :const:`AF_INET`, a ``(address, port, flowinfo, scope_id)`` 4-tuple for - :const:`AF_INET6`), and is meant to be passed to the :meth:`socket.connect` - method. - - .. audit-event:: socket.getaddrinfo host,port,family,type,protocol socket.getaddrinfo - - The following example fetches address information for a hypothetical TCP - connection to ``example.org`` on port 80 (results may differ on your - system if IPv6 isn't enabled):: - - >>> socket.getaddrinfo("example.org", 80, proto=socket.IPPROTO_TCP) - [(socket.AF_INET6, socket.SOCK_STREAM, - 6, '', ('2606:2800:220:1:248:1893:25c8:1946', 80, 0, 0)), - (socket.AF_INET, socket.SOCK_STREAM, - 6, '', ('93.184.216.34', 80))] - - .. versionchanged:: 3.2 - parameters can now be passed using keyword arguments. - - .. versionchanged:: 3.7 - for IPv6 multicast addresses, string representing an address will not - contain ``%scope_id`` part. - -.. function:: getfqdn([name]) - - Return a fully qualified domain name for *name*. If *name* is omitted or empty, - it is interpreted as the local host. To find the fully qualified name, the - hostname returned by :func:`gethostbyaddr` is checked, followed by aliases for the - host, if available. The first name which includes a period is selected. In - case no fully qualified domain name is available and *name* was provided, - it is returned unchanged. If *name* was empty or equal to ``'0.0.0.0'``, - the hostname from :func:`gethostname` is returned. - - -.. function:: gethostbyname(hostname) - - Translate a host name to IPv4 address format. The IPv4 address is returned as a - string, such as ``'100.50.200.5'``. If the host name is an IPv4 address itself - it is returned unchanged. See :func:`gethostbyname_ex` for a more complete - interface. :func:`gethostbyname` does not support IPv6 name resolution, and - :func:`getaddrinfo` should be used instead for IPv4/v6 dual stack support. - - .. audit-event:: socket.gethostbyname hostname socket.gethostbyname - - .. availability:: not WASI. - - -.. function:: gethostbyname_ex(hostname) - - Translate a host name to IPv4 address format, extended interface. Return a - 3-tuple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the host's - primary host name, *aliaslist* is a (possibly - empty) list of alternative host names for the same address, and *ipaddrlist* is - a list of IPv4 addresses for the same interface on the same host (often but not - always a single address). :func:`gethostbyname_ex` does not support IPv6 name - resolution, and :func:`getaddrinfo` should be used instead for IPv4/v6 dual - stack support. - - .. audit-event:: socket.gethostbyname hostname socket.gethostbyname_ex - - .. availability:: not WASI. - - -.. function:: gethostname() - - Return a string containing the hostname of the machine where the Python - interpreter is currently executing. - - .. audit-event:: socket.gethostname "" socket.gethostname - - Note: :func:`gethostname` doesn't always return the fully qualified domain - name; use :func:`getfqdn` for that. - - .. availability:: not WASI. - - -.. function:: gethostbyaddr(ip_address) - - Return a 3-tuple ``(hostname, aliaslist, ipaddrlist)`` where *hostname* is the - primary host name responding to the given *ip_address*, *aliaslist* is a - (possibly empty) list of alternative host names for the same address, and - *ipaddrlist* is a list of IPv4/v6 addresses for the same interface on the same - host (most likely containing only a single address). To find the fully qualified - domain name, use the function :func:`getfqdn`. :func:`gethostbyaddr` supports - both IPv4 and IPv6. - - .. audit-event:: socket.gethostbyaddr ip_address socket.gethostbyaddr - - .. availability:: not WASI. - - -.. function:: getnameinfo(sockaddr, flags) - - Translate a socket address *sockaddr* into a 2-tuple ``(host, port)``. Depending - on the settings of *flags*, the result can contain a fully qualified domain name - or numeric address representation in *host*. Similarly, *port* can contain a - string port name or a numeric port number. - - For IPv6 addresses, ``%scope_id`` is appended to the host part if *sockaddr* - contains meaningful *scope_id*. Usually this happens for multicast addresses. - - For more information about *flags* you can consult :manpage:`getnameinfo(3)`. - - .. audit-event:: socket.getnameinfo sockaddr socket.getnameinfo - - .. availability:: not WASI. - - -.. function:: getprotobyname(protocolname) - - Translate an internet protocol name (for example, ``'icmp'``) to a constant - suitable for passing as the (optional) third argument to the :func:`.socket` - function. This is usually only needed for sockets opened in "raw" mode - (:const:`SOCK_RAW`); for the normal socket modes, the correct protocol is chosen - automatically if the protocol is omitted or zero. - - .. availability:: not WASI. - - -.. function:: getservbyname(servicename[, protocolname]) - - Translate an internet service name and protocol name to a port number for that - service. The optional protocol name, if given, should be ``'tcp'`` or - ``'udp'``, otherwise any protocol will match. - - .. audit-event:: socket.getservbyname servicename,protocolname socket.getservbyname - - .. availability:: not WASI. - - -.. function:: getservbyport(port[, protocolname]) - - Translate an internet port number and protocol name to a service name for that - service. The optional protocol name, if given, should be ``'tcp'`` or - ``'udp'``, otherwise any protocol will match. - - .. audit-event:: socket.getservbyport port,protocolname socket.getservbyport - - .. availability:: not WASI. - - -.. function:: ntohl(x) - - Convert 32-bit positive integers from network to host byte order. On machines - where the host byte order is the same as network byte order, this is a no-op; - otherwise, it performs a 4-byte swap operation. - - -.. function:: ntohs(x) - - Convert 16-bit positive integers from network to host byte order. On machines - where the host byte order is the same as network byte order, this is a no-op; - otherwise, it performs a 2-byte swap operation. - - .. versionchanged:: 3.10 - Raises :exc:`OverflowError` if *x* does not fit in a 16-bit unsigned - integer. - - -.. function:: htonl(x) - - Convert 32-bit positive integers from host to network byte order. On machines - where the host byte order is the same as network byte order, this is a no-op; - otherwise, it performs a 4-byte swap operation. - - -.. function:: htons(x) - - Convert 16-bit positive integers from host to network byte order. On machines - where the host byte order is the same as network byte order, this is a no-op; - otherwise, it performs a 2-byte swap operation. - - .. versionchanged:: 3.10 - Raises :exc:`OverflowError` if *x* does not fit in a 16-bit unsigned - integer. - - -.. function:: inet_aton(ip_string) - - Convert an IPv4 address from dotted-quad string format (for example, - '123.45.67.89') to 32-bit packed binary format, as a bytes object four characters in - length. This is useful when conversing with a program that uses the standard C - library and needs objects of type :c:struct:`in_addr`, which is the C type - for the 32-bit packed binary this function returns. - - :func:`inet_aton` also accepts strings with less than three dots; see the - Unix manual page :manpage:`inet(3)` for details. - - If the IPv4 address string passed to this function is invalid, - :exc:`OSError` will be raised. Note that exactly what is valid depends on - the underlying C implementation of :c:func:`inet_aton`. - - :func:`inet_aton` does not support IPv6, and :func:`inet_pton` should be used - instead for IPv4/v6 dual stack support. - - -.. function:: inet_ntoa(packed_ip) - - Convert a 32-bit packed IPv4 address (a :term:`bytes-like object` four - bytes in length) to its standard dotted-quad string representation (for example, - '123.45.67.89'). This is useful when conversing with a program that uses the - standard C library and needs objects of type :c:struct:`in_addr`, which - is the C type for the 32-bit packed binary data this function takes as an - argument. - - If the byte sequence passed to this function is not exactly 4 bytes in - length, :exc:`OSError` will be raised. :func:`inet_ntoa` does not - support IPv6, and :func:`inet_ntop` should be used instead for IPv4/v6 dual - stack support. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. function:: inet_pton(address_family, ip_string) - - Convert an IP address from its family-specific string format to a packed, - binary format. :func:`inet_pton` is useful when a library or network protocol - calls for an object of type :c:struct:`in_addr` (similar to - :func:`inet_aton`) or :c:struct:`in6_addr`. - - Supported values for *address_family* are currently :const:`AF_INET` and - :const:`AF_INET6`. If the IP address string *ip_string* is invalid, - :exc:`OSError` will be raised. Note that exactly what is valid depends on - both the value of *address_family* and the underlying implementation of - :c:func:`inet_pton`. - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.4 - Windows support added - - -.. function:: inet_ntop(address_family, packed_ip) - - Convert a packed IP address (a :term:`bytes-like object` of some number of - bytes) to its standard, family-specific string representation (for - example, ``'7.10.0.5'`` or ``'5aef:2b::8'``). - :func:`inet_ntop` is useful when a library or network protocol returns an - object of type :c:struct:`in_addr` (similar to :func:`inet_ntoa`) or - :c:struct:`in6_addr`. - - Supported values for *address_family* are currently :const:`AF_INET` and - :const:`AF_INET6`. If the bytes object *packed_ip* is not the correct - length for the specified address family, :exc:`ValueError` will be raised. - :exc:`OSError` is raised for errors from the call to :func:`inet_ntop`. - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.4 - Windows support added - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - -.. - XXX: Are sendmsg(), recvmsg() and CMSG_*() available on any - non-Unix platforms? The old (obsolete?) 4.2BSD form of the - interface, in which struct msghdr has no msg_control or - msg_controllen members, is not currently supported. - -.. function:: CMSG_LEN(length) - - Return the total length, without trailing padding, of an ancillary - data item with associated data of the given *length*. This value - can often be used as the buffer size for :meth:`~socket.recvmsg` to - receive a single item of ancillary data, but :rfc:`3542` requires - portable applications to use :func:`CMSG_SPACE` and thus include - space for padding, even when the item will be the last in the - buffer. Raises :exc:`OverflowError` if *length* is outside the - permissible range of values. - - .. availability:: Unix, not Emscripten, not WASI. - - Most Unix platforms. - - .. versionadded:: 3.3 - - -.. function:: CMSG_SPACE(length) - - Return the buffer size needed for :meth:`~socket.recvmsg` to - receive an ancillary data item with associated data of the given - *length*, along with any trailing padding. The buffer space needed - to receive multiple items is the sum of the :func:`CMSG_SPACE` - values for their associated data lengths. Raises - :exc:`OverflowError` if *length* is outside the permissible range - of values. - - Note that some systems might support ancillary data without - providing this function. Also note that setting the buffer size - using the results of this function may not precisely limit the - amount of ancillary data that can be received, since additional - data may be able to fit into the padding area. - - .. availability:: Unix, not Emscripten, not WASI. - - most Unix platforms. - - .. versionadded:: 3.3 - - -.. function:: getdefaulttimeout() - - Return the default timeout in seconds (float) for new socket objects. A value - of ``None`` indicates that new socket objects have no timeout. When the socket - module is first imported, the default is ``None``. - - -.. function:: setdefaulttimeout(timeout) - - Set the default timeout in seconds (float) for new socket objects. When - the socket module is first imported, the default is ``None``. See - :meth:`~socket.settimeout` for possible values and their respective - meanings. - - -.. function:: sethostname(name) - - Set the machine's hostname to *name*. This will raise an - :exc:`OSError` if you don't have enough rights. - - .. audit-event:: socket.sethostname name socket.sethostname - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: if_nameindex() - - Return a list of network interface information - (index int, name string) tuples. - :exc:`OSError` if the system call fails. - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.8 - Windows support was added. - - .. note:: - - On Windows network interfaces have different names in different contexts - (all names are examples): - - * UUID: ``{FB605B73-AAC2-49A6-9A2F-25416AEA0573}`` - * name: ``ethernet_32770`` - * friendly name: ``vEthernet (nat)`` - * description: ``Hyper-V Virtual Ethernet Adapter`` - - This function returns names of the second form from the list, ``ethernet_32770`` - in this example case. - - -.. function:: if_nametoindex(if_name) - - Return a network interface index number corresponding to an - interface name. - :exc:`OSError` if no interface with the given name exists. - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.8 - Windows support was added. - - .. seealso:: - "Interface name" is a name as documented in :func:`if_nameindex`. - - -.. function:: if_indextoname(if_index) - - Return a network interface name corresponding to an - interface index number. - :exc:`OSError` if no interface with the given index exists. - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.8 - Windows support was added. - - .. seealso:: - "Interface name" is a name as documented in :func:`if_nameindex`. - - -.. function:: send_fds(sock, buffers, fds[, flags[, address]]) - - Send the list of file descriptors *fds* over an :const:`AF_UNIX` socket *sock*. - The *fds* parameter is a sequence of file descriptors. - Consult :meth:`sendmsg` for the documentation of these parameters. - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - Unix platforms supporting :meth:`~socket.sendmsg` - and :const:`SCM_RIGHTS` mechanism. - - .. versionadded:: 3.9 - - -.. function:: recv_fds(sock, bufsize, maxfds[, flags]) - - Receive up to *maxfds* file descriptors from an :const:`AF_UNIX` socket *sock*. - Return ``(msg, list(fds), flags, addr)``. - Consult :meth:`recvmsg` for the documentation of these parameters. - - .. availability:: Unix, Windows, not Emscripten, not WASI. - - Unix platforms supporting :meth:`~socket.sendmsg` - and :const:`SCM_RIGHTS` mechanism. - - .. versionadded:: 3.9 - - .. note:: - - Any truncated integers at the end of the list of file descriptors. - - -.. _socket-objects: - -Socket Objects --------------- - -Socket objects have the following methods. Except for -:meth:`~socket.makefile`, these correspond to Unix system calls applicable -to sockets. - -.. versionchanged:: 3.2 - Support for the :term:`context manager` protocol was added. Exiting the - context manager is equivalent to calling :meth:`~socket.close`. - - -.. method:: socket.accept() - - Accept a connection. The socket must be bound to an address and listening for - connections. The return value is a pair ``(conn, address)`` where *conn* is a - *new* socket object usable to send and receive data on the connection, and - *address* is the address bound to the socket on the other end of the connection. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The socket is now non-inheritable. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.bind(address) - - Bind the socket to *address*. The socket must not already be bound. (The format - of *address* depends on the address family --- see above.) - - .. audit-event:: socket.bind self,address socket.socket.bind - - .. availability:: not WASI. - - -.. method:: socket.close() - - Mark the socket closed. The underlying system resource (e.g. a file - descriptor) is also closed when all file objects from :meth:`makefile()` - are closed. Once that happens, all future operations on the socket - object will fail. The remote end will receive no more data (after - queued data is flushed). - - Sockets are automatically closed when they are garbage-collected, but - it is recommended to :meth:`close` them explicitly, or to use a - :keyword:`with` statement around them. - - .. versionchanged:: 3.6 - :exc:`OSError` is now raised if an error occurs when the underlying - :c:func:`close` call is made. - - .. note:: - - :meth:`close()` releases the resource associated with a connection but - does not necessarily close the connection immediately. If you want - to close the connection in a timely fashion, call :meth:`shutdown()` - before :meth:`close()`. - - -.. method:: socket.connect(address) - - Connect to a remote socket at *address*. (The format of *address* depends on the - address family --- see above.) - - If the connection is interrupted by a signal, the method waits until the - connection completes, or raise a :exc:`TimeoutError` on timeout, if the - signal handler doesn't raise an exception and the socket is blocking or has - a timeout. For non-blocking sockets, the method raises an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal (or the exception raised by the signal handler). - - .. audit-event:: socket.connect self,address socket.socket.connect - - .. versionchanged:: 3.5 - The method now waits until the connection completes instead of raising an - :exc:`InterruptedError` exception if the connection is interrupted by a - signal, the signal handler doesn't raise an exception and the socket is - blocking or has a timeout (see the :pep:`475` for the rationale). - - .. availability:: not WASI. - - -.. method:: socket.connect_ex(address) - - Like ``connect(address)``, but return an error indicator instead of raising an - exception for errors returned by the C-level :c:func:`connect` call (other - problems, such as "host not found," can still raise exceptions). The error - indicator is ``0`` if the operation succeeded, otherwise the value of the - :c:data:`errno` variable. This is useful to support, for example, asynchronous - connects. - - .. audit-event:: socket.connect self,address socket.socket.connect_ex - - .. availability:: not WASI. - -.. method:: socket.detach() - - Put the socket object into closed state without actually closing the - underlying file descriptor. The file descriptor is returned, and can - be reused for other purposes. - - .. versionadded:: 3.2 - - -.. method:: socket.dup() - - Duplicate the socket. - - The newly created socket is :ref:`non-inheritable `. - - .. versionchanged:: 3.4 - The socket is now non-inheritable. - - .. availability:: not WASI. - - -.. method:: socket.fileno() - - Return the socket's file descriptor (a small integer), or -1 on failure. This - is useful with :func:`select.select`. - - Under Windows the small integer returned by this method cannot be used where a - file descriptor can be used (such as :func:`os.fdopen`). Unix does not have - this limitation. - -.. method:: socket.get_inheritable() - - Get the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle: ``True`` if the socket can be inherited in - child processes, ``False`` if it cannot. - - .. versionadded:: 3.4 - - -.. method:: socket.getpeername() - - Return the remote address to which the socket is connected. This is useful to - find out the port number of a remote IPv4/v6 socket, for instance. (The format - of the address returned depends on the address family --- see above.) On some - systems this function is not supported. - - -.. method:: socket.getsockname() - - Return the socket's own address. This is useful to find out the port number of - an IPv4/v6 socket, for instance. (The format of the address returned depends on - the address family --- see above.) - - -.. method:: socket.getsockopt(level, optname[, buflen]) - - Return the value of the given socket option (see the Unix man page - :manpage:`getsockopt(2)`). The needed symbolic constants (:const:`SO_\*` etc.) - are defined in this module. If *buflen* is absent, an integer option is assumed - and its integer value is returned by the function. If *buflen* is present, it - specifies the maximum length of the buffer used to receive the option in, and - this buffer is returned as a bytes object. It is up to the caller to decode the - contents of the buffer (see the optional built-in module :mod:`struct` for a way - to decode C structures encoded as byte strings). - - .. availability:: not WASI. - - -.. method:: socket.getblocking() - - Return ``True`` if socket is in blocking mode, ``False`` if in - non-blocking. - - This is equivalent to checking ``socket.gettimeout() != 0``. - - .. versionadded:: 3.7 - - -.. method:: socket.gettimeout() - - Return the timeout in seconds (float) associated with socket operations, - or ``None`` if no timeout is set. This reflects the last call to - :meth:`setblocking` or :meth:`settimeout`. - - -.. method:: socket.ioctl(control, option) - - :platform: Windows - - The :meth:`ioctl` method is a limited interface to the WSAIoctl system - interface. Please refer to the `Win32 documentation - `_ for more - information. - - On other platforms, the generic :func:`fcntl.fcntl` and :func:`fcntl.ioctl` - functions may be used; they accept a socket object as their first argument. - - Currently only the following control codes are supported: - ``SIO_RCVALL``, ``SIO_KEEPALIVE_VALS``, and ``SIO_LOOPBACK_FAST_PATH``. - - .. versionchanged:: 3.6 - ``SIO_LOOPBACK_FAST_PATH`` was added. - -.. method:: socket.listen([backlog]) - - Enable a server to accept connections. If *backlog* is specified, it must - be at least 0 (if it is lower, it is set to 0); it specifies the number of - unaccepted connections that the system will allow before refusing new - connections. If not specified, a default reasonable value is chosen. - - .. availability:: not WASI. - - .. versionchanged:: 3.5 - The *backlog* parameter is now optional. - - -.. method:: socket.makefile(mode='r', buffering=None, *, encoding=None, \ - errors=None, newline=None) - - .. index:: single: I/O control; buffering - - Return a :term:`file object` associated with the socket. The exact returned - type depends on the arguments given to :meth:`makefile`. These arguments are - interpreted the same way as by the built-in :func:`open` function, except - the only supported *mode* values are ``'r'`` (default), ``'w'`` and ``'b'``. - - The socket must be in blocking mode; it can have a timeout, but the file - object's internal buffer may end up in an inconsistent state if a timeout - occurs. - - Closing the file object returned by :meth:`makefile` won't close the - original socket unless all other file objects have been closed and - :meth:`socket.close` has been called on the socket object. - - .. note:: - - On Windows, the file-like object created by :meth:`makefile` cannot be - used where a file object with a file descriptor is expected, such as the - stream arguments of :meth:`subprocess.Popen`. - - -.. method:: socket.recv(bufsize[, flags]) - - Receive data from the socket. The return value is a bytes object representing the - data received. The maximum amount of data to be received at once is specified - by *bufsize*. See the Unix manual page :manpage:`recv(2)` for the meaning of - the optional argument *flags*; it defaults to zero. - - .. note:: - - For best match with hardware and network realities, the value of *bufsize* - should be a relatively small power of 2, for example, 4096. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvfrom(bufsize[, flags]) - - Receive data from the socket. The return value is a pair ``(bytes, address)`` - where *bytes* is a bytes object representing the data received and *address* is the - address of the socket sending the data. See the Unix manual page - :manpage:`recv(2)` for the meaning of the optional argument *flags*; it defaults - to zero. (The format of *address* depends on the address family --- see above.) - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - .. versionchanged:: 3.7 - For multicast IPv6 address, first item of *address* does not contain - ``%scope_id`` part anymore. In order to get full IPv6 address use - :func:`getnameinfo`. - -.. method:: socket.recvmsg(bufsize[, ancbufsize[, flags]]) - - Receive normal data (up to *bufsize* bytes) and ancillary data from - the socket. The *ancbufsize* argument sets the size in bytes of - the internal buffer used to receive the ancillary data; it defaults - to 0, meaning that no ancillary data will be received. Appropriate - buffer sizes for ancillary data can be calculated using - :func:`CMSG_SPACE` or :func:`CMSG_LEN`, and items which do not fit - into the buffer might be truncated or discarded. The *flags* - argument defaults to 0 and has the same meaning as for - :meth:`recv`. - - The return value is a 4-tuple: ``(data, ancdata, msg_flags, - address)``. The *data* item is a :class:`bytes` object holding the - non-ancillary data received. The *ancdata* item is a list of zero - or more tuples ``(cmsg_level, cmsg_type, cmsg_data)`` representing - the ancillary data (control messages) received: *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - :class:`bytes` object holding the associated data. The *msg_flags* - item is the bitwise OR of various flags indicating conditions on - the received message; see your system documentation for details. - If the receiving socket is unconnected, *address* is the address of - the sending socket, if available; otherwise, its value is - unspecified. - - On some systems, :meth:`sendmsg` and :meth:`recvmsg` can be used to - pass file descriptors between processes over an :const:`AF_UNIX` - socket. When this facility is used (it is often restricted to - :const:`SOCK_STREAM` sockets), :meth:`recvmsg` will return, in its - ancillary data, items of the form ``(socket.SOL_SOCKET, - socket.SCM_RIGHTS, fds)``, where *fds* is a :class:`bytes` object - representing the new file descriptors as a binary array of the - native C :c:expr:`int` type. If :meth:`recvmsg` raises an - exception after the system call returns, it will first attempt to - close any file descriptors received via this mechanism. - - Some systems do not indicate the truncated length of ancillary data - items which have been only partially received. If an item appears - to extend beyond the end of the buffer, :meth:`recvmsg` will issue - a :exc:`RuntimeWarning`, and will return the part of it which is - inside the buffer provided it has not been truncated before the - start of its associated data. - - On systems which support the :const:`SCM_RIGHTS` mechanism, the - following function will receive up to *maxfds* file descriptors, - returning the message data and a list containing the descriptors - (while ignoring unexpected conditions such as unrelated control - messages being received). See also :meth:`sendmsg`. :: - - import socket, array - - def recv_fds(sock, msglen, maxfds): - fds = array.array("i") # Array of ints - msg, ancdata, flags, addr = sock.recvmsg(msglen, socket.CMSG_LEN(maxfds * fds.itemsize)) - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level == socket.SOL_SOCKET and cmsg_type == socket.SCM_RIGHTS: - # Append data, ignoring any truncated integers at the end. - fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - return msg, list(fds) - - .. availability:: Unix. - - Most Unix platforms. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.recvmsg_into(buffers[, ancbufsize[, flags]]) - - Receive normal data and ancillary data from the socket, behaving as - :meth:`recvmsg` would, but scatter the non-ancillary data into a - series of buffers instead of returning a new bytes object. The - *buffers* argument must be an iterable of objects that export - writable buffers (e.g. :class:`bytearray` objects); these will be - filled with successive chunks of the non-ancillary data until it - has all been written or there are no more buffers. The operating - system may set a limit (:func:`~os.sysconf` value ``SC_IOV_MAX``) - on the number of buffers that can be used. The *ancbufsize* and - *flags* arguments have the same meaning as for :meth:`recvmsg`. - - The return value is a 4-tuple: ``(nbytes, ancdata, msg_flags, - address)``, where *nbytes* is the total number of bytes of - non-ancillary data written into the buffers, and *ancdata*, - *msg_flags* and *address* are the same as for :meth:`recvmsg`. - - Example:: - - >>> import socket - >>> s1, s2 = socket.socketpair() - >>> b1 = bytearray(b'----') - >>> b2 = bytearray(b'0123456789') - >>> b3 = bytearray(b'--------------') - >>> s1.send(b'Mary had a little lamb') - 22 - >>> s2.recvmsg_into([b1, memoryview(b2)[2:9], b3]) - (22, [], 0, None) - >>> [b1, b2, b3] - [bytearray(b'Mary'), bytearray(b'01 had a 9'), bytearray(b'little lamb---')] - - .. availability:: Unix. - - Most Unix platforms. - - .. versionadded:: 3.3 - - -.. method:: socket.recvfrom_into(buffer[, nbytes[, flags]]) - - Receive data from the socket, writing it into *buffer* instead of creating a - new bytestring. The return value is a pair ``(nbytes, address)`` where *nbytes* is - the number of bytes received and *address* is the address of the socket sending - the data. See the Unix manual page :manpage:`recv(2)` for the meaning of the - optional argument *flags*; it defaults to zero. (The format of *address* - depends on the address family --- see above.) - - -.. method:: socket.recv_into(buffer[, nbytes[, flags]]) - - Receive up to *nbytes* bytes from the socket, storing the data into a buffer - rather than creating a new bytestring. If *nbytes* is not specified (or 0), - receive up to the size available in the given buffer. Returns the number of - bytes received. See the Unix manual page :manpage:`recv(2)` for the meaning - of the optional argument *flags*; it defaults to zero. - - -.. method:: socket.send(bytes[, flags]) - - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Returns the number of bytes sent. Applications are responsible for checking that - all data has been sent; if only some of the data was transmitted, the - application needs to attempt delivery of the remaining data. For further - information on this topic, consult the :ref:`socket-howto`. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendall(bytes[, flags]) - - Send data to the socket. The socket must be connected to a remote socket. The - optional *flags* argument has the same meaning as for :meth:`recv` above. - Unlike :meth:`send`, this method continues to send data from *bytes* until - either all data has been sent or an error occurs. ``None`` is returned on - success. On error, an exception is raised, and there is no way to determine how - much data, if any, was successfully sent. - - .. versionchanged:: 3.5 - The socket timeout is no longer reset each time data is sent successfully. - The socket timeout is now the maximum total duration to send all data. - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendto(bytes, address) - socket.sendto(bytes, flags, address) - - Send data to the socket. The socket should not be connected to a remote socket, - since the destination socket is specified by *address*. The optional *flags* - argument has the same meaning as for :meth:`recv` above. Return the number of - bytes sent. (The format of *address* depends on the address family --- see - above.) - - .. audit-event:: socket.sendto self,address socket.socket.sendto - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - - -.. method:: socket.sendmsg(buffers[, ancdata[, flags[, address]]]) - - Send normal and ancillary data to the socket, gathering the - non-ancillary data from a series of buffers and concatenating it - into a single message. The *buffers* argument specifies the - non-ancillary data as an iterable of - :term:`bytes-like objects ` - (e.g. :class:`bytes` objects); the operating system may set a limit - (:func:`~os.sysconf` value ``SC_IOV_MAX``) on the number of buffers - that can be used. The *ancdata* argument specifies the ancillary - data (control messages) as an iterable of zero or more tuples - ``(cmsg_level, cmsg_type, cmsg_data)``, where *cmsg_level* and - *cmsg_type* are integers specifying the protocol level and - protocol-specific type respectively, and *cmsg_data* is a - bytes-like object holding the associated data. Note that - some systems (in particular, systems without :func:`CMSG_SPACE`) - might support sending only one control message per call. The - *flags* argument defaults to 0 and has the same meaning as for - :meth:`send`. If *address* is supplied and not ``None``, it sets a - destination address for the message. The return value is the - number of bytes of non-ancillary data sent. - - The following function sends the list of file descriptors *fds* - over an :const:`AF_UNIX` socket, on systems which support the - :const:`SCM_RIGHTS` mechanism. See also :meth:`recvmsg`. :: - - import socket, array - - def send_fds(sock, msg, fds): - return sock.sendmsg([msg], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.array("i", fds))]) - - .. availability:: Unix, not WASI. - - Most Unix platforms. - - .. audit-event:: socket.sendmsg self,address socket.socket.sendmsg - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - If the system call is interrupted and the signal handler does not raise - an exception, the method now retries the system call instead of raising - an :exc:`InterruptedError` exception (see :pep:`475` for the rationale). - -.. method:: socket.sendmsg_afalg([msg], *, op[, iv[, assoclen[, flags]]]) - - Specialized version of :meth:`~socket.sendmsg` for :const:`AF_ALG` socket. - Set mode, IV, AEAD associated data length and flags for :const:`AF_ALG` socket. - - .. availability:: Linux >= 2.6.38. - - .. versionadded:: 3.6 - -.. method:: socket.sendfile(file, offset=0, count=None) - - Send a file until EOF is reached by using high-performance - :mod:`os.sendfile` and return the total number of bytes which were sent. - *file* must be a regular file object opened in binary mode. If - :mod:`os.sendfile` is not available (e.g. Windows) or *file* is not a - regular file :meth:`send` will be used instead. *offset* tells from where to - start reading the file. If specified, *count* is the total number of bytes - to transmit as opposed to sending the file until EOF is reached. File - position is updated on return or also in case of error in which case - :meth:`file.tell() ` can be used to figure out the number of - bytes which were sent. The socket must be of :const:`SOCK_STREAM` type. - Non-blocking sockets are not supported. - - .. versionadded:: 3.5 - -.. method:: socket.set_inheritable(inheritable) - - Set the :ref:`inheritable flag ` of the socket's file - descriptor or socket's handle. - - .. versionadded:: 3.4 - - -.. method:: socket.setblocking(flag) - - Set blocking or non-blocking mode of the socket: if *flag* is false, the - socket is set to non-blocking, else to blocking mode. - - This method is a shorthand for certain :meth:`~socket.settimeout` calls: - - * ``sock.setblocking(True)`` is equivalent to ``sock.settimeout(None)`` - - * ``sock.setblocking(False)`` is equivalent to ``sock.settimeout(0.0)`` - - .. versionchanged:: 3.7 - The method no longer applies :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. - - -.. method:: socket.settimeout(value) - - Set a timeout on blocking socket operations. The *value* argument can be a - nonnegative floating point number expressing seconds, or ``None``. - If a non-zero value is given, subsequent socket operations will raise a - :exc:`timeout` exception if the timeout period *value* has elapsed before - the operation has completed. If zero is given, the socket is put in - non-blocking mode. If ``None`` is given, the socket is put in blocking mode. - - For further information, please consult the :ref:`notes on socket timeouts `. - - .. versionchanged:: 3.7 - The method no longer toggles :const:`SOCK_NONBLOCK` flag on - :attr:`socket.type`. - - -.. method:: socket.setsockopt(level, optname, value: int) -.. method:: socket.setsockopt(level, optname, value: buffer) - :noindex: -.. method:: socket.setsockopt(level, optname, None, optlen: int) - :noindex: - - .. index:: pair: module; struct - - Set the value of the given socket option (see the Unix manual page - :manpage:`setsockopt(2)`). The needed symbolic constants are defined in the - :mod:`socket` module (:const:`SO_\*` etc.). The value can be an integer, - ``None`` or a :term:`bytes-like object` representing a buffer. In the later - case it is up to the caller to ensure that the bytestring contains the - proper bits (see the optional built-in module :mod:`struct` for a way to - encode C structures as bytestrings). When *value* is set to ``None``, - *optlen* argument is required. It's equivalent to call :c:func:`setsockopt` C - function with ``optval=NULL`` and ``optlen=optlen``. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - - .. versionchanged:: 3.6 - setsockopt(level, optname, None, optlen: int) form added. - - .. availability:: not WASI. - - -.. method:: socket.shutdown(how) - - Shut down one or both halves of the connection. If *how* is :const:`SHUT_RD`, - further receives are disallowed. If *how* is :const:`SHUT_WR`, further sends - are disallowed. If *how* is :const:`SHUT_RDWR`, further sends and receives are - disallowed. - - .. availability:: not WASI. - - -.. method:: socket.share(process_id) - - Duplicate a socket and prepare it for sharing with a target process. The - target process must be provided with *process_id*. The resulting bytes object - can then be passed to the target process using some form of interprocess - communication and the socket can be recreated there using :func:`fromshare`. - Once this method has been called, it is safe to close the socket since - the operating system has already duplicated it for the target process. - - .. availability:: Windows. - - .. versionadded:: 3.3 - - -Note that there are no methods :meth:`read` or :meth:`write`; use -:meth:`~socket.recv` and :meth:`~socket.send` without *flags* argument instead. - -Socket objects also have these (read-only) attributes that correspond to the -values given to the :class:`~socket.socket` constructor. - - -.. attribute:: socket.family - - The socket family. - - -.. attribute:: socket.type - - The socket type. - - -.. attribute:: socket.proto - - The socket protocol. - - - -.. _socket-timeouts: - -Notes on socket timeouts ------------------------- - -A socket object can be in one of three modes: blocking, non-blocking, or -timeout. Sockets are by default always created in blocking mode, but this -can be changed by calling :func:`setdefaulttimeout`. - -* In *blocking mode*, operations block until complete or the system returns - an error (such as connection timed out). - -* In *non-blocking mode*, operations fail (with an error that is unfortunately - system-dependent) if they cannot be completed immediately: functions from the - :mod:`select` module can be used to know when and whether a socket is available - for reading or writing. - -* In *timeout mode*, operations fail if they cannot be completed within the - timeout specified for the socket (they raise a :exc:`timeout` exception) - or if the system returns an error. - -.. note:: - At the operating system level, sockets in *timeout mode* are internally set - in non-blocking mode. Also, the blocking and timeout modes are shared between - file descriptors and socket objects that refer to the same network endpoint. - This implementation detail can have visible consequences if e.g. you decide - to use the :meth:`~socket.fileno()` of a socket. - -Timeouts and the ``connect`` method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :meth:`~socket.connect` operation is also subject to the timeout -setting, and in general it is recommended to call :meth:`~socket.settimeout` -before calling :meth:`~socket.connect` or pass a timeout parameter to -:meth:`create_connection`. However, the system network stack may also -return a connection timeout error of its own regardless of any Python socket -timeout setting. - -Timeouts and the ``accept`` method -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If :func:`getdefaulttimeout` is not :const:`None`, sockets returned by -the :meth:`~socket.accept` method inherit that timeout. Otherwise, the -behaviour depends on settings of the listening socket: - -* if the listening socket is in *blocking mode* or in *timeout mode*, - the socket returned by :meth:`~socket.accept` is in *blocking mode*; - -* if the listening socket is in *non-blocking mode*, whether the socket - returned by :meth:`~socket.accept` is in blocking or non-blocking mode - is operating system-dependent. If you want to ensure cross-platform - behaviour, it is recommended you manually override this setting. - - -.. _socket-example: - -Example -------- - -Here are four minimal example programs using the TCP/IP protocol: a server that -echoes all data that it receives back (servicing only one client), and a client -using it. Note that a server must perform the sequence :func:`.socket`, -:meth:`~socket.bind`, :meth:`~socket.listen`, :meth:`~socket.accept` (possibly -repeating the :meth:`~socket.accept` to service more than one client), while a -client only needs the sequence :func:`.socket`, :meth:`~socket.connect`. Also -note that the server does not :meth:`~socket.sendall`/:meth:`~socket.recv` on -the socket it is listening on but on the new socket returned by -:meth:`~socket.accept`. - -The first two examples support IPv4 only. :: - - # Echo server program - import socket - - HOST = '' # Symbolic name meaning all available interfaces - PORT = 50007 # Arbitrary non-privileged port - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind((HOST, PORT)) - s.listen(1) - conn, addr = s.accept() - with conn: - print('Connected by', addr) - while True: - data = conn.recv(1024) - if not data: break - conn.sendall(data) - -:: - - # Echo client program - import socket - - HOST = 'daring.cwi.nl' # The remote host - PORT = 50007 # The same port as used by the server - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.connect((HOST, PORT)) - s.sendall(b'Hello, world') - data = s.recv(1024) - print('Received', repr(data)) - -The next two examples are identical to the above two, but support both IPv4 and -IPv6. The server side will listen to the first address family available (it -should listen to both instead). On most of IPv6-ready systems, IPv6 will take -precedence and the server may not accept IPv4 traffic. The client side will try -to connect to all the addresses returned as a result of the name resolution, and -sends traffic to the first one connected successfully. :: - - # Echo server program - import socket - import sys - - HOST = None # Symbolic name meaning all available interfaces - PORT = 50007 # Arbitrary non-privileged port - s = None - for res in socket.getaddrinfo(HOST, PORT, socket.AF_UNSPEC, - socket.SOCK_STREAM, 0, socket.AI_PASSIVE): - af, socktype, proto, canonname, sa = res - try: - s = socket.socket(af, socktype, proto) - except OSError as msg: - s = None - continue - try: - s.bind(sa) - s.listen(1) - except OSError as msg: - s.close() - s = None - continue - break - if s is None: - print('could not open socket') - sys.exit(1) - conn, addr = s.accept() - with conn: - print('Connected by', addr) - while True: - data = conn.recv(1024) - if not data: break - conn.send(data) - -:: - - # Echo client program - import socket - import sys - - HOST = 'daring.cwi.nl' # The remote host - PORT = 50007 # The same port as used by the server - s = None - for res in socket.getaddrinfo(HOST, PORT, socket.AF_UNSPEC, socket.SOCK_STREAM): - af, socktype, proto, canonname, sa = res - try: - s = socket.socket(af, socktype, proto) - except OSError as msg: - s = None - continue - try: - s.connect(sa) - except OSError as msg: - s.close() - s = None - continue - break - if s is None: - print('could not open socket') - sys.exit(1) - with s: - s.sendall(b'Hello, world') - data = s.recv(1024) - print('Received', repr(data)) - -The next example shows how to write a very simple network sniffer with raw -sockets on Windows. The example requires administrator privileges to modify -the interface:: - - import socket - - # the public network interface - HOST = socket.gethostbyname(socket.gethostname()) - - # create a raw socket and bind it to the public interface - s = socket.socket(socket.AF_INET, socket.SOCK_RAW, socket.IPPROTO_IP) - s.bind((HOST, 0)) - - # Include IP headers - s.setsockopt(socket.IPPROTO_IP, socket.IP_HDRINCL, 1) - - # receive all packets - s.ioctl(socket.SIO_RCVALL, socket.RCVALL_ON) - - # receive a packet - print(s.recvfrom(65565)) - - # disabled promiscuous mode - s.ioctl(socket.SIO_RCVALL, socket.RCVALL_OFF) - -The next example shows how to use the socket interface to communicate to a CAN -network using the raw socket protocol. To use CAN with the broadcast -manager protocol instead, open a socket with:: - - socket.socket(socket.AF_CAN, socket.SOCK_DGRAM, socket.CAN_BCM) - -After binding (:const:`CAN_RAW`) or connecting (:const:`CAN_BCM`) the socket, you -can use the :meth:`socket.send` and :meth:`socket.recv` operations (and -their counterparts) on the socket object as usual. - -This last example might require special privileges:: - - import socket - import struct - - - # CAN frame packing/unpacking (see 'struct can_frame' in ) - - can_frame_fmt = "=IB3x8s" - can_frame_size = struct.calcsize(can_frame_fmt) - - def build_can_frame(can_id, data): - can_dlc = len(data) - data = data.ljust(8, b'\x00') - return struct.pack(can_frame_fmt, can_id, can_dlc, data) - - def dissect_can_frame(frame): - can_id, can_dlc, data = struct.unpack(can_frame_fmt, frame) - return (can_id, can_dlc, data[:can_dlc]) - - - # create a raw socket and bind it to the 'vcan0' interface - s = socket.socket(socket.AF_CAN, socket.SOCK_RAW, socket.CAN_RAW) - s.bind(('vcan0',)) - - while True: - cf, addr = s.recvfrom(can_frame_size) - - print('Received: can_id=%x, can_dlc=%x, data=%s' % dissect_can_frame(cf)) - - try: - s.send(cf) - except OSError: - print('Error sending CAN frame') - - try: - s.send(build_can_frame(0x01, b'\x01\x02\x03')) - except OSError: - print('Error sending CAN frame') - -Running an example several times with too small delay between executions, could -lead to this error:: - - OSError: [Errno 98] Address already in use - -This is because the previous execution has left the socket in a ``TIME_WAIT`` -state, and can't be immediately reused. - -There is a :mod:`socket` flag to set, in order to prevent this, -:const:`socket.SO_REUSEADDR`:: - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - s.bind((HOST, PORT)) - -the :data:`SO_REUSEADDR` flag tells the kernel to reuse a local socket in -``TIME_WAIT`` state, without waiting for its natural timeout to expire. - - -.. seealso:: - - For an introduction to socket programming (in C), see the following papers: - - - *An Introductory 4.3BSD Interprocess Communication Tutorial*, by Stuart Sechrest - - - *An Advanced 4.3BSD Interprocess Communication Tutorial*, by Samuel J. Leffler et - al, - - both in the UNIX Programmer's Manual, Supplementary Documents 1 (sections - PS1:7 and PS1:8). The platform-specific reference material for the various - socket-related system calls are also a valuable source of information on the - details of socket semantics. For Unix, refer to the manual pages; for Windows, - see the WinSock (or Winsock 2) specification. For IPv6-ready APIs, readers may - want to refer to :rfc:`3493` titled Basic Socket Interface Extensions for IPv6. diff --git a/Doc/library/socketserver.rst b/Doc/library/socketserver.rst deleted file mode 100644 index 7540375b8d1f50..00000000000000 --- a/Doc/library/socketserver.rst +++ /dev/null @@ -1,687 +0,0 @@ -:mod:`socketserver` --- A framework for network servers -======================================================= - -.. module:: socketserver - :synopsis: A framework for network servers. - -**Source code:** :source:`Lib/socketserver.py` - --------------- - -The :mod:`socketserver` module simplifies the task of writing network servers. - -.. include:: ../includes/wasm-notavail.rst - -There are four basic concrete server classes: - - -.. class:: TCPServer(server_address, RequestHandlerClass, bind_and_activate=True) - - This uses the internet TCP protocol, which provides for - continuous streams of data between the client and server. - If *bind_and_activate* is true, the constructor automatically attempts to - invoke :meth:`~BaseServer.server_bind` and - :meth:`~BaseServer.server_activate`. The other parameters are passed to - the :class:`BaseServer` base class. - - -.. class:: UDPServer(server_address, RequestHandlerClass, bind_and_activate=True) - - This uses datagrams, which are discrete packets of information that may - arrive out of order or be lost while in transit. The parameters are - the same as for :class:`TCPServer`. - - -.. class:: UnixStreamServer(server_address, RequestHandlerClass, bind_and_activate=True) - UnixDatagramServer(server_address, RequestHandlerClass, bind_and_activate=True) - - These more infrequently used classes are similar to the TCP and - UDP classes, but use Unix domain sockets; they're not available on - non-Unix platforms. The parameters are the same as for - :class:`TCPServer`. - - -These four classes process requests :dfn:`synchronously`; each request must be -completed before the next request can be started. This isn't suitable if each -request takes a long time to complete, because it requires a lot of computation, -or because it returns a lot of data which the client is slow to process. The -solution is to create a separate process or thread to handle each request; the -:class:`ForkingMixIn` and :class:`ThreadingMixIn` mix-in classes can be used to -support asynchronous behaviour. - -Creating a server requires several steps. First, you must create a request -handler class by subclassing the :class:`BaseRequestHandler` class and -overriding its :meth:`~BaseRequestHandler.handle` method; -this method will process incoming -requests. Second, you must instantiate one of the server classes, passing it -the server's address and the request handler class. It is recommended to use -the server in a :keyword:`with` statement. Then call the -:meth:`~BaseServer.handle_request` or -:meth:`~BaseServer.serve_forever` method of the server object to -process one or many requests. Finally, call :meth:`~BaseServer.server_close` -to close the socket (unless you used a :keyword:`!with` statement). - -When inheriting from :class:`ThreadingMixIn` for threaded connection behavior, -you should explicitly declare how you want your threads to behave on an abrupt -shutdown. The :class:`ThreadingMixIn` class defines an attribute -*daemon_threads*, which indicates whether or not the server should wait for -thread termination. You should set the flag explicitly if you would like -threads to behave autonomously; the default is :const:`False`, meaning that -Python will not exit until all threads created by :class:`ThreadingMixIn` have -exited. - -Server classes have the same external methods and attributes, no matter what -network protocol they use. - - -Server Creation Notes ---------------------- - -There are five classes in an inheritance diagram, four of which represent -synchronous servers of four types:: - - +------------+ - | BaseServer | - +------------+ - | - v - +-----------+ +------------------+ - | TCPServer |------->| UnixStreamServer | - +-----------+ +------------------+ - | - v - +-----------+ +--------------------+ - | UDPServer |------->| UnixDatagramServer | - +-----------+ +--------------------+ - -Note that :class:`UnixDatagramServer` derives from :class:`UDPServer`, not from -:class:`UnixStreamServer` --- the only difference between an IP and a Unix -server is the address family. - - -.. class:: ForkingMixIn - ThreadingMixIn - - Forking and threading versions of each type of server can be created - using these mix-in classes. For instance, :class:`ThreadingUDPServer` - is created as follows:: - - class ThreadingUDPServer(ThreadingMixIn, UDPServer): - pass - - The mix-in class comes first, since it overrides a method defined in - :class:`UDPServer`. Setting the various attributes also changes the - behavior of the underlying server mechanism. - - :class:`ForkingMixIn` and the Forking classes mentioned below are - only available on POSIX platforms that support :func:`~os.fork`. - - .. attribute:: block_on_close - - :meth:`ForkingMixIn.server_close ` - waits until all child processes complete, except if - :attr:`block_on_close` attribute is ``False``. - - :meth:`ThreadingMixIn.server_close ` - waits until all non-daemon threads complete, except if - :attr:`block_on_close` attribute is ``False``. - - .. attribute:: daemon_threads - - For :class:`ThreadingMixIn` use daemonic threads by setting - :data:`ThreadingMixIn.daemon_threads ` - to ``True`` to not wait until threads complete. - - .. versionchanged:: 3.7 - - :meth:`ForkingMixIn.server_close ` and - :meth:`ThreadingMixIn.server_close ` now waits until all - child processes and non-daemonic threads complete. - Add a new :attr:`ForkingMixIn.block_on_close ` class - attribute to opt-in for the pre-3.7 behaviour. - - -.. class:: ForkingTCPServer - ForkingUDPServer - ThreadingTCPServer - ThreadingUDPServer - - These classes are pre-defined using the mix-in classes. - - -To implement a service, you must derive a class from :class:`BaseRequestHandler` -and redefine its :meth:`~BaseRequestHandler.handle` method. -You can then run various versions of -the service by combining one of the server classes with your request handler -class. The request handler class must be different for datagram or stream -services. This can be hidden by using the handler subclasses -:class:`StreamRequestHandler` or :class:`DatagramRequestHandler`. - -Of course, you still have to use your head! For instance, it makes no sense to -use a forking server if the service contains state in memory that can be -modified by different requests, since the modifications in the child process -would never reach the initial state kept in the parent process and passed to -each child. In this case, you can use a threading server, but you will probably -have to use locks to protect the integrity of the shared data. - -On the other hand, if you are building an HTTP server where all data is stored -externally (for instance, in the file system), a synchronous class will -essentially render the service "deaf" while one request is being handled -- -which may be for a very long time if a client is slow to receive all the data it -has requested. Here a threading or forking server is appropriate. - -In some cases, it may be appropriate to process part of a request synchronously, -but to finish processing in a forked child depending on the request data. This -can be implemented by using a synchronous server and doing an explicit fork in -the request handler class :meth:`~BaseRequestHandler.handle` method. - -Another approach to handling multiple simultaneous requests in an environment -that supports neither threads nor :func:`~os.fork` (or where these are too -expensive or inappropriate for the service) is to maintain an explicit table of -partially finished requests and to use :mod:`selectors` to decide which -request to work on next (or whether to handle a new incoming request). This is -particularly important for stream services where each client can potentially be -connected for a long time (if threads or subprocesses cannot be used). See -:mod:`asyncore` for another way to manage this. - -.. XXX should data and methods be intermingled, or separate? - how should the distinction between class and instance variables be drawn? - - -Server Objects --------------- - -.. class:: BaseServer(server_address, RequestHandlerClass) - - This is the superclass of all Server objects in the module. It defines the - interface, given below, but does not implement most of the methods, which is - done in subclasses. The two parameters are stored in the respective - :attr:`server_address` and :attr:`RequestHandlerClass` attributes. - - - .. method:: fileno() - - Return an integer file descriptor for the socket on which the server is - listening. This function is most commonly passed to :mod:`selectors`, to - allow monitoring multiple servers in the same process. - - - .. method:: handle_request() - - Process a single request. This function calls the following methods in - order: :meth:`get_request`, :meth:`verify_request`, and - :meth:`process_request`. If the user-provided - :meth:`~BaseRequestHandler.handle` method of the - handler class raises an exception, the server's :meth:`handle_error` method - will be called. If no request is received within :attr:`timeout` - seconds, :meth:`handle_timeout` will be called and :meth:`handle_request` - will return. - - - .. method:: serve_forever(poll_interval=0.5) - - Handle requests until an explicit :meth:`shutdown` request. Poll for - shutdown every *poll_interval* seconds. - Ignores the :attr:`timeout` attribute. It - also calls :meth:`service_actions`, which may be used by a subclass or mixin - to provide actions specific to a given service. For example, the - :class:`ForkingMixIn` class uses :meth:`service_actions` to clean up zombie - child processes. - - .. versionchanged:: 3.3 - Added ``service_actions`` call to the ``serve_forever`` method. - - - .. method:: service_actions() - - This is called in the :meth:`serve_forever` loop. This method can be - overridden by subclasses or mixin classes to perform actions specific to - a given service, such as cleanup actions. - - .. versionadded:: 3.3 - - .. method:: shutdown() - - Tell the :meth:`serve_forever` loop to stop and wait until it does. - :meth:`shutdown` must be called while :meth:`serve_forever` is running in a - different thread otherwise it will deadlock. - - - .. method:: server_close() - - Clean up the server. May be overridden. - - - .. attribute:: address_family - - The family of protocols to which the server's socket belongs. - Common examples are :const:`socket.AF_INET` and :const:`socket.AF_UNIX`. - - - .. attribute:: RequestHandlerClass - - The user-provided request handler class; an instance of this class is created - for each request. - - - .. attribute:: server_address - - The address on which the server is listening. The format of addresses varies - depending on the protocol family; - see the documentation for the :mod:`socket` module - for details. For internet protocols, this is a tuple containing a string giving - the address, and an integer port number: ``('127.0.0.1', 80)``, for example. - - - .. attribute:: socket - - The socket object on which the server will listen for incoming requests. - - - The server classes support the following class variables: - - .. XXX should class variables be covered before instance variables, or vice versa? - - .. attribute:: allow_reuse_address - - Whether the server will allow the reuse of an address. This defaults to - :const:`False`, and can be set in subclasses to change the policy. - - - .. attribute:: request_queue_size - - The size of the request queue. If it takes a long time to process a single - request, any requests that arrive while the server is busy are placed into a - queue, up to :attr:`request_queue_size` requests. Once the queue is full, - further requests from clients will get a "Connection denied" error. The default - value is usually 5, but this can be overridden by subclasses. - - - .. attribute:: socket_type - - The type of socket used by the server; :const:`socket.SOCK_STREAM` and - :const:`socket.SOCK_DGRAM` are two common values. - - - .. attribute:: timeout - - Timeout duration, measured in seconds, or :const:`None` if no timeout is - desired. If :meth:`handle_request` receives no incoming requests within the - timeout period, the :meth:`handle_timeout` method is called. - - - There are various server methods that can be overridden by subclasses of base - server classes like :class:`TCPServer`; these methods aren't useful to external - users of the server object. - - .. XXX should the default implementations of these be documented, or should - it be assumed that the user will look at socketserver.py? - - .. method:: finish_request(request, client_address) - - Actually processes the request by instantiating :attr:`RequestHandlerClass` and - calling its :meth:`~BaseRequestHandler.handle` method. - - - .. method:: get_request() - - Must accept a request from the socket, and return a 2-tuple containing the *new* - socket object to be used to communicate with the client, and the client's - address. - - - .. method:: handle_error(request, client_address) - - This function is called if the :meth:`~BaseRequestHandler.handle` - method of a :attr:`RequestHandlerClass` instance raises - an exception. The default action is to print the traceback to - standard error and continue handling further requests. - - .. versionchanged:: 3.6 - Now only called for exceptions derived from the :exc:`Exception` - class. - - - .. method:: handle_timeout() - - This function is called when the :attr:`timeout` attribute has been set to a - value other than :const:`None` and the timeout period has passed with no - requests being received. The default action for forking servers is - to collect the status of any child processes that have exited, while - in threading servers this method does nothing. - - - .. method:: process_request(request, client_address) - - Calls :meth:`finish_request` to create an instance of the - :attr:`RequestHandlerClass`. If desired, this function can create a new process - or thread to handle the request; the :class:`ForkingMixIn` and - :class:`ThreadingMixIn` classes do this. - - - .. Is there any point in documenting the following two functions? - What would the purpose of overriding them be: initializing server - instance variables, adding new network families? - - .. method:: server_activate() - - Called by the server's constructor to activate the server. The default behavior - for a TCP server just invokes :meth:`~socket.socket.listen` - on the server's socket. May be overridden. - - - .. method:: server_bind() - - Called by the server's constructor to bind the socket to the desired address. - May be overridden. - - - .. method:: verify_request(request, client_address) - - Must return a Boolean value; if the value is :const:`True`, the request will - be processed, and if it's :const:`False`, the request will be denied. This - function can be overridden to implement access controls for a server. The - default implementation always returns :const:`True`. - - - .. versionchanged:: 3.6 - Support for the :term:`context manager` protocol was added. Exiting the - context manager is equivalent to calling :meth:`server_close`. - - -Request Handler Objects ------------------------ - -.. class:: BaseRequestHandler - - This is the superclass of all request handler objects. It defines - the interface, given below. A concrete request handler subclass must - define a new :meth:`handle` method, and can override any of - the other methods. A new instance of the subclass is created for each - request. - - - .. method:: setup() - - Called before the :meth:`handle` method to perform any initialization actions - required. The default implementation does nothing. - - - .. method:: handle() - - This function must do all the work required to service a request. The - default implementation does nothing. Several instance attributes are - available to it; the request is available as :attr:`request`; the client - address as :attr:`client_address`; and the server instance as - :attr:`server`, in case it needs access to per-server information. - - The type of :attr:`request` is different for datagram or stream - services. For stream services, :attr:`request` is a socket object; for - datagram services, :attr:`request` is a pair of string and socket. - - - .. method:: finish() - - Called after the :meth:`handle` method to perform any clean-up actions - required. The default implementation does nothing. If :meth:`setup` - raises an exception, this function will not be called. - - - .. attribute:: request - - The *new* :class:`socket.socket` object - to be used to communicate with the client. - - - .. attribute:: client_address - - Client address returned by :meth:`BaseServer.get_request`. - - - .. attribute:: server - - :class:`BaseServer` object used for handling the request. - - -.. class:: StreamRequestHandler - DatagramRequestHandler - - These :class:`BaseRequestHandler` subclasses override the - :meth:`~BaseRequestHandler.setup` and :meth:`~BaseRequestHandler.finish` - methods, and provide :attr:`rfile` and :attr:`wfile` attributes. - - .. attribute:: rfile - - A file object from which receives the request is read. - Support the :class:`io.BufferedIOBase` readable interface. - - .. attribute:: wfile - - A file object to which the reply is written. - Support the :class:`io.BufferedIOBase` writable interface - - - .. versionchanged:: 3.6 - :attr:`wfile` also supports the - :class:`io.BufferedIOBase` writable interface. - - -Examples --------- - -:class:`socketserver.TCPServer` Example -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This is the server side:: - - import socketserver - - class MyTCPHandler(socketserver.BaseRequestHandler): - """ - The request handler class for our server. - - It is instantiated once per connection to the server, and must - override the handle() method to implement communication to the - client. - """ - - def handle(self): - # self.request is the TCP socket connected to the client - self.data = self.request.recv(1024).strip() - print("{} wrote:".format(self.client_address[0])) - print(self.data) - # just send back the same data, but upper-cased - self.request.sendall(self.data.upper()) - - if __name__ == "__main__": - HOST, PORT = "localhost", 9999 - - # Create the server, binding to localhost on port 9999 - with socketserver.TCPServer((HOST, PORT), MyTCPHandler) as server: - # Activate the server; this will keep running until you - # interrupt the program with Ctrl-C - server.serve_forever() - -An alternative request handler class that makes use of streams (file-like -objects that simplify communication by providing the standard file interface):: - - class MyTCPHandler(socketserver.StreamRequestHandler): - - def handle(self): - # self.rfile is a file-like object created by the handler; - # we can now use e.g. readline() instead of raw recv() calls - self.data = self.rfile.readline().strip() - print("{} wrote:".format(self.client_address[0])) - print(self.data) - # Likewise, self.wfile is a file-like object used to write back - # to the client - self.wfile.write(self.data.upper()) - -The difference is that the ``readline()`` call in the second handler will call -``recv()`` multiple times until it encounters a newline character, while the -single ``recv()`` call in the first handler will just return what has been sent -from the client in one ``sendall()`` call. - - -This is the client side:: - - import socket - import sys - - HOST, PORT = "localhost", 9999 - data = " ".join(sys.argv[1:]) - - # Create a socket (SOCK_STREAM means a TCP socket) - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - # Connect to server and send data - sock.connect((HOST, PORT)) - sock.sendall(bytes(data + "\n", "utf-8")) - - # Receive data from the server and shut down - received = str(sock.recv(1024), "utf-8") - - print("Sent: {}".format(data)) - print("Received: {}".format(received)) - - -The output of the example should look something like this: - -Server: - -.. code-block:: shell-session - - $ python TCPServer.py - 127.0.0.1 wrote: - b'hello world with TCP' - 127.0.0.1 wrote: - b'python is nice' - -Client: - -.. code-block:: shell-session - - $ python TCPClient.py hello world with TCP - Sent: hello world with TCP - Received: HELLO WORLD WITH TCP - $ python TCPClient.py python is nice - Sent: python is nice - Received: PYTHON IS NICE - - -:class:`socketserver.UDPServer` Example -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This is the server side:: - - import socketserver - - class MyUDPHandler(socketserver.BaseRequestHandler): - """ - This class works similar to the TCP handler class, except that - self.request consists of a pair of data and client socket, and since - there is no connection the client address must be given explicitly - when sending data back via sendto(). - """ - - def handle(self): - data = self.request[0].strip() - socket = self.request[1] - print("{} wrote:".format(self.client_address[0])) - print(data) - socket.sendto(data.upper(), self.client_address) - - if __name__ == "__main__": - HOST, PORT = "localhost", 9999 - with socketserver.UDPServer((HOST, PORT), MyUDPHandler) as server: - server.serve_forever() - -This is the client side:: - - import socket - import sys - - HOST, PORT = "localhost", 9999 - data = " ".join(sys.argv[1:]) - - # SOCK_DGRAM is the socket type to use for UDP sockets - sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - - # As you can see, there is no connect() call; UDP has no connections. - # Instead, data is directly sent to the recipient via sendto(). - sock.sendto(bytes(data + "\n", "utf-8"), (HOST, PORT)) - received = str(sock.recv(1024), "utf-8") - - print("Sent: {}".format(data)) - print("Received: {}".format(received)) - -The output of the example should look exactly like for the TCP server example. - - -Asynchronous Mixins -~~~~~~~~~~~~~~~~~~~ - -To build asynchronous handlers, use the :class:`ThreadingMixIn` and -:class:`ForkingMixIn` classes. - -An example for the :class:`ThreadingMixIn` class:: - - import socket - import threading - import socketserver - - class ThreadedTCPRequestHandler(socketserver.BaseRequestHandler): - - def handle(self): - data = str(self.request.recv(1024), 'ascii') - cur_thread = threading.current_thread() - response = bytes("{}: {}".format(cur_thread.name, data), 'ascii') - self.request.sendall(response) - - class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): - pass - - def client(ip, port, message): - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.connect((ip, port)) - sock.sendall(bytes(message, 'ascii')) - response = str(sock.recv(1024), 'ascii') - print("Received: {}".format(response)) - - if __name__ == "__main__": - # Port 0 means to select an arbitrary unused port - HOST, PORT = "localhost", 0 - - server = ThreadedTCPServer((HOST, PORT), ThreadedTCPRequestHandler) - with server: - ip, port = server.server_address - - # Start a thread with the server -- that thread will then start one - # more thread for each request - server_thread = threading.Thread(target=server.serve_forever) - # Exit the server thread when the main thread terminates - server_thread.daemon = True - server_thread.start() - print("Server loop running in thread:", server_thread.name) - - client(ip, port, "Hello World 1") - client(ip, port, "Hello World 2") - client(ip, port, "Hello World 3") - - server.shutdown() - - -The output of the example should look something like this: - -.. code-block:: shell-session - - $ python ThreadedTCPServer.py - Server loop running in thread: Thread-1 - Received: Thread-2: Hello World 1 - Received: Thread-3: Hello World 2 - Received: Thread-4: Hello World 3 - - -The :class:`ForkingMixIn` class is used in the same way, except that the server -will spawn a new process for each request. -Available only on POSIX platforms that support :func:`~os.fork`. - diff --git a/Doc/library/spwd.rst b/Doc/library/spwd.rst deleted file mode 100644 index d1693ea67f0ceb..00000000000000 --- a/Doc/library/spwd.rst +++ /dev/null @@ -1,82 +0,0 @@ -:mod:`spwd` --- The shadow password database -============================================ - -.. module:: spwd - :platform: Unix - :synopsis: The shadow password database (getspnam() and friends). - :deprecated: - -.. deprecated-removed:: 3.11 3.13 - The :mod:`spwd` module is deprecated - (see :pep:`PEP 594 <594#spwd>` for details and alternatives). - --------------- - -This module provides access to the Unix shadow password database. It is -available on various Unix versions. - -.. include:: ../includes/wasm-notavail.rst - -You must have enough privileges to access the shadow password database (this -usually means you have to be root). - -Shadow password database entries are reported as a tuple-like object, whose -attributes correspond to the members of the ``spwd`` structure (Attribute field -below, see ````): - -+-------+---------------+---------------------------------+ -| Index | Attribute | Meaning | -+=======+===============+=================================+ -| 0 | ``sp_namp`` | Login name | -+-------+---------------+---------------------------------+ -| 1 | ``sp_pwdp`` | Encrypted password | -+-------+---------------+---------------------------------+ -| 2 | ``sp_lstchg`` | Date of last change | -+-------+---------------+---------------------------------+ -| 3 | ``sp_min`` | Minimal number of days between | -| | | changes | -+-------+---------------+---------------------------------+ -| 4 | ``sp_max`` | Maximum number of days between | -| | | changes | -+-------+---------------+---------------------------------+ -| 5 | ``sp_warn`` | Number of days before password | -| | | expires to warn user about it | -+-------+---------------+---------------------------------+ -| 6 | ``sp_inact`` | Number of days after password | -| | | expires until account is | -| | | disabled | -+-------+---------------+---------------------------------+ -| 7 | ``sp_expire`` | Number of days since 1970-01-01 | -| | | when account expires | -+-------+---------------+---------------------------------+ -| 8 | ``sp_flag`` | Reserved | -+-------+---------------+---------------------------------+ - -The sp_namp and sp_pwdp items are strings, all others are integers. -:exc:`KeyError` is raised if the entry asked for cannot be found. - -The following functions are defined: - - -.. function:: getspnam(name) - - Return the shadow password database entry for the given user name. - - .. versionchanged:: 3.6 - Raises a :exc:`PermissionError` instead of :exc:`KeyError` if the user - doesn't have privileges. - -.. function:: getspall() - - Return a list of all available shadow password database entries, in arbitrary - order. - - -.. seealso:: - - Module :mod:`grp` - An interface to the group database, similar to this. - - Module :mod:`pwd` - An interface to the normal password database, similar to this. - diff --git a/Doc/library/sqlite3.rst b/Doc/library/sqlite3.rst deleted file mode 100644 index 3e9b7a270ad5ef..00000000000000 --- a/Doc/library/sqlite3.rst +++ /dev/null @@ -1,2435 +0,0 @@ -:mod:`sqlite3` --- DB-API 2.0 interface for SQLite databases -============================================================ - -.. module:: sqlite3 - :synopsis: A DB-API 2.0 implementation using SQLite 3.x. - -.. sectionauthor:: Gerhard Häring - -**Source code:** :source:`Lib/sqlite3/` - -.. Make sure we always doctest the tutorial with an empty database. - -.. testsetup:: - - import sqlite3 - src = sqlite3.connect(":memory:", isolation_level=None) - dst = sqlite3.connect("tutorial.db", isolation_level=None) - src.backup(dst) - del src, dst - -.. _sqlite3-intro: - -SQLite is a C library that provides a lightweight disk-based database that -doesn't require a separate server process and allows accessing the database -using a nonstandard variant of the SQL query language. Some applications can use -SQLite for internal data storage. It's also possible to prototype an -application using SQLite and then port the code to a larger database such as -PostgreSQL or Oracle. - -The :mod:`!sqlite3` module was written by Gerhard Häring. It provides an SQL interface -compliant with the DB-API 2.0 specification described by :pep:`249`, and -requires SQLite 3.7.15 or newer. - -This document includes four main sections: - -* :ref:`sqlite3-tutorial` teaches how to use the :mod:`!sqlite3` module. -* :ref:`sqlite3-reference` describes the classes and functions this module - defines. -* :ref:`sqlite3-howtos` details how to handle specific tasks. -* :ref:`sqlite3-explanation` provides in-depth background on - transaction control. - -.. seealso:: - - https://www.sqlite.org - The SQLite web page; the documentation describes the syntax and the - available data types for the supported SQL dialect. - - https://www.w3schools.com/sql/ - Tutorial, reference and examples for learning SQL syntax. - - :pep:`249` - Database API Specification 2.0 - PEP written by Marc-André Lemburg. - - -.. We use the following practises for SQL code: - - UPPERCASE for keywords - - snake_case for schema - - single quotes for string literals - - singular for table names - - if needed, use double quotes for table and column names - -.. _sqlite3-tutorial: - -Tutorial --------- - -In this tutorial, you will create a database of Monty Python movies -using basic :mod:`!sqlite3` functionality. -It assumes a fundamental understanding of database concepts, -including `cursors`_ and `transactions`_. - -First, we need to create a new database and open -a database connection to allow :mod:`!sqlite3` to work with it. -Call :func:`sqlite3.connect` to create a connection to -the database :file:`tutorial.db` in the current working directory, -implicitly creating it if it does not exist: - -.. testcode:: - - import sqlite3 - con = sqlite3.connect("tutorial.db") - -The returned :class:`Connection` object ``con`` -represents the connection to the on-disk database. - -In order to execute SQL statements and fetch results from SQL queries, -we will need to use a database cursor. -Call :meth:`con.cursor() ` to create the :class:`Cursor`: - -.. testcode:: - - cur = con.cursor() - -Now that we've got a database connection and a cursor, -we can create a database table ``movie`` with columns for title, -release year, and review score. -For simplicity, we can just use column names in the table declaration -- -thanks to the `flexible typing`_ feature of SQLite, -specifying the data types is optional. -Execute the ``CREATE TABLE`` statement -by calling :meth:`cur.execute(...) `: - -.. testcode:: - - cur.execute("CREATE TABLE movie(title, year, score)") - -.. Ideally, we'd use sqlite_schema instead of sqlite_master below, - but SQLite versions older than 3.33.0 do not recognise that variant. - -We can verify that the new table has been created by querying -the ``sqlite_master`` table built-in to SQLite, -which should now contain an entry for the ``movie`` table definition -(see `The Schema Table`_ for details). -Execute that query by calling :meth:`cur.execute(...) `, -assign the result to ``res``, -and call :meth:`res.fetchone() ` to fetch the resulting row: - -.. doctest:: - - >>> res = cur.execute("SELECT name FROM sqlite_master") - >>> res.fetchone() - ('movie',) - -We can see that the table has been created, -as the query returns a :class:`tuple` containing the table's name. -If we query ``sqlite_master`` for a non-existent table ``spam``, -:meth:`!res.fetchone()` will return ``None``: - -.. doctest:: - - >>> res = cur.execute("SELECT name FROM sqlite_master WHERE name='spam'") - >>> res.fetchone() is None - True - -Now, add two rows of data supplied as SQL literals -by executing an ``INSERT`` statement, -once again by calling :meth:`cur.execute(...) `: - -.. testcode:: - - cur.execute(""" - INSERT INTO movie VALUES - ('Monty Python and the Holy Grail', 1975, 8.2), - ('And Now for Something Completely Different', 1971, 7.5) - """) - -The ``INSERT`` statement implicitly opens a transaction, -which needs to be committed before changes are saved in the database -(see :ref:`sqlite3-controlling-transactions` for details). -Call :meth:`con.commit() ` on the connection object -to commit the transaction: - -.. testcode:: - - con.commit() - -We can verify that the data was inserted correctly -by executing a ``SELECT`` query. -Use the now-familiar :meth:`cur.execute(...) ` to -assign the result to ``res``, -and call :meth:`res.fetchall() ` to return all resulting rows: - -.. doctest:: - - >>> res = cur.execute("SELECT score FROM movie") - >>> res.fetchall() - [(8.2,), (7.5,)] - -The result is a :class:`list` of two :class:`!tuple`\s, one per row, -each containing that row's ``score`` value. - -Now, insert three more rows by calling -:meth:`cur.executemany(...) `: - -.. testcode:: - - data = [ - ("Monty Python Live at the Hollywood Bowl", 1982, 7.9), - ("Monty Python's The Meaning of Life", 1983, 7.5), - ("Monty Python's Life of Brian", 1979, 8.0), - ] - cur.executemany("INSERT INTO movie VALUES(?, ?, ?)", data) - con.commit() # Remember to commit the transaction after executing INSERT. - -Notice that ``?`` placeholders are used to bind ``data`` to the query. -Always use placeholders instead of :ref:`string formatting ` -to bind Python values to SQL statements, -to avoid `SQL injection attacks`_ -(see :ref:`sqlite3-placeholders` for more details). - -We can verify that the new rows were inserted -by executing a ``SELECT`` query, -this time iterating over the results of the query: - -.. doctest:: - - >>> for row in cur.execute("SELECT year, title FROM movie ORDER BY year"): - ... print(row) - (1971, 'And Now for Something Completely Different') - (1975, 'Monty Python and the Holy Grail') - (1979, "Monty Python's Life of Brian") - (1982, 'Monty Python Live at the Hollywood Bowl') - (1983, "Monty Python's The Meaning of Life") - -Each row is a two-item :class:`tuple` of ``(year, title)``, -matching the columns selected in the query. - -Finally, verify that the database has been written to disk -by calling :meth:`con.close() ` -to close the existing connection, opening a new one, -creating a new cursor, then querying the database: - -.. doctest:: - - >>> con.close() - >>> new_con = sqlite3.connect("tutorial.db") - >>> new_cur = new_con.cursor() - >>> res = new_cur.execute("SELECT title, year FROM movie ORDER BY score DESC") - >>> title, year = res.fetchone() - >>> print(f'The highest scoring Monty Python movie is {title!r}, released in {year}') - The highest scoring Monty Python movie is 'Monty Python and the Holy Grail', released in 1975 - -You've now created an SQLite database using the :mod:`!sqlite3` module, -inserted data and retrieved values from it in multiple ways. - -.. _SQL injection attacks: https://en.wikipedia.org/wiki/SQL_injection -.. _The Schema Table: https://www.sqlite.org/schematab.html -.. _cursors: https://en.wikipedia.org/wiki/Cursor_(databases) -.. _flexible typing: https://www.sqlite.org/flextypegood.html -.. _sqlite_master: https://www.sqlite.org/schematab.html -.. _transactions: https://en.wikipedia.org/wiki/Database_transaction - -.. seealso:: - - * :ref:`sqlite3-howtos` for further reading: - - * :ref:`sqlite3-placeholders` - * :ref:`sqlite3-adapters` - * :ref:`sqlite3-converters` - * :ref:`sqlite3-connection-context-manager` - * :ref:`sqlite3-howto-row-factory` - - * :ref:`sqlite3-explanation` for in-depth background on transaction control. - -.. _sqlite3-reference: - -Reference ---------- - -.. We keep the old sqlite3-module-contents ref to prevent breaking links. -.. _sqlite3-module-contents: - -.. _sqlite3-module-functions: - -Module functions -^^^^^^^^^^^^^^^^ - -.. function:: connect(database, timeout=5.0, detect_types=0, \ - isolation_level="DEFERRED", check_same_thread=True, \ - factory=sqlite3.Connection, cached_statements=128, \ - uri=False) - - Open a connection to an SQLite database. - - :param database: - The path to the database file to be opened. - You can pass ``":memory:"`` to create an `SQLite database existing only - in memory `_, and open a connection - to it. - :type database: :term:`path-like object` - - :param float timeout: - How many seconds the connection should wait before raising - an :exc:`OperationalError` when a table is locked. - If another connection opens a transaction to modify a table, - that table will be locked until the transaction is committed. - Default five seconds. - - :param int detect_types: - Control whether and how data types not - :ref:`natively supported by SQLite ` - are looked up to be converted to Python types, - using the converters registered with :func:`register_converter`. - Set it to any combination (using ``|``, bitwise or) of - :const:`PARSE_DECLTYPES` and :const:`PARSE_COLNAMES` - to enable this. - Column names takes precedence over declared types if both flags are set. - Types cannot be detected for generated fields (for example ``max(data)``), - even when the *detect_types* parameter is set; :class:`str` will be - returned instead. - By default (``0``), type detection is disabled. - - :param isolation_level: - The :attr:`~Connection.isolation_level` of the connection, - controlling whether and how transactions are implicitly opened. - Can be ``"DEFERRED"`` (default), ``"EXCLUSIVE"`` or ``"IMMEDIATE"``; - or ``None`` to disable opening transactions implicitly. - See :ref:`sqlite3-controlling-transactions` for more. - :type isolation_level: str | None - - :param bool check_same_thread: - If ``True`` (default), :exc:`ProgrammingError` will be raised - if the database connection is used by a thread - other than the one that created it. - If ``False``, the connection may be accessed in multiple threads; - write operations may need to be serialized by the user - to avoid data corruption. - See :attr:`threadsafety` for more information. - - :param ~sqlite3.Connection factory: - A custom subclass of :class:`Connection` to create the connection with, - if not the default :class:`Connection` class. - - :param int cached_statements: - The number of statements that :mod:`!sqlite3` - should internally cache for this connection, to avoid parsing overhead. - By default, 128 statements. - - :param bool uri: - If set to ``True``, *database* is interpreted as a - :abbr:`URI (Uniform Resource Identifier)` with a file path - and an optional query string. - The scheme part *must* be ``"file:"``, - and the path can be relative or absolute. - The query string allows passing parameters to SQLite, - enabling various :ref:`sqlite3-uri-tricks`. - - :rtype: ~sqlite3.Connection - - .. audit-event:: sqlite3.connect database sqlite3.connect - .. audit-event:: sqlite3.connect/handle connection_handle sqlite3.connect - - .. versionadded:: 3.4 - The *uri* parameter. - - .. versionchanged:: 3.7 - *database* can now also be a :term:`path-like object`, not only a string. - - .. versionadded:: 3.10 - The ``sqlite3.connect/handle`` auditing event. - -.. function:: complete_statement(statement) - - Return ``True`` if the string *statement* appears to contain - one or more complete SQL statements. - No syntactic verification or parsing of any kind is performed, - other than checking that there are no unclosed string literals - and the statement is terminated by a semicolon. - - For example: - - .. doctest:: - - >>> sqlite3.complete_statement("SELECT foo FROM bar;") - True - >>> sqlite3.complete_statement("SELECT foo") - False - - This function may be useful during command-line input - to determine if the entered text seems to form a complete SQL statement, - or if additional input is needed before calling :meth:`~Cursor.execute`. - -.. function:: enable_callback_tracebacks(flag, /) - - Enable or disable callback tracebacks. - By default you will not get any tracebacks in user-defined functions, - aggregates, converters, authorizer callbacks etc. If you want to debug them, - you can call this function with *flag* set to ``True``. Afterwards, you - will get tracebacks from callbacks on :data:`sys.stderr`. Use ``False`` - to disable the feature again. - - Register an :func:`unraisable hook handler ` for an - improved debug experience: - - .. testsetup:: sqlite3.trace - - import sqlite3 - - .. doctest:: sqlite3.trace - - >>> sqlite3.enable_callback_tracebacks(True) - >>> con = sqlite3.connect(":memory:") - >>> def evil_trace(stmt): - ... 5/0 - >>> con.set_trace_callback(evil_trace) - >>> def debug(unraisable): - ... print(f"{unraisable.exc_value!r} in callback {unraisable.object.__name__}") - ... print(f"Error message: {unraisable.err_msg}") - >>> import sys - >>> sys.unraisablehook = debug - >>> cur = con.execute("SELECT 1") - ZeroDivisionError('division by zero') in callback evil_trace - Error message: None - -.. function:: register_adapter(type, adapter, /) - - Register an *adapter* :term:`callable` to adapt the Python type *type* - into an SQLite type. - The adapter is called with a Python object of type *type* as its sole - argument, and must return a value of a - :ref:`type that SQLite natively understands `. - -.. function:: register_converter(typename, converter, /) - - Register the *converter* :term:`callable` to convert SQLite objects of type - *typename* into a Python object of a specific type. - The converter is invoked for all SQLite values of type *typename*; - it is passed a :class:`bytes` object and should return an object of the - desired Python type. - Consult the parameter *detect_types* of - :func:`connect` for information regarding how type detection works. - - Note: *typename* and the name of the type in your query are matched - case-insensitively. - - -.. _sqlite3-module-constants: - -Module constants -^^^^^^^^^^^^^^^^ - -.. data:: PARSE_COLNAMES - - Pass this flag value to the *detect_types* parameter of - :func:`connect` to look up a converter function by - using the type name, parsed from the query column name, - as the converter dictionary key. - The type name must be wrapped in square brackets (``[]``). - - .. code-block:: sql - - SELECT p as "p [point]" FROM test; ! will look up converter "point" - - This flag may be combined with :const:`PARSE_DECLTYPES` using the ``|`` - (bitwise or) operator. - -.. data:: PARSE_DECLTYPES - - Pass this flag value to the *detect_types* parameter of - :func:`connect` to look up a converter function using - the declared types for each column. - The types are declared when the database table is created. - :mod:`!sqlite3` will look up a converter function using the first word of the - declared type as the converter dictionary key. - For example: - - .. code-block:: sql - - CREATE TABLE test( - i integer primary key, ! will look up a converter named "integer" - p point, ! will look up a converter named "point" - n number(10) ! will look up a converter named "number" - ) - - This flag may be combined with :const:`PARSE_COLNAMES` using the ``|`` - (bitwise or) operator. - -.. data:: SQLITE_OK - SQLITE_DENY - SQLITE_IGNORE - - Flags that should be returned by the *authorizer_callback* :term:`callable` - passed to :meth:`Connection.set_authorizer`, to indicate whether: - - * Access is allowed (:const:`!SQLITE_OK`), - * The SQL statement should be aborted with an error (:const:`!SQLITE_DENY`) - * The column should be treated as a ``NULL`` value (:const:`!SQLITE_IGNORE`) - -.. data:: apilevel - - String constant stating the supported DB-API level. Required by the DB-API. - Hard-coded to ``"2.0"``. - -.. data:: paramstyle - - String constant stating the type of parameter marker formatting expected by - the :mod:`!sqlite3` module. Required by the DB-API. Hard-coded to - ``"qmark"``. - - .. note:: - - The ``named`` DB-API parameter style is also supported. - -.. data:: sqlite_version - - Version number of the runtime SQLite library as a :class:`string `. - -.. data:: sqlite_version_info - - Version number of the runtime SQLite library as a :class:`tuple` of - :class:`integers `. - -.. data:: threadsafety - - Integer constant required by the DB-API 2.0, stating the level of thread - safety the :mod:`!sqlite3` module supports. This attribute is set based on - the default `threading mode `_ the - underlying SQLite library is compiled with. The SQLite threading modes are: - - 1. **Single-thread**: In this mode, all mutexes are disabled and SQLite is - unsafe to use in more than a single thread at once. - 2. **Multi-thread**: In this mode, SQLite can be safely used by multiple - threads provided that no single database connection is used - simultaneously in two or more threads. - 3. **Serialized**: In serialized mode, SQLite can be safely used by - multiple threads with no restriction. - - The mappings from SQLite threading modes to DB-API 2.0 threadsafety levels - are as follows: - - +------------------+-----------------+----------------------+-------------------------------+ - | SQLite threading | `threadsafety`_ | `SQLITE_THREADSAFE`_ | DB-API 2.0 meaning | - | mode | | | | - +==================+=================+======================+===============================+ - | single-thread | 0 | 0 | Threads may not share the | - | | | | module | - +------------------+-----------------+----------------------+-------------------------------+ - | multi-thread | 1 | 2 | Threads may share the module, | - | | | | but not connections | - +------------------+-----------------+----------------------+-------------------------------+ - | serialized | 3 | 1 | Threads may share the module, | - | | | | connections and cursors | - +------------------+-----------------+----------------------+-------------------------------+ - - .. _threadsafety: https://peps.python.org/pep-0249/#threadsafety - .. _SQLITE_THREADSAFE: https://sqlite.org/compile.html#threadsafe - - .. versionchanged:: 3.11 - Set *threadsafety* dynamically instead of hard-coding it to ``1``. - -.. data:: version - - Version number of this module as a :class:`string `. - This is not the version of the SQLite library. - -.. data:: version_info - - Version number of this module as a :class:`tuple` of :class:`integers `. - This is not the version of the SQLite library. - - -.. _sqlite3-connection-objects: - -Connection objects -^^^^^^^^^^^^^^^^^^ - -.. class:: Connection - - Each open SQLite database is represented by a ``Connection`` object, - which is created using :func:`sqlite3.connect`. - Their main purpose is creating :class:`Cursor` objects, - and :ref:`sqlite3-controlling-transactions`. - - .. seealso:: - - * :ref:`sqlite3-connection-shortcuts` - * :ref:`sqlite3-connection-context-manager` - - An SQLite database connection has the following attributes and methods: - - .. method:: cursor(factory=Cursor) - - Create and return a :class:`Cursor` object. - The cursor method accepts a single optional parameter *factory*. If - supplied, this must be a :term:`callable` returning - an instance of :class:`Cursor` or its subclasses. - - .. method:: blobopen(table, column, row, /, *, readonly=False, name="main") - - Open a :class:`Blob` handle to an existing - :abbr:`BLOB (Binary Large OBject)`. - - :param str table: - The name of the table where the blob is located. - - :param str column: - The name of the column where the blob is located. - - :param str row: - The name of the row where the blob is located. - - :param bool readonly: - Set to ``True`` if the blob should be opened without write - permissions. - Defaults to ``False``. - - :param str name: - The name of the database where the blob is located. - Defaults to ``"main"``. - - :raises OperationalError: - When trying to open a blob in a ``WITHOUT ROWID`` table. - - :rtype: Blob - - .. note:: - - The blob size cannot be changed using the :class:`Blob` class. - Use the SQL function ``zeroblob`` to create a blob with a fixed size. - - .. versionadded:: 3.11 - - .. method:: commit() - - Commit any pending transaction to the database. - If there is no open transaction, this method is a no-op. - - .. method:: rollback() - - Roll back to the start of any pending transaction. - If there is no open transaction, this method is a no-op. - - .. method:: close() - - Close the database connection. - Any pending transaction is not committed implicitly; - make sure to :meth:`commit` before closing - to avoid losing pending changes. - - .. method:: execute(sql, parameters=(), /) - - Create a new :class:`Cursor` object and call - :meth:`~Cursor.execute` on it with the given *sql* and *parameters*. - Return the new cursor object. - - .. method:: executemany(sql, parameters, /) - - Create a new :class:`Cursor` object and call - :meth:`~Cursor.executemany` on it with the given *sql* and *parameters*. - Return the new cursor object. - - .. method:: executescript(sql_script, /) - - Create a new :class:`Cursor` object and call - :meth:`~Cursor.executescript` on it with the given *sql_script*. - Return the new cursor object. - - .. method:: create_function(name, narg, func, *, deterministic=False) - - Create or remove a user-defined SQL function. - - :param str name: - The name of the SQL function. - - :param int narg: - The number of arguments the SQL function can accept. - If ``-1``, it may take any number of arguments. - - :param func: - A :term:`callable` that is called when the SQL function is invoked. - The callable must return :ref:`a type natively supported by SQLite - `. - Set to ``None`` to remove an existing SQL function. - :type func: :term:`callback` | None - - :param bool deterministic: - If ``True``, the created SQL function is marked as - `deterministic `_, - which allows SQLite to perform additional optimizations. - - :raises NotSupportedError: - If *deterministic* is used with SQLite versions older than 3.8.3. - - .. versionadded:: 3.8 - The *deterministic* parameter. - - Example: - - .. doctest:: - - >>> import hashlib - >>> def md5sum(t): - ... return hashlib.md5(t).hexdigest() - >>> con = sqlite3.connect(":memory:") - >>> con.create_function("md5", 1, md5sum) - >>> for row in con.execute("SELECT md5(?)", (b"foo",)): - ... print(row) - ('acbd18db4cc2f85cedef654fccc4a4d8',) - - - .. method:: create_aggregate(name, n_arg, aggregate_class) - - Create or remove a user-defined SQL aggregate function. - - :param str name: - The name of the SQL aggregate function. - - :param int n_arg: - The number of arguments the SQL aggregate function can accept. - If ``-1``, it may take any number of arguments. - - :param aggregate_class: - A class must implement the following methods: - - * ``step()``: Add a row to the aggregate. - * ``finalize()``: Return the final result of the aggregate as - :ref:`a type natively supported by SQLite `. - - The number of arguments that the ``step()`` method must accept - is controlled by *n_arg*. - - Set to ``None`` to remove an existing SQL aggregate function. - :type aggregate_class: :term:`class` | None - - Example: - - .. testcode:: - - class MySum: - def __init__(self): - self.count = 0 - - def step(self, value): - self.count += value - - def finalize(self): - return self.count - - con = sqlite3.connect(":memory:") - con.create_aggregate("mysum", 1, MySum) - cur = con.execute("CREATE TABLE test(i)") - cur.execute("INSERT INTO test(i) VALUES(1)") - cur.execute("INSERT INTO test(i) VALUES(2)") - cur.execute("SELECT mysum(i) FROM test") - print(cur.fetchone()[0]) - - con.close() - - .. testoutput:: - :hide: - - 3 - - - .. method:: create_window_function(name, num_params, aggregate_class, /) - - Create or remove a user-defined aggregate window function. - - :param str name: - The name of the SQL aggregate window function to create or remove. - - :param int num_params: - The number of arguments the SQL aggregate window function can accept. - If ``-1``, it may take any number of arguments. - - :param aggregate_class: - A class that must implement the following methods: - - * ``step()``: Add a row to the current window. - * ``value()``: Return the current value of the aggregate. - * ``inverse()``: Remove a row from the current window. - * ``finalize()``: Return the final result of the aggregate as - :ref:`a type natively supported by SQLite `. - - The number of arguments that the ``step()`` and ``value()`` methods - must accept is controlled by *num_params*. - - Set to ``None`` to remove an existing SQL aggregate window function. - - :raises NotSupportedError: - If used with a version of SQLite older than 3.25.0, - which does not support aggregate window functions. - - :type aggregate_class: :term:`class` | None - - .. versionadded:: 3.11 - - Example: - - .. testcode:: - - # Example taken from https://www.sqlite.org/windowfunctions.html#udfwinfunc - class WindowSumInt: - def __init__(self): - self.count = 0 - - def step(self, value): - """Add a row to the current window.""" - self.count += value - - def value(self): - """Return the current value of the aggregate.""" - return self.count - - def inverse(self, value): - """Remove a row from the current window.""" - self.count -= value - - def finalize(self): - """Return the final value of the aggregate. - - Any clean-up actions should be placed here. - """ - return self.count - - - con = sqlite3.connect(":memory:") - cur = con.execute("CREATE TABLE test(x, y)") - values = [ - ("a", 4), - ("b", 5), - ("c", 3), - ("d", 8), - ("e", 1), - ] - cur.executemany("INSERT INTO test VALUES(?, ?)", values) - con.create_window_function("sumint", 1, WindowSumInt) - cur.execute(""" - SELECT x, sumint(y) OVER ( - ORDER BY x ROWS BETWEEN 1 PRECEDING AND 1 FOLLOWING - ) AS sum_y - FROM test ORDER BY x - """) - print(cur.fetchall()) - - .. testoutput:: - :hide: - - [('a', 9), ('b', 12), ('c', 16), ('d', 12), ('e', 9)] - - .. method:: create_collation(name, callable, /) - - Create a collation named *name* using the collating function *callable*. - *callable* is passed two :class:`string ` arguments, - and it should return an :class:`integer `: - - * ``1`` if the first is ordered higher than the second - * ``-1`` if the first is ordered lower than the second - * ``0`` if they are ordered equal - - The following example shows a reverse sorting collation: - - .. testcode:: - - def collate_reverse(string1, string2): - if string1 == string2: - return 0 - elif string1 < string2: - return 1 - else: - return -1 - - con = sqlite3.connect(":memory:") - con.create_collation("reverse", collate_reverse) - - cur = con.execute("CREATE TABLE test(x)") - cur.executemany("INSERT INTO test(x) VALUES(?)", [("a",), ("b",)]) - cur.execute("SELECT x FROM test ORDER BY x COLLATE reverse") - for row in cur: - print(row) - con.close() - - .. testoutput:: - :hide: - - ('b',) - ('a',) - - Remove a collation function by setting *callable* to ``None``. - - .. versionchanged:: 3.11 - The collation name can contain any Unicode character. Earlier, only - ASCII characters were allowed. - - - .. method:: interrupt() - - Call this method from a different thread to abort any queries that might - be executing on the connection. - Aborted queries will raise an :exc:`OperationalError`. - - - .. method:: set_authorizer(authorizer_callback) - - Register :term:`callable` *authorizer_callback* to be invoked - for each attempt to access a column of a table in the database. - The callback should return one of :const:`SQLITE_OK`, - :const:`SQLITE_DENY`, or :const:`SQLITE_IGNORE` - to signal how access to the column should be handled - by the underlying SQLite library. - - The first argument to the callback signifies what kind of operation is to be - authorized. The second and third argument will be arguments or ``None`` - depending on the first argument. The 4th argument is the name of the database - ("main", "temp", etc.) if applicable. The 5th argument is the name of the - inner-most trigger or view that is responsible for the access attempt or - ``None`` if this access attempt is directly from input SQL code. - - Please consult the SQLite documentation about the possible values for the first - argument and the meaning of the second and third argument depending on the first - one. All necessary constants are available in the :mod:`!sqlite3` module. - - Passing ``None`` as *authorizer_callback* will disable the authorizer. - - .. versionchanged:: 3.11 - Added support for disabling the authorizer using ``None``. - - - .. method:: set_progress_handler(progress_handler, n) - - Register :term:`callable` *progress_handler* to be invoked for every *n* - instructions of the SQLite virtual machine. This is useful if you want to - get called from SQLite during long-running operations, for example to update - a GUI. - - If you want to clear any previously installed progress handler, call the - method with ``None`` for *progress_handler*. - - Returning a non-zero value from the handler function will terminate the - currently executing query and cause it to raise an :exc:`OperationalError` - exception. - - - .. method:: set_trace_callback(trace_callback) - - Register :term:`callable` *trace_callback* to be invoked - for each SQL statement that is actually executed by the SQLite backend. - - The only argument passed to the callback is the statement (as - :class:`str`) that is being executed. The return value of the callback is - ignored. Note that the backend does not only run statements passed to the - :meth:`Cursor.execute` methods. Other sources include the - :ref:`transaction management ` of the - :mod:`!sqlite3` module and the execution of triggers defined in the current - database. - - Passing ``None`` as *trace_callback* will disable the trace callback. - - .. note:: - Exceptions raised in the trace callback are not propagated. As a - development and debugging aid, use - :meth:`~sqlite3.enable_callback_tracebacks` to enable printing - tracebacks from exceptions raised in the trace callback. - - .. versionadded:: 3.3 - - - .. method:: enable_load_extension(enabled, /) - - Enable the SQLite engine to load SQLite extensions from shared libraries - if *enabled* is ``True``; - else, disallow loading SQLite extensions. - SQLite extensions can define new functions, - aggregates or whole new virtual table implementations. One well-known - extension is the fulltext-search extension distributed with SQLite. - - .. note:: - - The :mod:`!sqlite3` module is not built with loadable extension support by - default, because some platforms (notably macOS) have SQLite - libraries which are compiled without this feature. - To get loadable extension support, - you must pass the :option:`--enable-loadable-sqlite-extensions` option - to :program:`configure`. - - .. audit-event:: sqlite3.enable_load_extension connection,enabled sqlite3.Connection.enable_load_extension - - .. versionadded:: 3.2 - - .. versionchanged:: 3.10 - Added the ``sqlite3.enable_load_extension`` auditing event. - - .. testsetup:: sqlite3.loadext - - import sqlite3 - con = sqlite3.connect(":memory:") - - .. testcode:: sqlite3.loadext - :skipif: True # not testable at the moment - - con.enable_load_extension(True) - - # Load the fulltext search extension - con.execute("select load_extension('./fts3.so')") - - # alternatively you can load the extension using an API call: - # con.load_extension("./fts3.so") - - # disable extension loading again - con.enable_load_extension(False) - - # example from SQLite wiki - con.execute("CREATE VIRTUAL TABLE recipe USING fts3(name, ingredients)") - con.executescript(""" - INSERT INTO recipe (name, ingredients) VALUES('broccoli stew', 'broccoli peppers cheese tomatoes'); - INSERT INTO recipe (name, ingredients) VALUES('pumpkin stew', 'pumpkin onions garlic celery'); - INSERT INTO recipe (name, ingredients) VALUES('broccoli pie', 'broccoli cheese onions flour'); - INSERT INTO recipe (name, ingredients) VALUES('pumpkin pie', 'pumpkin sugar flour butter'); - """) - for row in con.execute("SELECT rowid, name, ingredients FROM recipe WHERE name MATCH 'pie'"): - print(row) - - con.close() - - .. testoutput:: sqlite3.loadext - :hide: - - (2, 'broccoli pie', 'broccoli cheese onions flour') - (3, 'pumpkin pie', 'pumpkin sugar flour butter') - - .. method:: load_extension(path, /) - - Load an SQLite extension from a shared library located at *path*. - Enable extension loading with :meth:`enable_load_extension` before - calling this method. - - .. audit-event:: sqlite3.load_extension connection,path sqlite3.Connection.load_extension - - .. versionadded:: 3.2 - - .. versionchanged:: 3.10 - Added the ``sqlite3.load_extension`` auditing event. - - .. method:: iterdump - - Return an :term:`iterator` to dump the database as SQL source code. - Useful when saving an in-memory database for later restoration. - Similar to the ``.dump`` command in the :program:`sqlite3` shell. - - Example: - - .. testcode:: - - # Convert file example.db to SQL dump file dump.sql - con = sqlite3.connect('example.db') - with open('dump.sql', 'w') as f: - for line in con.iterdump(): - f.write('%s\n' % line) - con.close() - - .. seealso:: - - :ref:`sqlite3-howto-encoding` - - - .. method:: backup(target, *, pages=-1, progress=None, name="main", sleep=0.250) - - Create a backup of an SQLite database. - - Works even if the database is being accessed by other clients - or concurrently by the same connection. - - :param ~sqlite3.Connection target: - The database connection to save the backup to. - - :param int pages: - The number of pages to copy at a time. - If equal to or less than ``0``, - the entire database is copied in a single step. - Defaults to ``-1``. - - :param progress: - If set to a :term:`callable`, - it is invoked with three integer arguments for every backup iteration: - the *status* of the last iteration, - the *remaining* number of pages still to be copied, - and the *total* number of pages. - Defaults to ``None``. - :type progress: :term:`callback` | None - - :param str name: - The name of the database to back up. - Either ``"main"`` (the default) for the main database, - ``"temp"`` for the temporary database, - or the name of a custom database as attached using the - ``ATTACH DATABASE`` SQL statement. - - :param float sleep: - The number of seconds to sleep between successive attempts - to back up remaining pages. - - Example 1, copy an existing database into another: - - .. testcode:: - - def progress(status, remaining, total): - print(f'Copied {total-remaining} of {total} pages...') - - src = sqlite3.connect('example.db') - dst = sqlite3.connect('backup.db') - with dst: - src.backup(dst, pages=1, progress=progress) - dst.close() - src.close() - - .. testoutput:: - :hide: - - Copied 0 of 0 pages... - - Example 2, copy an existing database into a transient copy: - - .. testcode:: - - src = sqlite3.connect('example.db') - dst = sqlite3.connect(':memory:') - src.backup(dst) - - .. versionadded:: 3.7 - - .. seealso:: - - :ref:`sqlite3-howto-encoding` - - .. method:: getlimit(category, /) - - Get a connection runtime limit. - - :param int category: - The `SQLite limit category`_ to be queried. - - :rtype: int - - :raises ProgrammingError: - If *category* is not recognised by the underlying SQLite library. - - Example, query the maximum length of an SQL statement - for :class:`Connection` ``con`` (the default is 1000000000): - - .. testsetup:: sqlite3.limits - - import sqlite3 - con = sqlite3.connect(":memory:") - con.setlimit(sqlite3.SQLITE_LIMIT_SQL_LENGTH, 1_000_000_000) - con.setlimit(sqlite3.SQLITE_LIMIT_ATTACHED, 10) - - .. doctest:: sqlite3.limits - - >>> con.getlimit(sqlite3.SQLITE_LIMIT_SQL_LENGTH) - 1000000000 - - .. versionadded:: 3.11 - - - .. method:: setlimit(category, limit, /) - - Set a connection runtime limit. - Attempts to increase a limit above its hard upper bound are silently - truncated to the hard upper bound. Regardless of whether or not the limit - was changed, the prior value of the limit is returned. - - :param int category: - The `SQLite limit category`_ to be set. - - :param int limit: - The value of the new limit. - If negative, the current limit is unchanged. - - :rtype: int - - :raises ProgrammingError: - If *category* is not recognised by the underlying SQLite library. - - Example, limit the number of attached databases to 1 - for :class:`Connection` ``con`` (the default limit is 10): - - .. doctest:: sqlite3.limits - - >>> con.setlimit(sqlite3.SQLITE_LIMIT_ATTACHED, 1) - 10 - >>> con.getlimit(sqlite3.SQLITE_LIMIT_ATTACHED) - 1 - - .. versionadded:: 3.11 - - .. _SQLite limit category: https://www.sqlite.org/c3ref/c_limit_attached.html - - - .. method:: serialize(*, name="main") - - Serialize a database into a :class:`bytes` object. For an - ordinary on-disk database file, the serialization is just a copy of the - disk file. For an in-memory database or a "temp" database, the - serialization is the same sequence of bytes which would be written to - disk if that database were backed up to disk. - - :param str name: - The database name to be serialized. - Defaults to ``"main"``. - - :rtype: bytes - - .. note:: - - This method is only available if the underlying SQLite library has the - serialize API. - - .. versionadded:: 3.11 - - - .. method:: deserialize(data, /, *, name="main") - - Deserialize a :meth:`serialized ` database into a - :class:`Connection`. - This method causes the database connection to disconnect from database - *name*, and reopen *name* as an in-memory database based on the - serialization contained in *data*. - - :param bytes data: - A serialized database. - - :param str name: - The database name to deserialize into. - Defaults to ``"main"``. - - :raises OperationalError: - If the database connection is currently involved in a read - transaction or a backup operation. - - :raises DatabaseError: - If *data* does not contain a valid SQLite database. - - :raises OverflowError: - If :func:`len(data) ` is larger than ``2**63 - 1``. - - .. note:: - - This method is only available if the underlying SQLite library has the - deserialize API. - - .. versionadded:: 3.11 - - .. attribute:: in_transaction - - This read-only attribute corresponds to the low-level SQLite - `autocommit mode`_. - - ``True`` if a transaction is active (there are uncommitted changes), - ``False`` otherwise. - - .. versionadded:: 3.2 - - .. attribute:: isolation_level - - This attribute controls the :ref:`transaction handling - ` performed by :mod:`!sqlite3`. - If set to ``None``, transactions are never implicitly opened. - If set to one of ``"DEFERRED"``, ``"IMMEDIATE"``, or ``"EXCLUSIVE"``, - corresponding to the underlying `SQLite transaction behaviour`_, - implicit :ref:`transaction management - ` is performed. - - If not overridden by the *isolation_level* parameter of :func:`connect`, - the default is ``""``, which is an alias for ``"DEFERRED"``. - - .. attribute:: row_factory - - The initial :attr:`~Cursor.row_factory` - for :class:`Cursor` objects created from this connection. - Assigning to this attribute does not affect the :attr:`!row_factory` - of existing cursors belonging to this connection, only new ones. - Is ``None`` by default, - meaning each row is returned as a :class:`tuple`. - - See :ref:`sqlite3-howto-row-factory` for more details. - - .. attribute:: text_factory - - A :term:`callable` that accepts a :class:`bytes` parameter - and returns a text representation of it. - The callable is invoked for SQLite values with the ``TEXT`` data type. - By default, this attribute is set to :class:`str`. - - See :ref:`sqlite3-howto-encoding` for more details. - - .. attribute:: total_changes - - Return the total number of database rows that have been modified, inserted, or - deleted since the database connection was opened. - - -.. _sqlite3-cursor-objects: - -Cursor objects -^^^^^^^^^^^^^^ - - A ``Cursor`` object represents a `database cursor`_ - which is used to execute SQL statements, - and manage the context of a fetch operation. - Cursors are created using :meth:`Connection.cursor`, - or by using any of the :ref:`connection shortcut methods - `. - - Cursor objects are :term:`iterators `, - meaning that if you :meth:`~Cursor.execute` a ``SELECT`` query, - you can simply iterate over the cursor to fetch the resulting rows: - - .. testsetup:: sqlite3.cursor - - import sqlite3 - con = sqlite3.connect(":memory:", isolation_level=None) - cur = con.execute("CREATE TABLE data(t)") - cur.execute("INSERT INTO data VALUES(1)") - - .. testcode:: sqlite3.cursor - - for row in cur.execute("SELECT t FROM data"): - print(row) - - .. testoutput:: sqlite3.cursor - :hide: - - (1,) - - .. _database cursor: https://en.wikipedia.org/wiki/Cursor_(databases) - -.. class:: Cursor - - A :class:`Cursor` instance has the following attributes and methods. - - .. index:: single: ? (question mark); in SQL statements - .. index:: single: : (colon); in SQL statements - - .. method:: execute(sql, parameters=(), /) - - Execute SQL a single SQL statement, - optionally binding Python values using - :ref:`placeholders `. - - :param str sql: - A single SQL statement. - - :param parameters: - Python values to bind to placeholders in *sql*. - A :class:`!dict` if named placeholders are used. - A :term:`!sequence` if unnamed placeholders are used. - See :ref:`sqlite3-placeholders`. - :type parameters: :class:`dict` | :term:`sequence` - - :raises ProgrammingError: - If *sql* contains more than one SQL statement. - - If :attr:`~Connection.isolation_level` is not ``None``, - *sql* is an ``INSERT``, ``UPDATE``, ``DELETE``, or ``REPLACE`` statement, - and there is no open transaction, - a transaction is implicitly opened before executing *sql*. - - Use :meth:`executescript` to execute multiple SQL statements. - - .. method:: executemany(sql, parameters, /) - - For every item in *parameters*, - repeatedly execute the :ref:`parameterized ` - :abbr:`DML (Data Manipulation Language)` SQL statement *sql*. - - Uses the same implicit transaction handling as :meth:`~Cursor.execute`. - - :param str sql: - A single SQL DML statement. - - :param parameters: - An :term:`!iterable` of parameters to bind with - the placeholders in *sql*. - See :ref:`sqlite3-placeholders`. - :type parameters: :term:`iterable` - - :raises ProgrammingError: - If *sql* contains more than one SQL statement, - or is not a DML statement. - - Example: - - .. testcode:: sqlite3.cursor - - rows = [ - ("row1",), - ("row2",), - ] - # cur is an sqlite3.Cursor object - cur.executemany("INSERT INTO data VALUES(?)", rows) - - .. note:: - - Any resulting rows are discarded, - including DML statements with `RETURNING clauses`_. - - .. _RETURNING clauses: https://www.sqlite.org/lang_returning.html - - .. method:: executescript(sql_script, /) - - Execute the SQL statements in *sql_script*. - If there is a pending transaction, - an implicit ``COMMIT`` statement is executed first. - No other implicit transaction control is performed; - any transaction control must be added to *sql_script*. - - *sql_script* must be a :class:`string `. - - Example: - - .. testcode:: sqlite3.cursor - - # cur is an sqlite3.Cursor object - cur.executescript(""" - BEGIN; - CREATE TABLE person(firstname, lastname, age); - CREATE TABLE book(title, author, published); - CREATE TABLE publisher(name, address); - COMMIT; - """) - - .. method:: fetchone() - - If :attr:`~Cursor.row_factory` is ``None``, - return the next row query result set as a :class:`tuple`. - Else, pass it to the row factory and return its result. - Return ``None`` if no more data is available. - - - .. method:: fetchmany(size=cursor.arraysize) - - Return the next set of rows of a query result as a :class:`list`. - Return an empty list if no more rows are available. - - The number of rows to fetch per call is specified by the *size* parameter. - If *size* is not given, :attr:`arraysize` determines the number of rows - to be fetched. - If fewer than *size* rows are available, - as many rows as are available are returned. - - Note there are performance considerations involved with the *size* parameter. - For optimal performance, it is usually best to use the arraysize attribute. - If the *size* parameter is used, then it is best for it to retain the same - value from one :meth:`fetchmany` call to the next. - - .. method:: fetchall() - - Return all (remaining) rows of a query result as a :class:`list`. - Return an empty list if no rows are available. - Note that the :attr:`arraysize` attribute can affect the performance of - this operation. - - .. method:: close() - - Close the cursor now (rather than whenever ``__del__`` is called). - - The cursor will be unusable from this point forward; a :exc:`ProgrammingError` - exception will be raised if any operation is attempted with the cursor. - - .. method:: setinputsizes(sizes, /) - - Required by the DB-API. Does nothing in :mod:`!sqlite3`. - - .. method:: setoutputsize(size, column=None, /) - - Required by the DB-API. Does nothing in :mod:`!sqlite3`. - - .. attribute:: arraysize - - Read/write attribute that controls the number of rows returned by :meth:`fetchmany`. - The default value is 1 which means a single row would be fetched per call. - - .. attribute:: connection - - Read-only attribute that provides the SQLite database :class:`Connection` - belonging to the cursor. A :class:`Cursor` object created by - calling :meth:`con.cursor() ` will have a - :attr:`connection` attribute that refers to *con*: - - .. doctest:: - - >>> con = sqlite3.connect(":memory:") - >>> cur = con.cursor() - >>> cur.connection == con - True - - .. attribute:: description - - Read-only attribute that provides the column names of the last query. To - remain compatible with the Python DB API, it returns a 7-tuple for each - column where the last six items of each tuple are ``None``. - - It is set for ``SELECT`` statements without any matching rows as well. - - .. attribute:: lastrowid - - Read-only attribute that provides the row id of the last inserted row. It - is only updated after successful ``INSERT`` or ``REPLACE`` statements - using the :meth:`execute` method. For other statements, after - :meth:`executemany` or :meth:`executescript`, or if the insertion failed, - the value of ``lastrowid`` is left unchanged. The initial value of - ``lastrowid`` is ``None``. - - .. note:: - Inserts into ``WITHOUT ROWID`` tables are not recorded. - - .. versionchanged:: 3.6 - Added support for the ``REPLACE`` statement. - - .. attribute:: rowcount - - Read-only attribute that provides the number of modified rows for - ``INSERT``, ``UPDATE``, ``DELETE``, and ``REPLACE`` statements; - is ``-1`` for other statements, - including :abbr:`CTE (Common Table Expression)` queries. - It is only updated by the :meth:`execute` and :meth:`executemany` methods, - after the statement has run to completion. - This means that any resulting rows must be fetched in order for - :attr:`!rowcount` to be updated. - - .. attribute:: row_factory - - Control how a row fetched from this :class:`!Cursor` is represented. - If ``None``, a row is represented as a :class:`tuple`. - Can be set to the included :class:`sqlite3.Row`; - or a :term:`callable` that accepts two arguments, - a :class:`Cursor` object and the :class:`!tuple` of row values, - and returns a custom object representing an SQLite row. - - Defaults to what :attr:`Connection.row_factory` was set to - when the :class:`!Cursor` was created. - Assigning to this attribute does not affect - :attr:`Connection.row_factory` of the parent connection. - - See :ref:`sqlite3-howto-row-factory` for more details. - - -.. The sqlite3.Row example used to be a how-to. It has now been incorporated - into the Row reference. We keep the anchor here in order not to break - existing links. - -.. _sqlite3-columns-by-name: -.. _sqlite3-row-objects: - -Row objects -^^^^^^^^^^^ - -.. class:: Row - - A :class:`!Row` instance serves as a highly optimized - :attr:`~Connection.row_factory` for :class:`Connection` objects. - It supports iteration, equality testing, :func:`len`, - and :term:`mapping` access by column name and index. - - Two :class:`!Row` objects compare equal - if they have identical column names and values. - - See :ref:`sqlite3-howto-row-factory` for more details. - - .. method:: keys - - Return a :class:`list` of column names as :class:`strings `. - Immediately after a query, - it is the first member of each tuple in :attr:`Cursor.description`. - - .. versionchanged:: 3.5 - Added support of slicing. - - -.. _sqlite3-blob-objects: - -Blob objects -^^^^^^^^^^^^ - -.. versionadded:: 3.11 - -.. class:: Blob - - A :class:`Blob` instance is a :term:`file-like object` - that can read and write data in an SQLite :abbr:`BLOB (Binary Large OBject)`. - Call :func:`len(blob) ` to get the size (number of bytes) of the blob. - Use indices and :term:`slices ` for direct access to the blob data. - - Use the :class:`Blob` as a :term:`context manager` to ensure that the blob - handle is closed after use. - - .. testcode:: - - con = sqlite3.connect(":memory:") - con.execute("CREATE TABLE test(blob_col blob)") - con.execute("INSERT INTO test(blob_col) VALUES(zeroblob(13))") - - # Write to our blob, using two write operations: - with con.blobopen("test", "blob_col", 1) as blob: - blob.write(b"hello, ") - blob.write(b"world.") - # Modify the first and last bytes of our blob - blob[0] = ord("H") - blob[-1] = ord("!") - - # Read the contents of our blob - with con.blobopen("test", "blob_col", 1) as blob: - greeting = blob.read() - - print(greeting) # outputs "b'Hello, world!'" - - .. testoutput:: - :hide: - - b'Hello, world!' - - .. method:: close() - - Close the blob. - - The blob will be unusable from this point onward. An - :class:`~sqlite3.Error` (or subclass) exception will be raised if any - further operation is attempted with the blob. - - .. method:: read(length=-1, /) - - Read *length* bytes of data from the blob at the current offset position. - If the end of the blob is reached, the data up to - :abbr:`EOF (End of File)` will be returned. When *length* is not - specified, or is negative, :meth:`~Blob.read` will read until the end of - the blob. - - .. method:: write(data, /) - - Write *data* to the blob at the current offset. This function cannot - change the blob length. Writing beyond the end of the blob will raise - :exc:`ValueError`. - - .. method:: tell() - - Return the current access position of the blob. - - .. method:: seek(offset, origin=os.SEEK_SET, /) - - Set the current access position of the blob to *offset*. The *origin* - argument defaults to :const:`os.SEEK_SET` (absolute blob positioning). - Other values for *origin* are :const:`os.SEEK_CUR` (seek relative to the - current position) and :const:`os.SEEK_END` (seek relative to the blob’s - end). - - -PrepareProtocol objects -^^^^^^^^^^^^^^^^^^^^^^^ - -.. class:: PrepareProtocol - - The PrepareProtocol type's single purpose is to act as a :pep:`246` style - adaption protocol for objects that can :ref:`adapt themselves - ` to :ref:`native SQLite types `. - - -.. _sqlite3-exceptions: - -Exceptions -^^^^^^^^^^ - -The exception hierarchy is defined by the DB-API 2.0 (:pep:`249`). - -.. exception:: Warning - - This exception is not currently raised by the :mod:`!sqlite3` module, - but may be raised by applications using :mod:`!sqlite3`, - for example if a user-defined function truncates data while inserting. - ``Warning`` is a subclass of :exc:`Exception`. - -.. exception:: Error - - The base class of the other exceptions in this module. - Use this to catch all errors with one single :keyword:`except` statement. - ``Error`` is a subclass of :exc:`Exception`. - - If the exception originated from within the SQLite library, - the following two attributes are added to the exception: - - .. attribute:: sqlite_errorcode - - The numeric error code from the - `SQLite API `_ - - .. versionadded:: 3.11 - - .. attribute:: sqlite_errorname - - The symbolic name of the numeric error code - from the `SQLite API `_ - - .. versionadded:: 3.11 - -.. exception:: InterfaceError - - Exception raised for misuse of the low-level SQLite C API. - In other words, if this exception is raised, it probably indicates a bug in the - :mod:`!sqlite3` module. - ``InterfaceError`` is a subclass of :exc:`Error`. - -.. exception:: DatabaseError - - Exception raised for errors that are related to the database. - This serves as the base exception for several types of database errors. - It is only raised implicitly through the specialised subclasses. - ``DatabaseError`` is a subclass of :exc:`Error`. - -.. exception:: DataError - - Exception raised for errors caused by problems with the processed data, - like numeric values out of range, and strings which are too long. - ``DataError`` is a subclass of :exc:`DatabaseError`. - -.. exception:: OperationalError - - Exception raised for errors that are related to the database's operation, - and not necessarily under the control of the programmer. - For example, the database path is not found, - or a transaction could not be processed. - ``OperationalError`` is a subclass of :exc:`DatabaseError`. - -.. exception:: IntegrityError - - Exception raised when the relational integrity of the database is affected, - e.g. a foreign key check fails. It is a subclass of :exc:`DatabaseError`. - -.. exception:: InternalError - - Exception raised when SQLite encounters an internal error. - If this is raised, it may indicate that there is a problem with the runtime - SQLite library. - ``InternalError`` is a subclass of :exc:`DatabaseError`. - -.. exception:: ProgrammingError - - Exception raised for :mod:`!sqlite3` API programming errors, - for example supplying the wrong number of bindings to a query, - or trying to operate on a closed :class:`Connection`. - ``ProgrammingError`` is a subclass of :exc:`DatabaseError`. - -.. exception:: NotSupportedError - - Exception raised in case a method or database API is not supported by the - underlying SQLite library. For example, setting *deterministic* to - ``True`` in :meth:`~Connection.create_function`, if the underlying SQLite library - does not support deterministic functions. - ``NotSupportedError`` is a subclass of :exc:`DatabaseError`. - - -.. _sqlite3-types: - -SQLite and Python types -^^^^^^^^^^^^^^^^^^^^^^^ - -SQLite natively supports the following types: ``NULL``, ``INTEGER``, -``REAL``, ``TEXT``, ``BLOB``. - -The following Python types can thus be sent to SQLite without any problem: - -+-------------------------------+-------------+ -| Python type | SQLite type | -+===============================+=============+ -| ``None`` | ``NULL`` | -+-------------------------------+-------------+ -| :class:`int` | ``INTEGER`` | -+-------------------------------+-------------+ -| :class:`float` | ``REAL`` | -+-------------------------------+-------------+ -| :class:`str` | ``TEXT`` | -+-------------------------------+-------------+ -| :class:`bytes` | ``BLOB`` | -+-------------------------------+-------------+ - - -This is how SQLite types are converted to Python types by default: - -+-------------+----------------------------------------------+ -| SQLite type | Python type | -+=============+==============================================+ -| ``NULL`` | ``None`` | -+-------------+----------------------------------------------+ -| ``INTEGER`` | :class:`int` | -+-------------+----------------------------------------------+ -| ``REAL`` | :class:`float` | -+-------------+----------------------------------------------+ -| ``TEXT`` | depends on :attr:`~Connection.text_factory`, | -| | :class:`str` by default | -+-------------+----------------------------------------------+ -| ``BLOB`` | :class:`bytes` | -+-------------+----------------------------------------------+ - -The type system of the :mod:`!sqlite3` module is extensible in two ways: you can -store additional Python types in an SQLite database via -:ref:`object adapters `, -and you can let the :mod:`!sqlite3` module convert SQLite types to -Python types via :ref:`converters `. - - -.. _sqlite3-default-converters: - -Default adapters and converters -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are default adapters for the date and datetime types in the datetime -module. They will be sent as ISO dates/ISO timestamps to SQLite. - -The default converters are registered under the name "date" for -:class:`datetime.date` and under the name "timestamp" for -:class:`datetime.datetime`. - -This way, you can use date/timestamps from Python without any additional -fiddling in most cases. The format of the adapters is also compatible with the -experimental SQLite date/time functions. - -The following example demonstrates this. - -.. literalinclude:: ../includes/sqlite3/pysqlite_datetime.py - -If a timestamp stored in SQLite has a fractional part longer than 6 -numbers, its value will be truncated to microsecond precision by the -timestamp converter. - -.. note:: - - The default "timestamp" converter ignores UTC offsets in the database and - always returns a naive :class:`datetime.datetime` object. To preserve UTC - offsets in timestamps, either leave converters disabled, or register an - offset-aware converter with :func:`register_converter`. - - -.. _sqlite3-howtos: - -How-to guides -------------- - -.. _sqlite3-placeholders: - -How to use placeholders to bind values in SQL queries -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -SQL operations usually need to use values from Python variables. However, -beware of using Python's string operations to assemble queries, as they -are vulnerable to `SQL injection attacks`_. For example, an attacker can simply -close the single quote and inject ``OR TRUE`` to select all rows:: - - >>> # Never do this -- insecure! - >>> symbol = input() - ' OR TRUE; -- - >>> sql = "SELECT * FROM stocks WHERE symbol = '%s'" % symbol - >>> print(sql) - SELECT * FROM stocks WHERE symbol = '' OR TRUE; --' - >>> cur.execute(sql) - -Instead, use the DB-API's parameter substitution. To insert a variable into a -query string, use a placeholder in the string, and substitute the actual values -into the query by providing them as a :class:`tuple` of values to the second -argument of the cursor's :meth:`~Cursor.execute` method. - -An SQL statement may use one of two kinds of placeholders: -question marks (qmark style) or named placeholders (named style). -For the qmark style, *parameters* must be a -:term:`sequence` whose length must match the number of placeholders, -or a :exc:`ProgrammingError` is raised. -For the named style, *parameters* should be -an instance of a :class:`dict` (or a subclass), -which must contain keys for all named parameters; -any extra items are ignored. -Here's an example of both styles: - -.. testcode:: - - con = sqlite3.connect(":memory:") - cur = con.execute("CREATE TABLE lang(name, first_appeared)") - - # This is the named style used with executemany(): - data = ( - {"name": "C", "year": 1972}, - {"name": "Fortran", "year": 1957}, - {"name": "Python", "year": 1991}, - {"name": "Go", "year": 2009}, - ) - cur.executemany("INSERT INTO lang VALUES(:name, :year)", data) - - # This is the qmark style used in a SELECT query: - params = (1972,) - cur.execute("SELECT * FROM lang WHERE first_appeared = ?", params) - print(cur.fetchall()) - -.. testoutput:: - :hide: - - [('C', 1972)] - -.. note:: - - :pep:`249` numeric placeholders are *not* supported. - If used, they will be interpreted as named placeholders. - - -.. _sqlite3-adapters: - -How to adapt custom Python types to SQLite values -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -SQLite supports only a limited set of data types natively. -To store custom Python types in SQLite databases, *adapt* them to one of the -:ref:`Python types SQLite natively understands `. - -There are two ways to adapt Python objects to SQLite types: -letting your object adapt itself, or using an *adapter callable*. -The latter will take precedence above the former. -For a library that exports a custom type, -it may make sense to enable that type to adapt itself. -As an application developer, it may make more sense to take direct control by -registering custom adapter functions. - - -.. _sqlite3-conform: - -How to write adaptable objects -"""""""""""""""""""""""""""""" - -Suppose we have a :class:`!Point` class that represents a pair of coordinates, -``x`` and ``y``, in a Cartesian coordinate system. -The coordinate pair will be stored as a text string in the database, -using a semicolon to separate the coordinates. -This can be implemented by adding a ``__conform__(self, protocol)`` -method which returns the adapted value. -The object passed to *protocol* will be of type :class:`PrepareProtocol`. - -.. testcode:: - - class Point: - def __init__(self, x, y): - self.x, self.y = x, y - - def __conform__(self, protocol): - if protocol is sqlite3.PrepareProtocol: - return f"{self.x};{self.y}" - - con = sqlite3.connect(":memory:") - cur = con.cursor() - - cur.execute("SELECT ?", (Point(4.0, -3.2),)) - print(cur.fetchone()[0]) - -.. testoutput:: - :hide: - - 4.0;-3.2 - - -How to register adapter callables -""""""""""""""""""""""""""""""""" - -The other possibility is to create a function that converts the Python object -to an SQLite-compatible type. -This function can then be registered using :func:`register_adapter`. - -.. testcode:: - - class Point: - def __init__(self, x, y): - self.x, self.y = x, y - - def adapt_point(point): - return f"{point.x};{point.y}" - - sqlite3.register_adapter(Point, adapt_point) - - con = sqlite3.connect(":memory:") - cur = con.cursor() - - cur.execute("SELECT ?", (Point(1.0, 2.5),)) - print(cur.fetchone()[0]) - -.. testoutput:: - :hide: - - 1.0;2.5 - - -.. _sqlite3-converters: - -How to convert SQLite values to custom Python types -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Writing an adapter lets you convert *from* custom Python types *to* SQLite -values. -To be able to convert *from* SQLite values *to* custom Python types, -we use *converters*. - -Let's go back to the :class:`!Point` class. We stored the x and y coordinates -separated via semicolons as strings in SQLite. - -First, we'll define a converter function that accepts the string as a parameter -and constructs a :class:`!Point` object from it. - -.. note:: - - Converter functions are **always** passed a :class:`bytes` object, - no matter the underlying SQLite data type. - -.. testcode:: - - def convert_point(s): - x, y = map(float, s.split(b";")) - return Point(x, y) - -We now need to tell :mod:`!sqlite3` when it should convert a given SQLite value. -This is done when connecting to a database, using the *detect_types* parameter -of :func:`connect`. There are three options: - -* Implicit: set *detect_types* to :const:`PARSE_DECLTYPES` -* Explicit: set *detect_types* to :const:`PARSE_COLNAMES` -* Both: set *detect_types* to - ``sqlite3.PARSE_DECLTYPES | sqlite3.PARSE_COLNAMES``. - Column names take precedence over declared types. - -The following example illustrates the implicit and explicit approaches: - -.. testcode:: - - class Point: - def __init__(self, x, y): - self.x, self.y = x, y - - def __repr__(self): - return f"Point({self.x}, {self.y})" - - def adapt_point(point): - return f"{point.x};{point.y}" - - def convert_point(s): - x, y = list(map(float, s.split(b";"))) - return Point(x, y) - - # Register the adapter and converter - sqlite3.register_adapter(Point, adapt_point) - sqlite3.register_converter("point", convert_point) - - # 1) Parse using declared types - p = Point(4.0, -3.2) - con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_DECLTYPES) - cur = con.execute("CREATE TABLE test(p point)") - - cur.execute("INSERT INTO test(p) VALUES(?)", (p,)) - cur.execute("SELECT p FROM test") - print("with declared types:", cur.fetchone()[0]) - cur.close() - con.close() - - # 2) Parse using column names - con = sqlite3.connect(":memory:", detect_types=sqlite3.PARSE_COLNAMES) - cur = con.execute("CREATE TABLE test(p)") - - cur.execute("INSERT INTO test(p) VALUES(?)", (p,)) - cur.execute('SELECT p AS "p [point]" FROM test') - print("with column names:", cur.fetchone()[0]) - -.. testoutput:: - :hide: - - with declared types: Point(4.0, -3.2) - with column names: Point(4.0, -3.2) - - -.. _sqlite3-adapter-converter-recipes: - -Adapter and converter recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This section shows recipes for common adapters and converters. - -.. testcode:: - - import datetime - import sqlite3 - - def adapt_date_iso(val): - """Adapt datetime.date to ISO 8601 date.""" - return val.isoformat() - - def adapt_datetime_iso(val): - """Adapt datetime.datetime to timezone-naive ISO 8601 date.""" - return val.isoformat() - - def adapt_datetime_epoch(val): - """Adapt datetime.datetime to Unix timestamp.""" - return int(val.timestamp()) - - sqlite3.register_adapter(datetime.date, adapt_date_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_iso) - sqlite3.register_adapter(datetime.datetime, adapt_datetime_epoch) - - def convert_date(val): - """Convert ISO 8601 date to datetime.date object.""" - return datetime.date.fromisoformat(val.decode()) - - def convert_datetime(val): - """Convert ISO 8601 datetime to datetime.datetime object.""" - return datetime.datetime.fromisoformat(val.decode()) - - def convert_timestamp(val): - """Convert Unix epoch timestamp to datetime.datetime object.""" - return datetime.datetime.fromtimestamp(int(val)) - - sqlite3.register_converter("date", convert_date) - sqlite3.register_converter("datetime", convert_datetime) - sqlite3.register_converter("timestamp", convert_timestamp) - -.. testcode:: - :hide: - - dt = datetime.datetime(2019, 5, 18, 15, 17, 8, 123456) - - assert adapt_date_iso(dt.date()) == "2019-05-18" - assert convert_date(b"2019-05-18") == dt.date() - - assert adapt_datetime_iso(dt) == "2019-05-18T15:17:08.123456" - assert convert_datetime(b"2019-05-18T15:17:08.123456") == dt - - # Using current time as fromtimestamp() returns local date/time. - # Droping microseconds as adapt_datetime_epoch truncates fractional second part. - now = datetime.datetime.now().replace(microsecond=0) - current_timestamp = int(now.timestamp()) - - assert adapt_datetime_epoch(now) == current_timestamp - assert convert_timestamp(str(current_timestamp).encode()) == now - - -.. _sqlite3-connection-shortcuts: - -How to use connection shortcut methods -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Using the :meth:`~Connection.execute`, -:meth:`~Connection.executemany`, and :meth:`~Connection.executescript` -methods of the :class:`Connection` class, your code can -be written more concisely because you don't have to create the (often -superfluous) :class:`Cursor` objects explicitly. Instead, the :class:`Cursor` -objects are created implicitly and these shortcut methods return the cursor -objects. This way, you can execute a ``SELECT`` statement and iterate over it -directly using only a single call on the :class:`Connection` object. - -.. testcode:: - - # Create and fill the table. - con = sqlite3.connect(":memory:") - con.execute("CREATE TABLE lang(name, first_appeared)") - data = [ - ("C++", 1985), - ("Objective-C", 1984), - ] - con.executemany("INSERT INTO lang(name, first_appeared) VALUES(?, ?)", data) - - # Print the table contents - for row in con.execute("SELECT name, first_appeared FROM lang"): - print(row) - - print("I just deleted", con.execute("DELETE FROM lang").rowcount, "rows") - - # close() is not a shortcut method and it's not called automatically; - # the connection object should be closed manually - con.close() - -.. testoutput:: - :hide: - - ('C++', 1985) - ('Objective-C', 1984) - I just deleted 2 rows - - -.. _sqlite3-connection-context-manager: - -How to use the connection context manager -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -A :class:`Connection` object can be used as a context manager that -automatically commits or rolls back open transactions when leaving the body of -the context manager. -If the body of the :keyword:`with` statement finishes without exceptions, -the transaction is committed. -If this commit fails, -or if the body of the ``with`` statement raises an uncaught exception, -the transaction is rolled back. - -If there is no open transaction upon leaving the body of the ``with`` statement, -the context manager is a no-op. - -.. note:: - The context manager neither implicitly opens a new transaction - nor closes the connection. If you need a closing context manager, consider - using :meth:`contextlib.closing`. - -.. testcode:: - - con = sqlite3.connect(":memory:") - con.execute("CREATE TABLE lang(id INTEGER PRIMARY KEY, name VARCHAR UNIQUE)") - - # Successful, con.commit() is called automatically afterwards - with con: - con.execute("INSERT INTO lang(name) VALUES(?)", ("Python",)) - - # con.rollback() is called after the with block finishes with an exception, - # the exception is still raised and must be caught - try: - with con: - con.execute("INSERT INTO lang(name) VALUES(?)", ("Python",)) - except sqlite3.IntegrityError: - print("couldn't add Python twice") - - # Connection object used as context manager only commits or rollbacks transactions, - # so the connection object should be closed manually - con.close() - -.. testoutput:: - :hide: - - couldn't add Python twice - - -.. _sqlite3-uri-tricks: - -How to work with SQLite URIs -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Some useful URI tricks include: - -* Open a database in read-only mode: - -.. doctest:: - - >>> con = sqlite3.connect("file:tutorial.db?mode=ro", uri=True) - >>> con.execute("CREATE TABLE readonly(data)") - Traceback (most recent call last): - OperationalError: attempt to write a readonly database - -* Do not implicitly create a new database file if it does not already exist; - will raise :exc:`~sqlite3.OperationalError` if unable to create a new file: - -.. doctest:: - - >>> con = sqlite3.connect("file:nosuchdb.db?mode=rw", uri=True) - Traceback (most recent call last): - OperationalError: unable to open database file - - -* Create a shared named in-memory database: - -.. testcode:: - - db = "file:mem1?mode=memory&cache=shared" - con1 = sqlite3.connect(db, uri=True) - con2 = sqlite3.connect(db, uri=True) - with con1: - con1.execute("CREATE TABLE shared(data)") - con1.execute("INSERT INTO shared VALUES(28)") - res = con2.execute("SELECT data FROM shared") - assert res.fetchone() == (28,) - - -More information about this feature, including a list of parameters, -can be found in the `SQLite URI documentation`_. - -.. _SQLite URI documentation: https://www.sqlite.org/uri.html - - -.. _sqlite3-howto-row-factory: - -How to create and use row factories -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, :mod:`!sqlite3` represents each row as a :class:`tuple`. -If a :class:`!tuple` does not suit your needs, -you can use the :class:`sqlite3.Row` class -or a custom :attr:`~Cursor.row_factory`. - -While :attr:`!row_factory` exists as an attribute both on the -:class:`Cursor` and the :class:`Connection`, -it is recommended to set :class:`Connection.row_factory`, -so all cursors created from the connection will use the same row factory. - -:class:`!Row` provides indexed and case-insensitive named access to columns, -with minimal memory overhead and performance impact over a :class:`!tuple`. -To use :class:`!Row` as a row factory, -assign it to the :attr:`!row_factory` attribute: - -.. doctest:: - - >>> con = sqlite3.connect(":memory:") - >>> con.row_factory = sqlite3.Row - -Queries now return :class:`!Row` objects: - -.. doctest:: - - >>> res = con.execute("SELECT 'Earth' AS name, 6378 AS radius") - >>> row = res.fetchone() - >>> row.keys() - ['name', 'radius'] - >>> row[0] # Access by index. - 'Earth' - >>> row["name"] # Access by name. - 'Earth' - >>> row["RADIUS"] # Column names are case-insensitive. - 6378 - -.. note:: - - The ``FROM`` clause can be omitted in the ``SELECT`` statement, as in the - above example. In such cases, SQLite returns a single row with columns - defined by expressions, e.g. literals, with the given aliases - ``expr AS alias``. - -You can create a custom :attr:`~Cursor.row_factory` -that returns each row as a :class:`dict`, with column names mapped to values: - -.. testcode:: - - def dict_factory(cursor, row): - fields = [column[0] for column in cursor.description] - return {key: value for key, value in zip(fields, row)} - -Using it, queries now return a :class:`!dict` instead of a :class:`!tuple`: - -.. doctest:: - - >>> con = sqlite3.connect(":memory:") - >>> con.row_factory = dict_factory - >>> for row in con.execute("SELECT 1 AS a, 2 AS b"): - ... print(row) - {'a': 1, 'b': 2} - -The following row factory returns a :term:`named tuple`: - -.. testcode:: - - from collections import namedtuple - - def namedtuple_factory(cursor, row): - fields = [column[0] for column in cursor.description] - cls = namedtuple("Row", fields) - return cls._make(row) - -:func:`!namedtuple_factory` can be used as follows: - -.. doctest:: - - >>> con = sqlite3.connect(":memory:") - >>> con.row_factory = namedtuple_factory - >>> cur = con.execute("SELECT 1 AS a, 2 AS b") - >>> row = cur.fetchone() - >>> row - Row(a=1, b=2) - >>> row[0] # Indexed access. - 1 - >>> row.b # Attribute access. - 2 - -With some adjustments, the above recipe can be adapted to use a -:class:`~dataclasses.dataclass`, or any other custom class, -instead of a :class:`~collections.namedtuple`. - - -.. _sqlite3-howto-encoding: - -How to handle non-UTF-8 text encodings -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, :mod:`!sqlite3` uses :class:`str` to adapt SQLite values -with the ``TEXT`` data type. -This works well for UTF-8 encoded text, but it might fail for other encodings -and invalid UTF-8. -You can use a custom :attr:`~Connection.text_factory` to handle such cases. - -Because of SQLite's `flexible typing`_, it is not uncommon to encounter table -columns with the ``TEXT`` data type containing non-UTF-8 encodings, -or even arbitrary data. -To demonstrate, let's assume we have a database with ISO-8859-2 (Latin-2) -encoded text, for example a table of Czech-English dictionary entries. -Assuming we now have a :class:`Connection` instance :py:data:`!con` -connected to this database, -we can decode the Latin-2 encoded text using this :attr:`~Connection.text_factory`: - -.. testcode:: - - con.text_factory = lambda data: str(data, encoding="latin2") - -For invalid UTF-8 or arbitrary data in stored in ``TEXT`` table columns, -you can use the following technique, borrowed from the :ref:`unicode-howto`: - -.. testcode:: - - con.text_factory = lambda data: str(data, errors="surrogateescape") - -.. note:: - - The :mod:`!sqlite3` module API does not support strings - containing surrogates. - -.. seealso:: - - :ref:`unicode-howto` - - -.. _sqlite3-explanation: - -Explanation ------------ - -.. _sqlite3-controlling-transactions: - -Transaction control -^^^^^^^^^^^^^^^^^^^ - -The :mod:`!sqlite3` module does not adhere to the transaction handling recommended -by :pep:`249`. - -If the connection attribute :attr:`~Connection.isolation_level` -is not ``None``, -new transactions are implicitly opened before -:meth:`~Cursor.execute` and :meth:`~Cursor.executemany` executes -``INSERT``, ``UPDATE``, ``DELETE``, or ``REPLACE`` statements; -for other statements, no implicit transaction handling is performed. -Use the :meth:`~Connection.commit` and :meth:`~Connection.rollback` methods -to respectively commit and roll back pending transactions. -You can choose the underlying `SQLite transaction behaviour`_ — -that is, whether and what type of ``BEGIN`` statements :mod:`!sqlite3` -implicitly executes – -via the :attr:`~Connection.isolation_level` attribute. - -If :attr:`~Connection.isolation_level` is set to ``None``, -no transactions are implicitly opened at all. -This leaves the underlying SQLite library in `autocommit mode`_, -but also allows the user to perform their own transaction handling -using explicit SQL statements. -The underlying SQLite library autocommit mode can be queried using the -:attr:`~Connection.in_transaction` attribute. - -The :meth:`~Cursor.executescript` method implicitly commits -any pending transaction before execution of the given SQL script, -regardless of the value of :attr:`~Connection.isolation_level`. - -.. versionchanged:: 3.6 - :mod:`!sqlite3` used to implicitly commit an open transaction before DDL - statements. This is no longer the case. - -.. _autocommit mode: - https://www.sqlite.org/lang_transaction.html#implicit_versus_explicit_transactions - -.. _SQLite transaction behaviour: - https://www.sqlite.org/lang_transaction.html#deferred_immediate_and_exclusive_transactions diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst deleted file mode 100644 index e7694d70163f76..00000000000000 --- a/Doc/library/ssl.rst +++ /dev/null @@ -1,2777 +0,0 @@ -:mod:`ssl` --- TLS/SSL wrapper for socket objects -================================================= - -.. module:: ssl - :synopsis: TLS/SSL wrapper for socket objects - -.. moduleauthor:: Bill Janssen -.. sectionauthor:: Bill Janssen - -**Source code:** :source:`Lib/ssl.py` - -.. index:: single: OpenSSL; (use in module ssl) - -.. index:: TLS, SSL, Transport Layer Security, Secure Sockets Layer - --------------- - -This module provides access to Transport Layer Security (often known as "Secure -Sockets Layer") encryption and peer authentication facilities for network -sockets, both client-side and server-side. This module uses the OpenSSL -library. It is available on all modern Unix systems, Windows, macOS, and -probably additional platforms, as long as OpenSSL is installed on that platform. - -.. note:: - - Some behavior may be platform dependent, since calls are made to the - operating system socket APIs. The installed version of OpenSSL may also - cause variations in behavior. For example, TLSv1.3 with OpenSSL version - 1.1.1. - -.. warning:: - Don't use this module without reading the :ref:`ssl-security`. Doing so - may lead to a false sense of security, as the default settings of the - ssl module are not necessarily appropriate for your application. - -.. include:: ../includes/wasm-notavail.rst - -This section documents the objects and functions in the ``ssl`` module; for more -general information about TLS, SSL, and certificates, the reader is referred to -the documents in the "See Also" section at the bottom. - -This module provides a class, :class:`ssl.SSLSocket`, which is derived from the -:class:`socket.socket` type, and provides a socket-like wrapper that also -encrypts and decrypts the data going over the socket with SSL. It supports -additional methods such as :meth:`getpeercert`, which retrieves the -certificate of the other side of the connection, and :meth:`cipher`, which -retrieves the cipher being used for the secure connection. - -For more sophisticated applications, the :class:`ssl.SSLContext` class -helps manage settings and certificates, which can then be inherited -by SSL sockets created through the :meth:`SSLContext.wrap_socket` method. - -.. versionchanged:: 3.5.3 - Updated to support linking with OpenSSL 1.1.0 - -.. versionchanged:: 3.6 - - OpenSSL 0.9.8, 1.0.0 and 1.0.1 are deprecated and no longer supported. - In the future the ssl module will require at least OpenSSL 1.0.2 or - 1.1.0. - -.. versionchanged:: 3.10 - - :pep:`644` has been implemented. The ssl module requires OpenSSL 1.1.1 - or newer. - - Use of deprecated constants and functions result in deprecation warnings. - - -Functions, Constants, and Exceptions ------------------------------------- - - -Socket creation -^^^^^^^^^^^^^^^ - -Since Python 3.2 and 2.7.9, it is recommended to use the -:meth:`SSLContext.wrap_socket` of an :class:`SSLContext` instance to wrap -sockets as :class:`SSLSocket` objects. The helper functions -:func:`create_default_context` returns a new context with secure default -settings. The old :func:`wrap_socket` function is deprecated since it is -both inefficient and has no support for server name indication (SNI) and -hostname matching. - -Client socket example with default context and IPv4/IPv6 dual stack:: - - import socket - import ssl - - hostname = 'www.python.org' - context = ssl.create_default_context() - - with socket.create_connection((hostname, 443)) as sock: - with context.wrap_socket(sock, server_hostname=hostname) as ssock: - print(ssock.version()) - - -Client socket example with custom context and IPv4:: - - hostname = 'www.python.org' - # PROTOCOL_TLS_CLIENT requires valid cert chain and hostname - context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - context.load_verify_locations('path/to/cabundle.pem') - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: - with context.wrap_socket(sock, server_hostname=hostname) as ssock: - print(ssock.version()) - - -Server socket example listening on localhost IPv4:: - - context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - context.load_cert_chain('/path/to/certchain.pem', '/path/to/private.key') - - with socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) as sock: - sock.bind(('127.0.0.1', 8443)) - sock.listen(5) - with context.wrap_socket(sock, server_side=True) as ssock: - conn, addr = ssock.accept() - ... - - -Context creation -^^^^^^^^^^^^^^^^ - -A convenience function helps create :class:`SSLContext` objects for common -purposes. - -.. function:: create_default_context(purpose=Purpose.SERVER_AUTH, cafile=None, capath=None, cadata=None) - - Return a new :class:`SSLContext` object with default settings for - the given *purpose*. The settings are chosen by the :mod:`ssl` module, - and usually represent a higher security level than when calling the - :class:`SSLContext` constructor directly. - - *cafile*, *capath*, *cadata* represent optional CA certificates to - trust for certificate verification, as in - :meth:`SSLContext.load_verify_locations`. If all three are - :const:`None`, this function can choose to trust the system's default - CA certificates instead. - - The settings are: :data:`PROTOCOL_TLS_CLIENT` or - :data:`PROTOCOL_TLS_SERVER`, :data:`OP_NO_SSLv2`, and :data:`OP_NO_SSLv3` - with high encryption cipher suites without RC4 and - without unauthenticated cipher suites. Passing :const:`~Purpose.SERVER_AUTH` - as *purpose* sets :data:`~SSLContext.verify_mode` to :data:`CERT_REQUIRED` - and either loads CA certificates (when at least one of *cafile*, *capath* or - *cadata* is given) or uses :meth:`SSLContext.load_default_certs` to load - default CA certificates. - - When :attr:`~SSLContext.keylog_filename` is supported and the environment - variable :envvar:`SSLKEYLOGFILE` is set, :func:`create_default_context` - enables key logging. - - .. note:: - The protocol, options, cipher and other settings may change to more - restrictive values anytime without prior deprecation. The values - represent a fair balance between compatibility and security. - - If your application needs specific settings, you should create a - :class:`SSLContext` and apply the settings yourself. - - .. note:: - If you find that when certain older clients or servers attempt to connect - with a :class:`SSLContext` created by this function that they get an error - stating "Protocol or cipher suite mismatch", it may be that they only - support SSL3.0 which this function excludes using the - :data:`OP_NO_SSLv3`. SSL3.0 is widely considered to be `completely broken - `_. If you still wish to continue to - use this function but still allow SSL 3.0 connections you can re-enable - them using:: - - ctx = ssl.create_default_context(Purpose.CLIENT_AUTH) - ctx.options &= ~ssl.OP_NO_SSLv3 - - .. versionadded:: 3.4 - - .. versionchanged:: 3.4.4 - - RC4 was dropped from the default cipher string. - - .. versionchanged:: 3.6 - - ChaCha20/Poly1305 was added to the default cipher string. - - 3DES was dropped from the default cipher string. - - .. versionchanged:: 3.8 - - Support for key logging to :envvar:`SSLKEYLOGFILE` was added. - - .. versionchanged:: 3.10 - - The context now uses :data:`PROTOCOL_TLS_CLIENT` or - :data:`PROTOCOL_TLS_SERVER` protocol instead of generic - :data:`PROTOCOL_TLS`. - - -Exceptions -^^^^^^^^^^ - -.. exception:: SSLError - - Raised to signal an error from the underlying SSL implementation - (currently provided by the OpenSSL library). This signifies some - problem in the higher-level encryption and authentication layer that's - superimposed on the underlying network connection. This error - is a subtype of :exc:`OSError`. The error code and message of - :exc:`SSLError` instances are provided by the OpenSSL library. - - .. versionchanged:: 3.3 - :exc:`SSLError` used to be a subtype of :exc:`socket.error`. - - .. attribute:: library - - A string mnemonic designating the OpenSSL submodule in which the error - occurred, such as ``SSL``, ``PEM`` or ``X509``. The range of possible - values depends on the OpenSSL version. - - .. versionadded:: 3.3 - - .. attribute:: reason - - A string mnemonic designating the reason this error occurred, for - example ``CERTIFICATE_VERIFY_FAILED``. The range of possible - values depends on the OpenSSL version. - - .. versionadded:: 3.3 - -.. exception:: SSLZeroReturnError - - A subclass of :exc:`SSLError` raised when trying to read or write and - the SSL connection has been closed cleanly. Note that this doesn't - mean that the underlying transport (read TCP) has been closed. - - .. versionadded:: 3.3 - -.. exception:: SSLWantReadError - - A subclass of :exc:`SSLError` raised by a :ref:`non-blocking SSL socket - ` when trying to read or write data, but more data needs - to be received on the underlying TCP transport before the request can be - fulfilled. - - .. versionadded:: 3.3 - -.. exception:: SSLWantWriteError - - A subclass of :exc:`SSLError` raised by a :ref:`non-blocking SSL socket - ` when trying to read or write data, but more data needs - to be sent on the underlying TCP transport before the request can be - fulfilled. - - .. versionadded:: 3.3 - -.. exception:: SSLSyscallError - - A subclass of :exc:`SSLError` raised when a system error was encountered - while trying to fulfill an operation on a SSL socket. Unfortunately, - there is no easy way to inspect the original errno number. - - .. versionadded:: 3.3 - -.. exception:: SSLEOFError - - A subclass of :exc:`SSLError` raised when the SSL connection has been - terminated abruptly. Generally, you shouldn't try to reuse the underlying - transport when this error is encountered. - - .. versionadded:: 3.3 - -.. exception:: SSLCertVerificationError - - A subclass of :exc:`SSLError` raised when certificate validation has - failed. - - .. versionadded:: 3.7 - - .. attribute:: verify_code - - A numeric error number that denotes the verification error. - - .. attribute:: verify_message - - A human readable string of the verification error. - -.. exception:: CertificateError - - An alias for :exc:`SSLCertVerificationError`. - - .. versionchanged:: 3.7 - The exception is now an alias for :exc:`SSLCertVerificationError`. - - -Random generation -^^^^^^^^^^^^^^^^^ - -.. function:: RAND_bytes(num) - - Return *num* cryptographically strong pseudo-random bytes. Raises an - :class:`SSLError` if the PRNG has not been seeded with enough data or if the - operation is not supported by the current RAND method. :func:`RAND_status` - can be used to check the status of the PRNG and :func:`RAND_add` can be used - to seed the PRNG. - - For almost all applications :func:`os.urandom` is preferable. - - Read the Wikipedia article, `Cryptographically secure pseudorandom number - generator (CSPRNG) - `_, - to get the requirements of a cryptographically strong generator. - - .. versionadded:: 3.3 - -.. function:: RAND_pseudo_bytes(num) - - Return (bytes, is_cryptographic): bytes are *num* pseudo-random bytes, - is_cryptographic is ``True`` if the bytes generated are cryptographically - strong. Raises an :class:`SSLError` if the operation is not supported by the - current RAND method. - - Generated pseudo-random byte sequences will be unique if they are of - sufficient length, but are not necessarily unpredictable. They can be used - for non-cryptographic purposes and for certain purposes in cryptographic - protocols, but usually not for key generation etc. - - For almost all applications :func:`os.urandom` is preferable. - - .. versionadded:: 3.3 - - .. deprecated:: 3.6 - - OpenSSL has deprecated :func:`ssl.RAND_pseudo_bytes`, use - :func:`ssl.RAND_bytes` instead. - -.. function:: RAND_status() - - Return ``True`` if the SSL pseudo-random number generator has been seeded - with 'enough' randomness, and ``False`` otherwise. You can use - :func:`ssl.RAND_egd` and :func:`ssl.RAND_add` to increase the randomness of - the pseudo-random number generator. - -.. function:: RAND_add(bytes, entropy) - - Mix the given *bytes* into the SSL pseudo-random number generator. The - parameter *entropy* (a float) is a lower bound on the entropy contained in - string (so you can always use ``0.0``). See :rfc:`1750` for more - information on sources of entropy. - - .. versionchanged:: 3.5 - Writable :term:`bytes-like object` is now accepted. - -Certificate handling -^^^^^^^^^^^^^^^^^^^^ - -.. testsetup:: - - import ssl - -.. function:: match_hostname(cert, hostname) - - Verify that *cert* (in decoded format as returned by - :meth:`SSLSocket.getpeercert`) matches the given *hostname*. The rules - applied are those for checking the identity of HTTPS servers as outlined - in :rfc:`2818`, :rfc:`5280` and :rfc:`6125`. In addition to HTTPS, this - function should be suitable for checking the identity of servers in - various SSL-based protocols such as FTPS, IMAPS, POPS and others. - - :exc:`CertificateError` is raised on failure. On success, the function - returns nothing:: - - >>> cert = {'subject': ((('commonName', 'example.com'),),)} - >>> ssl.match_hostname(cert, "example.com") - >>> ssl.match_hostname(cert, "example.org") - Traceback (most recent call last): - File "", line 1, in - File "/home/py3k/Lib/ssl.py", line 130, in match_hostname - ssl.CertificateError: hostname 'example.org' doesn't match 'example.com' - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3.3 - The function now follows :rfc:`6125`, section 6.4.3 and does neither - match multiple wildcards (e.g. ``*.*.com`` or ``*a*.example.org``) nor - a wildcard inside an internationalized domain names (IDN) fragment. - IDN A-labels such as ``www*.xn--pthon-kva.org`` are still supported, - but ``x*.python.org`` no longer matches ``xn--tda.python.org``. - - .. versionchanged:: 3.5 - Matching of IP addresses, when present in the subjectAltName field - of the certificate, is now supported. - - .. versionchanged:: 3.7 - The function is no longer used to TLS connections. Hostname matching - is now performed by OpenSSL. - - Allow wildcard when it is the leftmost and the only character - in that segment. Partial wildcards like ``www*.example.com`` are no - longer supported. - - .. deprecated:: 3.7 - -.. function:: cert_time_to_seconds(cert_time) - - Return the time in seconds since the Epoch, given the ``cert_time`` - string representing the "notBefore" or "notAfter" date from a - certificate in ``"%b %d %H:%M:%S %Y %Z"`` strptime format (C - locale). - - Here's an example: - - .. doctest:: newcontext - - >>> import ssl - >>> timestamp = ssl.cert_time_to_seconds("Jan 5 09:34:43 2018 GMT") - >>> timestamp # doctest: +SKIP - 1515144883 - >>> from datetime import datetime - >>> print(datetime.utcfromtimestamp(timestamp)) # doctest: +SKIP - 2018-01-05 09:34:43 - - "notBefore" or "notAfter" dates must use GMT (:rfc:`5280`). - - .. versionchanged:: 3.5 - Interpret the input time as a time in UTC as specified by 'GMT' - timezone in the input string. Local timezone was used - previously. Return an integer (no fractions of a second in the - input format) - -.. function:: get_server_certificate(addr, ssl_version=PROTOCOL_TLS_CLIENT, \ - ca_certs=None[, timeout]) - - Given the address ``addr`` of an SSL-protected server, as a (*hostname*, - *port-number*) pair, fetches the server's certificate, and returns it as a - PEM-encoded string. If ``ssl_version`` is specified, uses that version of - the SSL protocol to attempt to connect to the server. If ``ca_certs`` is - specified, it should be a file containing a list of root certificates, the - same format as used for the same parameter in - :meth:`SSLContext.wrap_socket`. The call will attempt to validate the - server certificate against that set of root certificates, and will fail - if the validation attempt fails. A timeout can be specified with the - ``timeout`` parameter. - - .. versionchanged:: 3.3 - This function is now IPv6-compatible. - - .. versionchanged:: 3.5 - The default *ssl_version* is changed from :data:`PROTOCOL_SSLv3` to - :data:`PROTOCOL_TLS` for maximum compatibility with modern servers. - - .. versionchanged:: 3.10 - The *timeout* parameter was added. - -.. function:: DER_cert_to_PEM_cert(DER_cert_bytes) - - Given a certificate as a DER-encoded blob of bytes, returns a PEM-encoded - string version of the same certificate. - -.. function:: PEM_cert_to_DER_cert(PEM_cert_string) - - Given a certificate as an ASCII PEM string, returns a DER-encoded sequence of - bytes for that same certificate. - -.. function:: get_default_verify_paths() - - Returns a named tuple with paths to OpenSSL's default cafile and capath. - The paths are the same as used by - :meth:`SSLContext.set_default_verify_paths`. The return value is a - :term:`named tuple` ``DefaultVerifyPaths``: - - * :attr:`cafile` - resolved path to cafile or ``None`` if the file doesn't exist, - * :attr:`capath` - resolved path to capath or ``None`` if the directory doesn't exist, - * :attr:`openssl_cafile_env` - OpenSSL's environment key that points to a cafile, - * :attr:`openssl_cafile` - hard coded path to a cafile, - * :attr:`openssl_capath_env` - OpenSSL's environment key that points to a capath, - * :attr:`openssl_capath` - hard coded path to a capath directory - - .. versionadded:: 3.4 - -.. function:: enum_certificates(store_name) - - Retrieve certificates from Windows' system cert store. *store_name* may be - one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert - stores, too. - - The function returns a list of (cert_bytes, encoding_type, trust) tuples. - The encoding_type specifies the encoding of cert_bytes. It is either - :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for - PKCS#7 ASN.1 data. Trust specifies the purpose of the certificate as a set - of OIDS or exactly ``True`` if the certificate is trustworthy for all - purposes. - - Example:: - - >>> ssl.enum_certificates("CA") - [(b'data...', 'x509_asn', {'1.3.6.1.5.5.7.3.1', '1.3.6.1.5.5.7.3.2'}), - (b'data...', 'x509_asn', True)] - - .. availability:: Windows. - - .. versionadded:: 3.4 - -.. function:: enum_crls(store_name) - - Retrieve CRLs from Windows' system cert store. *store_name* may be - one of ``CA``, ``ROOT`` or ``MY``. Windows may provide additional cert - stores, too. - - The function returns a list of (cert_bytes, encoding_type, trust) tuples. - The encoding_type specifies the encoding of cert_bytes. It is either - :const:`x509_asn` for X.509 ASN.1 data or :const:`pkcs_7_asn` for - PKCS#7 ASN.1 data. - - .. availability:: Windows. - - .. versionadded:: 3.4 - -.. function:: wrap_socket(sock, keyfile=None, certfile=None, \ - server_side=False, cert_reqs=CERT_NONE, ssl_version=PROTOCOL_TLS, \ - ca_certs=None, do_handshake_on_connect=True, \ - suppress_ragged_eofs=True, ciphers=None) - - Takes an instance ``sock`` of :class:`socket.socket`, and returns an instance - of :class:`ssl.SSLSocket`, a subtype of :class:`socket.socket`, which wraps - the underlying socket in an SSL context. ``sock`` must be a - :data:`~socket.SOCK_STREAM` socket; other socket types are unsupported. - - Internally, function creates a :class:`SSLContext` with protocol - *ssl_version* and :attr:`SSLContext.options` set to *cert_reqs*. If - parameters *keyfile*, *certfile*, *ca_certs* or *ciphers* are set, then - the values are passed to :meth:`SSLContext.load_cert_chain`, - :meth:`SSLContext.load_verify_locations`, and - :meth:`SSLContext.set_ciphers`. - - The arguments *server_side*, *do_handshake_on_connect*, and - *suppress_ragged_eofs* have the same meaning as - :meth:`SSLContext.wrap_socket`. - - .. deprecated:: 3.7 - - Since Python 3.2 and 2.7.9, it is recommended to use the - :meth:`SSLContext.wrap_socket` instead of :func:`wrap_socket`. The - top-level function is limited and creates an insecure client socket - without server name indication or hostname matching. - -Constants -^^^^^^^^^ - - All constants are now :class:`enum.IntEnum` or :class:`enum.IntFlag` collections. - - .. versionadded:: 3.6 - -.. data:: CERT_NONE - - Possible value for :attr:`SSLContext.verify_mode`, or the ``cert_reqs`` - parameter to :func:`wrap_socket`. Except for :const:`PROTOCOL_TLS_CLIENT`, - it is the default mode. With client-side sockets, just about any - cert is accepted. Validation errors, such as untrusted or expired cert, - are ignored and do not abort the TLS/SSL handshake. - - In server mode, no certificate is requested from the client, so the client - does not send any for client cert authentication. - - See the discussion of :ref:`ssl-security` below. - -.. data:: CERT_OPTIONAL - - Possible value for :attr:`SSLContext.verify_mode`, or the ``cert_reqs`` - parameter to :func:`wrap_socket`. In client mode, :const:`CERT_OPTIONAL` - has the same meaning as :const:`CERT_REQUIRED`. It is recommended to - use :const:`CERT_REQUIRED` for client-side sockets instead. - - In server mode, a client certificate request is sent to the client. The - client may either ignore the request or send a certificate in order - perform TLS client cert authentication. If the client chooses to send - a certificate, it is verified. Any verification error immediately aborts - the TLS handshake. - - Use of this setting requires a valid set of CA certificates to - be passed, either to :meth:`SSLContext.load_verify_locations` or as a - value of the ``ca_certs`` parameter to :func:`wrap_socket`. - -.. data:: CERT_REQUIRED - - Possible value for :attr:`SSLContext.verify_mode`, or the ``cert_reqs`` - parameter to :func:`wrap_socket`. In this mode, certificates are - required from the other side of the socket connection; an :class:`SSLError` - will be raised if no certificate is provided, or if its validation fails. - This mode is **not** sufficient to verify a certificate in client mode as - it does not match hostnames. :attr:`~SSLContext.check_hostname` must be - enabled as well to verify the authenticity of a cert. - :const:`PROTOCOL_TLS_CLIENT` uses :const:`CERT_REQUIRED` and - enables :attr:`~SSLContext.check_hostname` by default. - - With server socket, this mode provides mandatory TLS client cert - authentication. A client certificate request is sent to the client and - the client must provide a valid and trusted certificate. - - Use of this setting requires a valid set of CA certificates to - be passed, either to :meth:`SSLContext.load_verify_locations` or as a - value of the ``ca_certs`` parameter to :func:`wrap_socket`. - -.. class:: VerifyMode - - :class:`enum.IntEnum` collection of CERT_* constants. - - .. versionadded:: 3.6 - -.. data:: VERIFY_DEFAULT - - Possible value for :attr:`SSLContext.verify_flags`. In this mode, certificate - revocation lists (CRLs) are not checked. By default OpenSSL does neither - require nor verify CRLs. - - .. versionadded:: 3.4 - -.. data:: VERIFY_CRL_CHECK_LEAF - - Possible value for :attr:`SSLContext.verify_flags`. In this mode, only the - peer cert is checked but none of the intermediate CA certificates. The mode - requires a valid CRL that is signed by the peer cert's issuer (its direct - ancestor CA). If no proper CRL has been loaded with - :attr:`SSLContext.load_verify_locations`, validation will fail. - - .. versionadded:: 3.4 - -.. data:: VERIFY_CRL_CHECK_CHAIN - - Possible value for :attr:`SSLContext.verify_flags`. In this mode, CRLs of - all certificates in the peer cert chain are checked. - - .. versionadded:: 3.4 - -.. data:: VERIFY_X509_STRICT - - Possible value for :attr:`SSLContext.verify_flags` to disable workarounds - for broken X.509 certificates. - - .. versionadded:: 3.4 - -.. data:: VERIFY_ALLOW_PROXY_CERTS - - Possible value for :attr:`SSLContext.verify_flags` to enables proxy - certificate verification. - - .. versionadded:: 3.10 - -.. data:: VERIFY_X509_TRUSTED_FIRST - - Possible value for :attr:`SSLContext.verify_flags`. It instructs OpenSSL to - prefer trusted certificates when building the trust chain to validate a - certificate. This flag is enabled by default. - - .. versionadded:: 3.4.4 - -.. data:: VERIFY_X509_PARTIAL_CHAIN - - Possible value for :attr:`SSLContext.verify_flags`. It instructs OpenSSL to - accept intermediate CAs in the trust store to be treated as trust-anchors, - in the same way as the self-signed root CA certificates. This makes it - possible to trust certificates issued by an intermediate CA without having - to trust its ancestor root CA. - - .. versionadded:: 3.10 - - -.. class:: VerifyFlags - - :class:`enum.IntFlag` collection of VERIFY_* constants. - - .. versionadded:: 3.6 - -.. data:: PROTOCOL_TLS - - Selects the highest protocol version that both the client and server support. - Despite the name, this option can select both "SSL" and "TLS" protocols. - - .. versionadded:: 3.6 - - .. deprecated:: 3.10 - - TLS clients and servers require different default settings for secure - communication. The generic TLS protocol constant is deprecated in - favor of :data:`PROTOCOL_TLS_CLIENT` and :data:`PROTOCOL_TLS_SERVER`. - -.. data:: PROTOCOL_TLS_CLIENT - - Auto-negotiate the highest protocol version that both the client and - server support, and configure the context client-side connections. The - protocol enables :data:`CERT_REQUIRED` and - :attr:`~SSLContext.check_hostname` by default. - - .. versionadded:: 3.6 - -.. data:: PROTOCOL_TLS_SERVER - - Auto-negotiate the highest protocol version that both the client and - server support, and configure the context server-side connections. - - .. versionadded:: 3.6 - -.. data:: PROTOCOL_SSLv23 - - Alias for :data:`PROTOCOL_TLS`. - - .. deprecated:: 3.6 - - Use :data:`PROTOCOL_TLS` instead. - -.. data:: PROTOCOL_SSLv2 - - Selects SSL version 2 as the channel encryption protocol. - - This protocol is not available if OpenSSL is compiled with the - ``no-ssl2`` option. - - .. warning:: - - SSL version 2 is insecure. Its use is highly discouraged. - - .. deprecated:: 3.6 - - OpenSSL has removed support for SSLv2. - -.. data:: PROTOCOL_SSLv3 - - Selects SSL version 3 as the channel encryption protocol. - - This protocol is not available if OpenSSL is compiled with the - ``no-ssl3`` option. - - .. warning:: - - SSL version 3 is insecure. Its use is highly discouraged. - - .. deprecated:: 3.6 - - OpenSSL has deprecated all version specific protocols. Use the default - protocol :data:`PROTOCOL_TLS_SERVER` or :data:`PROTOCOL_TLS_CLIENT` - with :attr:`SSLContext.minimum_version` and - :attr:`SSLContext.maximum_version` instead. - - -.. data:: PROTOCOL_TLSv1 - - Selects TLS version 1.0 as the channel encryption protocol. - - .. deprecated:: 3.6 - - OpenSSL has deprecated all version specific protocols. - -.. data:: PROTOCOL_TLSv1_1 - - Selects TLS version 1.1 as the channel encryption protocol. - Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - - .. deprecated:: 3.6 - - OpenSSL has deprecated all version specific protocols. - -.. data:: PROTOCOL_TLSv1_2 - - Selects TLS version 1.2 as the channel encryption protocol. - Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - - .. deprecated:: 3.6 - - OpenSSL has deprecated all version specific protocols. - -.. data:: OP_ALL - - Enables workarounds for various bugs present in other SSL implementations. - This option is set by default. It does not necessarily set the same - flags as OpenSSL's ``SSL_OP_ALL`` constant. - - .. versionadded:: 3.2 - -.. data:: OP_NO_SSLv2 - - Prevents an SSLv2 connection. This option is only applicable in - conjunction with :const:`PROTOCOL_TLS`. It prevents the peers from - choosing SSLv2 as the protocol version. - - .. versionadded:: 3.2 - - .. deprecated:: 3.6 - - SSLv2 is deprecated - -.. data:: OP_NO_SSLv3 - - Prevents an SSLv3 connection. This option is only applicable in - conjunction with :const:`PROTOCOL_TLS`. It prevents the peers from - choosing SSLv3 as the protocol version. - - .. versionadded:: 3.2 - - .. deprecated:: 3.6 - - SSLv3 is deprecated - -.. data:: OP_NO_TLSv1 - - Prevents a TLSv1 connection. This option is only applicable in - conjunction with :const:`PROTOCOL_TLS`. It prevents the peers from - choosing TLSv1 as the protocol version. - - .. versionadded:: 3.2 - - .. deprecated:: 3.7 - The option is deprecated since OpenSSL 1.1.0, use the new - :attr:`SSLContext.minimum_version` and - :attr:`SSLContext.maximum_version` instead. - -.. data:: OP_NO_TLSv1_1 - - Prevents a TLSv1.1 connection. This option is only applicable in conjunction - with :const:`PROTOCOL_TLS`. It prevents the peers from choosing TLSv1.1 as - the protocol version. Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - - .. deprecated:: 3.7 - The option is deprecated since OpenSSL 1.1.0. - -.. data:: OP_NO_TLSv1_2 - - Prevents a TLSv1.2 connection. This option is only applicable in conjunction - with :const:`PROTOCOL_TLS`. It prevents the peers from choosing TLSv1.2 as - the protocol version. Available only with openssl version 1.0.1+. - - .. versionadded:: 3.4 - - .. deprecated:: 3.7 - The option is deprecated since OpenSSL 1.1.0. - -.. data:: OP_NO_TLSv1_3 - - Prevents a TLSv1.3 connection. This option is only applicable in conjunction - with :const:`PROTOCOL_TLS`. It prevents the peers from choosing TLSv1.3 as - the protocol version. TLS 1.3 is available with OpenSSL 1.1.1 or later. - When Python has been compiled against an older version of OpenSSL, the - flag defaults to *0*. - - .. versionadded:: 3.7 - - .. deprecated:: 3.7 - The option is deprecated since OpenSSL 1.1.0. It was added to 2.7.15, - 3.6.3 and 3.7.0 for backwards compatibility with OpenSSL 1.0.2. - -.. data:: OP_NO_RENEGOTIATION - - Disable all renegotiation in TLSv1.2 and earlier. Do not send - HelloRequest messages, and ignore renegotiation requests via ClientHello. - - This option is only available with OpenSSL 1.1.0h and later. - - .. versionadded:: 3.7 - -.. data:: OP_CIPHER_SERVER_PREFERENCE - - Use the server's cipher ordering preference, rather than the client's. - This option has no effect on client sockets and SSLv2 server sockets. - - .. versionadded:: 3.3 - -.. data:: OP_SINGLE_DH_USE - - Prevents re-use of the same DH key for distinct SSL sessions. This - improves forward secrecy but requires more computational resources. - This option only applies to server sockets. - - .. versionadded:: 3.3 - -.. data:: OP_SINGLE_ECDH_USE - - Prevents re-use of the same ECDH key for distinct SSL sessions. This - improves forward secrecy but requires more computational resources. - This option only applies to server sockets. - - .. versionadded:: 3.3 - -.. data:: OP_ENABLE_MIDDLEBOX_COMPAT - - Send dummy Change Cipher Spec (CCS) messages in TLS 1.3 handshake to make - a TLS 1.3 connection look more like a TLS 1.2 connection. - - This option is only available with OpenSSL 1.1.1 and later. - - .. versionadded:: 3.8 - -.. data:: OP_NO_COMPRESSION - - Disable compression on the SSL channel. This is useful if the application - protocol supports its own compression scheme. - - .. versionadded:: 3.3 - -.. class:: Options - - :class:`enum.IntFlag` collection of OP_* constants. - -.. data:: OP_NO_TICKET - - Prevent client side from requesting a session ticket. - - .. versionadded:: 3.6 - -.. data:: OP_IGNORE_UNEXPECTED_EOF - - Ignore unexpected shutdown of TLS connections. - - This option is only available with OpenSSL 3.0.0 and later. - - .. versionadded:: 3.10 - -.. data:: HAS_ALPN - - Whether the OpenSSL library has built-in support for the *Application-Layer - Protocol Negotiation* TLS extension as described in :rfc:`7301`. - - .. versionadded:: 3.5 - -.. data:: HAS_NEVER_CHECK_COMMON_NAME - - Whether the OpenSSL library has built-in support not checking subject - common name and :attr:`SSLContext.hostname_checks_common_name` is - writeable. - - .. versionadded:: 3.7 - -.. data:: HAS_ECDH - - Whether the OpenSSL library has built-in support for the Elliptic Curve-based - Diffie-Hellman key exchange. This should be true unless the feature was - explicitly disabled by the distributor. - - .. versionadded:: 3.3 - -.. data:: HAS_SNI - - Whether the OpenSSL library has built-in support for the *Server Name - Indication* extension (as defined in :rfc:`6066`). - - .. versionadded:: 3.2 - -.. data:: HAS_NPN - - Whether the OpenSSL library has built-in support for the *Next Protocol - Negotiation* as described in the `Application Layer Protocol - Negotiation `_. - When true, you can use the :meth:`SSLContext.set_npn_protocols` method to advertise - which protocols you want to support. - - .. versionadded:: 3.3 - -.. data:: HAS_SSLv2 - - Whether the OpenSSL library has built-in support for the SSL 2.0 protocol. - - .. versionadded:: 3.7 - -.. data:: HAS_SSLv3 - - Whether the OpenSSL library has built-in support for the SSL 3.0 protocol. - - .. versionadded:: 3.7 - -.. data:: HAS_TLSv1 - - Whether the OpenSSL library has built-in support for the TLS 1.0 protocol. - - .. versionadded:: 3.7 - -.. data:: HAS_TLSv1_1 - - Whether the OpenSSL library has built-in support for the TLS 1.1 protocol. - - .. versionadded:: 3.7 - -.. data:: HAS_TLSv1_2 - - Whether the OpenSSL library has built-in support for the TLS 1.2 protocol. - - .. versionadded:: 3.7 - -.. data:: HAS_TLSv1_3 - - Whether the OpenSSL library has built-in support for the TLS 1.3 protocol. - - .. versionadded:: 3.7 - -.. data:: CHANNEL_BINDING_TYPES - - List of supported TLS channel binding types. Strings in this list - can be used as arguments to :meth:`SSLSocket.get_channel_binding`. - - .. versionadded:: 3.3 - -.. data:: OPENSSL_VERSION - - The version string of the OpenSSL library loaded by the interpreter:: - - >>> ssl.OPENSSL_VERSION - 'OpenSSL 1.0.2k 26 Jan 2017' - - .. versionadded:: 3.2 - -.. data:: OPENSSL_VERSION_INFO - - A tuple of five integers representing version information about the - OpenSSL library:: - - >>> ssl.OPENSSL_VERSION_INFO - (1, 0, 2, 11, 15) - - .. versionadded:: 3.2 - -.. data:: OPENSSL_VERSION_NUMBER - - The raw version number of the OpenSSL library, as a single integer:: - - >>> ssl.OPENSSL_VERSION_NUMBER - 268443839 - >>> hex(ssl.OPENSSL_VERSION_NUMBER) - '0x100020bf' - - .. versionadded:: 3.2 - -.. data:: ALERT_DESCRIPTION_HANDSHAKE_FAILURE - ALERT_DESCRIPTION_INTERNAL_ERROR - ALERT_DESCRIPTION_* - - Alert Descriptions from :rfc:`5246` and others. The `IANA TLS Alert Registry - `_ - contains this list and references to the RFCs where their meaning is defined. - - Used as the return value of the callback function in - :meth:`SSLContext.set_servername_callback`. - - .. versionadded:: 3.4 - -.. class:: AlertDescription - - :class:`enum.IntEnum` collection of ALERT_DESCRIPTION_* constants. - - .. versionadded:: 3.6 - -.. data:: Purpose.SERVER_AUTH - - Option for :func:`create_default_context` and - :meth:`SSLContext.load_default_certs`. This value indicates that the - context may be used to authenticate web servers (therefore, it will - be used to create client-side sockets). - - .. versionadded:: 3.4 - -.. data:: Purpose.CLIENT_AUTH - - Option for :func:`create_default_context` and - :meth:`SSLContext.load_default_certs`. This value indicates that the - context may be used to authenticate web clients (therefore, it will - be used to create server-side sockets). - - .. versionadded:: 3.4 - -.. class:: SSLErrorNumber - - :class:`enum.IntEnum` collection of SSL_ERROR_* constants. - - .. versionadded:: 3.6 - -.. class:: TLSVersion - - :class:`enum.IntEnum` collection of SSL and TLS versions for - :attr:`SSLContext.maximum_version` and :attr:`SSLContext.minimum_version`. - - .. versionadded:: 3.7 - -.. attribute:: TLSVersion.MINIMUM_SUPPORTED -.. attribute:: TLSVersion.MAXIMUM_SUPPORTED - - The minimum or maximum supported SSL or TLS version. These are magic - constants. Their values don't reflect the lowest and highest available - TLS/SSL versions. - -.. attribute:: TLSVersion.SSLv3 -.. attribute:: TLSVersion.TLSv1 -.. attribute:: TLSVersion.TLSv1_1 -.. attribute:: TLSVersion.TLSv1_2 -.. attribute:: TLSVersion.TLSv1_3 - - SSL 3.0 to TLS 1.3. - - .. deprecated:: 3.10 - - All :class:`TLSVersion` members except :attr:`TLSVersion.TLSv1_2` and - :attr:`TLSVersion.TLSv1_3` are deprecated. - - -SSL Sockets ------------ - -.. class:: SSLSocket(socket.socket) - - SSL sockets provide the following methods of :ref:`socket-objects`: - - - :meth:`~socket.socket.accept()` - - :meth:`~socket.socket.bind()` - - :meth:`~socket.socket.close()` - - :meth:`~socket.socket.connect()` - - :meth:`~socket.socket.detach()` - - :meth:`~socket.socket.fileno()` - - :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` - - :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` - - :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, - :meth:`~socket.socket.setblocking()` - - :meth:`~socket.socket.listen()` - - :meth:`~socket.socket.makefile()` - - :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` - (but passing a non-zero ``flags`` argument is not allowed) - - :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with - the same limitation) - - :meth:`~socket.socket.sendfile()` (but :mod:`os.sendfile` will be used - for plain-text sockets only, else :meth:`~socket.socket.send()` will be used) - - :meth:`~socket.socket.shutdown()` - - However, since the SSL (and TLS) protocol has its own framing atop - of TCP, the SSL sockets abstraction can, in certain respects, diverge from - the specification of normal, OS-level sockets. See especially the - :ref:`notes on non-blocking sockets `. - - Instances of :class:`SSLSocket` must be created using the - :meth:`SSLContext.wrap_socket` method. - - .. versionchanged:: 3.5 - The :meth:`sendfile` method was added. - - .. versionchanged:: 3.5 - The :meth:`shutdown` does not reset the socket timeout each time bytes - are received or sent. The socket timeout is now the maximum total duration - of the shutdown. - - .. deprecated:: 3.6 - It is deprecated to create a :class:`SSLSocket` instance directly, use - :meth:`SSLContext.wrap_socket` to wrap a socket. - - .. versionchanged:: 3.7 - :class:`SSLSocket` instances must to created with - :meth:`~SSLContext.wrap_socket`. In earlier versions, it was possible - to create instances directly. This was never documented or officially - supported. - - .. versionchanged:: 3.10 - Python now uses ``SSL_read_ex`` and ``SSL_write_ex`` internally. The - functions support reading and writing of data larger than 2 GB. Writing - zero-length data no longer fails with a protocol violation error. - -SSL sockets also have the following additional methods and attributes: - -.. method:: SSLSocket.read(len=1024, buffer=None) - - Read up to *len* bytes of data from the SSL socket and return the result as - a ``bytes`` instance. If *buffer* is specified, then read into the buffer - instead, and return the number of bytes read. - - Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is - :ref:`non-blocking ` and the read would block. - - As at any time a re-negotiation is possible, a call to :meth:`read` can also - cause write operations. - - .. versionchanged:: 3.5 - The socket timeout is no longer reset each time bytes are received or sent. - The socket timeout is now the maximum total duration to read up to *len* - bytes. - - .. deprecated:: 3.6 - Use :meth:`~SSLSocket.recv` instead of :meth:`~SSLSocket.read`. - -.. method:: SSLSocket.write(buf) - - Write *buf* to the SSL socket and return the number of bytes written. The - *buf* argument must be an object supporting the buffer interface. - - Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is - :ref:`non-blocking ` and the write would block. - - As at any time a re-negotiation is possible, a call to :meth:`write` can - also cause read operations. - - .. versionchanged:: 3.5 - The socket timeout is no longer reset each time bytes are received or sent. - The socket timeout is now the maximum total duration to write *buf*. - - .. deprecated:: 3.6 - Use :meth:`~SSLSocket.send` instead of :meth:`~SSLSocket.write`. - -.. note:: - - The :meth:`~SSLSocket.read` and :meth:`~SSLSocket.write` methods are the - low-level methods that read and write unencrypted, application-level data - and decrypt/encrypt it to encrypted, wire-level data. These methods - require an active SSL connection, i.e. the handshake was completed and - :meth:`SSLSocket.unwrap` was not called. - - Normally you should use the socket API methods like - :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` instead of these - methods. - -.. method:: SSLSocket.do_handshake() - - Perform the SSL setup handshake. - - .. versionchanged:: 3.4 - The handshake method also performs :func:`match_hostname` when the - :attr:`~SSLContext.check_hostname` attribute of the socket's - :attr:`~SSLSocket.context` is true. - - .. versionchanged:: 3.5 - The socket timeout is no longer reset each time bytes are received or sent. - The socket timeout is now the maximum total duration of the handshake. - - .. versionchanged:: 3.7 - Hostname or IP address is matched by OpenSSL during handshake. The - function :func:`match_hostname` is no longer used. In case OpenSSL - refuses a hostname or IP address, the handshake is aborted early and - a TLS alert message is sent to the peer. - -.. method:: SSLSocket.getpeercert(binary_form=False) - - If there is no certificate for the peer on the other end of the connection, - return ``None``. If the SSL handshake hasn't been done yet, raise - :exc:`ValueError`. - - If the ``binary_form`` parameter is :const:`False`, and a certificate was - received from the peer, this method returns a :class:`dict` instance. If the - certificate was not validated, the dict is empty. If the certificate was - validated, it returns a dict with several keys, amongst them ``subject`` - (the principal for which the certificate was issued) and ``issuer`` - (the principal issuing the certificate). If a certificate contains an - instance of the *Subject Alternative Name* extension (see :rfc:`3280`), - there will also be a ``subjectAltName`` key in the dictionary. - - The ``subject`` and ``issuer`` fields are tuples containing the sequence - of relative distinguished names (RDNs) given in the certificate's data - structure for the respective fields, and each RDN is a sequence of - name-value pairs. Here is a real-world example:: - - {'issuer': ((('countryName', 'IL'),), - (('organizationName', 'StartCom Ltd.'),), - (('organizationalUnitName', - 'Secure Digital Certificate Signing'),), - (('commonName', - 'StartCom Class 2 Primary Intermediate Server CA'),)), - 'notAfter': 'Nov 22 08:15:19 2013 GMT', - 'notBefore': 'Nov 21 03:09:52 2011 GMT', - 'serialNumber': '95F0', - 'subject': ((('description', '571208-SLe257oHY9fVQ07Z'),), - (('countryName', 'US'),), - (('stateOrProvinceName', 'California'),), - (('localityName', 'San Francisco'),), - (('organizationName', 'Electronic Frontier Foundation, Inc.'),), - (('commonName', '*.eff.org'),), - (('emailAddress', 'hostmaster@eff.org'),)), - 'subjectAltName': (('DNS', '*.eff.org'), ('DNS', 'eff.org')), - 'version': 3} - - .. note:: - - To validate a certificate for a particular service, you can use the - :func:`match_hostname` function. - - If the ``binary_form`` parameter is :const:`True`, and a certificate was - provided, this method returns the DER-encoded form of the entire certificate - as a sequence of bytes, or :const:`None` if the peer did not provide a - certificate. Whether the peer provides a certificate depends on the SSL - socket's role: - - * for a client SSL socket, the server will always provide a certificate, - regardless of whether validation was required; - - * for a server SSL socket, the client will only provide a certificate - when requested by the server; therefore :meth:`getpeercert` will return - :const:`None` if you used :const:`CERT_NONE` (rather than - :const:`CERT_OPTIONAL` or :const:`CERT_REQUIRED`). - - .. versionchanged:: 3.2 - The returned dictionary includes additional items such as ``issuer`` - and ``notBefore``. - - .. versionchanged:: 3.4 - :exc:`ValueError` is raised when the handshake isn't done. - The returned dictionary includes additional X509v3 extension items - such as ``crlDistributionPoints``, ``caIssuers`` and ``OCSP`` URIs. - - .. versionchanged:: 3.9 - IPv6 address strings no longer have a trailing new line. - -.. method:: SSLSocket.cipher() - - Returns a three-value tuple containing the name of the cipher being used, the - version of the SSL protocol that defines its use, and the number of secret - bits being used. If no connection has been established, returns ``None``. - -.. method:: SSLSocket.shared_ciphers() - - Return the list of ciphers available in both the client and server. Each - entry of the returned list is a three-value tuple containing the name of the - cipher, the version of the SSL protocol that defines its use, and the number - of secret bits the cipher uses. :meth:`~SSLSocket.shared_ciphers` returns - ``None`` if no connection has been established or the socket is a client - socket. - - .. versionadded:: 3.5 - -.. method:: SSLSocket.compression() - - Return the compression algorithm being used as a string, or ``None`` - if the connection isn't compressed. - - If the higher-level protocol supports its own compression mechanism, - you can use :data:`OP_NO_COMPRESSION` to disable SSL-level compression. - - .. versionadded:: 3.3 - -.. method:: SSLSocket.get_channel_binding(cb_type="tls-unique") - - Get channel binding data for current connection, as a bytes object. Returns - ``None`` if not connected or the handshake has not been completed. - - The *cb_type* parameter allow selection of the desired channel binding - type. Valid channel binding types are listed in the - :data:`CHANNEL_BINDING_TYPES` list. Currently only the 'tls-unique' channel - binding, defined by :rfc:`5929`, is supported. :exc:`ValueError` will be - raised if an unsupported channel binding type is requested. - - .. versionadded:: 3.3 - -.. method:: SSLSocket.selected_alpn_protocol() - - Return the protocol that was selected during the TLS handshake. If - :meth:`SSLContext.set_alpn_protocols` was not called, if the other party does - not support ALPN, if this socket does not support any of the client's - proposed protocols, or if the handshake has not happened yet, ``None`` is - returned. - - .. versionadded:: 3.5 - -.. method:: SSLSocket.selected_npn_protocol() - - Return the higher-level protocol that was selected during the TLS/SSL - handshake. If :meth:`SSLContext.set_npn_protocols` was not called, or - if the other party does not support NPN, or if the handshake has not yet - happened, this will return ``None``. - - .. versionadded:: 3.3 - - .. deprecated:: 3.10 - - NPN has been superseded by ALPN - -.. method:: SSLSocket.unwrap() - - Performs the SSL shutdown handshake, which removes the TLS layer from the - underlying socket, and returns the underlying socket object. This can be - used to go from encrypted operation over a connection to unencrypted. The - returned socket should always be used for further communication with the - other side of the connection, rather than the original socket. - -.. method:: SSLSocket.verify_client_post_handshake() - - Requests post-handshake authentication (PHA) from a TLS 1.3 client. PHA - can only be initiated for a TLS 1.3 connection from a server-side socket, - after the initial TLS handshake and with PHA enabled on both sides, see - :attr:`SSLContext.post_handshake_auth`. - - The method does not perform a cert exchange immediately. The server-side - sends a CertificateRequest during the next write event and expects the - client to respond with a certificate on the next read event. - - If any precondition isn't met (e.g. not TLS 1.3, PHA not enabled), an - :exc:`SSLError` is raised. - - .. note:: - Only available with OpenSSL 1.1.1 and TLS 1.3 enabled. Without TLS 1.3 - support, the method raises :exc:`NotImplementedError`. - - .. versionadded:: 3.8 - -.. method:: SSLSocket.version() - - Return the actual SSL protocol version negotiated by the connection - as a string, or ``None`` if no secure connection is established. - As of this writing, possible return values include ``"SSLv2"``, - ``"SSLv3"``, ``"TLSv1"``, ``"TLSv1.1"`` and ``"TLSv1.2"``. - Recent OpenSSL versions may define more return values. - - .. versionadded:: 3.5 - -.. method:: SSLSocket.pending() - - Returns the number of already decrypted bytes available for read, pending on - the connection. - -.. attribute:: SSLSocket.context - - The :class:`SSLContext` object this SSL socket is tied to. If the SSL - socket was created using the deprecated :func:`wrap_socket` function - (rather than :meth:`SSLContext.wrap_socket`), this is a custom context - object created for this SSL socket. - - .. versionadded:: 3.2 - -.. attribute:: SSLSocket.server_side - - A boolean which is ``True`` for server-side sockets and ``False`` for - client-side sockets. - - .. versionadded:: 3.2 - -.. attribute:: SSLSocket.server_hostname - - Hostname of the server: :class:`str` type, or ``None`` for server-side - socket or if the hostname was not specified in the constructor. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.7 - The attribute is now always ASCII text. When ``server_hostname`` is - an internationalized domain name (IDN), this attribute now stores the - A-label form (``"xn--pythn-mua.org"``), rather than the U-label form - (``"pythön.org"``). - -.. attribute:: SSLSocket.session - - The :class:`SSLSession` for this SSL connection. The session is available - for client and server side sockets after the TLS handshake has been - performed. For client sockets the session can be set before - :meth:`~SSLSocket.do_handshake` has been called to reuse a session. - - .. versionadded:: 3.6 - -.. attribute:: SSLSocket.session_reused - - .. versionadded:: 3.6 - - -SSL Contexts ------------- - -.. versionadded:: 3.2 - -An SSL context holds various data longer-lived than single SSL connections, -such as SSL configuration options, certificate(s) and private key(s). -It also manages a cache of SSL sessions for server-side sockets, in order -to speed up repeated connections from the same clients. - -.. class:: SSLContext(protocol=None) - - Create a new SSL context. You may pass *protocol* which must be one - of the ``PROTOCOL_*`` constants defined in this module. The parameter - specifies which version of the SSL protocol to use. Typically, the - server chooses a particular protocol version, and the client must adapt - to the server's choice. Most of the versions are not interoperable - with the other versions. If not specified, the default is - :data:`PROTOCOL_TLS`; it provides the most compatibility with other - versions. - - Here's a table showing which versions in a client (down the side) can connect - to which versions in a server (along the top): - - .. table:: - - ======================== ============ ============ ============= ========= =========== =========== - *client* / **server** **SSLv2** **SSLv3** **TLS** [3]_ **TLSv1** **TLSv1.1** **TLSv1.2** - ------------------------ ------------ ------------ ------------- --------- ----------- ----------- - *SSLv2* yes no no [1]_ no no no - *SSLv3* no yes no [2]_ no no no - *TLS* (*SSLv23*) [3]_ no [1]_ no [2]_ yes yes yes yes - *TLSv1* no no yes yes no no - *TLSv1.1* no no yes no yes no - *TLSv1.2* no no yes no no yes - ======================== ============ ============ ============= ========= =========== =========== - - .. rubric:: Footnotes - .. [1] :class:`SSLContext` disables SSLv2 with :data:`OP_NO_SSLv2` by default. - .. [2] :class:`SSLContext` disables SSLv3 with :data:`OP_NO_SSLv3` by default. - .. [3] TLS 1.3 protocol will be available with :data:`PROTOCOL_TLS` in - OpenSSL >= 1.1.1. There is no dedicated PROTOCOL constant for just - TLS 1.3. - - .. seealso:: - :func:`create_default_context` lets the :mod:`ssl` module choose - security settings for a given purpose. - - .. versionchanged:: 3.6 - - The context is created with secure default values. The options - :data:`OP_NO_COMPRESSION`, :data:`OP_CIPHER_SERVER_PREFERENCE`, - :data:`OP_SINGLE_DH_USE`, :data:`OP_SINGLE_ECDH_USE`, - :data:`OP_NO_SSLv2` (except for :data:`PROTOCOL_SSLv2`), - and :data:`OP_NO_SSLv3` (except for :data:`PROTOCOL_SSLv3`) are - set by default. The initial cipher suite list contains only ``HIGH`` - ciphers, no ``NULL`` ciphers and no ``MD5`` ciphers (except for - :data:`PROTOCOL_SSLv2`). - - .. deprecated:: 3.10 - - :class:`SSLContext` without protocol argument is deprecated. The - context class will either require :data:`PROTOCOL_TLS_CLIENT` or - :data:`PROTOCOL_TLS_SERVER` protocol in the future. - - .. versionchanged:: 3.10 - - The default cipher suites now include only secure AES and ChaCha20 - ciphers with forward secrecy and security level 2. RSA and DH keys with - less than 2048 bits and ECC keys with less than 224 bits are prohibited. - :data:`PROTOCOL_TLS`, :data:`PROTOCOL_TLS_CLIENT`, and - :data:`PROTOCOL_TLS_SERVER` use TLS 1.2 as minimum TLS version. - - -:class:`SSLContext` objects have the following methods and attributes: - -.. method:: SSLContext.cert_store_stats() - - Get statistics about quantities of loaded X.509 certificates, count of - X.509 certificates flagged as CA certificates and certificate revocation - lists as dictionary. - - Example for a context with one CA cert and one other cert:: - - >>> context.cert_store_stats() - {'crl': 0, 'x509_ca': 1, 'x509': 2} - - .. versionadded:: 3.4 - - -.. method:: SSLContext.load_cert_chain(certfile, keyfile=None, password=None) - - Load a private key and the corresponding certificate. The *certfile* - string must be the path to a single file in PEM format containing the - certificate as well as any number of CA certificates needed to establish - the certificate's authenticity. The *keyfile* string, if present, must - point to a file containing the private key. Otherwise the private - key will be taken from *certfile* as well. See the discussion of - :ref:`ssl-certificates` for more information on how the certificate - is stored in the *certfile*. - - The *password* argument may be a function to call to get the password for - decrypting the private key. It will only be called if the private key is - encrypted and a password is necessary. It will be called with no arguments, - and it should return a string, bytes, or bytearray. If the return value is - a string it will be encoded as UTF-8 before using it to decrypt the key. - Alternatively a string, bytes, or bytearray value may be supplied directly - as the *password* argument. It will be ignored if the private key is not - encrypted and no password is needed. - - If the *password* argument is not specified and a password is required, - OpenSSL's built-in password prompting mechanism will be used to - interactively prompt the user for a password. - - An :class:`SSLError` is raised if the private key doesn't - match with the certificate. - - .. versionchanged:: 3.3 - New optional argument *password*. - -.. method:: SSLContext.load_default_certs(purpose=Purpose.SERVER_AUTH) - - Load a set of default "certification authority" (CA) certificates from - default locations. On Windows it loads CA certs from the ``CA`` and - ``ROOT`` system stores. On all systems it calls - :meth:`SSLContext.set_default_verify_paths`. In the future the method may - load CA certificates from other locations, too. - - The *purpose* flag specifies what kind of CA certificates are loaded. The - default settings :const:`Purpose.SERVER_AUTH` loads certificates, that are - flagged and trusted for TLS web server authentication (client side - sockets). :const:`Purpose.CLIENT_AUTH` loads CA certificates for client - certificate verification on the server side. - - .. versionadded:: 3.4 - -.. method:: SSLContext.load_verify_locations(cafile=None, capath=None, cadata=None) - - Load a set of "certification authority" (CA) certificates used to validate - other peers' certificates when :data:`verify_mode` is other than - :data:`CERT_NONE`. At least one of *cafile* or *capath* must be specified. - - This method can also load certification revocation lists (CRLs) in PEM or - DER format. In order to make use of CRLs, :attr:`SSLContext.verify_flags` - must be configured properly. - - The *cafile* string, if present, is the path to a file of concatenated - CA certificates in PEM format. See the discussion of - :ref:`ssl-certificates` for more information about how to arrange the - certificates in this file. - - The *capath* string, if present, is - the path to a directory containing several CA certificates in PEM format, - following an `OpenSSL specific layout - `_. - - The *cadata* object, if present, is either an ASCII string of one or more - PEM-encoded certificates or a :term:`bytes-like object` of DER-encoded - certificates. Like with *capath* extra lines around PEM-encoded - certificates are ignored but at least one certificate must be present. - - .. versionchanged:: 3.4 - New optional argument *cadata* - -.. method:: SSLContext.get_ca_certs(binary_form=False) - - Get a list of loaded "certification authority" (CA) certificates. If the - ``binary_form`` parameter is :const:`False` each list - entry is a dict like the output of :meth:`SSLSocket.getpeercert`. Otherwise - the method returns a list of DER-encoded certificates. The returned list - does not contain certificates from *capath* unless a certificate was - requested and loaded by a SSL connection. - - .. note:: - Certificates in a capath directory aren't loaded unless they have - been used at least once. - - .. versionadded:: 3.4 - -.. method:: SSLContext.get_ciphers() - - Get a list of enabled ciphers. The list is in order of cipher priority. - See :meth:`SSLContext.set_ciphers`. - - Example:: - - >>> ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - >>> ctx.set_ciphers('ECDHE+AESGCM:!ECDSA') - >>> ctx.get_ciphers() - [{'aead': True, - 'alg_bits': 256, - 'auth': 'auth-rsa', - 'description': 'ECDHE-RSA-AES256-GCM-SHA384 TLSv1.2 Kx=ECDH Au=RSA ' - 'Enc=AESGCM(256) Mac=AEAD', - 'digest': None, - 'id': 50380848, - 'kea': 'kx-ecdhe', - 'name': 'ECDHE-RSA-AES256-GCM-SHA384', - 'protocol': 'TLSv1.2', - 'strength_bits': 256, - 'symmetric': 'aes-256-gcm'}, - {'aead': True, - 'alg_bits': 128, - 'auth': 'auth-rsa', - 'description': 'ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 Kx=ECDH Au=RSA ' - 'Enc=AESGCM(128) Mac=AEAD', - 'digest': None, - 'id': 50380847, - 'kea': 'kx-ecdhe', - 'name': 'ECDHE-RSA-AES128-GCM-SHA256', - 'protocol': 'TLSv1.2', - 'strength_bits': 128, - 'symmetric': 'aes-128-gcm'}] - - .. versionadded:: 3.6 - -.. method:: SSLContext.set_default_verify_paths() - - Load a set of default "certification authority" (CA) certificates from - a filesystem path defined when building the OpenSSL library. Unfortunately, - there's no easy way to know whether this method succeeds: no error is - returned if no certificates are to be found. When the OpenSSL library is - provided as part of the operating system, though, it is likely to be - configured properly. - -.. method:: SSLContext.set_ciphers(ciphers) - - Set the available ciphers for sockets created with this context. - It should be a string in the `OpenSSL cipher list format - `_. - If no cipher can be selected (because compile-time options or other - configuration forbids use of all the specified ciphers), an - :class:`SSLError` will be raised. - - .. note:: - when connected, the :meth:`SSLSocket.cipher` method of SSL sockets will - give the currently selected cipher. - - TLS 1.3 cipher suites cannot be disabled with - :meth:`~SSLContext.set_ciphers`. - -.. method:: SSLContext.set_alpn_protocols(protocols) - - Specify which protocols the socket should advertise during the SSL/TLS - handshake. It should be a list of ASCII strings, like ``['http/1.1', - 'spdy/2']``, ordered by preference. The selection of a protocol will happen - during the handshake, and will play out according to :rfc:`7301`. After a - successful handshake, the :meth:`SSLSocket.selected_alpn_protocol` method will - return the agreed-upon protocol. - - This method will raise :exc:`NotImplementedError` if :data:`HAS_ALPN` is - ``False``. - - .. versionadded:: 3.5 - -.. method:: SSLContext.set_npn_protocols(protocols) - - Specify which protocols the socket should advertise during the SSL/TLS - handshake. It should be a list of strings, like ``['http/1.1', 'spdy/2']``, - ordered by preference. The selection of a protocol will happen during the - handshake, and will play out according to the `Application Layer Protocol Negotiation - `_. After a - successful handshake, the :meth:`SSLSocket.selected_npn_protocol` method will - return the agreed-upon protocol. - - This method will raise :exc:`NotImplementedError` if :data:`HAS_NPN` is - ``False``. - - .. versionadded:: 3.3 - - .. deprecated:: 3.10 - - NPN has been superseded by ALPN - -.. attribute:: SSLContext.sni_callback - - Register a callback function that will be called after the TLS Client Hello - handshake message has been received by the SSL/TLS server when the TLS client - specifies a server name indication. The server name indication mechanism - is specified in :rfc:`6066` section 3 - Server Name Indication. - - Only one callback can be set per ``SSLContext``. If *sni_callback* - is set to ``None`` then the callback is disabled. Calling this function a - subsequent time will disable the previously registered callback. - - The callback function will be called with three - arguments; the first being the :class:`ssl.SSLSocket`, the second is a string - that represents the server name that the client is intending to communicate - (or :const:`None` if the TLS Client Hello does not contain a server name) - and the third argument is the original :class:`SSLContext`. The server name - argument is text. For internationalized domain name, the server - name is an IDN A-label (``"xn--pythn-mua.org"``). - - A typical use of this callback is to change the :class:`ssl.SSLSocket`'s - :attr:`SSLSocket.context` attribute to a new object of type - :class:`SSLContext` representing a certificate chain that matches the server - name. - - Due to the early negotiation phase of the TLS connection, only limited - methods and attributes are usable like - :meth:`SSLSocket.selected_alpn_protocol` and :attr:`SSLSocket.context`. - The :meth:`SSLSocket.getpeercert`, - :meth:`SSLSocket.cipher` and :meth:`SSLSocket.compression` methods require that - the TLS connection has progressed beyond the TLS Client Hello and therefore - will not return meaningful values nor can they be called safely. - - The *sni_callback* function must return ``None`` to allow the - TLS negotiation to continue. If a TLS failure is required, a constant - :const:`ALERT_DESCRIPTION_* ` can be - returned. Other return values will result in a TLS fatal error with - :const:`ALERT_DESCRIPTION_INTERNAL_ERROR`. - - If an exception is raised from the *sni_callback* function the TLS - connection will terminate with a fatal TLS alert message - :const:`ALERT_DESCRIPTION_HANDSHAKE_FAILURE`. - - This method will raise :exc:`NotImplementedError` if the OpenSSL library - had OPENSSL_NO_TLSEXT defined when it was built. - - .. versionadded:: 3.7 - -.. attribute:: SSLContext.set_servername_callback(server_name_callback) - - This is a legacy API retained for backwards compatibility. When possible, - you should use :attr:`sni_callback` instead. The given *server_name_callback* - is similar to *sni_callback*, except that when the server hostname is an - IDN-encoded internationalized domain name, the *server_name_callback* - receives a decoded U-label (``"pythön.org"``). - - If there is an decoding error on the server name, the TLS connection will - terminate with an :const:`ALERT_DESCRIPTION_INTERNAL_ERROR` fatal TLS - alert message to the client. - - .. versionadded:: 3.4 - -.. method:: SSLContext.load_dh_params(dhfile) - - Load the key generation parameters for Diffie-Hellman (DH) key exchange. - Using DH key exchange improves forward secrecy at the expense of - computational resources (both on the server and on the client). - The *dhfile* parameter should be the path to a file containing DH - parameters in PEM format. - - This setting doesn't apply to client sockets. You can also use the - :data:`OP_SINGLE_DH_USE` option to further improve security. - - .. versionadded:: 3.3 - -.. method:: SSLContext.set_ecdh_curve(curve_name) - - Set the curve name for Elliptic Curve-based Diffie-Hellman (ECDH) key - exchange. ECDH is significantly faster than regular DH while arguably - as secure. The *curve_name* parameter should be a string describing - a well-known elliptic curve, for example ``prime256v1`` for a widely - supported curve. - - This setting doesn't apply to client sockets. You can also use the - :data:`OP_SINGLE_ECDH_USE` option to further improve security. - - This method is not available if :data:`HAS_ECDH` is ``False``. - - .. versionadded:: 3.3 - - .. seealso:: - `SSL/TLS & Perfect Forward Secrecy `_ - Vincent Bernat. - -.. method:: SSLContext.wrap_socket(sock, server_side=False, \ - do_handshake_on_connect=True, suppress_ragged_eofs=True, \ - server_hostname=None, session=None) - - Wrap an existing Python socket *sock* and return an instance of - :attr:`SSLContext.sslsocket_class` (default :class:`SSLSocket`). The - returned SSL socket is tied to the context, its settings and certificates. - *sock* must be a :const:`~socket.SOCK_STREAM` socket; other - socket types are unsupported. - - The parameter ``server_side`` is a boolean which identifies whether - server-side or client-side behavior is desired from this socket. - - For client-side sockets, the context construction is lazy; if the - underlying socket isn't connected yet, the context construction will be - performed after :meth:`connect` is called on the socket. For - server-side sockets, if the socket has no remote peer, it is assumed - to be a listening socket, and the server-side SSL wrapping is - automatically performed on client connections accepted via the - :meth:`accept` method. The method may raise :exc:`SSLError`. - - On client connections, the optional parameter *server_hostname* specifies - the hostname of the service which we are connecting to. This allows a - single server to host multiple SSL-based services with distinct certificates, - quite similarly to HTTP virtual hosts. Specifying *server_hostname* will - raise a :exc:`ValueError` if *server_side* is true. - - The parameter ``do_handshake_on_connect`` specifies whether to do the SSL - handshake automatically after doing a :meth:`socket.connect`, or whether the - application program will call it explicitly, by invoking the - :meth:`SSLSocket.do_handshake` method. Calling - :meth:`SSLSocket.do_handshake` explicitly gives the program control over the - blocking behavior of the socket I/O involved in the handshake. - - The parameter ``suppress_ragged_eofs`` specifies how the - :meth:`SSLSocket.recv` method should signal unexpected EOF from the other end - of the connection. If specified as :const:`True` (the default), it returns a - normal EOF (an empty bytes object) in response to unexpected EOF errors - raised from the underlying socket; if :const:`False`, it will raise the - exceptions back to the caller. - - *session*, see :attr:`~SSLSocket.session`. - - .. versionchanged:: 3.5 - Always allow a server_hostname to be passed, even if OpenSSL does not - have SNI. - - .. versionchanged:: 3.6 - *session* argument was added. - - .. versionchanged:: 3.7 - The method returns an instance of :attr:`SSLContext.sslsocket_class` - instead of hard-coded :class:`SSLSocket`. - -.. attribute:: SSLContext.sslsocket_class - - The return type of :meth:`SSLContext.wrap_socket`, defaults to - :class:`SSLSocket`. The attribute can be overridden on instance of class - in order to return a custom subclass of :class:`SSLSocket`. - - .. versionadded:: 3.7 - -.. method:: SSLContext.wrap_bio(incoming, outgoing, server_side=False, \ - server_hostname=None, session=None) - - Wrap the BIO objects *incoming* and *outgoing* and return an instance of - :attr:`SSLContext.sslobject_class` (default :class:`SSLObject`). The SSL - routines will read input data from the incoming BIO and write data to the - outgoing BIO. - - The *server_side*, *server_hostname* and *session* parameters have the - same meaning as in :meth:`SSLContext.wrap_socket`. - - .. versionchanged:: 3.6 - *session* argument was added. - - .. versionchanged:: 3.7 - The method returns an instance of :attr:`SSLContext.sslobject_class` - instead of hard-coded :class:`SSLObject`. - -.. attribute:: SSLContext.sslobject_class - - The return type of :meth:`SSLContext.wrap_bio`, defaults to - :class:`SSLObject`. The attribute can be overridden on instance of class - in order to return a custom subclass of :class:`SSLObject`. - - .. versionadded:: 3.7 - -.. method:: SSLContext.session_stats() - - Get statistics about the SSL sessions created or managed by this context. - A dictionary is returned which maps the names of each `piece of information `_ to their - numeric values. For example, here is the total number of hits and misses - in the session cache since the context was created:: - - >>> stats = context.session_stats() - >>> stats['hits'], stats['misses'] - (0, 0) - -.. attribute:: SSLContext.check_hostname - - Whether to match the peer cert's hostname in - :meth:`SSLSocket.do_handshake`. The context's - :attr:`~SSLContext.verify_mode` must be set to :data:`CERT_OPTIONAL` or - :data:`CERT_REQUIRED`, and you must pass *server_hostname* to - :meth:`~SSLContext.wrap_socket` in order to match the hostname. Enabling - hostname checking automatically sets :attr:`~SSLContext.verify_mode` from - :data:`CERT_NONE` to :data:`CERT_REQUIRED`. It cannot be set back to - :data:`CERT_NONE` as long as hostname checking is enabled. The - :data:`PROTOCOL_TLS_CLIENT` protocol enables hostname checking by default. - With other protocols, hostname checking must be enabled explicitly. - - Example:: - - import socket, ssl - - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - context.verify_mode = ssl.CERT_REQUIRED - context.check_hostname = True - context.load_default_certs() - - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ssl_sock = context.wrap_socket(s, server_hostname='www.verisign.com') - ssl_sock.connect(('www.verisign.com', 443)) - - .. versionadded:: 3.4 - - .. versionchanged:: 3.7 - - :attr:`~SSLContext.verify_mode` is now automatically changed - to :data:`CERT_REQUIRED` when hostname checking is enabled and - :attr:`~SSLContext.verify_mode` is :data:`CERT_NONE`. Previously - the same operation would have failed with a :exc:`ValueError`. - -.. attribute:: SSLContext.keylog_filename - - Write TLS keys to a keylog file, whenever key material is generated or - received. The keylog file is designed for debugging purposes only. The - file format is specified by NSS and used by many traffic analyzers such - as Wireshark. The log file is opened in append-only mode. Writes are - synchronized between threads, but not between processes. - - .. versionadded:: 3.8 - -.. attribute:: SSLContext.maximum_version - - A :class:`TLSVersion` enum member representing the highest supported - TLS version. The value defaults to :attr:`TLSVersion.MAXIMUM_SUPPORTED`. - The attribute is read-only for protocols other than :attr:`PROTOCOL_TLS`, - :attr:`PROTOCOL_TLS_CLIENT`, and :attr:`PROTOCOL_TLS_SERVER`. - - The attributes :attr:`~SSLContext.maximum_version`, - :attr:`~SSLContext.minimum_version` and - :attr:`SSLContext.options` all affect the supported SSL - and TLS versions of the context. The implementation does not prevent - invalid combination. For example a context with - :attr:`OP_NO_TLSv1_2` in :attr:`~SSLContext.options` and - :attr:`~SSLContext.maximum_version` set to :attr:`TLSVersion.TLSv1_2` - will not be able to establish a TLS 1.2 connection. - - .. versionadded:: 3.7 - -.. attribute:: SSLContext.minimum_version - - Like :attr:`SSLContext.maximum_version` except it is the lowest - supported version or :attr:`TLSVersion.MINIMUM_SUPPORTED`. - - .. versionadded:: 3.7 - -.. attribute:: SSLContext.num_tickets - - Control the number of TLS 1.3 session tickets of a - :attr:`PROTOCOL_TLS_SERVER` context. The setting has no impact on TLS - 1.0 to 1.2 connections. - - .. versionadded:: 3.8 - -.. attribute:: SSLContext.options - - An integer representing the set of SSL options enabled on this context. - The default value is :data:`OP_ALL`, but you can specify other options - such as :data:`OP_NO_SSLv2` by ORing them together. - - .. versionchanged:: 3.6 - :attr:`SSLContext.options` returns :class:`Options` flags: - - >>> ssl.create_default_context().options # doctest: +SKIP - - - .. deprecated:: 3.7 - - All ``OP_NO_SSL*`` and ``OP_NO_TLS*`` options have been deprecated since - Python 3.7. Use :attr:`SSLContext.minimum_version` and - :attr:`SSLContext.maximum_version` instead. - -.. attribute:: SSLContext.post_handshake_auth - - Enable TLS 1.3 post-handshake client authentication. Post-handshake auth - is disabled by default and a server can only request a TLS client - certificate during the initial handshake. When enabled, a server may - request a TLS client certificate at any time after the handshake. - - When enabled on client-side sockets, the client signals the server that - it supports post-handshake authentication. - - When enabled on server-side sockets, :attr:`SSLContext.verify_mode` must - be set to :data:`CERT_OPTIONAL` or :data:`CERT_REQUIRED`, too. The - actual client cert exchange is delayed until - :meth:`SSLSocket.verify_client_post_handshake` is called and some I/O is - performed. - - .. versionadded:: 3.8 - -.. attribute:: SSLContext.protocol - - The protocol version chosen when constructing the context. This attribute - is read-only. - -.. attribute:: SSLContext.hostname_checks_common_name - - Whether :attr:`~SSLContext.check_hostname` falls back to verify the cert's - subject common name in the absence of a subject alternative name - extension (default: true). - - .. versionadded:: 3.7 - - .. versionchanged:: 3.10 - - The flag had no effect with OpenSSL before version 1.1.1k. Python 3.8.9, - 3.9.3, and 3.10 include workarounds for previous versions. - -.. attribute:: SSLContext.security_level - - An integer representing the `security level - `_ - for the context. This attribute is read-only. - - .. versionadded:: 3.10 - -.. attribute:: SSLContext.verify_flags - - The flags for certificate verification operations. You can set flags like - :data:`VERIFY_CRL_CHECK_LEAF` by ORing them together. By default OpenSSL - does neither require nor verify certificate revocation lists (CRLs). - - .. versionadded:: 3.4 - - .. versionchanged:: 3.6 - :attr:`SSLContext.verify_flags` returns :class:`VerifyFlags` flags: - - >>> ssl.create_default_context().verify_flags # doctest: +SKIP - - -.. attribute:: SSLContext.verify_mode - - Whether to try to verify other peers' certificates and how to behave - if verification fails. This attribute must be one of - :data:`CERT_NONE`, :data:`CERT_OPTIONAL` or :data:`CERT_REQUIRED`. - - .. versionchanged:: 3.6 - :attr:`SSLContext.verify_mode` returns :class:`VerifyMode` enum: - - >>> ssl.create_default_context().verify_mode # doctest: +SKIP - - -.. index:: single: certificates - -.. index:: single: X509 certificate - -.. _ssl-certificates: - -Certificates ------------- - -Certificates in general are part of a public-key / private-key system. In this -system, each *principal*, (which may be a machine, or a person, or an -organization) is assigned a unique two-part encryption key. One part of the key -is public, and is called the *public key*; the other part is kept secret, and is -called the *private key*. The two parts are related, in that if you encrypt a -message with one of the parts, you can decrypt it with the other part, and -**only** with the other part. - -A certificate contains information about two principals. It contains the name -of a *subject*, and the subject's public key. It also contains a statement by a -second principal, the *issuer*, that the subject is who they claim to be, and -that this is indeed the subject's public key. The issuer's statement is signed -with the issuer's private key, which only the issuer knows. However, anyone can -verify the issuer's statement by finding the issuer's public key, decrypting the -statement with it, and comparing it to the other information in the certificate. -The certificate also contains information about the time period over which it is -valid. This is expressed as two fields, called "notBefore" and "notAfter". - -In the Python use of certificates, a client or server can use a certificate to -prove who they are. The other side of a network connection can also be required -to produce a certificate, and that certificate can be validated to the -satisfaction of the client or server that requires such validation. The -connection attempt can be set to raise an exception if the validation fails. -Validation is done automatically, by the underlying OpenSSL framework; the -application need not concern itself with its mechanics. But the application -does usually need to provide sets of certificates to allow this process to take -place. - -Python uses files to contain certificates. They should be formatted as "PEM" -(see :rfc:`1422`), which is a base-64 encoded form wrapped with a header line -and a footer line:: - - -----BEGIN CERTIFICATE----- - ... (certificate in base64 PEM encoding) ... - -----END CERTIFICATE----- - -Certificate chains -^^^^^^^^^^^^^^^^^^ - -The Python files which contain certificates can contain a sequence of -certificates, sometimes called a *certificate chain*. This chain should start -with the specific certificate for the principal who "is" the client or server, -and then the certificate for the issuer of that certificate, and then the -certificate for the issuer of *that* certificate, and so on up the chain till -you get to a certificate which is *self-signed*, that is, a certificate which -has the same subject and issuer, sometimes called a *root certificate*. The -certificates should just be concatenated together in the certificate file. For -example, suppose we had a three certificate chain, from our server certificate -to the certificate of the certification authority that signed our server -certificate, to the root certificate of the agency which issued the -certification authority's certificate:: - - -----BEGIN CERTIFICATE----- - ... (certificate for your server)... - -----END CERTIFICATE----- - -----BEGIN CERTIFICATE----- - ... (the certificate for the CA)... - -----END CERTIFICATE----- - -----BEGIN CERTIFICATE----- - ... (the root certificate for the CA's issuer)... - -----END CERTIFICATE----- - -CA certificates -^^^^^^^^^^^^^^^ - -If you are going to require validation of the other side of the connection's -certificate, you need to provide a "CA certs" file, filled with the certificate -chains for each issuer you are willing to trust. Again, this file just contains -these chains concatenated together. For validation, Python will use the first -chain it finds in the file which matches. The platform's certificates file can -be used by calling :meth:`SSLContext.load_default_certs`, this is done -automatically with :func:`.create_default_context`. - -Combined key and certificate -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Often the private key is stored in the same file as the certificate; in this -case, only the ``certfile`` parameter to :meth:`SSLContext.load_cert_chain` -and :func:`wrap_socket` needs to be passed. If the private key is stored -with the certificate, it should come before the first certificate in -the certificate chain:: - - -----BEGIN RSA PRIVATE KEY----- - ... (private key in base64 encoding) ... - -----END RSA PRIVATE KEY----- - -----BEGIN CERTIFICATE----- - ... (certificate in base64 PEM encoding) ... - -----END CERTIFICATE----- - -Self-signed certificates -^^^^^^^^^^^^^^^^^^^^^^^^ - -If you are going to create a server that provides SSL-encrypted connection -services, you will need to acquire a certificate for that service. There are -many ways of acquiring appropriate certificates, such as buying one from a -certification authority. Another common practice is to generate a self-signed -certificate. The simplest way to do this is with the OpenSSL package, using -something like the following:: - - % openssl req -new -x509 -days 365 -nodes -out cert.pem -keyout cert.pem - Generating a 1024 bit RSA private key - .......++++++ - .............................++++++ - writing new private key to 'cert.pem' - ----- - You are about to be asked to enter information that will be incorporated - into your certificate request. - What you are about to enter is what is called a Distinguished Name or a DN. - There are quite a few fields but you can leave some blank - For some fields there will be a default value, - If you enter '.', the field will be left blank. - ----- - Country Name (2 letter code) [AU]:US - State or Province Name (full name) [Some-State]:MyState - Locality Name (eg, city) []:Some City - Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc. - Organizational Unit Name (eg, section) []:My Group - Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com - Email Address []:ops@myserver.mygroup.myorganization.com - % - -The disadvantage of a self-signed certificate is that it is its own root -certificate, and no one else will have it in their cache of known (and trusted) -root certificates. - - -Examples --------- - -Testing for SSL support -^^^^^^^^^^^^^^^^^^^^^^^ - -To test for the presence of SSL support in a Python installation, user code -should use the following idiom:: - - try: - import ssl - except ImportError: - pass - else: - ... # do something that requires SSL support - -Client-side operation -^^^^^^^^^^^^^^^^^^^^^ - -This example creates a SSL context with the recommended security settings -for client sockets, including automatic certificate verification:: - - >>> context = ssl.create_default_context() - -If you prefer to tune security settings yourself, you might create -a context from scratch (but beware that you might not get the settings -right):: - - >>> context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> context.load_verify_locations("/etc/ssl/certs/ca-bundle.crt") - -(this snippet assumes your operating system places a bundle of all CA -certificates in ``/etc/ssl/certs/ca-bundle.crt``; if not, you'll get an -error and have to adjust the location) - -The :data:`PROTOCOL_TLS_CLIENT` protocol configures the context for cert -validation and hostname verification. :attr:`~SSLContext.verify_mode` is -set to :data:`CERT_REQUIRED` and :attr:`~SSLContext.check_hostname` is set -to ``True``. All other protocols create SSL contexts with insecure defaults. - -When you use the context to connect to a server, :const:`CERT_REQUIRED` -and :attr:`~SSLContext.check_hostname` validate the server certificate: it -ensures that the server certificate was signed with one of the CA -certificates, checks the signature for correctness, and verifies other -properties like validity and identity of the hostname:: - - >>> conn = context.wrap_socket(socket.socket(socket.AF_INET), - ... server_hostname="www.python.org") - >>> conn.connect(("www.python.org", 443)) - -You may then fetch the certificate:: - - >>> cert = conn.getpeercert() - -Visual inspection shows that the certificate does identify the desired service -(that is, the HTTPS host ``www.python.org``):: - - >>> pprint.pprint(cert) - {'OCSP': ('http://ocsp.digicert.com',), - 'caIssuers': ('http://cacerts.digicert.com/DigiCertSHA2ExtendedValidationServerCA.crt',), - 'crlDistributionPoints': ('http://crl3.digicert.com/sha2-ev-server-g1.crl', - 'http://crl4.digicert.com/sha2-ev-server-g1.crl'), - 'issuer': ((('countryName', 'US'),), - (('organizationName', 'DigiCert Inc'),), - (('organizationalUnitName', 'www.digicert.com'),), - (('commonName', 'DigiCert SHA2 Extended Validation Server CA'),)), - 'notAfter': 'Sep 9 12:00:00 2016 GMT', - 'notBefore': 'Sep 5 00:00:00 2014 GMT', - 'serialNumber': '01BB6F00122B177F36CAB49CEA8B6B26', - 'subject': ((('businessCategory', 'Private Organization'),), - (('1.3.6.1.4.1.311.60.2.1.3', 'US'),), - (('1.3.6.1.4.1.311.60.2.1.2', 'Delaware'),), - (('serialNumber', '3359300'),), - (('streetAddress', '16 Allen Rd'),), - (('postalCode', '03894-4801'),), - (('countryName', 'US'),), - (('stateOrProvinceName', 'NH'),), - (('localityName', 'Wolfeboro'),), - (('organizationName', 'Python Software Foundation'),), - (('commonName', 'www.python.org'),)), - 'subjectAltName': (('DNS', 'www.python.org'), - ('DNS', 'python.org'), - ('DNS', 'pypi.org'), - ('DNS', 'docs.python.org'), - ('DNS', 'testpypi.org'), - ('DNS', 'bugs.python.org'), - ('DNS', 'wiki.python.org'), - ('DNS', 'hg.python.org'), - ('DNS', 'mail.python.org'), - ('DNS', 'packaging.python.org'), - ('DNS', 'pythonhosted.org'), - ('DNS', 'www.pythonhosted.org'), - ('DNS', 'test.pythonhosted.org'), - ('DNS', 'us.pycon.org'), - ('DNS', 'id.python.org')), - 'version': 3} - -Now the SSL channel is established and the certificate verified, you can -proceed to talk with the server:: - - >>> conn.sendall(b"HEAD / HTTP/1.0\r\nHost: linuxfr.org\r\n\r\n") - >>> pprint.pprint(conn.recv(1024).split(b"\r\n")) - [b'HTTP/1.1 200 OK', - b'Date: Sat, 18 Oct 2014 18:27:20 GMT', - b'Server: nginx', - b'Content-Type: text/html; charset=utf-8', - b'X-Frame-Options: SAMEORIGIN', - b'Content-Length: 45679', - b'Accept-Ranges: bytes', - b'Via: 1.1 varnish', - b'Age: 2188', - b'X-Served-By: cache-lcy1134-LCY', - b'X-Cache: HIT', - b'X-Cache-Hits: 11', - b'Vary: Cookie', - b'Strict-Transport-Security: max-age=63072000; includeSubDomains', - b'Connection: close', - b'', - b''] - -See the discussion of :ref:`ssl-security` below. - - -Server-side operation -^^^^^^^^^^^^^^^^^^^^^ - -For server operation, typically you'll need to have a server certificate, and -private key, each in a file. You'll first create a context holding the key -and the certificate, so that clients can check your authenticity. Then -you'll open a socket, bind it to a port, call :meth:`listen` on it, and start -waiting for clients to connect:: - - import socket, ssl - - context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) - context.load_cert_chain(certfile="mycertfile", keyfile="mykeyfile") - - bindsocket = socket.socket() - bindsocket.bind(('myaddr.example.com', 10023)) - bindsocket.listen(5) - -When a client connects, you'll call :meth:`accept` on the socket to get the -new socket from the other end, and use the context's :meth:`SSLContext.wrap_socket` -method to create a server-side SSL socket for the connection:: - - while True: - newsocket, fromaddr = bindsocket.accept() - connstream = context.wrap_socket(newsocket, server_side=True) - try: - deal_with_client(connstream) - finally: - connstream.shutdown(socket.SHUT_RDWR) - connstream.close() - -Then you'll read data from the ``connstream`` and do something with it till you -are finished with the client (or the client is finished with you):: - - def deal_with_client(connstream): - data = connstream.recv(1024) - # empty data means the client is finished with us - while data: - if not do_something(connstream, data): - # we'll assume do_something returns False - # when we're finished with client - break - data = connstream.recv(1024) - # finished with client - -And go back to listening for new client connections (of course, a real server -would probably handle each client connection in a separate thread, or put -the sockets in :ref:`non-blocking mode ` and use an event loop). - - -.. _ssl-nonblocking: - -Notes on non-blocking sockets ------------------------------ - -SSL sockets behave slightly different than regular sockets in -non-blocking mode. When working with non-blocking sockets, there are -thus several things you need to be aware of: - -- Most :class:`SSLSocket` methods will raise either - :exc:`SSLWantWriteError` or :exc:`SSLWantReadError` instead of - :exc:`BlockingIOError` if an I/O operation would - block. :exc:`SSLWantReadError` will be raised if a read operation on - the underlying socket is necessary, and :exc:`SSLWantWriteError` for - a write operation on the underlying socket. Note that attempts to - *write* to an SSL socket may require *reading* from the underlying - socket first, and attempts to *read* from the SSL socket may require - a prior *write* to the underlying socket. - - .. versionchanged:: 3.5 - - In earlier Python versions, the :meth:`!SSLSocket.send` method - returned zero instead of raising :exc:`SSLWantWriteError` or - :exc:`SSLWantReadError`. - -- Calling :func:`~select.select` tells you that the OS-level socket can be - read from (or written to), but it does not imply that there is sufficient - data at the upper SSL layer. For example, only part of an SSL frame might - have arrived. Therefore, you must be ready to handle :meth:`SSLSocket.recv` - and :meth:`SSLSocket.send` failures, and retry after another call to - :func:`~select.select`. - -- Conversely, since the SSL layer has its own framing, a SSL socket may - still have data available for reading without :func:`~select.select` - being aware of it. Therefore, you should first call - :meth:`SSLSocket.recv` to drain any potentially available data, and then - only block on a :func:`~select.select` call if still necessary. - - (of course, similar provisions apply when using other primitives such as - :func:`~select.poll`, or those in the :mod:`selectors` module) - -- The SSL handshake itself will be non-blocking: the - :meth:`SSLSocket.do_handshake` method has to be retried until it returns - successfully. Here is a synopsis using :func:`~select.select` to wait for - the socket's readiness:: - - while True: - try: - sock.do_handshake() - break - except ssl.SSLWantReadError: - select.select([sock], [], []) - except ssl.SSLWantWriteError: - select.select([], [sock], []) - -.. seealso:: - - The :mod:`asyncio` module supports :ref:`non-blocking SSL sockets - ` and provides a - higher level API. It polls for events using the :mod:`selectors` module and - handles :exc:`SSLWantWriteError`, :exc:`SSLWantReadError` and - :exc:`BlockingIOError` exceptions. It runs the SSL handshake asynchronously - as well. - - -Memory BIO Support ------------------- - -.. versionadded:: 3.5 - -Ever since the SSL module was introduced in Python 2.6, the :class:`SSLSocket` -class has provided two related but distinct areas of functionality: - -- SSL protocol handling -- Network IO - -The network IO API is identical to that provided by :class:`socket.socket`, -from which :class:`SSLSocket` also inherits. This allows an SSL socket to be -used as a drop-in replacement for a regular socket, making it very easy to add -SSL support to an existing application. - -Combining SSL protocol handling and network IO usually works well, but there -are some cases where it doesn't. An example is async IO frameworks that want to -use a different IO multiplexing model than the "select/poll on a file -descriptor" (readiness based) model that is assumed by :class:`socket.socket` -and by the internal OpenSSL socket IO routines. This is mostly relevant for -platforms like Windows where this model is not efficient. For this purpose, a -reduced scope variant of :class:`SSLSocket` called :class:`SSLObject` is -provided. - -.. class:: SSLObject - - A reduced-scope variant of :class:`SSLSocket` representing an SSL protocol - instance that does not contain any network IO methods. This class is - typically used by framework authors that want to implement asynchronous IO - for SSL through memory buffers. - - This class implements an interface on top of a low-level SSL object as - implemented by OpenSSL. This object captures the state of an SSL connection - but does not provide any network IO itself. IO needs to be performed through - separate "BIO" objects which are OpenSSL's IO abstraction layer. - - This class has no public constructor. An :class:`SSLObject` instance - must be created using the :meth:`~SSLContext.wrap_bio` method. This - method will create the :class:`SSLObject` instance and bind it to a - pair of BIOs. The *incoming* BIO is used to pass data from Python to the - SSL protocol instance, while the *outgoing* BIO is used to pass data the - other way around. - - The following methods are available: - - - :attr:`~SSLSocket.context` - - :attr:`~SSLSocket.server_side` - - :attr:`~SSLSocket.server_hostname` - - :attr:`~SSLSocket.session` - - :attr:`~SSLSocket.session_reused` - - :meth:`~SSLSocket.read` - - :meth:`~SSLSocket.write` - - :meth:`~SSLSocket.getpeercert` - - :meth:`~SSLSocket.selected_alpn_protocol` - - :meth:`~SSLSocket.selected_npn_protocol` - - :meth:`~SSLSocket.cipher` - - :meth:`~SSLSocket.shared_ciphers` - - :meth:`~SSLSocket.compression` - - :meth:`~SSLSocket.pending` - - :meth:`~SSLSocket.do_handshake` - - :meth:`~SSLSocket.verify_client_post_handshake` - - :meth:`~SSLSocket.unwrap` - - :meth:`~SSLSocket.get_channel_binding` - - :meth:`~SSLSocket.version` - - When compared to :class:`SSLSocket`, this object lacks the following - features: - - - Any form of network IO; ``recv()`` and ``send()`` read and write only to - the underlying :class:`MemoryBIO` buffers. - - - There is no *do_handshake_on_connect* machinery. You must always manually - call :meth:`~SSLSocket.do_handshake` to start the handshake. - - - There is no handling of *suppress_ragged_eofs*. All end-of-file conditions - that are in violation of the protocol are reported via the - :exc:`SSLEOFError` exception. - - - The method :meth:`~SSLSocket.unwrap` call does not return anything, - unlike for an SSL socket where it returns the underlying socket. - - - The *server_name_callback* callback passed to - :meth:`SSLContext.set_servername_callback` will get an :class:`SSLObject` - instance instead of a :class:`SSLSocket` instance as its first parameter. - - Some notes related to the use of :class:`SSLObject`: - - - All IO on an :class:`SSLObject` is :ref:`non-blocking `. - This means that for example :meth:`~SSLSocket.read` will raise an - :exc:`SSLWantReadError` if it needs more data than the incoming BIO has - available. - - - There is no module-level ``wrap_bio()`` call like there is for - :meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created - via an :class:`SSLContext`. - - .. versionchanged:: 3.7 - :class:`SSLObject` instances must to created with - :meth:`~SSLContext.wrap_bio`. In earlier versions, it was possible to - create instances directly. This was never documented or officially - supported. - -An SSLObject communicates with the outside world using memory buffers. The -class :class:`MemoryBIO` provides a memory buffer that can be used for this -purpose. It wraps an OpenSSL memory BIO (Basic IO) object: - -.. class:: MemoryBIO - - A memory buffer that can be used to pass data between Python and an SSL - protocol instance. - - .. attribute:: MemoryBIO.pending - - Return the number of bytes currently in the memory buffer. - - .. attribute:: MemoryBIO.eof - - A boolean indicating whether the memory BIO is current at the end-of-file - position. - - .. method:: MemoryBIO.read(n=-1) - - Read up to *n* bytes from the memory buffer. If *n* is not specified or - negative, all bytes are returned. - - .. method:: MemoryBIO.write(buf) - - Write the bytes from *buf* to the memory BIO. The *buf* argument must be an - object supporting the buffer protocol. - - The return value is the number of bytes written, which is always equal to - the length of *buf*. - - .. method:: MemoryBIO.write_eof() - - Write an EOF marker to the memory BIO. After this method has been called, it - is illegal to call :meth:`~MemoryBIO.write`. The attribute :attr:`eof` will - become true after all data currently in the buffer has been read. - - -SSL session ------------ - -.. versionadded:: 3.6 - -.. class:: SSLSession - - Session object used by :attr:`~SSLSocket.session`. - - .. attribute:: id - .. attribute:: time - .. attribute:: timeout - .. attribute:: ticket_lifetime_hint - .. attribute:: has_ticket - - -.. _ssl-security: - -Security considerations ------------------------ - -Best defaults -^^^^^^^^^^^^^ - -For **client use**, if you don't have any special requirements for your -security policy, it is highly recommended that you use the -:func:`create_default_context` function to create your SSL context. -It will load the system's trusted CA certificates, enable certificate -validation and hostname checking, and try to choose reasonably secure -protocol and cipher settings. - -For example, here is how you would use the :class:`smtplib.SMTP` class to -create a trusted, secure connection to a SMTP server:: - - >>> import ssl, smtplib - >>> smtp = smtplib.SMTP("mail.python.org", port=587) - >>> context = ssl.create_default_context() - >>> smtp.starttls(context=context) - (220, b'2.0.0 Ready to start TLS') - -If a client certificate is needed for the connection, it can be added with -:meth:`SSLContext.load_cert_chain`. - -By contrast, if you create the SSL context by calling the :class:`SSLContext` -constructor yourself, it will not have certificate validation nor hostname -checking enabled by default. If you do so, please read the paragraphs below -to achieve a good security level. - -Manual settings -^^^^^^^^^^^^^^^ - -Verifying certificates -'''''''''''''''''''''' - -When calling the :class:`SSLContext` constructor directly, -:const:`CERT_NONE` is the default. Since it does not authenticate the other -peer, it can be insecure, especially in client mode where most of time you -would like to ensure the authenticity of the server you're talking to. -Therefore, when in client mode, it is highly recommended to use -:const:`CERT_REQUIRED`. However, it is in itself not sufficient; you also -have to check that the server certificate, which can be obtained by calling -:meth:`SSLSocket.getpeercert`, matches the desired service. For many -protocols and applications, the service can be identified by the hostname; -in this case, the :func:`match_hostname` function can be used. This common -check is automatically performed when :attr:`SSLContext.check_hostname` is -enabled. - -.. versionchanged:: 3.7 - Hostname matchings is now performed by OpenSSL. Python no longer uses - :func:`match_hostname`. - -In server mode, if you want to authenticate your clients using the SSL layer -(rather than using a higher-level authentication mechanism), you'll also have -to specify :const:`CERT_REQUIRED` and similarly check the client certificate. - - -Protocol versions -''''''''''''''''' - -SSL versions 2 and 3 are considered insecure and are therefore dangerous to -use. If you want maximum compatibility between clients and servers, it is -recommended to use :const:`PROTOCOL_TLS_CLIENT` or -:const:`PROTOCOL_TLS_SERVER` as the protocol version. SSLv2 and SSLv3 are -disabled by default. - -:: - - >>> client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) - >>> client_context.minimum_version = ssl.TLSVersion.TLSv1_3 - >>> client_context.maximum_version = ssl.TLSVersion.TLSv1_3 - - -The SSL context created above will only allow TLSv1.3 and later (if -supported by your system) connections to a server. :const:`PROTOCOL_TLS_CLIENT` -implies certificate validation and hostname checks by default. You have to -load certificates into the context. - - -Cipher selection -'''''''''''''''' - -If you have advanced security requirements, fine-tuning of the ciphers -enabled when negotiating a SSL session is possible through the -:meth:`SSLContext.set_ciphers` method. Starting from Python 3.2.3, the -ssl module disables certain weak ciphers by default, but you may want -to further restrict the cipher choice. Be sure to read OpenSSL's documentation -about the `cipher list format `_. -If you want to check which ciphers are enabled by a given cipher list, use -:meth:`SSLContext.get_ciphers` or the ``openssl ciphers`` command on your -system. - -Multi-processing -^^^^^^^^^^^^^^^^ - -If using this module as part of a multi-processed application (using, -for example the :mod:`multiprocessing` or :mod:`concurrent.futures` modules), -be aware that OpenSSL's internal random number generator does not properly -handle forked processes. Applications must change the PRNG state of the -parent process if they use any SSL feature with :func:`os.fork`. Any -successful call of :func:`~ssl.RAND_add`, :func:`~ssl.RAND_bytes` or -:func:`~ssl.RAND_pseudo_bytes` is sufficient. - - -.. _ssl-tlsv1_3: - -TLS 1.3 -------- - -.. versionadded:: 3.7 - -The TLS 1.3 protocol behaves slightly differently than previous version -of TLS/SSL. Some new TLS 1.3 features are not yet available. - -- TLS 1.3 uses a disjunct set of cipher suites. All AES-GCM and - ChaCha20 cipher suites are enabled by default. The method - :meth:`SSLContext.set_ciphers` cannot enable or disable any TLS 1.3 - ciphers yet, but :meth:`SSLContext.get_ciphers` returns them. -- Session tickets are no longer sent as part of the initial handshake and - are handled differently. :attr:`SSLSocket.session` and :class:`SSLSession` - are not compatible with TLS 1.3. -- Client-side certificates are also no longer verified during the initial - handshake. A server can request a certificate at any time. Clients - process certificate requests while they send or receive application data - from the server. -- TLS 1.3 features like early data, deferred TLS client cert request, - signature algorithm configuration, and rekeying are not supported yet. - - -.. seealso:: - - Class :class:`socket.socket` - Documentation of underlying :mod:`socket` class - - `SSL/TLS Strong Encryption: An Introduction `_ - Intro from the Apache HTTP Server documentation - - :rfc:`RFC 1422: Privacy Enhancement for Internet Electronic Mail: Part II: Certificate-Based Key Management <1422>` - Steve Kent - - :rfc:`RFC 4086: Randomness Requirements for Security <4086>` - Donald E., Jeffrey I. Schiller - - :rfc:`RFC 5280: Internet X.509 Public Key Infrastructure Certificate and Certificate Revocation List (CRL) Profile <5280>` - D. Cooper - - :rfc:`RFC 5246: The Transport Layer Security (TLS) Protocol Version 1.2 <5246>` - T. Dierks et. al. - - :rfc:`RFC 6066: Transport Layer Security (TLS) Extensions <6066>` - D. Eastlake - - `IANA TLS: Transport Layer Security (TLS) Parameters `_ - IANA - - :rfc:`RFC 7525: Recommendations for Secure Use of Transport Layer Security (TLS) and Datagram Transport Layer Security (DTLS) <7525>` - IETF - - `Mozilla's Server Side TLS recommendations `_ - Mozilla diff --git a/Doc/library/stat.rst b/Doc/library/stat.rst deleted file mode 100644 index 77538514598a50..00000000000000 --- a/Doc/library/stat.rst +++ /dev/null @@ -1,437 +0,0 @@ -:mod:`stat` --- Interpreting :func:`~os.stat` results -===================================================== - -.. module:: stat - :synopsis: Utilities for interpreting the results of os.stat(), - os.lstat() and os.fstat(). - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/stat.py` - --------------- - -The :mod:`stat` module defines constants and functions for interpreting the -results of :func:`os.stat`, :func:`os.fstat` and :func:`os.lstat` (if they -exist). For complete details about the :c:func:`stat`, :c:func:`!fstat` and -:c:func:`!lstat` calls, consult the documentation for your system. - -.. versionchanged:: 3.4 - The stat module is backed by a C implementation. - -The :mod:`stat` module defines the following functions to test for specific file -types: - - -.. function:: S_ISDIR(mode) - - Return non-zero if the mode is from a directory. - - -.. function:: S_ISCHR(mode) - - Return non-zero if the mode is from a character special device file. - - -.. function:: S_ISBLK(mode) - - Return non-zero if the mode is from a block special device file. - - -.. function:: S_ISREG(mode) - - Return non-zero if the mode is from a regular file. - - -.. function:: S_ISFIFO(mode) - - Return non-zero if the mode is from a FIFO (named pipe). - - -.. function:: S_ISLNK(mode) - - Return non-zero if the mode is from a symbolic link. - - -.. function:: S_ISSOCK(mode) - - Return non-zero if the mode is from a socket. - -.. function:: S_ISDOOR(mode) - - Return non-zero if the mode is from a door. - - .. versionadded:: 3.4 - -.. function:: S_ISPORT(mode) - - Return non-zero if the mode is from an event port. - - .. versionadded:: 3.4 - -.. function:: S_ISWHT(mode) - - Return non-zero if the mode is from a whiteout. - - .. versionadded:: 3.4 - -Two additional functions are defined for more general manipulation of the file's -mode: - - -.. function:: S_IMODE(mode) - - Return the portion of the file's mode that can be set by - :func:`os.chmod`\ ---that is, the file's permission bits, plus the sticky - bit, set-group-id, and set-user-id bits (on systems that support them). - - -.. function:: S_IFMT(mode) - - Return the portion of the file's mode that describes the file type (used by the - :func:`!S_IS\*` functions above). - -Normally, you would use the :func:`!os.path.is\*` functions for testing the type -of a file; the functions here are useful when you are doing multiple tests of -the same file and wish to avoid the overhead of the :c:func:`stat` system call -for each test. These are also useful when checking for information about a file -that isn't handled by :mod:`os.path`, like the tests for block and character -devices. - -Example:: - - import os, sys - from stat import * - - def walktree(top, callback): - '''recursively descend the directory tree rooted at top, - calling the callback function for each regular file''' - - for f in os.listdir(top): - pathname = os.path.join(top, f) - mode = os.lstat(pathname).st_mode - if S_ISDIR(mode): - # It's a directory, recurse into it - walktree(pathname, callback) - elif S_ISREG(mode): - # It's a file, call the callback function - callback(pathname) - else: - # Unknown file type, print a message - print('Skipping %s' % pathname) - - def visitfile(file): - print('visiting', file) - - if __name__ == '__main__': - walktree(sys.argv[1], visitfile) - -An additional utility function is provided to convert a file's mode in a human -readable string: - -.. function:: filemode(mode) - - Convert a file's mode to a string of the form '-rwxrwxrwx'. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - The function supports :data:`S_IFDOOR`, :data:`S_IFPORT` and - :data:`S_IFWHT`. - - -All the variables below are simply symbolic indexes into the 10-tuple returned -by :func:`os.stat`, :func:`os.fstat` or :func:`os.lstat`. - - -.. data:: ST_MODE - - Inode protection mode. - - -.. data:: ST_INO - - Inode number. - - -.. data:: ST_DEV - - Device inode resides on. - - -.. data:: ST_NLINK - - Number of links to the inode. - - -.. data:: ST_UID - - User id of the owner. - - -.. data:: ST_GID - - Group id of the owner. - - -.. data:: ST_SIZE - - Size in bytes of a plain file; amount of data waiting on some special files. - - -.. data:: ST_ATIME - - Time of last access. - - -.. data:: ST_MTIME - - Time of last modification. - - -.. data:: ST_CTIME - - The "ctime" as reported by the operating system. On some systems (like Unix) is - the time of the last metadata change, and, on others (like Windows), is the - creation time (see platform documentation for details). - -The interpretation of "file size" changes according to the file type. For plain -files this is the size of the file in bytes. For FIFOs and sockets under most -flavors of Unix (including Linux in particular), the "size" is the number of -bytes waiting to be read at the time of the call to :func:`os.stat`, -:func:`os.fstat`, or :func:`os.lstat`; this can sometimes be useful, especially -for polling one of these special files after a non-blocking open. The meaning -of the size field for other character and block devices varies more, depending -on the implementation of the underlying system call. - -The variables below define the flags used in the :data:`ST_MODE` field. - -Use of the functions above is more portable than use of the first set of flags: - -.. data:: S_IFSOCK - - Socket. - -.. data:: S_IFLNK - - Symbolic link. - -.. data:: S_IFREG - - Regular file. - -.. data:: S_IFBLK - - Block device. - -.. data:: S_IFDIR - - Directory. - -.. data:: S_IFCHR - - Character device. - -.. data:: S_IFIFO - - FIFO. - -.. data:: S_IFDOOR - - Door. - - .. versionadded:: 3.4 - -.. data:: S_IFPORT - - Event port. - - .. versionadded:: 3.4 - -.. data:: S_IFWHT - - Whiteout. - - .. versionadded:: 3.4 - -.. note:: - - :data:`S_IFDOOR`, :data:`S_IFPORT` or :data:`S_IFWHT` are defined as - 0 when the platform does not have support for the file types. - -The following flags can also be used in the *mode* argument of :func:`os.chmod`: - -.. data:: S_ISUID - - Set UID bit. - -.. data:: S_ISGID - - Set-group-ID bit. This bit has several special uses. For a directory - it indicates that BSD semantics is to be used for that directory: - files created there inherit their group ID from the directory, not - from the effective group ID of the creating process, and directories - created there will also get the :data:`S_ISGID` bit set. For a - file that does not have the group execution bit (:data:`S_IXGRP`) - set, the set-group-ID bit indicates mandatory file/record locking - (see also :data:`S_ENFMT`). - -.. data:: S_ISVTX - - Sticky bit. When this bit is set on a directory it means that a file - in that directory can be renamed or deleted only by the owner of the - file, by the owner of the directory, or by a privileged process. - -.. data:: S_IRWXU - - Mask for file owner permissions. - -.. data:: S_IRUSR - - Owner has read permission. - -.. data:: S_IWUSR - - Owner has write permission. - -.. data:: S_IXUSR - - Owner has execute permission. - -.. data:: S_IRWXG - - Mask for group permissions. - -.. data:: S_IRGRP - - Group has read permission. - -.. data:: S_IWGRP - - Group has write permission. - -.. data:: S_IXGRP - - Group has execute permission. - -.. data:: S_IRWXO - - Mask for permissions for others (not in group). - -.. data:: S_IROTH - - Others have read permission. - -.. data:: S_IWOTH - - Others have write permission. - -.. data:: S_IXOTH - - Others have execute permission. - -.. data:: S_ENFMT - - System V file locking enforcement. This flag is shared with :data:`S_ISGID`: - file/record locking is enforced on files that do not have the group - execution bit (:data:`S_IXGRP`) set. - -.. data:: S_IREAD - - Unix V7 synonym for :data:`S_IRUSR`. - -.. data:: S_IWRITE - - Unix V7 synonym for :data:`S_IWUSR`. - -.. data:: S_IEXEC - - Unix V7 synonym for :data:`S_IXUSR`. - -The following flags can be used in the *flags* argument of :func:`os.chflags`: - -.. data:: UF_NODUMP - - Do not dump the file. - -.. data:: UF_IMMUTABLE - - The file may not be changed. - -.. data:: UF_APPEND - - The file may only be appended to. - -.. data:: UF_OPAQUE - - The directory is opaque when viewed through a union stack. - -.. data:: UF_NOUNLINK - - The file may not be renamed or deleted. - -.. data:: UF_COMPRESSED - - The file is stored compressed (macOS 10.6+). - -.. data:: UF_HIDDEN - - The file should not be displayed in a GUI (macOS 10.5+). - -.. data:: SF_ARCHIVED - - The file may be archived. - -.. data:: SF_IMMUTABLE - - The file may not be changed. - -.. data:: SF_APPEND - - The file may only be appended to. - -.. data:: SF_NOUNLINK - - The file may not be renamed or deleted. - -.. data:: SF_SNAPSHOT - - The file is a snapshot file. - -See the \*BSD or macOS systems man page :manpage:`chflags(2)` for more information. - -On Windows, the following file attribute constants are available for use when -testing bits in the ``st_file_attributes`` member returned by :func:`os.stat`. -See the `Windows API documentation -`_ -for more detail on the meaning of these constants. - -.. data:: FILE_ATTRIBUTE_ARCHIVE - FILE_ATTRIBUTE_COMPRESSED - FILE_ATTRIBUTE_DEVICE - FILE_ATTRIBUTE_DIRECTORY - FILE_ATTRIBUTE_ENCRYPTED - FILE_ATTRIBUTE_HIDDEN - FILE_ATTRIBUTE_INTEGRITY_STREAM - FILE_ATTRIBUTE_NORMAL - FILE_ATTRIBUTE_NOT_CONTENT_INDEXED - FILE_ATTRIBUTE_NO_SCRUB_DATA - FILE_ATTRIBUTE_OFFLINE - FILE_ATTRIBUTE_READONLY - FILE_ATTRIBUTE_REPARSE_POINT - FILE_ATTRIBUTE_SPARSE_FILE - FILE_ATTRIBUTE_SYSTEM - FILE_ATTRIBUTE_TEMPORARY - FILE_ATTRIBUTE_VIRTUAL - - .. versionadded:: 3.5 - -On Windows, the following constants are available for comparing against the -``st_reparse_tag`` member returned by :func:`os.lstat`. These are well-known -constants, but are not an exhaustive list. - -.. data:: IO_REPARSE_TAG_SYMLINK - IO_REPARSE_TAG_MOUNT_POINT - IO_REPARSE_TAG_APPEXECLINK - - .. versionadded:: 3.8 diff --git a/Doc/library/statistics.rst b/Doc/library/statistics.rst deleted file mode 100644 index 03867b3d391a49..00000000000000 --- a/Doc/library/statistics.rst +++ /dev/null @@ -1,1032 +0,0 @@ -:mod:`statistics` --- Mathematical statistics functions -======================================================= - -.. module:: statistics - :synopsis: Mathematical statistics functions - -.. moduleauthor:: Steven D'Aprano -.. sectionauthor:: Steven D'Aprano - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/statistics.py` - -.. testsetup:: * - - from statistics import * - __name__ = '' - --------------- - -This module provides functions for calculating mathematical statistics of -numeric (:class:`~numbers.Real`-valued) data. - -The module is not intended to be a competitor to third-party libraries such -as `NumPy `_, `SciPy `_, or -proprietary full-featured statistics packages aimed at professional -statisticians such as Minitab, SAS and Matlab. It is aimed at the level of -graphing and scientific calculators. - -Unless explicitly noted, these functions support :class:`int`, -:class:`float`, :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. -Behaviour with other types (whether in the numeric tower or not) is -currently unsupported. Collections with a mix of types are also undefined -and implementation-dependent. If your input data consists of mixed types, -you may be able to use :func:`map` to ensure a consistent result, for -example: ``map(float, input_data)``. - -Some datasets use ``NaN`` (not a number) values to represent missing data. -Since NaNs have unusual comparison semantics, they cause surprising or -undefined behaviors in the statistics functions that sort data or that count -occurrences. The functions affected are ``median()``, ``median_low()``, -``median_high()``, ``median_grouped()``, ``mode()``, ``multimode()``, and -``quantiles()``. The ``NaN`` values should be stripped before calling these -functions:: - - >>> from statistics import median - >>> from math import isnan - >>> from itertools import filterfalse - - >>> data = [20.7, float('NaN'),19.2, 18.3, float('NaN'), 14.4] - >>> sorted(data) # This has surprising behavior - [20.7, nan, 14.4, 18.3, 19.2, nan] - >>> median(data) # This result is unexpected - 16.35 - - >>> sum(map(isnan, data)) # Number of missing values - 2 - >>> clean = list(filterfalse(isnan, data)) # Strip NaN values - >>> clean - [20.7, 19.2, 18.3, 14.4] - >>> sorted(clean) # Sorting now works as expected - [14.4, 18.3, 19.2, 20.7] - >>> median(clean) # This result is now well defined - 18.75 - - -Averages and measures of central location ------------------------------------------ - -These functions calculate an average or typical value from a population -or sample. - -======================= =============================================================== -:func:`mean` Arithmetic mean ("average") of data. -:func:`fmean` Fast, floating point arithmetic mean, with optional weighting. -:func:`geometric_mean` Geometric mean of data. -:func:`harmonic_mean` Harmonic mean of data. -:func:`median` Median (middle value) of data. -:func:`median_low` Low median of data. -:func:`median_high` High median of data. -:func:`median_grouped` Median, or 50th percentile, of grouped data. -:func:`mode` Single mode (most common value) of discrete or nominal data. -:func:`multimode` List of modes (most common values) of discrete or nominal data. -:func:`quantiles` Divide data into intervals with equal probability. -======================= =============================================================== - -Measures of spread ------------------- - -These functions calculate a measure of how much the population or sample -tends to deviate from the typical or average values. - -======================= ============================================= -:func:`pstdev` Population standard deviation of data. -:func:`pvariance` Population variance of data. -:func:`stdev` Sample standard deviation of data. -:func:`variance` Sample variance of data. -======================= ============================================= - -Statistics for relations between two inputs -------------------------------------------- - -These functions calculate statistics regarding relations between two inputs. - -========================= ===================================================== -:func:`covariance` Sample covariance for two variables. -:func:`correlation` Pearson's correlation coefficient for two variables. -:func:`linear_regression` Slope and intercept for simple linear regression. -========================= ===================================================== - - -Function details ----------------- - -Note: The functions do not require the data given to them to be sorted. -However, for reading convenience, most of the examples show sorted sequences. - -.. function:: mean(data) - - Return the sample arithmetic mean of *data* which can be a sequence or iterable. - - The arithmetic mean is the sum of the data divided by the number of data - points. It is commonly called "the average", although it is only one of many - different mathematical averages. It is a measure of the central location of - the data. - - If *data* is empty, :exc:`StatisticsError` will be raised. - - Some examples of use: - - .. doctest:: - - >>> mean([1, 2, 3, 4, 4]) - 2.8 - >>> mean([-1.0, 2.5, 3.25, 5.75]) - 2.625 - - >>> from fractions import Fraction as F - >>> mean([F(3, 7), F(1, 21), F(5, 3), F(1, 3)]) - Fraction(13, 21) - - >>> from decimal import Decimal as D - >>> mean([D("0.5"), D("0.75"), D("0.625"), D("0.375")]) - Decimal('0.5625') - - .. note:: - - The mean is strongly affected by `outliers - `_ and is not necessarily a - typical example of the data points. For a more robust, although less - efficient, measure of `central tendency - `_, see :func:`median`. - - The sample mean gives an unbiased estimate of the true population mean, - so that when taken on average over all the possible samples, - ``mean(sample)`` converges on the true mean of the entire population. If - *data* represents the entire population rather than a sample, then - ``mean(data)`` is equivalent to calculating the true population mean μ. - - -.. function:: fmean(data, weights=None) - - Convert *data* to floats and compute the arithmetic mean. - - This runs faster than the :func:`mean` function and it always returns a - :class:`float`. The *data* may be a sequence or iterable. If the input - dataset is empty, raises a :exc:`StatisticsError`. - - .. doctest:: - - >>> fmean([3.5, 4.0, 5.25]) - 4.25 - - Optional weighting is supported. For example, a professor assigns a - grade for a course by weighting quizzes at 20%, homework at 20%, a - midterm exam at 30%, and a final exam at 30%: - - .. doctest:: - - >>> grades = [85, 92, 83, 91] - >>> weights = [0.20, 0.20, 0.30, 0.30] - >>> fmean(grades, weights) - 87.6 - - If *weights* is supplied, it must be the same length as the *data* or - a :exc:`ValueError` will be raised. - - .. versionadded:: 3.8 - - .. versionchanged:: 3.11 - Added support for *weights*. - - -.. function:: geometric_mean(data) - - Convert *data* to floats and compute the geometric mean. - - The geometric mean indicates the central tendency or typical value of the - *data* using the product of the values (as opposed to the arithmetic mean - which uses their sum). - - Raises a :exc:`StatisticsError` if the input dataset is empty, - if it contains a zero, or if it contains a negative value. - The *data* may be a sequence or iterable. - - No special efforts are made to achieve exact results. - (However, this may change in the future.) - - .. doctest:: - - >>> round(geometric_mean([54, 24, 36]), 1) - 36.0 - - .. versionadded:: 3.8 - - -.. function:: harmonic_mean(data, weights=None) - - Return the harmonic mean of *data*, a sequence or iterable of - real-valued numbers. If *weights* is omitted or *None*, then - equal weighting is assumed. - - The harmonic mean is the reciprocal of the arithmetic :func:`mean` of the - reciprocals of the data. For example, the harmonic mean of three values *a*, - *b* and *c* will be equivalent to ``3/(1/a + 1/b + 1/c)``. If one of the - values is zero, the result will be zero. - - The harmonic mean is a type of average, a measure of the central - location of the data. It is often appropriate when averaging - ratios or rates, for example speeds. - - Suppose a car travels 10 km at 40 km/hr, then another 10 km at 60 km/hr. - What is the average speed? - - .. doctest:: - - >>> harmonic_mean([40, 60]) - 48.0 - - Suppose a car travels 40 km/hr for 5 km, and when traffic clears, - speeds-up to 60 km/hr for the remaining 30 km of the journey. What - is the average speed? - - .. doctest:: - - >>> harmonic_mean([40, 60], weights=[5, 30]) - 56.0 - - :exc:`StatisticsError` is raised if *data* is empty, any element - is less than zero, or if the weighted sum isn't positive. - - The current algorithm has an early-out when it encounters a zero - in the input. This means that the subsequent inputs are not tested - for validity. (This behavior may change in the future.) - - .. versionadded:: 3.6 - - .. versionchanged:: 3.10 - Added support for *weights*. - -.. function:: median(data) - - Return the median (middle value) of numeric data, using the common "mean of - middle two" method. If *data* is empty, :exc:`StatisticsError` is raised. - *data* can be a sequence or iterable. - - The median is a robust measure of central location and is less affected by - the presence of outliers. When the number of data points is odd, the - middle data point is returned: - - .. doctest:: - - >>> median([1, 3, 5]) - 3 - - When the number of data points is even, the median is interpolated by taking - the average of the two middle values: - - .. doctest:: - - >>> median([1, 3, 5, 7]) - 4.0 - - This is suited for when your data is discrete, and you don't mind that the - median may not be an actual data point. - - If the data is ordinal (supports order operations) but not numeric (doesn't - support addition), consider using :func:`median_low` or :func:`median_high` - instead. - -.. function:: median_low(data) - - Return the low median of numeric data. If *data* is empty, - :exc:`StatisticsError` is raised. *data* can be a sequence or iterable. - - The low median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the smaller of - the two middle values is returned. - - .. doctest:: - - >>> median_low([1, 3, 5]) - 3 - >>> median_low([1, 3, 5, 7]) - 3 - - Use the low median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_high(data) - - Return the high median of data. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. - - The high median is always a member of the data set. When the number of data - points is odd, the middle value is returned. When it is even, the larger of - the two middle values is returned. - - .. doctest:: - - >>> median_high([1, 3, 5]) - 3 - >>> median_high([1, 3, 5, 7]) - 5 - - Use the high median when your data are discrete and you prefer the median to - be an actual data point rather than interpolated. - - -.. function:: median_grouped(data, interval=1) - - Return the median of grouped continuous data, calculated as the 50th - percentile, using interpolation. If *data* is empty, :exc:`StatisticsError` - is raised. *data* can be a sequence or iterable. - - .. doctest:: - - >>> median_grouped([52, 52, 53, 54]) - 52.5 - - In the following example, the data are rounded, so that each value represents - the midpoint of data classes, e.g. 1 is the midpoint of the class 0.5--1.5, 2 - is the midpoint of 1.5--2.5, 3 is the midpoint of 2.5--3.5, etc. With the data - given, the middle value falls somewhere in the class 3.5--4.5, and - interpolation is used to estimate it: - - .. doctest:: - - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - - Optional argument *interval* represents the class interval, and defaults - to 1. Changing the class interval naturally will change the interpolation: - - .. doctest:: - - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 - - This function does not check whether the data points are at least - *interval* apart. - - .. impl-detail:: - - Under some circumstances, :func:`median_grouped` may coerce data points to - floats. This behaviour is likely to change in the future. - - .. seealso:: - - * "Statistics for the Behavioral Sciences", Frederick J Gravetter and - Larry B Wallnau (8th Edition). - - * The `SSMEDIAN - `_ - function in the Gnome Gnumeric spreadsheet, including `this discussion - `_. - - -.. function:: mode(data) - - Return the single most common data point from discrete or nominal *data*. - The mode (when it exists) is the most typical value and serves as a - measure of central location. - - If there are multiple modes with the same frequency, returns the first one - encountered in the *data*. If the smallest or largest of those is - desired instead, use ``min(multimode(data))`` or ``max(multimode(data))``. - If the input *data* is empty, :exc:`StatisticsError` is raised. - - ``mode`` assumes discrete data and returns a single value. This is the - standard treatment of the mode as commonly taught in schools: - - .. doctest:: - - >>> mode([1, 1, 2, 3, 3, 3, 3, 4]) - 3 - - The mode is unique in that it is the only statistic in this package that - also applies to nominal (non-numeric) data: - - .. doctest:: - - >>> mode(["red", "blue", "blue", "red", "green", "red", "red"]) - 'red' - - .. versionchanged:: 3.8 - Now handles multimodal datasets by returning the first mode encountered. - Formerly, it raised :exc:`StatisticsError` when more than one mode was - found. - - -.. function:: multimode(data) - - Return a list of the most frequently occurring values in the order they - were first encountered in the *data*. Will return more than one result if - there are multiple modes or an empty list if the *data* is empty: - - .. doctest:: - - >>> multimode('aabbbbccddddeeffffgg') - ['b', 'd', 'f'] - >>> multimode('') - [] - - .. versionadded:: 3.8 - - -.. function:: pstdev(data, mu=None) - - Return the population standard deviation (the square root of the population - variance). See :func:`pvariance` for arguments and other details. - - .. doctest:: - - >>> pstdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 0.986893273527251 - - -.. function:: pvariance(data, mu=None) - - Return the population variance of *data*, a non-empty sequence or iterable - of real-valued numbers. Variance, or second moment about the mean, is a - measure of the variability (spread or dispersion) of data. A large - variance indicates that the data is spread out; a small variance indicates - it is clustered closely around the mean. - - If the optional second argument *mu* is given, it is typically the mean of - the *data*. It can also be used to compute the second moment around a - point that is not the mean. If it is missing or ``None`` (the default), - the arithmetic mean is automatically calculated. - - Use this function to calculate the variance from the entire population. To - estimate the variance from a sample, the :func:`variance` function is usually - a better choice. - - Raises :exc:`StatisticsError` if *data* is empty. - - Examples: - - .. doctest:: - - >>> data = [0.0, 0.25, 0.25, 1.25, 1.5, 1.75, 2.75, 3.25] - >>> pvariance(data) - 1.25 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *mu* to avoid recalculation: - - .. doctest:: - - >>> mu = mean(data) - >>> pvariance(data, mu) - 1.25 - - Decimals and Fractions are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> pvariance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('24.815') - - >>> from fractions import Fraction as F - >>> pvariance([F(1, 4), F(5, 4), F(1, 2)]) - Fraction(13, 72) - - .. note:: - - When called with the entire population, this gives the population variance - σ². When called on a sample instead, this is the biased sample variance - s², also known as variance with N degrees of freedom. - - If you somehow know the true population mean μ, you may use this - function to calculate the variance of a sample, giving the known - population mean as the second argument. Provided the data points are a - random sample of the population, the result will be an unbiased estimate - of the population variance. - - -.. function:: stdev(data, xbar=None) - - Return the sample standard deviation (the square root of the sample - variance). See :func:`variance` for arguments and other details. - - .. doctest:: - - >>> stdev([1.5, 2.5, 2.5, 2.75, 3.25, 4.75]) - 1.0810874155219827 - - -.. function:: variance(data, xbar=None) - - Return the sample variance of *data*, an iterable of at least two real-valued - numbers. Variance, or second moment about the mean, is a measure of the - variability (spread or dispersion) of data. A large variance indicates that - the data is spread out; a small variance indicates it is clustered closely - around the mean. - - If the optional second argument *xbar* is given, it should be the mean of - *data*. If it is missing or ``None`` (the default), the mean is - automatically calculated. - - Use this function when your data is a sample from a population. To calculate - the variance from the entire population, see :func:`pvariance`. - - Raises :exc:`StatisticsError` if *data* has fewer than two values. - - Examples: - - .. doctest:: - - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> variance(data) - 1.3720238095238095 - - If you have already calculated the mean of your data, you can pass it as the - optional second argument *xbar* to avoid recalculation: - - .. doctest:: - - >>> m = mean(data) - >>> variance(data, m) - 1.3720238095238095 - - This function does not attempt to verify that you have passed the actual mean - as *xbar*. Using arbitrary values for *xbar* can lead to invalid or - impossible results. - - Decimal and Fraction values are supported: - - .. doctest:: - - >>> from decimal import Decimal as D - >>> variance([D("27.5"), D("30.25"), D("30.25"), D("34.5"), D("41.75")]) - Decimal('31.01875') - - >>> from fractions import Fraction as F - >>> variance([F(1, 6), F(1, 2), F(5, 3)]) - Fraction(67, 108) - - .. note:: - - This is the sample variance s² with Bessel's correction, also known as - variance with N-1 degrees of freedom. Provided that the data points are - representative (e.g. independent and identically distributed), the result - should be an unbiased estimate of the true population variance. - - If you somehow know the actual population mean μ you should pass it to the - :func:`pvariance` function as the *mu* parameter to get the variance of a - sample. - -.. function:: quantiles(data, *, n=4, method='exclusive') - - Divide *data* into *n* continuous intervals with equal probability. - Returns a list of ``n - 1`` cut points separating the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. Set - *n* to 100 for percentiles which gives the 99 cuts points that separate - *data* into 100 equal sized groups. Raises :exc:`StatisticsError` if *n* - is not least 1. - - The *data* can be any iterable containing sample data. For meaningful - results, the number of data points in *data* should be larger than *n*. - Raises :exc:`StatisticsError` if there are not at least two data points. - - The cut points are linearly interpolated from the - two nearest data points. For example, if a cut point falls one-third - of the distance between two sample values, ``100`` and ``112``, the - cut-point will evaluate to ``104``. - - The *method* for computing quantiles can be varied depending on - whether the *data* includes or excludes the lowest and - highest possible values from the population. - - The default *method* is "exclusive" and is used for data sampled from - a population that can have more extreme values than found in the - samples. The portion of the population falling below the *i-th* of - *m* sorted data points is computed as ``i / (m + 1)``. Given nine - sample values, the method sorts them and assigns the following - percentiles: 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%. - - Setting the *method* to "inclusive" is used for describing population - data or for samples that are known to include the most extreme values - from the population. The minimum value in *data* is treated as the 0th - percentile and the maximum value is treated as the 100th percentile. - The portion of the population falling below the *i-th* of *m* sorted - data points is computed as ``(i - 1) / (m - 1)``. Given 11 sample - values, the method sorts them and assigns the following percentiles: - 0%, 10%, 20%, 30%, 40%, 50%, 60%, 70%, 80%, 90%, 100%. - - .. doctest:: - - # Decile cut points for empirically sampled data - >>> data = [105, 129, 87, 86, 111, 111, 89, 81, 108, 92, 110, - ... 100, 75, 105, 103, 109, 76, 119, 99, 91, 103, 129, - ... 106, 101, 84, 111, 74, 87, 86, 103, 103, 106, 86, - ... 111, 75, 87, 102, 121, 111, 88, 89, 101, 106, 95, - ... 103, 107, 101, 81, 109, 104] - >>> [round(q, 1) for q in quantiles(data, n=10)] - [81.0, 86.2, 89.0, 99.4, 102.5, 103.6, 106.0, 109.8, 111.0] - - .. versionadded:: 3.8 - -.. function:: covariance(x, y, /) - - Return the sample covariance of two inputs *x* and *y*. Covariance - is a measure of the joint variability of two inputs. - - Both inputs must be of the same length (no less than two), otherwise - :exc:`StatisticsError` is raised. - - Examples: - - .. doctest:: - - >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] - >>> y = [1, 2, 3, 1, 2, 3, 1, 2, 3] - >>> covariance(x, y) - 0.75 - >>> z = [9, 8, 7, 6, 5, 4, 3, 2, 1] - >>> covariance(x, z) - -7.5 - >>> covariance(z, x) - -7.5 - - .. versionadded:: 3.10 - -.. function:: correlation(x, y, /) - - Return the `Pearson's correlation coefficient - `_ - for two inputs. Pearson's correlation coefficient *r* takes values - between -1 and +1. It measures the strength and direction of the linear - relationship, where +1 means very strong, positive linear relationship, - -1 very strong, negative linear relationship, and 0 no linear relationship. - - Both inputs must be of the same length (no less than two), and need - not to be constant, otherwise :exc:`StatisticsError` is raised. - - Examples: - - .. doctest:: - - >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] - >>> y = [9, 8, 7, 6, 5, 4, 3, 2, 1] - >>> correlation(x, x) - 1.0 - >>> correlation(x, y) - -1.0 - - .. versionadded:: 3.10 - -.. function:: linear_regression(x, y, /, *, proportional=False) - - Return the slope and intercept of `simple linear regression - `_ - parameters estimated using ordinary least squares. Simple linear - regression describes the relationship between an independent variable *x* and - a dependent variable *y* in terms of this linear function: - - *y = slope \* x + intercept + noise* - - where ``slope`` and ``intercept`` are the regression parameters that are - estimated, and ``noise`` represents the - variability of the data that was not explained by the linear regression - (it is equal to the difference between predicted and actual values - of the dependent variable). - - Both inputs must be of the same length (no less than two), and - the independent variable *x* cannot be constant; - otherwise a :exc:`StatisticsError` is raised. - - For example, we can use the `release dates of the Monty - Python films `_ - to predict the cumulative number of Monty Python films - that would have been produced by 2019 - assuming that they had kept the pace. - - .. doctest:: - - >>> year = [1971, 1975, 1979, 1982, 1983] - >>> films_total = [1, 2, 3, 4, 5] - >>> slope, intercept = linear_regression(year, films_total) - >>> round(slope * 2019 + intercept) - 16 - - If *proportional* is true, the independent variable *x* and the - dependent variable *y* are assumed to be directly proportional. - The data is fit to a line passing through the origin. - Since the *intercept* will always be 0.0, the underlying linear - function simplifies to: - - *y = slope \* x + noise* - - .. versionadded:: 3.10 - - .. versionchanged:: 3.11 - Added support for *proportional*. - -Exceptions ----------- - -A single exception is defined: - -.. exception:: StatisticsError - - Subclass of :exc:`ValueError` for statistics-related exceptions. - - -:class:`NormalDist` objects ---------------------------- - -:class:`NormalDist` is a tool for creating and manipulating normal -distributions of a `random variable -`_. It is a -class that treats the mean and standard deviation of data -measurements as a single entity. - -Normal distributions arise from the `Central Limit Theorem -`_ and have a wide range -of applications in statistics. - -.. class:: NormalDist(mu=0.0, sigma=1.0) - - Returns a new *NormalDist* object where *mu* represents the `arithmetic - mean `_ and *sigma* - represents the `standard deviation - `_. - - If *sigma* is negative, raises :exc:`StatisticsError`. - - .. attribute:: mean - - A read-only property for the `arithmetic mean - `_ of a normal - distribution. - - .. attribute:: median - - A read-only property for the `median - `_ of a normal - distribution. - - .. attribute:: mode - - A read-only property for the `mode - `_ of a normal - distribution. - - .. attribute:: stdev - - A read-only property for the `standard deviation - `_ of a normal - distribution. - - .. attribute:: variance - - A read-only property for the `variance - `_ of a normal - distribution. Equal to the square of the standard deviation. - - .. classmethod:: NormalDist.from_samples(data) - - Makes a normal distribution instance with *mu* and *sigma* parameters - estimated from the *data* using :func:`fmean` and :func:`stdev`. - - The *data* can be any :term:`iterable` and should consist of values - that can be converted to type :class:`float`. If *data* does not - contain at least two elements, raises :exc:`StatisticsError` because it - takes at least one point to estimate a central value and at least two - points to estimate dispersion. - - .. method:: NormalDist.samples(n, *, seed=None) - - Generates *n* random samples for a given mean and standard deviation. - Returns a :class:`list` of :class:`float` values. - - If *seed* is given, creates a new instance of the underlying random - number generator. This is useful for creating reproducible results, - even in a multi-threading context. - - .. method:: NormalDist.pdf(x) - - Using a `probability density function (pdf) - `_, compute - the relative likelihood that a random variable *X* will be near the - given value *x*. Mathematically, it is the limit of the ratio ``P(x <= - X < x+dx) / dx`` as *dx* approaches zero. - - The relative likelihood is computed as the probability of a sample - occurring in a narrow range divided by the width of the range (hence - the word "density"). Since the likelihood is relative to other points, - its value can be greater than ``1.0``. - - .. method:: NormalDist.cdf(x) - - Using a `cumulative distribution function (cdf) - `_, - compute the probability that a random variable *X* will be less than or - equal to *x*. Mathematically, it is written ``P(X <= x)``. - - .. method:: NormalDist.inv_cdf(p) - - Compute the inverse cumulative distribution function, also known as the - `quantile function `_ - or the `percent-point - `_ - function. Mathematically, it is written ``x : P(X <= x) = p``. - - Finds the value *x* of the random variable *X* such that the - probability of the variable being less than or equal to that value - equals the given probability *p*. - - .. method:: NormalDist.overlap(other) - - Measures the agreement between two normal probability distributions. - Returns a value between 0.0 and 1.0 giving `the overlapping area for - the two probability density functions - `_. - - .. method:: NormalDist.quantiles(n=4) - - Divide the normal distribution into *n* continuous intervals with - equal probability. Returns a list of (n - 1) cut points separating - the intervals. - - Set *n* to 4 for quartiles (the default). Set *n* to 10 for deciles. - Set *n* to 100 for percentiles which gives the 99 cuts points that - separate the normal distribution into 100 equal sized groups. - - .. method:: NormalDist.zscore(x) - - Compute the - `Standard Score `_ - describing *x* in terms of the number of standard deviations - above or below the mean of the normal distribution: - ``(x - mean) / stdev``. - - .. versionadded:: 3.9 - - Instances of :class:`NormalDist` support addition, subtraction, - multiplication and division by a constant. These operations - are used for translation and scaling. For example: - - .. doctest:: - - >>> temperature_february = NormalDist(5, 2.5) # Celsius - >>> temperature_february * (9/5) + 32 # Fahrenheit - NormalDist(mu=41.0, sigma=4.5) - - Dividing a constant by an instance of :class:`NormalDist` is not supported - because the result wouldn't be normally distributed. - - Since normal distributions arise from additive effects of independent - variables, it is possible to `add and subtract two independent normally - distributed random variables - `_ - represented as instances of :class:`NormalDist`. For example: - - .. doctest:: - - >>> birth_weights = NormalDist.from_samples([2.5, 3.1, 2.1, 2.4, 2.7, 3.5]) - >>> drug_effects = NormalDist(0.4, 0.15) - >>> combined = birth_weights + drug_effects - >>> round(combined.mean, 1) - 3.1 - >>> round(combined.stdev, 1) - 0.5 - - .. versionadded:: 3.8 - - -:class:`NormalDist` Examples and Recipes -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`NormalDist` readily solves classic probability problems. - -For example, given `historical data for SAT exams -`_ showing -that scores are normally distributed with a mean of 1060 and a standard -deviation of 195, determine the percentage of students with test scores -between 1100 and 1200, after rounding to the nearest whole number: - -.. doctest:: - - >>> sat = NormalDist(1060, 195) - >>> fraction = sat.cdf(1200 + 0.5) - sat.cdf(1100 - 0.5) - >>> round(fraction * 100.0, 1) - 18.4 - -Find the `quartiles `_ and `deciles -`_ for the SAT scores: - -.. doctest:: - - >>> list(map(round, sat.quantiles())) - [928, 1060, 1192] - >>> list(map(round, sat.quantiles(n=10))) - [810, 896, 958, 1011, 1060, 1109, 1162, 1224, 1310] - -To estimate the distribution for a model than isn't easy to solve -analytically, :class:`NormalDist` can generate input samples for a `Monte -Carlo simulation `_: - -.. doctest:: - - >>> def model(x, y, z): - ... return (3*x + 7*x*y - 5*y) / (11 * z) - ... - >>> n = 100_000 - >>> X = NormalDist(10, 2.5).samples(n, seed=3652260728) - >>> Y = NormalDist(15, 1.75).samples(n, seed=4582495471) - >>> Z = NormalDist(50, 1.25).samples(n, seed=6582483453) - >>> quantiles(map(model, X, Y, Z)) # doctest: +SKIP - [1.4591308524824727, 1.8035946855390597, 2.175091447274739] - -Normal distributions can be used to approximate `Binomial -distributions `_ -when the sample size is large and when the probability of a successful -trial is near 50%. - -For example, an open source conference has 750 attendees and two rooms with a -500 person capacity. There is a talk about Python and another about Ruby. -In previous conferences, 65% of the attendees preferred to listen to Python -talks. Assuming the population preferences haven't changed, what is the -probability that the Python room will stay within its capacity limits? - -.. doctest:: - - >>> n = 750 # Sample size - >>> p = 0.65 # Preference for Python - >>> q = 1.0 - p # Preference for Ruby - >>> k = 500 # Room capacity - - >>> # Approximation using the cumulative normal distribution - >>> from math import sqrt - >>> round(NormalDist(mu=n*p, sigma=sqrt(n*p*q)).cdf(k + 0.5), 4) - 0.8402 - - >>> # Solution using the cumulative binomial distribution - >>> from math import comb, fsum - >>> round(fsum(comb(n, r) * p**r * q**(n-r) for r in range(k+1)), 4) - 0.8402 - - >>> # Approximation using a simulation - >>> from random import seed, choices - >>> seed(8675309) - >>> def trial(): - ... return choices(('Python', 'Ruby'), (p, q), k=n).count('Python') - >>> mean(trial() <= k for i in range(10_000)) - 0.8398 - -Normal distributions commonly arise in machine learning problems. - -Wikipedia has a `nice example of a Naive Bayesian Classifier -`_. -The challenge is to predict a person's gender from measurements of normally -distributed features including height, weight, and foot size. - -We're given a training dataset with measurements for eight people. The -measurements are assumed to be normally distributed, so we summarize the data -with :class:`NormalDist`: - -.. doctest:: - - >>> height_male = NormalDist.from_samples([6, 5.92, 5.58, 5.92]) - >>> height_female = NormalDist.from_samples([5, 5.5, 5.42, 5.75]) - >>> weight_male = NormalDist.from_samples([180, 190, 170, 165]) - >>> weight_female = NormalDist.from_samples([100, 150, 130, 150]) - >>> foot_size_male = NormalDist.from_samples([12, 11, 12, 10]) - >>> foot_size_female = NormalDist.from_samples([6, 8, 7, 9]) - -Next, we encounter a new person whose feature measurements are known but whose -gender is unknown: - -.. doctest:: - - >>> ht = 6.0 # height - >>> wt = 130 # weight - >>> fs = 8 # foot size - -Starting with a 50% `prior probability -`_ of being male or female, -we compute the posterior as the prior times the product of likelihoods for the -feature measurements given the gender: - -.. doctest:: - - >>> prior_male = 0.5 - >>> prior_female = 0.5 - >>> posterior_male = (prior_male * height_male.pdf(ht) * - ... weight_male.pdf(wt) * foot_size_male.pdf(fs)) - - >>> posterior_female = (prior_female * height_female.pdf(ht) * - ... weight_female.pdf(wt) * foot_size_female.pdf(fs)) - -The final prediction goes to the largest posterior. This is known as the -`maximum a posteriori -`_ or MAP: - -.. doctest:: - - >>> 'male' if posterior_male > posterior_female else 'female' - 'female' - - -.. - # This modelines must appear within the last ten lines of the file. - kate: indent-width 3; remove-trailing-space on; replace-tabs on; encoding utf-8; diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst deleted file mode 100644 index fd3218a89c7cd2..00000000000000 --- a/Doc/library/stdtypes.rst +++ /dev/null @@ -1,5658 +0,0 @@ -.. XXX: reference/datamodel and this have quite a few overlaps! - - -.. _bltin-types: - -************** -Built-in Types -************** - -The following sections describe the standard types that are built into the -interpreter. - -.. index:: pair: built-in; types - -The principal built-in types are numerics, sequences, mappings, classes, -instances and exceptions. - -Some collection classes are mutable. The methods that add, subtract, or -rearrange their members in place, and don't return a specific item, never return -the collection instance itself but ``None``. - -Some operations are supported by several object types; in particular, -practically all objects can be compared for equality, tested for truth -value, and converted to a string (with the :func:`repr` function or the -slightly different :func:`str` function). The latter function is implicitly -used when an object is written by the :func:`print` function. - - -.. _truth: - -Truth Value Testing -=================== - -.. index:: - pair: statement; if - pair: statement; while - pair: truth; value - pair: Boolean; operations - single: false - -Any object can be tested for truth value, for use in an :keyword:`if` or -:keyword:`while` condition or as operand of the Boolean operations below. - -.. index:: single: true - -By default, an object is considered true unless its class defines either a -:meth:`~object.__bool__` method that returns ``False`` or a :meth:`__len__` method that -returns zero, when called with the object. [1]_ Here are most of the built-in -objects considered false: - -.. index:: - single: None (Built-in object) - single: False (Built-in object) - -* constants defined to be false: ``None`` and ``False`` - -* zero of any numeric type: ``0``, ``0.0``, ``0j``, ``Decimal(0)``, - ``Fraction(0, 1)`` - -* empty sequences and collections: ``''``, ``()``, ``[]``, ``{}``, ``set()``, - ``range(0)`` - -.. index:: - pair: operator; or - pair: operator; and - single: False - single: True - -Operations and built-in functions that have a Boolean result always return ``0`` -or ``False`` for false and ``1`` or ``True`` for true, unless otherwise stated. -(Important exception: the Boolean operations ``or`` and ``and`` always return -one of their operands.) - - -.. _boolean: - -Boolean Operations --- :keyword:`!and`, :keyword:`!or`, :keyword:`!not` -======================================================================= - -.. index:: pair: Boolean; operations - -These are the Boolean operations, ordered by ascending priority: - -+-------------+---------------------------------+-------+ -| Operation | Result | Notes | -+=============+=================================+=======+ -| ``x or y`` | if *x* is true, then *x*, else | \(1) | -| | *y* | | -+-------------+---------------------------------+-------+ -| ``x and y`` | if *x* is false, then *x*, else | \(2) | -| | *y* | | -+-------------+---------------------------------+-------+ -| ``not x`` | if *x* is false, then ``True``, | \(3) | -| | else ``False`` | | -+-------------+---------------------------------+-------+ - -.. index:: - pair: operator; and - pair: operator; or - pair: operator; not - -Notes: - -(1) - This is a short-circuit operator, so it only evaluates the second - argument if the first one is false. - -(2) - This is a short-circuit operator, so it only evaluates the second - argument if the first one is true. - -(3) - ``not`` has a lower priority than non-Boolean operators, so ``not a == b`` is - interpreted as ``not (a == b)``, and ``a == not b`` is a syntax error. - - -.. _stdcomparisons: - -Comparisons -=========== - -.. index:: - pair: chaining; comparisons - pair: operator; comparison - pair: operator; == - pair: operator; < (less) - pair: operator; <= - pair: operator; > (greater) - pair: operator; >= - pair: operator; != - pair: operator; is - pair: operator; is not - -There are eight comparison operations in Python. They all have the same -priority (which is higher than that of the Boolean operations). Comparisons can -be chained arbitrarily; for example, ``x < y <= z`` is equivalent to ``x < y and -y <= z``, except that *y* is evaluated only once (but in both cases *z* is not -evaluated at all when ``x < y`` is found to be false). - -This table summarizes the comparison operations: - -+------------+-------------------------+ -| Operation | Meaning | -+============+=========================+ -| ``<`` | strictly less than | -+------------+-------------------------+ -| ``<=`` | less than or equal | -+------------+-------------------------+ -| ``>`` | strictly greater than | -+------------+-------------------------+ -| ``>=`` | greater than or equal | -+------------+-------------------------+ -| ``==`` | equal | -+------------+-------------------------+ -| ``!=`` | not equal | -+------------+-------------------------+ -| ``is`` | object identity | -+------------+-------------------------+ -| ``is not`` | negated object identity | -+------------+-------------------------+ - -.. index:: - pair: object; numeric - pair: objects; comparing - -Objects of different types, except different numeric types, never compare equal. -The ``==`` operator is always defined but for some object types (for example, -class objects) is equivalent to :keyword:`is`. The ``<``, ``<=``, ``>`` and ``>=`` -operators are only defined where they make sense; for example, they raise a -:exc:`TypeError` exception when one of the arguments is a complex number. - -.. index:: - single: __eq__() (instance method) - single: __ne__() (instance method) - single: __lt__() (instance method) - single: __le__() (instance method) - single: __gt__() (instance method) - single: __ge__() (instance method) - -Non-identical instances of a class normally compare as non-equal unless the -class defines the :meth:`~object.__eq__` method. - -Instances of a class cannot be ordered with respect to other instances of the -same class, or other types of object, unless the class defines enough of the -methods :meth:`~object.__lt__`, :meth:`~object.__le__`, :meth:`~object.__gt__`, and -:meth:`~object.__ge__` (in general, :meth:`~object.__lt__` and -:meth:`~object.__eq__` are sufficient, if you want the conventional meanings of the -comparison operators). - -The behavior of the :keyword:`is` and :keyword:`is not` operators cannot be -customized; also they can be applied to any two objects and never raise an -exception. - -.. index:: - pair: operator; in - pair: operator; not in - -Two more operations with the same syntactic priority, :keyword:`in` and -:keyword:`not in`, are supported by types that are :term:`iterable` or -implement the :meth:`__contains__` method. - -.. _typesnumeric: - -Numeric Types --- :class:`int`, :class:`float`, :class:`complex` -================================================================ - -.. index:: - pair: object; numeric - pair: object; Boolean - pair: object; integer - pair: object; floating point - pair: object; complex number - pair: C; language - -There are three distinct numeric types: :dfn:`integers`, :dfn:`floating -point numbers`, and :dfn:`complex numbers`. In addition, Booleans are a -subtype of integers. Integers have unlimited precision. Floating point -numbers are usually implemented using :c:expr:`double` in C; information -about the precision and internal representation of floating point -numbers for the machine on which your program is running is available -in :data:`sys.float_info`. Complex numbers have a real and imaginary -part, which are each a floating point number. To extract these parts -from a complex number *z*, use ``z.real`` and ``z.imag``. (The standard -library includes the additional numeric types :mod:`fractions.Fraction`, for -rationals, and :mod:`decimal.Decimal`, for floating-point numbers with -user-definable precision.) - -.. index:: - pair: numeric; literals - pair: integer; literals - pair: floating point; literals - pair: complex number; literals - pair: hexadecimal; literals - pair: octal; literals - pair: binary; literals - -Numbers are created by numeric literals or as the result of built-in functions -and operators. Unadorned integer literals (including hex, octal and binary -numbers) yield integers. Numeric literals containing a decimal point or an -exponent sign yield floating point numbers. Appending ``'j'`` or ``'J'`` to a -numeric literal yields an imaginary number (a complex number with a zero real -part) which you can add to an integer or float to get a complex number with real -and imaginary parts. - -.. index:: - single: arithmetic - pair: built-in function; int - pair: built-in function; float - pair: built-in function; complex - single: operator; + (plus) - single: + (plus); unary operator - single: + (plus); binary operator - single: operator; - (minus) - single: - (minus); unary operator - single: - (minus); binary operator - pair: operator; * (asterisk) - pair: operator; / (slash) - pair: operator; // - pair: operator; % (percent) - pair: operator; ** - -Python fully supports mixed arithmetic: when a binary arithmetic operator has -operands of different numeric types, the operand with the "narrower" type is -widened to that of the other, where integer is narrower than floating point, -which is narrower than complex. A comparison between numbers of different types -behaves as though the exact values of those numbers were being compared. [2]_ - -The constructors :func:`int`, :func:`float`, and -:func:`complex` can be used to produce numbers of a specific type. - -All numeric types (except complex) support the following operations (for priorities of -the operations, see :ref:`operator-summary`): - -+---------------------+---------------------------------+---------+--------------------+ -| Operation | Result | Notes | Full documentation | -+=====================+=================================+=========+====================+ -| ``x + y`` | sum of *x* and *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x - y`` | difference of *x* and *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x * y`` | product of *x* and *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x / y`` | quotient of *x* and *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x // y`` | floored quotient of *x* and | \(1)\(2)| | -| | *y* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``x % y`` | remainder of ``x / y`` | \(2) | | -+---------------------+---------------------------------+---------+--------------------+ -| ``-x`` | *x* negated | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``+x`` | *x* unchanged | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``abs(x)`` | absolute value or magnitude of | | :func:`abs` | -| | *x* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``int(x)`` | *x* converted to integer | \(3)\(6)| :func:`int` | -+---------------------+---------------------------------+---------+--------------------+ -| ``float(x)`` | *x* converted to floating point | \(4)\(6)| :func:`float` | -+---------------------+---------------------------------+---------+--------------------+ -| ``complex(re, im)`` | a complex number with real part | \(6) | :func:`complex` | -| | *re*, imaginary part *im*. | | | -| | *im* defaults to zero. | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``c.conjugate()`` | conjugate of the complex number | | | -| | *c* | | | -+---------------------+---------------------------------+---------+--------------------+ -| ``divmod(x, y)`` | the pair ``(x // y, x % y)`` | \(2) | :func:`divmod` | -+---------------------+---------------------------------+---------+--------------------+ -| ``pow(x, y)`` | *x* to the power *y* | \(5) | :func:`pow` | -+---------------------+---------------------------------+---------+--------------------+ -| ``x ** y`` | *x* to the power *y* | \(5) | | -+---------------------+---------------------------------+---------+--------------------+ - -.. index:: - triple: operations on; numeric; types - single: conjugate() (complex number method) - -Notes: - -(1) - Also referred to as integer division. For operands of type :class:`int`, - the result has type :class:`int`. For operands of type :class:`float`, - the result has type :class:`float`. In general, the result is a whole - integer, though the result's type is not necessarily :class:`int`. The result is - always rounded towards minus infinity: ``1//2`` is ``0``, ``(-1)//2`` is - ``-1``, ``1//(-2)`` is ``-1``, and ``(-1)//(-2)`` is ``0``. - -(2) - Not for complex numbers. Instead convert to floats using :func:`abs` if - appropriate. - -(3) - .. index:: - pair: module; math - single: floor() (in module math) - single: ceil() (in module math) - single: trunc() (in module math) - pair: numeric; conversions - - Conversion from :class:`float` to :class:`int` truncates, discarding the - fractional part. See functions :func:`math.floor` and :func:`math.ceil` for - alternative conversions. - -(4) - float also accepts the strings "nan" and "inf" with an optional prefix "+" - or "-" for Not a Number (NaN) and positive or negative infinity. - -(5) - Python defines ``pow(0, 0)`` and ``0 ** 0`` to be ``1``, as is common for - programming languages. - -(6) - The numeric literals accepted include the digits ``0`` to ``9`` or any - Unicode equivalent (code points with the ``Nd`` property). - - See https://www.unicode.org/Public/14.0.0/ucd/extracted/DerivedNumericType.txt - for a complete list of code points with the ``Nd`` property. - - -All :class:`numbers.Real` types (:class:`int` and :class:`float`) also include -the following operations: - -+--------------------+---------------------------------------------+ -| Operation | Result | -+====================+=============================================+ -| :func:`math.trunc(\| *x* truncated to :class:`~numbers.Integral` | -| x) ` | | -+--------------------+---------------------------------------------+ -| :func:`round(x[, | *x* rounded to *n* digits, | -| n]) ` | rounding half to even. If *n* is | -| | omitted, it defaults to 0. | -+--------------------+---------------------------------------------+ -| :func:`math.floor(\| the greatest :class:`~numbers.Integral` | -| x) ` | <= *x* | -+--------------------+---------------------------------------------+ -| :func:`math.ceil(x)| the least :class:`~numbers.Integral` >= *x* | -| ` | | -+--------------------+---------------------------------------------+ - -For additional numeric operations see the :mod:`math` and :mod:`cmath` -modules. - -.. XXXJH exceptions: overflow (when? what operations?) zerodivision - - -.. _bitstring-ops: - -Bitwise Operations on Integer Types ------------------------------------ - -.. index:: - triple: operations on; integer; types - pair: bitwise; operations - pair: shifting; operations - pair: masking; operations - pair: operator; | (vertical bar) - pair: operator; ^ (caret) - pair: operator; & (ampersand) - pair: operator; << - pair: operator; >> - pair: operator; ~ (tilde) - -Bitwise operations only make sense for integers. The result of bitwise -operations is calculated as though carried out in two's complement with an -infinite number of sign bits. - -The priorities of the binary bitwise operations are all lower than the numeric -operations and higher than the comparisons; the unary operation ``~`` has the -same priority as the other unary numeric operations (``+`` and ``-``). - -This table lists the bitwise operations sorted in ascending priority: - -+------------+--------------------------------+----------+ -| Operation | Result | Notes | -+============+================================+==========+ -| ``x | y`` | bitwise :dfn:`or` of *x* and | \(4) | -| | *y* | | -+------------+--------------------------------+----------+ -| ``x ^ y`` | bitwise :dfn:`exclusive or` of | \(4) | -| | *x* and *y* | | -+------------+--------------------------------+----------+ -| ``x & y`` | bitwise :dfn:`and` of *x* and | \(4) | -| | *y* | | -+------------+--------------------------------+----------+ -| ``x << n`` | *x* shifted left by *n* bits | (1)(2) | -+------------+--------------------------------+----------+ -| ``x >> n`` | *x* shifted right by *n* bits | (1)(3) | -+------------+--------------------------------+----------+ -| ``~x`` | the bits of *x* inverted | | -+------------+--------------------------------+----------+ - -Notes: - -(1) - Negative shift counts are illegal and cause a :exc:`ValueError` to be raised. - -(2) - A left shift by *n* bits is equivalent to multiplication by ``pow(2, n)``. - -(3) - A right shift by *n* bits is equivalent to floor division by ``pow(2, n)``. - -(4) - Performing these calculations with at least one extra sign extension bit in - a finite two's complement representation (a working bit-width of - ``1 + max(x.bit_length(), y.bit_length())`` or more) is sufficient to get the - same result as if there were an infinite number of sign bits. - - -Additional Methods on Integer Types ------------------------------------ - -The int type implements the :class:`numbers.Integral` :term:`abstract base -class`. In addition, it provides a few more methods: - -.. method:: int.bit_length() - - Return the number of bits necessary to represent an integer in binary, - excluding the sign and leading zeros:: - - >>> n = -37 - >>> bin(n) - '-0b100101' - >>> n.bit_length() - 6 - - More precisely, if ``x`` is nonzero, then ``x.bit_length()`` is the - unique positive integer ``k`` such that ``2**(k-1) <= abs(x) < 2**k``. - Equivalently, when ``abs(x)`` is small enough to have a correctly - rounded logarithm, then ``k = 1 + int(log(abs(x), 2))``. - If ``x`` is zero, then ``x.bit_length()`` returns ``0``. - - Equivalent to:: - - def bit_length(self): - s = bin(self) # binary representation: bin(-37) --> '-0b100101' - s = s.lstrip('-0b') # remove leading zeros and minus sign - return len(s) # len('100101') --> 6 - - .. versionadded:: 3.1 - -.. method:: int.bit_count() - - Return the number of ones in the binary representation of the absolute - value of the integer. This is also known as the population count. - Example:: - - >>> n = 19 - >>> bin(n) - '0b10011' - >>> n.bit_count() - 3 - >>> (-n).bit_count() - 3 - - Equivalent to:: - - def bit_count(self): - return bin(self).count("1") - - .. versionadded:: 3.10 - -.. method:: int.to_bytes(length=1, byteorder='big', *, signed=False) - - Return an array of bytes representing an integer. - - >>> (1024).to_bytes(2, byteorder='big') - b'\x04\x00' - >>> (1024).to_bytes(10, byteorder='big') - b'\x00\x00\x00\x00\x00\x00\x00\x00\x04\x00' - >>> (-1024).to_bytes(10, byteorder='big', signed=True) - b'\xff\xff\xff\xff\xff\xff\xff\xff\xfc\x00' - >>> x = 1000 - >>> x.to_bytes((x.bit_length() + 7) // 8, byteorder='little') - b'\xe8\x03' - - The integer is represented using *length* bytes, and defaults to 1. An - :exc:`OverflowError` is raised if the integer is not representable with - the given number of bytes. - - The *byteorder* argument determines the byte order used to represent the - integer, and defaults to ``"big"``. If *byteorder* is - ``"big"``, the most significant byte is at the beginning of the byte - array. If *byteorder* is ``"little"``, the most significant byte is at - the end of the byte array. - - The *signed* argument determines whether two's complement is used to - represent the integer. If *signed* is ``False`` and a negative integer is - given, an :exc:`OverflowError` is raised. The default value for *signed* - is ``False``. - - The default values can be used to conveniently turn an integer into a - single byte object:: - - >>> (65).to_bytes() - b'A' - - However, when using the default arguments, don't try - to convert a value greater than 255 or you'll get an :exc:`OverflowError`. - - Equivalent to:: - - def to_bytes(n, length=1, byteorder='big', signed=False): - if byteorder == 'little': - order = range(length) - elif byteorder == 'big': - order = reversed(range(length)) - else: - raise ValueError("byteorder must be either 'little' or 'big'") - - return bytes((n >> i*8) & 0xff for i in order) - - .. versionadded:: 3.2 - .. versionchanged:: 3.11 - Added default argument values for ``length`` and ``byteorder``. - -.. classmethod:: int.from_bytes(bytes, byteorder='big', *, signed=False) - - Return the integer represented by the given array of bytes. - - >>> int.from_bytes(b'\x00\x10', byteorder='big') - 16 - >>> int.from_bytes(b'\x00\x10', byteorder='little') - 4096 - >>> int.from_bytes(b'\xfc\x00', byteorder='big', signed=True) - -1024 - >>> int.from_bytes(b'\xfc\x00', byteorder='big', signed=False) - 64512 - >>> int.from_bytes([255, 0, 0], byteorder='big') - 16711680 - - The argument *bytes* must either be a :term:`bytes-like object` or an - iterable producing bytes. - - The *byteorder* argument determines the byte order used to represent the - integer, and defaults to ``"big"``. If *byteorder* is - ``"big"``, the most significant byte is at the beginning of the byte - array. If *byteorder* is ``"little"``, the most significant byte is at - the end of the byte array. To request the native byte order of the host - system, use :data:`sys.byteorder` as the byte order value. - - The *signed* argument indicates whether two's complement is used to - represent the integer. - - Equivalent to:: - - def from_bytes(bytes, byteorder='big', signed=False): - if byteorder == 'little': - little_ordered = list(bytes) - elif byteorder == 'big': - little_ordered = list(reversed(bytes)) - else: - raise ValueError("byteorder must be either 'little' or 'big'") - - n = sum(b << i*8 for i, b in enumerate(little_ordered)) - if signed and little_ordered and (little_ordered[-1] & 0x80): - n -= 1 << 8*len(little_ordered) - - return n - - .. versionadded:: 3.2 - .. versionchanged:: 3.11 - Added default argument value for ``byteorder``. - -.. method:: int.as_integer_ratio() - - Return a pair of integers whose ratio is exactly equal to the original - integer and with a positive denominator. The integer ratio of integers - (whole numbers) is always the integer as the numerator and ``1`` as the - denominator. - - .. versionadded:: 3.8 - -Additional Methods on Float ---------------------------- - -The float type implements the :class:`numbers.Real` :term:`abstract base -class`. float also has the following additional methods. - -.. method:: float.as_integer_ratio() - - Return a pair of integers whose ratio is exactly equal to the - original float and with a positive denominator. Raises - :exc:`OverflowError` on infinities and a :exc:`ValueError` on - NaNs. - -.. method:: float.is_integer() - - Return ``True`` if the float instance is finite with integral - value, and ``False`` otherwise:: - - >>> (-2.0).is_integer() - True - >>> (3.2).is_integer() - False - -Two methods support conversion to -and from hexadecimal strings. Since Python's floats are stored -internally as binary numbers, converting a float to or from a -*decimal* string usually involves a small rounding error. In -contrast, hexadecimal strings allow exact representation and -specification of floating-point numbers. This can be useful when -debugging, and in numerical work. - - -.. method:: float.hex() - - Return a representation of a floating-point number as a hexadecimal - string. For finite floating-point numbers, this representation - will always include a leading ``0x`` and a trailing ``p`` and - exponent. - - -.. classmethod:: float.fromhex(s) - - Class method to return the float represented by a hexadecimal - string *s*. The string *s* may have leading and trailing - whitespace. - - -Note that :meth:`float.hex` is an instance method, while -:meth:`float.fromhex` is a class method. - -A hexadecimal string takes the form:: - - [sign] ['0x'] integer ['.' fraction] ['p' exponent] - -where the optional ``sign`` may by either ``+`` or ``-``, ``integer`` -and ``fraction`` are strings of hexadecimal digits, and ``exponent`` -is a decimal integer with an optional leading sign. Case is not -significant, and there must be at least one hexadecimal digit in -either the integer or the fraction. This syntax is similar to the -syntax specified in section 6.4.4.2 of the C99 standard, and also to -the syntax used in Java 1.5 onwards. In particular, the output of -:meth:`float.hex` is usable as a hexadecimal floating-point literal in -C or Java code, and hexadecimal strings produced by C's ``%a`` format -character or Java's ``Double.toHexString`` are accepted by -:meth:`float.fromhex`. - - -Note that the exponent is written in decimal rather than hexadecimal, -and that it gives the power of 2 by which to multiply the coefficient. -For example, the hexadecimal string ``0x3.a7p10`` represents the -floating-point number ``(3 + 10./16 + 7./16**2) * 2.0**10``, or -``3740.0``:: - - >>> float.fromhex('0x3.a7p10') - 3740.0 - - -Applying the reverse conversion to ``3740.0`` gives a different -hexadecimal string representing the same number:: - - >>> float.hex(3740.0) - '0x1.d380000000000p+11' - - -.. _numeric-hash: - -Hashing of numeric types ------------------------- - -For numbers ``x`` and ``y``, possibly of different types, it's a requirement -that ``hash(x) == hash(y)`` whenever ``x == y`` (see the :meth:`~object.__hash__` -method documentation for more details). For ease of implementation and -efficiency across a variety of numeric types (including :class:`int`, -:class:`float`, :class:`decimal.Decimal` and :class:`fractions.Fraction`) -Python's hash for numeric types is based on a single mathematical function -that's defined for any rational number, and hence applies to all instances of -:class:`int` and :class:`fractions.Fraction`, and all finite instances of -:class:`float` and :class:`decimal.Decimal`. Essentially, this function is -given by reduction modulo ``P`` for a fixed prime ``P``. The value of ``P`` is -made available to Python as the :attr:`modulus` attribute of -:data:`sys.hash_info`. - -.. impl-detail:: - - Currently, the prime used is ``P = 2**31 - 1`` on machines with 32-bit C - longs and ``P = 2**61 - 1`` on machines with 64-bit C longs. - -Here are the rules in detail: - -- If ``x = m / n`` is a nonnegative rational number and ``n`` is not divisible - by ``P``, define ``hash(x)`` as ``m * invmod(n, P) % P``, where ``invmod(n, - P)`` gives the inverse of ``n`` modulo ``P``. - -- If ``x = m / n`` is a nonnegative rational number and ``n`` is - divisible by ``P`` (but ``m`` is not) then ``n`` has no inverse - modulo ``P`` and the rule above doesn't apply; in this case define - ``hash(x)`` to be the constant value ``sys.hash_info.inf``. - -- If ``x = m / n`` is a negative rational number define ``hash(x)`` - as ``-hash(-x)``. If the resulting hash is ``-1``, replace it with - ``-2``. - -- The particular values ``sys.hash_info.inf`` and ``-sys.hash_info.inf`` - are used as hash values for positive - infinity or negative infinity (respectively). - -- For a :class:`complex` number ``z``, the hash values of the real - and imaginary parts are combined by computing ``hash(z.real) + - sys.hash_info.imag * hash(z.imag)``, reduced modulo - ``2**sys.hash_info.width`` so that it lies in - ``range(-2**(sys.hash_info.width - 1), 2**(sys.hash_info.width - - 1))``. Again, if the result is ``-1``, it's replaced with ``-2``. - - -To clarify the above rules, here's some example Python code, -equivalent to the built-in hash, for computing the hash of a rational -number, :class:`float`, or :class:`complex`:: - - - import sys, math - - def hash_fraction(m, n): - """Compute the hash of a rational number m / n. - - Assumes m and n are integers, with n positive. - Equivalent to hash(fractions.Fraction(m, n)). - - """ - P = sys.hash_info.modulus - # Remove common factors of P. (Unnecessary if m and n already coprime.) - while m % P == n % P == 0: - m, n = m // P, n // P - - if n % P == 0: - hash_value = sys.hash_info.inf - else: - # Fermat's Little Theorem: pow(n, P-1, P) is 1, so - # pow(n, P-2, P) gives the inverse of n modulo P. - hash_value = (abs(m) % P) * pow(n, P - 2, P) % P - if m < 0: - hash_value = -hash_value - if hash_value == -1: - hash_value = -2 - return hash_value - - def hash_float(x): - """Compute the hash of a float x.""" - - if math.isnan(x): - return object.__hash__(x) - elif math.isinf(x): - return sys.hash_info.inf if x > 0 else -sys.hash_info.inf - else: - return hash_fraction(*x.as_integer_ratio()) - - def hash_complex(z): - """Compute the hash of a complex number z.""" - - hash_value = hash_float(z.real) + sys.hash_info.imag * hash_float(z.imag) - # do a signed reduction modulo 2**sys.hash_info.width - M = 2**(sys.hash_info.width - 1) - hash_value = (hash_value & (M - 1)) - (hash_value & M) - if hash_value == -1: - hash_value = -2 - return hash_value - -.. _typeiter: - -Iterator Types -============== - -.. index:: - single: iterator protocol - single: protocol; iterator - single: sequence; iteration - single: container; iteration over - -Python supports a concept of iteration over containers. This is implemented -using two distinct methods; these are used to allow user-defined classes to -support iteration. Sequences, described below in more detail, always support -the iteration methods. - -One method needs to be defined for container objects to provide :term:`iterable` -support: - -.. XXX duplicated in reference/datamodel! - -.. method:: container.__iter__() - - Return an :term:`iterator` object. The object is required to support the - iterator protocol described below. If a container supports different types - of iteration, additional methods can be provided to specifically request - iterators for those iteration types. (An example of an object supporting - multiple forms of iteration would be a tree structure which supports both - breadth-first and depth-first traversal.) This method corresponds to the - :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python - objects in the Python/C API. - -The iterator objects themselves are required to support the following two -methods, which together form the :dfn:`iterator protocol`: - - -.. method:: iterator.__iter__() - - Return the :term:`iterator` object itself. This is required to allow both - containers and iterators to be used with the :keyword:`for` and - :keyword:`in` statements. This method corresponds to the - :c:member:`~PyTypeObject.tp_iter` slot of the type structure for Python - objects in the Python/C API. - - -.. method:: iterator.__next__() - - Return the next item from the :term:`iterator`. If there are no further - items, raise the :exc:`StopIteration` exception. This method corresponds to - the :c:member:`~PyTypeObject.tp_iternext` slot of the type structure for - Python objects in the Python/C API. - -Python defines several iterator objects to support iteration over general and -specific sequence types, dictionaries, and other more specialized forms. The -specific types are not important beyond their implementation of the iterator -protocol. - -Once an iterator's :meth:`~iterator.__next__` method raises -:exc:`StopIteration`, it must continue to do so on subsequent calls. -Implementations that do not obey this property are deemed broken. - - -.. _generator-types: - -Generator Types ---------------- - -Python's :term:`generator`\s provide a convenient way to implement the iterator -protocol. If a container object's :meth:`__iter__` method is implemented as a -generator, it will automatically return an iterator object (technically, a -generator object) supplying the :meth:`__iter__` and :meth:`~generator.__next__` -methods. -More information about generators can be found in :ref:`the documentation for -the yield expression `. - - -.. _typesseq: - -Sequence Types --- :class:`list`, :class:`tuple`, :class:`range` -================================================================ - -There are three basic sequence types: lists, tuples, and range objects. -Additional sequence types tailored for processing of -:ref:`binary data ` and :ref:`text strings ` are -described in dedicated sections. - - -.. _typesseq-common: - -Common Sequence Operations --------------------------- - -.. index:: pair: object; sequence - -The operations in the following table are supported by most sequence types, -both mutable and immutable. The :class:`collections.abc.Sequence` ABC is -provided to make it easier to correctly implement these operations on -custom sequence types. - -This table lists the sequence operations sorted in ascending priority. In the -table, *s* and *t* are sequences of the same type, *n*, *i*, *j* and *k* are -integers and *x* is an arbitrary object that meets any type and value -restrictions imposed by *s*. - -The ``in`` and ``not in`` operations have the same priorities as the -comparison operations. The ``+`` (concatenation) and ``*`` (repetition) -operations have the same priority as the corresponding numeric operations. [3]_ - -.. index:: - triple: operations on; sequence; types - pair: built-in function; len - pair: built-in function; min - pair: built-in function; max - pair: concatenation; operation - pair: repetition; operation - pair: subscript; operation - pair: slice; operation - pair: operator; in - pair: operator; not in - single: count() (sequence method) - single: index() (sequence method) - -+--------------------------+--------------------------------+----------+ -| Operation | Result | Notes | -+==========================+================================+==========+ -| ``x in s`` | ``True`` if an item of *s* is | \(1) | -| | equal to *x*, else ``False`` | | -+--------------------------+--------------------------------+----------+ -| ``x not in s`` | ``False`` if an item of *s* is | \(1) | -| | equal to *x*, else ``True`` | | -+--------------------------+--------------------------------+----------+ -| ``s + t`` | the concatenation of *s* and | (6)(7) | -| | *t* | | -+--------------------------+--------------------------------+----------+ -| ``s * n`` or | equivalent to adding *s* to | (2)(7) | -| ``n * s`` | itself *n* times | | -+--------------------------+--------------------------------+----------+ -| ``s[i]`` | *i*\ th item of *s*, origin 0 | \(3) | -+--------------------------+--------------------------------+----------+ -| ``s[i:j]`` | slice of *s* from *i* to *j* | (3)(4) | -+--------------------------+--------------------------------+----------+ -| ``s[i:j:k]`` | slice of *s* from *i* to *j* | (3)(5) | -| | with step *k* | | -+--------------------------+--------------------------------+----------+ -| ``len(s)`` | length of *s* | | -+--------------------------+--------------------------------+----------+ -| ``min(s)`` | smallest item of *s* | | -+--------------------------+--------------------------------+----------+ -| ``max(s)`` | largest item of *s* | | -+--------------------------+--------------------------------+----------+ -| ``s.index(x[, i[, j]])`` | index of the first occurrence | \(8) | -| | of *x* in *s* (at or after | | -| | index *i* and before index *j*)| | -+--------------------------+--------------------------------+----------+ -| ``s.count(x)`` | total number of occurrences of | | -| | *x* in *s* | | -+--------------------------+--------------------------------+----------+ - -Sequences of the same type also support comparisons. In particular, tuples -and lists are compared lexicographically by comparing corresponding elements. -This means that to compare equal, every element must compare equal and the -two sequences must be of the same type and have the same length. (For full -details see :ref:`comparisons` in the language reference.) - -.. index:: - single: loop; over mutable sequence - single: mutable sequence; loop over - -Forward and reversed iterators over mutable sequences access values using an -index. That index will continue to march forward (or backward) even if the -underlying sequence is mutated. The iterator terminates only when an -:exc:`IndexError` or a :exc:`StopIteration` is encountered (or when the index -drops below zero). - -Notes: - -(1) - While the ``in`` and ``not in`` operations are used only for simple - containment testing in the general case, some specialised sequences - (such as :class:`str`, :class:`bytes` and :class:`bytearray`) also use - them for subsequence testing:: - - >>> "gg" in "eggs" - True - -(2) - Values of *n* less than ``0`` are treated as ``0`` (which yields an empty - sequence of the same type as *s*). Note that items in the sequence *s* - are not copied; they are referenced multiple times. This often haunts - new Python programmers; consider:: - - >>> lists = [[]] * 3 - >>> lists - [[], [], []] - >>> lists[0].append(3) - >>> lists - [[3], [3], [3]] - - What has happened is that ``[[]]`` is a one-element list containing an empty - list, so all three elements of ``[[]] * 3`` are references to this single empty - list. Modifying any of the elements of ``lists`` modifies this single list. - You can create a list of different lists this way:: - - >>> lists = [[] for i in range(3)] - >>> lists[0].append(3) - >>> lists[1].append(5) - >>> lists[2].append(7) - >>> lists - [[3], [5], [7]] - - Further explanation is available in the FAQ entry - :ref:`faq-multidimensional-list`. - -(3) - If *i* or *j* is negative, the index is relative to the end of sequence *s*: - ``len(s) + i`` or ``len(s) + j`` is substituted. But note that ``-0`` is - still ``0``. - -(4) - The slice of *s* from *i* to *j* is defined as the sequence of items with index - *k* such that ``i <= k < j``. If *i* or *j* is greater than ``len(s)``, use - ``len(s)``. If *i* is omitted or ``None``, use ``0``. If *j* is omitted or - ``None``, use ``len(s)``. If *i* is greater than or equal to *j*, the slice is - empty. - -(5) - The slice of *s* from *i* to *j* with step *k* is defined as the sequence of - items with index ``x = i + n*k`` such that ``0 <= n < (j-i)/k``. In other words, - the indices are ``i``, ``i+k``, ``i+2*k``, ``i+3*k`` and so on, stopping when - *j* is reached (but never including *j*). When *k* is positive, - *i* and *j* are reduced to ``len(s)`` if they are greater. - When *k* is negative, *i* and *j* are reduced to ``len(s) - 1`` if - they are greater. If *i* or *j* are omitted or ``None``, they become - "end" values (which end depends on the sign of *k*). Note, *k* cannot be zero. - If *k* is ``None``, it is treated like ``1``. - -(6) - Concatenating immutable sequences always results in a new object. This - means that building up a sequence by repeated concatenation will have a - quadratic runtime cost in the total sequence length. To get a linear - runtime cost, you must switch to one of the alternatives below: - - * if concatenating :class:`str` objects, you can build a list and use - :meth:`str.join` at the end or else write to an :class:`io.StringIO` - instance and retrieve its value when complete - - * if concatenating :class:`bytes` objects, you can similarly use - :meth:`bytes.join` or :class:`io.BytesIO`, or you can do in-place - concatenation with a :class:`bytearray` object. :class:`bytearray` - objects are mutable and have an efficient overallocation mechanism - - * if concatenating :class:`tuple` objects, extend a :class:`list` instead - - * for other types, investigate the relevant class documentation - - -(7) - Some sequence types (such as :class:`range`) only support item sequences - that follow specific patterns, and hence don't support sequence - concatenation or repetition. - -(8) - ``index`` raises :exc:`ValueError` when *x* is not found in *s*. - Not all implementations support passing the additional arguments *i* and *j*. - These arguments allow efficient searching of subsections of the sequence. Passing - the extra arguments is roughly equivalent to using ``s[i:j].index(x)``, only - without copying any data and with the returned index being relative to - the start of the sequence rather than the start of the slice. - - -.. _typesseq-immutable: - -Immutable Sequence Types ------------------------- - -.. index:: - triple: immutable; sequence; types - pair: object; tuple - pair: built-in function; hash - -The only operation that immutable sequence types generally implement that is -not also implemented by mutable sequence types is support for the :func:`hash` -built-in. - -This support allows immutable sequences, such as :class:`tuple` instances, to -be used as :class:`dict` keys and stored in :class:`set` and :class:`frozenset` -instances. - -Attempting to hash an immutable sequence that contains unhashable values will -result in :exc:`TypeError`. - - -.. _typesseq-mutable: - -Mutable Sequence Types ----------------------- - -.. index:: - triple: mutable; sequence; types - pair: object; list - pair: object; bytearray - -The operations in the following table are defined on mutable sequence types. -The :class:`collections.abc.MutableSequence` ABC is provided to make it -easier to correctly implement these operations on custom sequence types. - -In the table *s* is an instance of a mutable sequence type, *t* is any -iterable object and *x* is an arbitrary object that meets any type -and value restrictions imposed by *s* (for example, :class:`bytearray` only -accepts integers that meet the value restriction ``0 <= x <= 255``). - - -.. index:: - triple: operations on; sequence; types - triple: operations on; list; type - pair: subscript; assignment - pair: slice; assignment - pair: statement; del - single: append() (sequence method) - single: clear() (sequence method) - single: copy() (sequence method) - single: extend() (sequence method) - single: insert() (sequence method) - single: pop() (sequence method) - single: remove() (sequence method) - single: reverse() (sequence method) - -+------------------------------+--------------------------------+---------------------+ -| Operation | Result | Notes | -+==============================+================================+=====================+ -| ``s[i] = x`` | item *i* of *s* is replaced by | | -| | *x* | | -+------------------------------+--------------------------------+---------------------+ -| ``s[i:j] = t`` | slice of *s* from *i* to *j* | | -| | is replaced by the contents of | | -| | the iterable *t* | | -+------------------------------+--------------------------------+---------------------+ -| ``del s[i:j]`` | same as ``s[i:j] = []`` | | -+------------------------------+--------------------------------+---------------------+ -| ``s[i:j:k] = t`` | the elements of ``s[i:j:k]`` | \(1) | -| | are replaced by those of *t* | | -+------------------------------+--------------------------------+---------------------+ -| ``del s[i:j:k]`` | removes the elements of | | -| | ``s[i:j:k]`` from the list | | -+------------------------------+--------------------------------+---------------------+ -| ``s.append(x)`` | appends *x* to the end of the | | -| | sequence (same as | | -| | ``s[len(s):len(s)] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.clear()`` | removes all items from *s* | \(5) | -| | (same as ``del s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.copy()`` | creates a shallow copy of *s* | \(5) | -| | (same as ``s[:]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.extend(t)`` or | extends *s* with the | | -| ``s += t`` | contents of *t* (for the | | -| | most part the same as | | -| | ``s[len(s):len(s)] = t``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s *= n`` | updates *s* with its contents | \(6) | -| | repeated *n* times | | -+------------------------------+--------------------------------+---------------------+ -| ``s.insert(i, x)`` | inserts *x* into *s* at the | | -| | index given by *i* | | -| | (same as ``s[i:i] = [x]``) | | -+------------------------------+--------------------------------+---------------------+ -| ``s.pop()`` or ``s.pop(i)`` | retrieves the item at *i* and | \(2) | -| | also removes it from *s* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.remove(x)`` | remove the first item from *s* | \(3) | -| | where ``s[i]`` is equal to *x* | | -+------------------------------+--------------------------------+---------------------+ -| ``s.reverse()`` | reverses the items of *s* in | \(4) | -| | place | | -+------------------------------+--------------------------------+---------------------+ - - -Notes: - -(1) - *t* must have the same length as the slice it is replacing. - -(2) - The optional argument *i* defaults to ``-1``, so that by default the last - item is removed and returned. - -(3) - :meth:`remove` raises :exc:`ValueError` when *x* is not found in *s*. - -(4) - The :meth:`reverse` method modifies the sequence in place for economy of - space when reversing a large sequence. To remind users that it operates by - side effect, it does not return the reversed sequence. - -(5) - :meth:`clear` and :meth:`!copy` are included for consistency with the - interfaces of mutable containers that don't support slicing operations - (such as :class:`dict` and :class:`set`). :meth:`!copy` is not part of the - :class:`collections.abc.MutableSequence` ABC, but most concrete - mutable sequence classes provide it. - - .. versionadded:: 3.3 - :meth:`clear` and :meth:`!copy` methods. - -(6) - The value *n* is an integer, or an object implementing - :meth:`~object.__index__`. Zero and negative values of *n* clear - the sequence. Items in the sequence are not copied; they are referenced - multiple times, as explained for ``s * n`` under :ref:`typesseq-common`. - - -.. _typesseq-list: - -Lists ------ - -.. index:: pair: object; list - -Lists are mutable sequences, typically used to store collections of -homogeneous items (where the precise degree of similarity will vary by -application). - -.. class:: list([iterable]) - - Lists may be constructed in several ways: - - * Using a pair of square brackets to denote the empty list: ``[]`` - * Using square brackets, separating items with commas: ``[a]``, ``[a, b, c]`` - * Using a list comprehension: ``[x for x in iterable]`` - * Using the type constructor: ``list()`` or ``list(iterable)`` - - The constructor builds a list whose items are the same and in the same - order as *iterable*'s items. *iterable* may be either a sequence, a - container that supports iteration, or an iterator object. If *iterable* - is already a list, a copy is made and returned, similar to ``iterable[:]``. - For example, ``list('abc')`` returns ``['a', 'b', 'c']`` and - ``list( (1, 2, 3) )`` returns ``[1, 2, 3]``. - If no argument is given, the constructor creates a new empty list, ``[]``. - - - Many other operations also produce lists, including the :func:`sorted` - built-in. - - Lists implement all of the :ref:`common ` and - :ref:`mutable ` sequence operations. Lists also provide the - following additional method: - - .. method:: list.sort(*, key=None, reverse=False) - - This method sorts the list in place, using only ``<`` comparisons - between items. Exceptions are not suppressed - if any comparison operations - fail, the entire sort operation will fail (and the list will likely be left - in a partially modified state). - - :meth:`sort` accepts two arguments that can only be passed by keyword - (:ref:`keyword-only arguments `): - - *key* specifies a function of one argument that is used to extract a - comparison key from each list element (for example, ``key=str.lower``). - The key corresponding to each item in the list is calculated once and - then used for the entire sorting process. The default value of ``None`` - means that list items are sorted directly without calculating a separate - key value. - - The :func:`functools.cmp_to_key` utility is available to convert a 2.x - style *cmp* function to a *key* function. - - *reverse* is a boolean value. If set to ``True``, then the list elements - are sorted as if each comparison were reversed. - - This method modifies the sequence in place for economy of space when - sorting a large sequence. To remind users that it operates by side - effect, it does not return the sorted sequence (use :func:`sorted` to - explicitly request a new sorted list instance). - - The :meth:`sort` method is guaranteed to be stable. A sort is stable if it - guarantees not to change the relative order of elements that compare equal - --- this is helpful for sorting in multiple passes (for example, sort by - department, then by salary grade). - - For sorting examples and a brief sorting tutorial, see :ref:`sortinghowto`. - - .. impl-detail:: - - While a list is being sorted, the effect of attempting to mutate, or even - inspect, the list is undefined. The C implementation of Python makes the - list appear empty for the duration, and raises :exc:`ValueError` if it can - detect that the list has been mutated during a sort. - - -.. _typesseq-tuple: - -Tuples ------- - -.. index:: pair: object; tuple - -Tuples are immutable sequences, typically used to store collections of -heterogeneous data (such as the 2-tuples produced by the :func:`enumerate` -built-in). Tuples are also used for cases where an immutable sequence of -homogeneous data is needed (such as allowing storage in a :class:`set` or -:class:`dict` instance). - -.. class:: tuple([iterable]) - - Tuples may be constructed in a number of ways: - - * Using a pair of parentheses to denote the empty tuple: ``()`` - * Using a trailing comma for a singleton tuple: ``a,`` or ``(a,)`` - * Separating items with commas: ``a, b, c`` or ``(a, b, c)`` - * Using the :func:`tuple` built-in: ``tuple()`` or ``tuple(iterable)`` - - The constructor builds a tuple whose items are the same and in the same - order as *iterable*'s items. *iterable* may be either a sequence, a - container that supports iteration, or an iterator object. If *iterable* - is already a tuple, it is returned unchanged. For example, - ``tuple('abc')`` returns ``('a', 'b', 'c')`` and - ``tuple( [1, 2, 3] )`` returns ``(1, 2, 3)``. - If no argument is given, the constructor creates a new empty tuple, ``()``. - - Note that it is actually the comma which makes a tuple, not the parentheses. - The parentheses are optional, except in the empty tuple case, or - when they are needed to avoid syntactic ambiguity. For example, - ``f(a, b, c)`` is a function call with three arguments, while - ``f((a, b, c))`` is a function call with a 3-tuple as the sole argument. - - Tuples implement all of the :ref:`common ` sequence - operations. - -For heterogeneous collections of data where access by name is clearer than -access by index, :func:`collections.namedtuple` may be a more appropriate -choice than a simple tuple object. - - -.. _typesseq-range: - -Ranges ------- - -.. index:: pair: object; range - -The :class:`range` type represents an immutable sequence of numbers and is -commonly used for looping a specific number of times in :keyword:`for` -loops. - -.. class:: range(stop) - range(start, stop[, step]) - - The arguments to the range constructor must be integers (either built-in - :class:`int` or any object that implements the :meth:`~object.__index__` special - method). If the *step* argument is omitted, it defaults to ``1``. - If the *start* argument is omitted, it defaults to ``0``. - If *step* is zero, :exc:`ValueError` is raised. - - For a positive *step*, the contents of a range ``r`` are determined by the - formula ``r[i] = start + step*i`` where ``i >= 0`` and - ``r[i] < stop``. - - For a negative *step*, the contents of the range are still determined by - the formula ``r[i] = start + step*i``, but the constraints are ``i >= 0`` - and ``r[i] > stop``. - - A range object will be empty if ``r[0]`` does not meet the value - constraint. Ranges do support negative indices, but these are interpreted - as indexing from the end of the sequence determined by the positive - indices. - - Ranges containing absolute values larger than :data:`sys.maxsize` are - permitted but some features (such as :func:`len`) may raise - :exc:`OverflowError`. - - Range examples:: - - >>> list(range(10)) - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - >>> list(range(1, 11)) - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - >>> list(range(0, 30, 5)) - [0, 5, 10, 15, 20, 25] - >>> list(range(0, 10, 3)) - [0, 3, 6, 9] - >>> list(range(0, -10, -1)) - [0, -1, -2, -3, -4, -5, -6, -7, -8, -9] - >>> list(range(0)) - [] - >>> list(range(1, 0)) - [] - - Ranges implement all of the :ref:`common ` sequence operations - except concatenation and repetition (due to the fact that range objects can - only represent sequences that follow a strict pattern and repetition and - concatenation will usually violate that pattern). - - .. attribute:: start - - The value of the *start* parameter (or ``0`` if the parameter was - not supplied) - - .. attribute:: stop - - The value of the *stop* parameter - - .. attribute:: step - - The value of the *step* parameter (or ``1`` if the parameter was - not supplied) - -The advantage of the :class:`range` type over a regular :class:`list` or -:class:`tuple` is that a :class:`range` object will always take the same -(small) amount of memory, no matter the size of the range it represents (as it -only stores the ``start``, ``stop`` and ``step`` values, calculating individual -items and subranges as needed). - -Range objects implement the :class:`collections.abc.Sequence` ABC, and provide -features such as containment tests, element index lookup, slicing and -support for negative indices (see :ref:`typesseq`): - - >>> r = range(0, 20, 2) - >>> r - range(0, 20, 2) - >>> 11 in r - False - >>> 10 in r - True - >>> r.index(10) - 5 - >>> r[5] - 10 - >>> r[:5] - range(0, 10, 2) - >>> r[-1] - 18 - -Testing range objects for equality with ``==`` and ``!=`` compares -them as sequences. That is, two range objects are considered equal if -they represent the same sequence of values. (Note that two range -objects that compare equal might have different :attr:`~range.start`, -:attr:`~range.stop` and :attr:`~range.step` attributes, for example -``range(0) == range(2, 1, 3)`` or ``range(0, 3, 2) == range(0, 4, 2)``.) - -.. versionchanged:: 3.2 - Implement the Sequence ABC. - Support slicing and negative indices. - Test :class:`int` objects for membership in constant time instead of - iterating through all items. - -.. versionchanged:: 3.3 - Define '==' and '!=' to compare range objects based on the - sequence of values they define (instead of comparing based on - object identity). - -.. versionadded:: 3.3 - The :attr:`~range.start`, :attr:`~range.stop` and :attr:`~range.step` - attributes. - -.. seealso:: - - * The `linspace recipe `_ - shows how to implement a lazy version of range suitable for floating - point applications. - -.. index:: - single: string; text sequence type - single: str (built-in class); (see also string) - pair: object; string - -.. _textseq: - -Text Sequence Type --- :class:`str` -=================================== - -Textual data in Python is handled with :class:`str` objects, or :dfn:`strings`. -Strings are immutable -:ref:`sequences ` of Unicode code points. String literals are -written in a variety of ways: - -* Single quotes: ``'allows embedded "double" quotes'`` -* Double quotes: ``"allows embedded 'single' quotes"`` -* Triple quoted: ``'''Three single quotes'''``, ``"""Three double quotes"""`` - -Triple quoted strings may span multiple lines - all associated whitespace will -be included in the string literal. - -String literals that are part of a single expression and have only whitespace -between them will be implicitly converted to a single string literal. That -is, ``("spam " "eggs") == "spam eggs"``. - -See :ref:`strings` for more about the various forms of string literal, -including supported escape sequences, and the ``r`` ("raw") prefix that -disables most escape sequence processing. - -Strings may also be created from other objects using the :class:`str` -constructor. - -Since there is no separate "character" type, indexing a string produces -strings of length 1. That is, for a non-empty string *s*, ``s[0] == s[0:1]``. - -.. index:: - pair: object; io.StringIO - -There is also no mutable string type, but :meth:`str.join` or -:class:`io.StringIO` can be used to efficiently construct strings from -multiple fragments. - -.. versionchanged:: 3.3 - For backwards compatibility with the Python 2 series, the ``u`` prefix is - once again permitted on string literals. It has no effect on the meaning - of string literals and cannot be combined with the ``r`` prefix. - - -.. index:: - single: string; str (built-in class) - -.. class:: str(object='') - str(object=b'', encoding='utf-8', errors='strict') - - Return a :ref:`string ` version of *object*. If *object* is not - provided, returns the empty string. Otherwise, the behavior of ``str()`` - depends on whether *encoding* or *errors* is given, as follows. - - If neither *encoding* nor *errors* is given, ``str(object)`` returns - :meth:`type(object).__str__(object) `, - which is the "informal" or nicely - printable string representation of *object*. For string objects, this is - the string itself. If *object* does not have a :meth:`~object.__str__` - method, then :func:`str` falls back to returning - :meth:`repr(object) `. - - .. index:: - single: buffer protocol; str (built-in class) - single: bytes; str (built-in class) - - If at least one of *encoding* or *errors* is given, *object* should be a - :term:`bytes-like object` (e.g. :class:`bytes` or :class:`bytearray`). In - this case, if *object* is a :class:`bytes` (or :class:`bytearray`) object, - then ``str(bytes, encoding, errors)`` is equivalent to - :meth:`bytes.decode(encoding, errors) `. Otherwise, the bytes - object underlying the buffer object is obtained before calling - :meth:`bytes.decode`. See :ref:`binaryseq` and - :ref:`bufferobjects` for information on buffer objects. - - Passing a :class:`bytes` object to :func:`str` without the *encoding* - or *errors* arguments falls under the first case of returning the informal - string representation (see also the :option:`-b` command-line option to - Python). For example:: - - >>> str(b'Zoot!') - "b'Zoot!'" - - For more information on the ``str`` class and its methods, see - :ref:`textseq` and the :ref:`string-methods` section below. To output - formatted strings, see the :ref:`f-strings` and :ref:`formatstrings` - sections. In addition, see the :ref:`stringservices` section. - - -.. index:: - pair: string; methods - -.. _string-methods: - -String Methods --------------- - -.. index:: - pair: module; re - -Strings implement all of the :ref:`common ` sequence -operations, along with the additional methods described below. - -Strings also support two styles of string formatting, one providing a large -degree of flexibility and customization (see :meth:`str.format`, -:ref:`formatstrings` and :ref:`string-formatting`) and the other based on C -``printf`` style formatting that handles a narrower range of types and is -slightly harder to use correctly, but is often faster for the cases it can -handle (:ref:`old-string-formatting`). - -The :ref:`textservices` section of the standard library covers a number of -other modules that provide various text related utilities (including regular -expression support in the :mod:`re` module). - -.. method:: str.capitalize() - - Return a copy of the string with its first character capitalized and the - rest lowercased. - - .. versionchanged:: 3.8 - The first character is now put into titlecase rather than uppercase. - This means that characters like digraphs will only have their first - letter capitalized, instead of the full character. - -.. method:: str.casefold() - - Return a casefolded copy of the string. Casefolded strings may be used for - caseless matching. - - Casefolding is similar to lowercasing but more aggressive because it is - intended to remove all case distinctions in a string. For example, the German - lowercase letter ``'ß'`` is equivalent to ``"ss"``. Since it is already - lowercase, :meth:`lower` would do nothing to ``'ß'``; :meth:`casefold` - converts it to ``"ss"``. - - The casefolding algorithm is described in section 3.13 of the Unicode - Standard. - - .. versionadded:: 3.3 - - -.. method:: str.center(width[, fillchar]) - - Return centered in a string of length *width*. Padding is done using the - specified *fillchar* (default is an ASCII space). The original string is - returned if *width* is less than or equal to ``len(s)``. - - - -.. method:: str.count(sub[, start[, end]]) - - Return the number of non-overlapping occurrences of substring *sub* in the - range [*start*, *end*]. Optional arguments *start* and *end* are - interpreted as in slice notation. - - If *sub* is empty, returns the number of empty strings between characters - which is the length of the string plus one. - - -.. method:: str.encode(encoding="utf-8", errors="strict") - - Return the string encoded to :class:`bytes`. - - *encoding* defaults to ``'utf-8'``; - see :ref:`standard-encodings` for possible values. - - *errors* controls how encoding errors are handled. - If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. - Other possible values are ``'ignore'``, - ``'replace'``, ``'xmlcharrefreplace'``, ``'backslashreplace'`` and any - other name registered via :func:`codecs.register_error`. - See :ref:`error-handlers` for details. - - For performance reasons, the value of *errors* is not checked for validity - unless an encoding error actually occurs, - :ref:`devmode` is enabled - or a :ref:`debug build ` is used. - - .. versionchanged:: 3.1 - Added support for keyword arguments. - - .. versionchanged:: 3.9 - The value of the *errors* argument is now checked in :ref:`devmode` and - in :ref:`debug mode `. - - -.. method:: str.endswith(suffix[, start[, end]]) - - Return ``True`` if the string ends with the specified *suffix*, otherwise return - ``False``. *suffix* can also be a tuple of suffixes to look for. With optional - *start*, test beginning at that position. With optional *end*, stop comparing - at that position. - - -.. method:: str.expandtabs(tabsize=8) - - Return a copy of the string where all tab characters are replaced by one or - more spaces, depending on the current column and the given tab size. Tab - positions occur every *tabsize* characters (default is 8, giving tab - positions at columns 0, 8, 16 and so on). To expand the string, the current - column is set to zero and the string is examined character by character. If - the character is a tab (``\t``), one or more space characters are inserted - in the result until the current column is equal to the next tab position. - (The tab character itself is not copied.) If the character is a newline - (``\n``) or return (``\r``), it is copied and the current column is reset to - zero. Any other character is copied unchanged and the current column is - incremented by one regardless of how the character is represented when - printed. - - >>> '01\t012\t0123\t01234'.expandtabs() - '01 012 0123 01234' - >>> '01\t012\t0123\t01234'.expandtabs(4) - '01 012 0123 01234' - - -.. method:: str.find(sub[, start[, end]]) - - Return the lowest index in the string where substring *sub* is found within - the slice ``s[start:end]``. Optional arguments *start* and *end* are - interpreted as in slice notation. Return ``-1`` if *sub* is not found. - - .. note:: - - The :meth:`~str.find` method should be used only if you need to know the - position of *sub*. To check if *sub* is a substring or not, use the - :keyword:`in` operator:: - - >>> 'Py' in 'Python' - True - - -.. method:: str.format(*args, **kwargs) - - Perform a string formatting operation. The string on which this method is - called can contain literal text or replacement fields delimited by braces - ``{}``. Each replacement field contains either the numeric index of a - positional argument, or the name of a keyword argument. Returns a copy of - the string where each replacement field is replaced with the string value of - the corresponding argument. - - >>> "The sum of 1 + 2 is {0}".format(1+2) - 'The sum of 1 + 2 is 3' - - See :ref:`formatstrings` for a description of the various formatting options - that can be specified in format strings. - - .. note:: - When formatting a number (:class:`int`, :class:`float`, :class:`complex`, - :class:`decimal.Decimal` and subclasses) with the ``n`` type - (ex: ``'{:n}'.format(1234)``), the function temporarily sets the - ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale to decode - ``decimal_point`` and ``thousands_sep`` fields of :c:func:`localeconv` if - they are non-ASCII or longer than 1 byte, and the ``LC_NUMERIC`` locale is - different than the ``LC_CTYPE`` locale. This temporary change affects - other threads. - - .. versionchanged:: 3.7 - When formatting a number with the ``n`` type, the function sets - temporarily the ``LC_CTYPE`` locale to the ``LC_NUMERIC`` locale in some - cases. - - -.. method:: str.format_map(mapping) - - Similar to ``str.format(**mapping)``, except that ``mapping`` is - used directly and not copied to a :class:`dict`. This is useful - if for example ``mapping`` is a dict subclass: - - >>> class Default(dict): - ... def __missing__(self, key): - ... return key - ... - >>> '{name} was born in {country}'.format_map(Default(name='Guido')) - 'Guido was born in country' - - .. versionadded:: 3.2 - - -.. method:: str.index(sub[, start[, end]]) - - Like :meth:`~str.find`, but raise :exc:`ValueError` when the substring is - not found. - - -.. method:: str.isalnum() - - Return ``True`` if all characters in the string are alphanumeric and there is at - least one character, ``False`` otherwise. A character ``c`` is alphanumeric if one - of the following returns ``True``: ``c.isalpha()``, ``c.isdecimal()``, - ``c.isdigit()``, or ``c.isnumeric()``. - - -.. method:: str.isalpha() - - Return ``True`` if all characters in the string are alphabetic and there is at least - one character, ``False`` otherwise. Alphabetic characters are those characters defined - in the Unicode character database as "Letter", i.e., those with general category - property being one of "Lm", "Lt", "Lu", "Ll", or "Lo". Note that this is different - from the "Alphabetic" property defined in the Unicode Standard. - - -.. method:: str.isascii() - - Return ``True`` if the string is empty or all characters in the string are ASCII, - ``False`` otherwise. - ASCII characters have code points in the range U+0000-U+007F. - - .. versionadded:: 3.7 - - -.. method:: str.isdecimal() - - Return ``True`` if all characters in the string are decimal - characters and there is at least one character, ``False`` - otherwise. Decimal characters are those that can be used to form - numbers in base 10, e.g. U+0660, ARABIC-INDIC DIGIT - ZERO. Formally a decimal character is a character in the Unicode - General Category "Nd". - - -.. method:: str.isdigit() - - Return ``True`` if all characters in the string are digits and there is at least one - character, ``False`` otherwise. Digits include decimal characters and digits that need - special handling, such as the compatibility superscript digits. - This covers digits which cannot be used to form numbers in base 10, - like the Kharosthi numbers. Formally, a digit is a character that has the - property value Numeric_Type=Digit or Numeric_Type=Decimal. - - -.. method:: str.isidentifier() - - Return ``True`` if the string is a valid identifier according to the language - definition, section :ref:`identifiers`. - - Call :func:`keyword.iskeyword` to test whether string ``s`` is a reserved - identifier, such as :keyword:`def` and :keyword:`class`. - - Example: - :: - - >>> from keyword import iskeyword - - >>> 'hello'.isidentifier(), iskeyword('hello') - (True, False) - >>> 'def'.isidentifier(), iskeyword('def') - (True, True) - - -.. method:: str.islower() - - Return ``True`` if all cased characters [4]_ in the string are lowercase and - there is at least one cased character, ``False`` otherwise. - - -.. method:: str.isnumeric() - - Return ``True`` if all characters in the string are numeric - characters, and there is at least one character, ``False`` - otherwise. Numeric characters include digit characters, and all characters - that have the Unicode numeric value property, e.g. U+2155, - VULGAR FRACTION ONE FIFTH. Formally, numeric characters are those with the property - value Numeric_Type=Digit, Numeric_Type=Decimal or Numeric_Type=Numeric. - - -.. method:: str.isprintable() - - Return ``True`` if all characters in the string are printable or the string is - empty, ``False`` otherwise. Nonprintable characters are those characters defined - in the Unicode character database as "Other" or "Separator", excepting the - ASCII space (0x20) which is considered printable. (Note that printable - characters in this context are those which should not be escaped when - :func:`repr` is invoked on a string. It has no bearing on the handling of - strings written to :data:`sys.stdout` or :data:`sys.stderr`.) - - -.. method:: str.isspace() - - Return ``True`` if there are only whitespace characters in the string and there is - at least one character, ``False`` otherwise. - - A character is *whitespace* if in the Unicode character database - (see :mod:`unicodedata`), either its general category is ``Zs`` - ("Separator, space"), or its bidirectional class is one of ``WS``, - ``B``, or ``S``. - - -.. method:: str.istitle() - - Return ``True`` if the string is a titlecased string and there is at least one - character, for example uppercase characters may only follow uncased characters - and lowercase characters only cased ones. Return ``False`` otherwise. - - -.. method:: str.isupper() - - Return ``True`` if all cased characters [4]_ in the string are uppercase and - there is at least one cased character, ``False`` otherwise. - - >>> 'BANANA'.isupper() - True - >>> 'banana'.isupper() - False - >>> 'baNana'.isupper() - False - >>> ' '.isupper() - False - - - -.. _meth-str-join: - -.. method:: str.join(iterable) - - Return a string which is the concatenation of the strings in *iterable*. - A :exc:`TypeError` will be raised if there are any non-string values in - *iterable*, including :class:`bytes` objects. The separator between - elements is the string providing this method. - - -.. method:: str.ljust(width[, fillchar]) - - Return the string left justified in a string of length *width*. Padding is - done using the specified *fillchar* (default is an ASCII space). The - original string is returned if *width* is less than or equal to ``len(s)``. - - -.. method:: str.lower() - - Return a copy of the string with all the cased characters [4]_ converted to - lowercase. - - The lowercasing algorithm used is described in section 3.13 of the Unicode - Standard. - - -.. method:: str.lstrip([chars]) - - Return a copy of the string with leading characters removed. The *chars* - argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a prefix; rather, all combinations of its values are stripped:: - - >>> ' spacious '.lstrip() - 'spacious ' - >>> 'www.example.com'.lstrip('cmowz.') - 'example.com' - - See :meth:`str.removeprefix` for a method that will remove a single prefix - string rather than all of a set of characters. For example:: - - >>> 'Arthur: three!'.lstrip('Arthur: ') - 'ee!' - >>> 'Arthur: three!'.removeprefix('Arthur: ') - 'three!' - - -.. staticmethod:: str.maketrans(x[, y[, z]]) - - This static method returns a translation table usable for :meth:`str.translate`. - - If there is only one argument, it must be a dictionary mapping Unicode - ordinals (integers) or characters (strings of length 1) to Unicode ordinals, - strings (of arbitrary lengths) or ``None``. Character keys will then be - converted to ordinals. - - If there are two arguments, they must be strings of equal length, and in the - resulting dictionary, each character in x will be mapped to the character at - the same position in y. If there is a third argument, it must be a string, - whose characters will be mapped to ``None`` in the result. - - -.. method:: str.partition(sep) - - Split the string at the first occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator itself, and the part - after the separator. If the separator is not found, return a 3-tuple containing - the string itself, followed by two empty strings. - - -.. method:: str.removeprefix(prefix, /) - - If the string starts with the *prefix* string, return - ``string[len(prefix):]``. Otherwise, return a copy of the original - string:: - - >>> 'TestHook'.removeprefix('Test') - 'Hook' - >>> 'BaseTestCase'.removeprefix('Test') - 'BaseTestCase' - - .. versionadded:: 3.9 - - -.. method:: str.removesuffix(suffix, /) - - If the string ends with the *suffix* string and that *suffix* is not empty, - return ``string[:-len(suffix)]``. Otherwise, return a copy of the - original string:: - - >>> 'MiscTests'.removesuffix('Tests') - 'Misc' - >>> 'TmpDirMixin'.removesuffix('Tests') - 'TmpDirMixin' - - .. versionadded:: 3.9 - - -.. method:: str.replace(old, new[, count]) - - Return a copy of the string with all occurrences of substring *old* replaced by - *new*. If the optional argument *count* is given, only the first *count* - occurrences are replaced. - - -.. method:: str.rfind(sub[, start[, end]]) - - Return the highest index in the string where substring *sub* is found, such - that *sub* is contained within ``s[start:end]``. Optional arguments *start* - and *end* are interpreted as in slice notation. Return ``-1`` on failure. - - -.. method:: str.rindex(sub[, start[, end]]) - - Like :meth:`rfind` but raises :exc:`ValueError` when the substring *sub* is not - found. - - -.. method:: str.rjust(width[, fillchar]) - - Return the string right justified in a string of length *width*. Padding is - done using the specified *fillchar* (default is an ASCII space). The - original string is returned if *width* is less than or equal to ``len(s)``. - - -.. method:: str.rpartition(sep) - - Split the string at the last occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator itself, and the part - after the separator. If the separator is not found, return a 3-tuple containing - two empty strings, followed by the string itself. - - -.. method:: str.rsplit(sep=None, maxsplit=-1) - - Return a list of the words in the string, using *sep* as the delimiter string. - If *maxsplit* is given, at most *maxsplit* splits are done, the *rightmost* - ones. If *sep* is not specified or ``None``, any whitespace string is a - separator. Except for splitting from the right, :meth:`rsplit` behaves like - :meth:`split` which is described in detail below. - - -.. method:: str.rstrip([chars]) - - Return a copy of the string with trailing characters removed. The *chars* - argument is a string specifying the set of characters to be removed. If omitted - or ``None``, the *chars* argument defaults to removing whitespace. The *chars* - argument is not a suffix; rather, all combinations of its values are stripped:: - - >>> ' spacious '.rstrip() - ' spacious' - >>> 'mississippi'.rstrip('ipz') - 'mississ' - - See :meth:`str.removesuffix` for a method that will remove a single suffix - string rather than all of a set of characters. For example:: - - >>> 'Monty Python'.rstrip(' Python') - 'M' - >>> 'Monty Python'.removesuffix(' Python') - 'Monty' - -.. method:: str.split(sep=None, maxsplit=-1) - - Return a list of the words in the string, using *sep* as the delimiter - string. If *maxsplit* is given, at most *maxsplit* splits are done (thus, - the list will have at most ``maxsplit+1`` elements). If *maxsplit* is not - specified or ``-1``, then there is no limit on the number of splits - (all possible splits are made). - - If *sep* is given, consecutive delimiters are not grouped together and are - deemed to delimit empty strings (for example, ``'1,,2'.split(',')`` returns - ``['1', '', '2']``). The *sep* argument may consist of multiple characters - (for example, ``'1<>2<>3'.split('<>')`` returns ``['1', '2', '3']``). - Splitting an empty string with a specified separator returns ``['']``. - - For example:: - - >>> '1,2,3'.split(',') - ['1', '2', '3'] - >>> '1,2,3'.split(',', maxsplit=1) - ['1', '2,3'] - >>> '1,2,,3,'.split(',') - ['1', '2', '', '3', ''] - - If *sep* is not specified or is ``None``, a different splitting algorithm is - applied: runs of consecutive whitespace are regarded as a single separator, - and the result will contain no empty strings at the start or end if the - string has leading or trailing whitespace. Consequently, splitting an empty - string or a string consisting of just whitespace with a ``None`` separator - returns ``[]``. - - For example:: - - >>> '1 2 3'.split() - ['1', '2', '3'] - >>> '1 2 3'.split(maxsplit=1) - ['1', '2 3'] - >>> ' 1 2 3 '.split() - ['1', '2', '3'] - - -.. index:: - single: universal newlines; str.splitlines method - -.. method:: str.splitlines(keepends=False) - - Return a list of the lines in the string, breaking at line boundaries. Line - breaks are not included in the resulting list unless *keepends* is given and - true. - - This method splits on the following line boundaries. In particular, the - boundaries are a superset of :term:`universal newlines`. - - +-----------------------+-----------------------------+ - | Representation | Description | - +=======================+=============================+ - | ``\n`` | Line Feed | - +-----------------------+-----------------------------+ - | ``\r`` | Carriage Return | - +-----------------------+-----------------------------+ - | ``\r\n`` | Carriage Return + Line Feed | - +-----------------------+-----------------------------+ - | ``\v`` or ``\x0b`` | Line Tabulation | - +-----------------------+-----------------------------+ - | ``\f`` or ``\x0c`` | Form Feed | - +-----------------------+-----------------------------+ - | ``\x1c`` | File Separator | - +-----------------------+-----------------------------+ - | ``\x1d`` | Group Separator | - +-----------------------+-----------------------------+ - | ``\x1e`` | Record Separator | - +-----------------------+-----------------------------+ - | ``\x85`` | Next Line (C1 Control Code) | - +-----------------------+-----------------------------+ - | ``\u2028`` | Line Separator | - +-----------------------+-----------------------------+ - | ``\u2029`` | Paragraph Separator | - +-----------------------+-----------------------------+ - - .. versionchanged:: 3.2 - - ``\v`` and ``\f`` added to list of line boundaries. - - For example:: - - >>> 'ab c\n\nde fg\rkl\r\n'.splitlines() - ['ab c', '', 'de fg', 'kl'] - >>> 'ab c\n\nde fg\rkl\r\n'.splitlines(keepends=True) - ['ab c\n', '\n', 'de fg\r', 'kl\r\n'] - - Unlike :meth:`~str.split` when a delimiter string *sep* is given, this - method returns an empty list for the empty string, and a terminal line - break does not result in an extra line:: - - >>> "".splitlines() - [] - >>> "One line\n".splitlines() - ['One line'] - - For comparison, ``split('\n')`` gives:: - - >>> ''.split('\n') - [''] - >>> 'Two lines\n'.split('\n') - ['Two lines', ''] - - -.. method:: str.startswith(prefix[, start[, end]]) - - Return ``True`` if string starts with the *prefix*, otherwise return ``False``. - *prefix* can also be a tuple of prefixes to look for. With optional *start*, - test string beginning at that position. With optional *end*, stop comparing - string at that position. - - -.. method:: str.strip([chars]) - - Return a copy of the string with the leading and trailing characters removed. - The *chars* argument is a string specifying the set of characters to be removed. - If omitted or ``None``, the *chars* argument defaults to removing whitespace. - The *chars* argument is not a prefix or suffix; rather, all combinations of its - values are stripped:: - - >>> ' spacious '.strip() - 'spacious' - >>> 'www.example.com'.strip('cmowz.') - 'example' - - The outermost leading and trailing *chars* argument values are stripped - from the string. Characters are removed from the leading end until - reaching a string character that is not contained in the set of - characters in *chars*. A similar action takes place on the trailing end. - For example:: - - >>> comment_string = '#....... Section 3.2.1 Issue #32 .......' - >>> comment_string.strip('.#! ') - 'Section 3.2.1 Issue #32' - - -.. method:: str.swapcase() - - Return a copy of the string with uppercase characters converted to lowercase and - vice versa. Note that it is not necessarily true that - ``s.swapcase().swapcase() == s``. - - -.. method:: str.title() - - Return a titlecased version of the string where words start with an uppercase - character and the remaining characters are lowercase. - - For example:: - - >>> 'Hello world'.title() - 'Hello World' - - The algorithm uses a simple language-independent definition of a word as - groups of consecutive letters. The definition works in many contexts but - it means that apostrophes in contractions and possessives form word - boundaries, which may not be the desired result:: - - >>> "they're bill's friends from the UK".title() - "They'Re Bill'S Friends From The Uk" - - The :func:`string.capwords` function does not have this problem, as it - splits words on spaces only. - - Alternatively, a workaround for apostrophes can be constructed using regular - expressions:: - - >>> import re - >>> def titlecase(s): - ... return re.sub(r"[A-Za-z]+('[A-Za-z]+)?", - ... lambda mo: mo.group(0).capitalize(), - ... s) - ... - >>> titlecase("they're bill's friends.") - "They're Bill's Friends." - - -.. method:: str.translate(table) - - Return a copy of the string in which each character has been mapped through - the given translation table. The table must be an object that implements - indexing via :meth:`~object.__getitem__`, typically a :term:`mapping` or - :term:`sequence`. When indexed by a Unicode ordinal (an integer), the - table object can do any of the following: return a Unicode ordinal or a - string, to map the character to one or more other characters; return - ``None``, to delete the character from the return string; or raise a - :exc:`LookupError` exception, to map the character to itself. - - You can use :meth:`str.maketrans` to create a translation map from - character-to-character mappings in different formats. - - See also the :mod:`codecs` module for a more flexible approach to custom - character mappings. - - -.. method:: str.upper() - - Return a copy of the string with all the cased characters [4]_ converted to - uppercase. Note that ``s.upper().isupper()`` might be ``False`` if ``s`` - contains uncased characters or if the Unicode category of the resulting - character(s) is not "Lu" (Letter, uppercase), but e.g. "Lt" (Letter, - titlecase). - - The uppercasing algorithm used is described in section 3.13 of the Unicode - Standard. - - -.. method:: str.zfill(width) - - Return a copy of the string left filled with ASCII ``'0'`` digits to - make a string of length *width*. A leading sign prefix (``'+'``/``'-'``) - is handled by inserting the padding *after* the sign character rather - than before. The original string is returned if *width* is less than - or equal to ``len(s)``. - - For example:: - - >>> "42".zfill(5) - '00042' - >>> "-42".zfill(5) - '-0042' - - - -.. _old-string-formatting: - -``printf``-style String Formatting ----------------------------------- - -.. index:: - single: formatting, string (%) - single: interpolation, string (%) - single: string; formatting, printf - single: string; interpolation, printf - single: printf-style formatting - single: sprintf-style formatting - single: % (percent); printf-style formatting - -.. note:: - - The formatting operations described here exhibit a variety of quirks that - lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). Using the newer :ref:`formatted string literals - `, the :meth:`str.format` interface, or :ref:`template strings - ` may help avoid these errors. Each of these - alternatives provides their own trade-offs and benefits of simplicity, - flexibility, and/or extensibility. - -String objects have one unique built-in operation: the ``%`` operator (modulo). -This is also known as the string *formatting* or *interpolation* operator. -Given ``format % values`` (where *format* is a string), ``%`` conversion -specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. - -If *format* requires a single argument, *values* may be a single non-tuple -object. [5]_ Otherwise, *values* must be a tuple with exactly the number of -items specified by the format string, or a single mapping object (for example, a -dictionary). - -.. index:: - single: () (parentheses); in printf-style formatting - single: * (asterisk); in printf-style formatting - single: . (dot); in printf-style formatting - -A conversion specifier contains two or more characters and has the following -components, which must occur in this order: - -#. The ``'%'`` character, which marks the start of the specifier. - -#. Mapping key (optional), consisting of a parenthesised sequence of characters - (for example, ``(somename)``). - -#. Conversion flags (optional), which affect the result of some conversion - types. - -#. Minimum field width (optional). If specified as an ``'*'`` (asterisk), the - actual width is read from the next element of the tuple in *values*, and the - object to convert comes after the minimum field width and optional precision. - -#. Precision (optional), given as a ``'.'`` (dot) followed by the precision. If - specified as ``'*'`` (an asterisk), the actual precision is read from the next - element of the tuple in *values*, and the value to convert comes after the - precision. - -#. Length modifier (optional). - -#. Conversion type. - -When the right argument is a dictionary (or other mapping type), then the -formats in the string *must* include a parenthesised mapping key into that -dictionary inserted immediately after the ``'%'`` character. The mapping key -selects the value to be formatted from the mapping. For example: - - >>> print('%(language)s has %(number)03d quote types.' % - ... {'language': "Python", "number": 2}) - Python has 002 quote types. - -In this case no ``*`` specifiers may occur in a format (since they require a -sequential parameter list). - -The conversion flag characters are: - -.. index:: - single: # (hash); in printf-style formatting - single: - (minus); in printf-style formatting - single: + (plus); in printf-style formatting - single: space; in printf-style formatting - -+---------+---------------------------------------------------------------------+ -| Flag | Meaning | -+=========+=====================================================================+ -| ``'#'`` | The value conversion will use the "alternate form" (where defined | -| | below). | -+---------+---------------------------------------------------------------------+ -| ``'0'`` | The conversion will be zero padded for numeric values. | -+---------+---------------------------------------------------------------------+ -| ``'-'`` | The converted value is left adjusted (overrides the ``'0'`` | -| | conversion if both are given). | -+---------+---------------------------------------------------------------------+ -| ``' '`` | (a space) A blank should be left before a positive number (or empty | -| | string) produced by a signed conversion. | -+---------+---------------------------------------------------------------------+ -| ``'+'`` | A sign character (``'+'`` or ``'-'``) will precede the conversion | -| | (overrides a "space" flag). | -+---------+---------------------------------------------------------------------+ - -A length modifier (``h``, ``l``, or ``L``) may be present, but is ignored as it -is not necessary for Python -- so e.g. ``%ld`` is identical to ``%d``. - -The conversion types are: - -+------------+-----------------------------------------------------+-------+ -| Conversion | Meaning | Notes | -+============+=====================================================+=======+ -| ``'d'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'i'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'o'`` | Signed octal value. | \(1) | -+------------+-----------------------------------------------------+-------+ -| ``'u'`` | Obsolete type -- it is identical to ``'d'``. | \(6) | -+------------+-----------------------------------------------------+-------+ -| ``'x'`` | Signed hexadecimal (lowercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'X'`` | Signed hexadecimal (uppercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'c'`` | Single character (accepts integer or single | | -| | character string). | | -+------------+-----------------------------------------------------+-------+ -| ``'r'`` | String (converts any Python object using | \(5) | -| | :func:`repr`). | | -+------------+-----------------------------------------------------+-------+ -| ``'s'`` | String (converts any Python object using | \(5) | -| | :func:`str`). | | -+------------+-----------------------------------------------------+-------+ -| ``'a'`` | String (converts any Python object using | \(5) | -| | :func:`ascii`). | | -+------------+-----------------------------------------------------+-------+ -| ``'%'`` | No argument is converted, results in a ``'%'`` | | -| | character in the result. | | -+------------+-----------------------------------------------------+-------+ - -Notes: - -(1) - The alternate form causes a leading octal specifier (``'0o'``) to be - inserted before the first digit. - -(2) - The alternate form causes a leading ``'0x'`` or ``'0X'`` (depending on whether - the ``'x'`` or ``'X'`` format was used) to be inserted before the first digit. - -(3) - The alternate form causes the result to always contain a decimal point, even if - no digits follow it. - - The precision determines the number of digits after the decimal point and - defaults to 6. - -(4) - The alternate form causes the result to always contain a decimal point, and - trailing zeroes are not removed as they would otherwise be. - - The precision determines the number of significant digits before and after the - decimal point and defaults to 6. - -(5) - If precision is ``N``, the output is truncated to ``N`` characters. - -(6) - See :pep:`237`. - -Since Python strings have an explicit length, ``%s`` conversions do not assume -that ``'\0'`` is the end of the string. - -.. XXX Examples? - -.. versionchanged:: 3.1 - ``%f`` conversions for numbers whose absolute value is over 1e50 are no - longer replaced by ``%g`` conversions. - - -.. index:: - single: buffer protocol; binary sequence types - -.. _binaryseq: - -Binary Sequence Types --- :class:`bytes`, :class:`bytearray`, :class:`memoryview` -================================================================================= - -.. index:: - pair: object; bytes - pair: object; bytearray - pair: object; memoryview - pair: module; array - -The core built-in types for manipulating binary data are :class:`bytes` and -:class:`bytearray`. They are supported by :class:`memoryview` which uses -the :ref:`buffer protocol ` to access the memory of other -binary objects without needing to make a copy. - -The :mod:`array` module supports efficient storage of basic data types like -32-bit integers and IEEE754 double-precision floating values. - -.. _typebytes: - -Bytes Objects -------------- - -.. index:: pair: object; bytes - -Bytes objects are immutable sequences of single bytes. Since many major -binary protocols are based on the ASCII text encoding, bytes objects offer -several methods that are only valid when working with ASCII compatible -data and are closely related to string objects in a variety of other ways. - -.. class:: bytes([source[, encoding[, errors]]]) - - Firstly, the syntax for bytes literals is largely the same as that for string - literals, except that a ``b`` prefix is added: - - * Single quotes: ``b'still allows embedded "double" quotes'`` - * Double quotes: ``b"still allows embedded 'single' quotes"`` - * Triple quoted: ``b'''3 single quotes'''``, ``b"""3 double quotes"""`` - - Only ASCII characters are permitted in bytes literals (regardless of the - declared source code encoding). Any binary values over 127 must be entered - into bytes literals using the appropriate escape sequence. - - As with string literals, bytes literals may also use a ``r`` prefix to disable - processing of escape sequences. See :ref:`strings` for more about the various - forms of bytes literal, including supported escape sequences. - - While bytes literals and representations are based on ASCII text, bytes - objects actually behave like immutable sequences of integers, with each - value in the sequence restricted such that ``0 <= x < 256`` (attempts to - violate this restriction will trigger :exc:`ValueError`). This is done - deliberately to emphasise that while many binary formats include ASCII based - elements and can be usefully manipulated with some text-oriented algorithms, - this is not generally the case for arbitrary binary data (blindly applying - text processing algorithms to binary data formats that are not ASCII - compatible will usually lead to data corruption). - - In addition to the literal forms, bytes objects can be created in a number of - other ways: - - * A zero-filled bytes object of a specified length: ``bytes(10)`` - * From an iterable of integers: ``bytes(range(20))`` - * Copying existing binary data via the buffer protocol: ``bytes(obj)`` - - Also see the :ref:`bytes ` built-in. - - Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal - numbers are a commonly used format for describing binary data. Accordingly, - the bytes type has an additional class method to read data in that format: - - .. classmethod:: fromhex(string) - - This :class:`bytes` class method returns a bytes object, decoding the - given string object. The string must contain two hexadecimal digits per - byte, with ASCII whitespace being ignored. - - >>> bytes.fromhex('2Ef0 F1f2 ') - b'.\xf0\xf1\xf2' - - .. versionchanged:: 3.7 - :meth:`bytes.fromhex` now skips all ASCII whitespace in the string, - not just spaces. - - A reverse conversion function exists to transform a bytes object into its - hexadecimal representation. - - .. method:: hex([sep[, bytes_per_sep]]) - - Return a string object containing two hexadecimal digits for each - byte in the instance. - - >>> b'\xf0\xf1\xf2'.hex() - 'f0f1f2' - - If you want to make the hex string easier to read, you can specify a - single character separator *sep* parameter to include in the output. - By default, this separator will be included between each byte. - A second optional *bytes_per_sep* parameter controls the spacing. - Positive values calculate the separator position from the right, - negative values from the left. - - >>> value = b'\xf0\xf1\xf2' - >>> value.hex('-') - 'f0-f1-f2' - >>> value.hex('_', 2) - 'f0_f1f2' - >>> b'UUDDLRLRAB'.hex(' ', -4) - '55554444 4c524c52 4142' - - .. versionadded:: 3.5 - - .. versionchanged:: 3.8 - :meth:`bytes.hex` now supports optional *sep* and *bytes_per_sep* - parameters to insert separators between bytes in the hex output. - -Since bytes objects are sequences of integers (akin to a tuple), for a bytes -object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytes -object of length 1. (This contrasts with text strings, where both indexing -and slicing will produce a string of length 1) - -The representation of bytes objects uses the literal format (``b'...'``) -since it is often more useful than e.g. ``bytes([46, 46, 46])``. You can -always convert a bytes object into a list of integers using ``list(b)``. - - -.. _typebytearray: - -Bytearray Objects ------------------ - -.. index:: pair: object; bytearray - -:class:`bytearray` objects are a mutable counterpart to :class:`bytes` -objects. - -.. class:: bytearray([source[, encoding[, errors]]]) - - There is no dedicated literal syntax for bytearray objects, instead - they are always created by calling the constructor: - - * Creating an empty instance: ``bytearray()`` - * Creating a zero-filled instance with a given length: ``bytearray(10)`` - * From an iterable of integers: ``bytearray(range(20))`` - * Copying existing binary data via the buffer protocol: ``bytearray(b'Hi!')`` - - As bytearray objects are mutable, they support the - :ref:`mutable ` sequence operations in addition to the - common bytes and bytearray operations described in :ref:`bytes-methods`. - - Also see the :ref:`bytearray ` built-in. - - Since 2 hexadecimal digits correspond precisely to a single byte, hexadecimal - numbers are a commonly used format for describing binary data. Accordingly, - the bytearray type has an additional class method to read data in that format: - - .. classmethod:: fromhex(string) - - This :class:`bytearray` class method returns bytearray object, decoding - the given string object. The string must contain two hexadecimal digits - per byte, with ASCII whitespace being ignored. - - >>> bytearray.fromhex('2Ef0 F1f2 ') - bytearray(b'.\xf0\xf1\xf2') - - .. versionchanged:: 3.7 - :meth:`bytearray.fromhex` now skips all ASCII whitespace in the string, - not just spaces. - - A reverse conversion function exists to transform a bytearray object into its - hexadecimal representation. - - .. method:: hex([sep[, bytes_per_sep]]) - - Return a string object containing two hexadecimal digits for each - byte in the instance. - - >>> bytearray(b'\xf0\xf1\xf2').hex() - 'f0f1f2' - - .. versionadded:: 3.5 - - .. versionchanged:: 3.8 - Similar to :meth:`bytes.hex`, :meth:`bytearray.hex` now supports - optional *sep* and *bytes_per_sep* parameters to insert separators - between bytes in the hex output. - -Since bytearray objects are sequences of integers (akin to a list), for a -bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be -a bytearray object of length 1. (This contrasts with text strings, where -both indexing and slicing will produce a string of length 1) - -The representation of bytearray objects uses the bytes literal format -(``bytearray(b'...')``) since it is often more useful than e.g. -``bytearray([46, 46, 46])``. You can always convert a bytearray object into -a list of integers using ``list(b)``. - - -.. _bytes-methods: - -Bytes and Bytearray Operations ------------------------------- - -.. index:: pair: bytes; methods - pair: bytearray; methods - -Both bytes and bytearray objects support the :ref:`common ` -sequence operations. They interoperate not just with operands of the same -type, but with any :term:`bytes-like object`. Due to this flexibility, they can be -freely mixed in operations without causing errors. However, the return type -of the result may depend on the order of operands. - -.. note:: - - The methods on bytes and bytearray objects don't accept strings as their - arguments, just as the methods on strings don't accept bytes as their - arguments. For example, you have to write:: - - a = "abc" - b = a.replace("a", "f") - - and:: - - a = b"abc" - b = a.replace(b"a", b"f") - -Some bytes and bytearray operations assume the use of ASCII compatible -binary formats, and hence should be avoided when working with arbitrary -binary data. These restrictions are covered below. - -.. note:: - Using these ASCII based operations to manipulate binary data that is not - stored in an ASCII based format may lead to data corruption. - -The following methods on bytes and bytearray objects can be used with -arbitrary binary data. - -.. method:: bytes.count(sub[, start[, end]]) - bytearray.count(sub[, start[, end]]) - - Return the number of non-overlapping occurrences of subsequence *sub* in - the range [*start*, *end*]. Optional arguments *start* and *end* are - interpreted as in slice notation. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - If *sub* is empty, returns the number of empty slices between characters - which is the length of the bytes object plus one. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.removeprefix(prefix, /) - bytearray.removeprefix(prefix, /) - - If the binary data starts with the *prefix* string, return - ``bytes[len(prefix):]``. Otherwise, return a copy of the original - binary data:: - - >>> b'TestHook'.removeprefix(b'Test') - b'Hook' - >>> b'BaseTestCase'.removeprefix(b'Test') - b'BaseTestCase' - - The *prefix* may be any :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - .. versionadded:: 3.9 - - -.. method:: bytes.removesuffix(suffix, /) - bytearray.removesuffix(suffix, /) - - If the binary data ends with the *suffix* string and that *suffix* is - not empty, return ``bytes[:-len(suffix)]``. Otherwise, return a copy of - the original binary data:: - - >>> b'MiscTests'.removesuffix(b'Tests') - b'Misc' - >>> b'TmpDirMixin'.removesuffix(b'Tests') - b'TmpDirMixin' - - The *suffix* may be any :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - .. versionadded:: 3.9 - - -.. method:: bytes.decode(encoding="utf-8", errors="strict") - bytearray.decode(encoding="utf-8", errors="strict") - - Return the bytes decoded to a :class:`str`. - - *encoding* defaults to ``'utf-8'``; - see :ref:`standard-encodings` for possible values. - - *errors* controls how decoding errors are handled. - If ``'strict'`` (the default), a :exc:`UnicodeError` exception is raised. - Other possible values are ``'ignore'``, ``'replace'``, - and any other name registered via :func:`codecs.register_error`. - See :ref:`error-handlers` for details. - - For performance reasons, the value of *errors* is not checked for validity - unless a decoding error actually occurs, - :ref:`devmode` is enabled or a :ref:`debug build ` is used. - - .. note:: - - Passing the *encoding* argument to :class:`str` allows decoding any - :term:`bytes-like object` directly, without needing to make a temporary - :class:`!bytes` or :class:`!bytearray` object. - - .. versionchanged:: 3.1 - Added support for keyword arguments. - - .. versionchanged:: 3.9 - The value of the *errors* argument is now checked in :ref:`devmode` and - in :ref:`debug mode `. - - -.. method:: bytes.endswith(suffix[, start[, end]]) - bytearray.endswith(suffix[, start[, end]]) - - Return ``True`` if the binary data ends with the specified *suffix*, - otherwise return ``False``. *suffix* can also be a tuple of suffixes to - look for. With optional *start*, test beginning at that position. With - optional *end*, stop comparing at that position. - - The suffix(es) to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.find(sub[, start[, end]]) - bytearray.find(sub[, start[, end]]) - - Return the lowest index in the data where the subsequence *sub* is found, - such that *sub* is contained in the slice ``s[start:end]``. Optional - arguments *start* and *end* are interpreted as in slice notation. Return - ``-1`` if *sub* is not found. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. note:: - - The :meth:`~bytes.find` method should be used only if you need to know the - position of *sub*. To check if *sub* is a substring or not, use the - :keyword:`in` operator:: - - >>> b'Py' in b'Python' - True - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.index(sub[, start[, end]]) - bytearray.index(sub[, start[, end]]) - - Like :meth:`~bytes.find`, but raise :exc:`ValueError` when the - subsequence is not found. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.join(iterable) - bytearray.join(iterable) - - Return a bytes or bytearray object which is the concatenation of the - binary data sequences in *iterable*. A :exc:`TypeError` will be raised - if there are any values in *iterable* that are not :term:`bytes-like - objects `, including :class:`str` objects. The - separator between elements is the contents of the bytes or - bytearray object providing this method. - - -.. staticmethod:: bytes.maketrans(from, to) - bytearray.maketrans(from, to) - - This static method returns a translation table usable for - :meth:`bytes.translate` that will map each character in *from* into the - character at the same position in *to*; *from* and *to* must both be - :term:`bytes-like objects ` and have the same length. - - .. versionadded:: 3.1 - - -.. method:: bytes.partition(sep) - bytearray.partition(sep) - - Split the sequence at the first occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator itself or its - bytearray copy, and the part after the separator. - If the separator is not found, return a 3-tuple - containing a copy of the original sequence, followed by two empty bytes or - bytearray objects. - - The separator to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.replace(old, new[, count]) - bytearray.replace(old, new[, count]) - - Return a copy of the sequence with all occurrences of subsequence *old* - replaced by *new*. If the optional argument *count* is given, only the - first *count* occurrences are replaced. - - The subsequence to search for and its replacement may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.rfind(sub[, start[, end]]) - bytearray.rfind(sub[, start[, end]]) - - Return the highest index in the sequence where the subsequence *sub* is - found, such that *sub* is contained within ``s[start:end]``. Optional - arguments *start* and *end* are interpreted as in slice notation. Return - ``-1`` on failure. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.rindex(sub[, start[, end]]) - bytearray.rindex(sub[, start[, end]]) - - Like :meth:`~bytes.rfind` but raises :exc:`ValueError` when the - subsequence *sub* is not found. - - The subsequence to search for may be any :term:`bytes-like object` or an - integer in the range 0 to 255. - - .. versionchanged:: 3.3 - Also accept an integer in the range 0 to 255 as the subsequence. - - -.. method:: bytes.rpartition(sep) - bytearray.rpartition(sep) - - Split the sequence at the last occurrence of *sep*, and return a 3-tuple - containing the part before the separator, the separator itself or its - bytearray copy, and the part after the separator. - If the separator is not found, return a 3-tuple - containing two empty bytes or bytearray objects, followed by a copy of the - original sequence. - - The separator to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.startswith(prefix[, start[, end]]) - bytearray.startswith(prefix[, start[, end]]) - - Return ``True`` if the binary data starts with the specified *prefix*, - otherwise return ``False``. *prefix* can also be a tuple of prefixes to - look for. With optional *start*, test beginning at that position. With - optional *end*, stop comparing at that position. - - The prefix(es) to search for may be any :term:`bytes-like object`. - - -.. method:: bytes.translate(table, /, delete=b'') - bytearray.translate(table, /, delete=b'') - - Return a copy of the bytes or bytearray object where all bytes occurring in - the optional argument *delete* are removed, and the remaining bytes have - been mapped through the given translation table, which must be a bytes - object of length 256. - - You can use the :func:`bytes.maketrans` method to create a translation - table. - - Set the *table* argument to ``None`` for translations that only delete - characters:: - - >>> b'read this short text'.translate(None, b'aeiou') - b'rd ths shrt txt' - - .. versionchanged:: 3.6 - *delete* is now supported as a keyword argument. - - -The following methods on bytes and bytearray objects have default behaviours -that assume the use of ASCII compatible binary formats, but can still be used -with arbitrary binary data by passing appropriate arguments. Note that all of -the bytearray methods in this section do *not* operate in place, and instead -produce new objects. - -.. method:: bytes.center(width[, fillbyte]) - bytearray.center(width[, fillbyte]) - - Return a copy of the object centered in a sequence of length *width*. - Padding is done using the specified *fillbyte* (default is an ASCII - space). For :class:`bytes` objects, the original sequence is returned if - *width* is less than or equal to ``len(s)``. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.ljust(width[, fillbyte]) - bytearray.ljust(width[, fillbyte]) - - Return a copy of the object left justified in a sequence of length *width*. - Padding is done using the specified *fillbyte* (default is an ASCII - space). For :class:`bytes` objects, the original sequence is returned if - *width* is less than or equal to ``len(s)``. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.lstrip([chars]) - bytearray.lstrip([chars]) - - Return a copy of the sequence with specified leading bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults - to removing ASCII whitespace. The *chars* argument is not a prefix; - rather, all combinations of its values are stripped:: - - >>> b' spacious '.lstrip() - b'spacious ' - >>> b'www.example.com'.lstrip(b'cmowz.') - b'example.com' - - The binary sequence of byte values to remove may be any - :term:`bytes-like object`. See :meth:`~bytes.removeprefix` for a method - that will remove a single prefix string rather than all of a set of - characters. For example:: - - >>> b'Arthur: three!'.lstrip(b'Arthur: ') - b'ee!' - >>> b'Arthur: three!'.removeprefix(b'Arthur: ') - b'three!' - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.rjust(width[, fillbyte]) - bytearray.rjust(width[, fillbyte]) - - Return a copy of the object right justified in a sequence of length *width*. - Padding is done using the specified *fillbyte* (default is an ASCII - space). For :class:`bytes` objects, the original sequence is returned if - *width* is less than or equal to ``len(s)``. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.rsplit(sep=None, maxsplit=-1) - bytearray.rsplit(sep=None, maxsplit=-1) - - Split the binary sequence into subsequences of the same type, using *sep* - as the delimiter string. If *maxsplit* is given, at most *maxsplit* splits - are done, the *rightmost* ones. If *sep* is not specified or ``None``, - any subsequence consisting solely of ASCII whitespace is a separator. - Except for splitting from the right, :meth:`rsplit` behaves like - :meth:`split` which is described in detail below. - - -.. method:: bytes.rstrip([chars]) - bytearray.rstrip([chars]) - - Return a copy of the sequence with specified trailing bytes removed. The - *chars* argument is a binary sequence specifying the set of byte values to - be removed - the name refers to the fact this method is usually used with - ASCII characters. If omitted or ``None``, the *chars* argument defaults to - removing ASCII whitespace. The *chars* argument is not a suffix; rather, - all combinations of its values are stripped:: - - >>> b' spacious '.rstrip() - b' spacious' - >>> b'mississippi'.rstrip(b'ipz') - b'mississ' - - The binary sequence of byte values to remove may be any - :term:`bytes-like object`. See :meth:`~bytes.removesuffix` for a method - that will remove a single suffix string rather than all of a set of - characters. For example:: - - >>> b'Monty Python'.rstrip(b' Python') - b'M' - >>> b'Monty Python'.removesuffix(b' Python') - b'Monty' - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -.. method:: bytes.split(sep=None, maxsplit=-1) - bytearray.split(sep=None, maxsplit=-1) - - Split the binary sequence into subsequences of the same type, using *sep* - as the delimiter string. If *maxsplit* is given and non-negative, at most - *maxsplit* splits are done (thus, the list will have at most ``maxsplit+1`` - elements). If *maxsplit* is not specified or is ``-1``, then there is no - limit on the number of splits (all possible splits are made). - - If *sep* is given, consecutive delimiters are not grouped together and are - deemed to delimit empty subsequences (for example, ``b'1,,2'.split(b',')`` - returns ``[b'1', b'', b'2']``). The *sep* argument may consist of a - multibyte sequence (for example, ``b'1<>2<>3'.split(b'<>')`` returns - ``[b'1', b'2', b'3']``). Splitting an empty sequence with a specified - separator returns ``[b'']`` or ``[bytearray(b'')]`` depending on the type - of object being split. The *sep* argument may be any - :term:`bytes-like object`. - - For example:: - - >>> b'1,2,3'.split(b',') - [b'1', b'2', b'3'] - >>> b'1,2,3'.split(b',', maxsplit=1) - [b'1', b'2,3'] - >>> b'1,2,,3,'.split(b',') - [b'1', b'2', b'', b'3', b''] - - If *sep* is not specified or is ``None``, a different splitting algorithm - is applied: runs of consecutive ASCII whitespace are regarded as a single - separator, and the result will contain no empty strings at the start or - end if the sequence has leading or trailing whitespace. Consequently, - splitting an empty sequence or a sequence consisting solely of ASCII - whitespace without a specified separator returns ``[]``. - - For example:: - - - >>> b'1 2 3'.split() - [b'1', b'2', b'3'] - >>> b'1 2 3'.split(maxsplit=1) - [b'1', b'2 3'] - >>> b' 1 2 3 '.split() - [b'1', b'2', b'3'] - - -.. method:: bytes.strip([chars]) - bytearray.strip([chars]) - - Return a copy of the sequence with specified leading and trailing bytes - removed. The *chars* argument is a binary sequence specifying the set of - byte values to be removed - the name refers to the fact this method is - usually used with ASCII characters. If omitted or ``None``, the *chars* - argument defaults to removing ASCII whitespace. The *chars* argument is - not a prefix or suffix; rather, all combinations of its values are - stripped:: - - >>> b' spacious '.strip() - b'spacious' - >>> b'www.example.com'.strip(b'cmowz.') - b'example' - - The binary sequence of byte values to remove may be any - :term:`bytes-like object`. - - .. note:: - - The bytearray version of this method does *not* operate in place - - it always produces a new object, even if no changes were made. - - -The following methods on bytes and bytearray objects assume the use of ASCII -compatible binary formats and should not be applied to arbitrary binary data. -Note that all of the bytearray methods in this section do *not* operate in -place, and instead produce new objects. - -.. method:: bytes.capitalize() - bytearray.capitalize() - - Return a copy of the sequence with each byte interpreted as an ASCII - character, and the first byte capitalized and the rest lowercased. - Non-ASCII byte values are passed through unchanged. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.expandtabs(tabsize=8) - bytearray.expandtabs(tabsize=8) - - Return a copy of the sequence where all ASCII tab characters are replaced - by one or more ASCII spaces, depending on the current column and the given - tab size. Tab positions occur every *tabsize* bytes (default is 8, - giving tab positions at columns 0, 8, 16 and so on). To expand the - sequence, the current column is set to zero and the sequence is examined - byte by byte. If the byte is an ASCII tab character (``b'\t'``), one or - more space characters are inserted in the result until the current column - is equal to the next tab position. (The tab character itself is not - copied.) If the current byte is an ASCII newline (``b'\n'``) or - carriage return (``b'\r'``), it is copied and the current column is reset - to zero. Any other byte value is copied unchanged and the current column - is incremented by one regardless of how the byte value is represented when - printed:: - - >>> b'01\t012\t0123\t01234'.expandtabs() - b'01 012 0123 01234' - >>> b'01\t012\t0123\t01234'.expandtabs(4) - b'01 012 0123 01234' - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.isalnum() - bytearray.isalnum() - - Return ``True`` if all bytes in the sequence are alphabetical ASCII characters - or ASCII decimal digits and the sequence is not empty, ``False`` otherwise. - Alphabetic ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'``. ASCII decimal - digits are those byte values in the sequence ``b'0123456789'``. - - For example:: - - >>> b'ABCabc1'.isalnum() - True - >>> b'ABC abc1'.isalnum() - False - - -.. method:: bytes.isalpha() - bytearray.isalpha() - - Return ``True`` if all bytes in the sequence are alphabetic ASCII characters - and the sequence is not empty, ``False`` otherwise. Alphabetic ASCII - characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - For example:: - - >>> b'ABCabc'.isalpha() - True - >>> b'ABCabc1'.isalpha() - False - - -.. method:: bytes.isascii() - bytearray.isascii() - - Return ``True`` if the sequence is empty or all bytes in the sequence are ASCII, - ``False`` otherwise. - ASCII bytes are in the range 0-0x7F. - - .. versionadded:: 3.7 - - -.. method:: bytes.isdigit() - bytearray.isdigit() - - Return ``True`` if all bytes in the sequence are ASCII decimal digits - and the sequence is not empty, ``False`` otherwise. ASCII decimal digits are - those byte values in the sequence ``b'0123456789'``. - - For example:: - - >>> b'1234'.isdigit() - True - >>> b'1.23'.isdigit() - False - - -.. method:: bytes.islower() - bytearray.islower() - - Return ``True`` if there is at least one lowercase ASCII character - in the sequence and no uppercase ASCII characters, ``False`` otherwise. - - For example:: - - >>> b'hello world'.islower() - True - >>> b'Hello world'.islower() - False - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - -.. method:: bytes.isspace() - bytearray.isspace() - - Return ``True`` if all bytes in the sequence are ASCII whitespace and the - sequence is not empty, ``False`` otherwise. ASCII whitespace characters are - those byte values in the sequence ``b' \t\n\r\x0b\f'`` (space, tab, newline, - carriage return, vertical tab, form feed). - - -.. method:: bytes.istitle() - bytearray.istitle() - - Return ``True`` if the sequence is ASCII titlecase and the sequence is not - empty, ``False`` otherwise. See :meth:`bytes.title` for more details on the - definition of "titlecase". - - For example:: - - >>> b'Hello World'.istitle() - True - >>> b'Hello world'.istitle() - False - - -.. method:: bytes.isupper() - bytearray.isupper() - - Return ``True`` if there is at least one uppercase alphabetic ASCII character - in the sequence and no lowercase ASCII characters, ``False`` otherwise. - - For example:: - - >>> b'HELLO WORLD'.isupper() - True - >>> b'Hello world'.isupper() - False - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - -.. method:: bytes.lower() - bytearray.lower() - - Return a copy of the sequence with all the uppercase ASCII characters - converted to their corresponding lowercase counterpart. - - For example:: - - >>> b'Hello World'.lower() - b'hello world' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. index:: - single: universal newlines; bytes.splitlines method - single: universal newlines; bytearray.splitlines method - -.. method:: bytes.splitlines(keepends=False) - bytearray.splitlines(keepends=False) - - Return a list of the lines in the binary sequence, breaking at ASCII - line boundaries. This method uses the :term:`universal newlines` approach - to splitting lines. Line breaks are not included in the resulting list - unless *keepends* is given and true. - - For example:: - - >>> b'ab c\n\nde fg\rkl\r\n'.splitlines() - [b'ab c', b'', b'de fg', b'kl'] - >>> b'ab c\n\nde fg\rkl\r\n'.splitlines(keepends=True) - [b'ab c\n', b'\n', b'de fg\r', b'kl\r\n'] - - Unlike :meth:`~bytes.split` when a delimiter string *sep* is given, this - method returns an empty list for the empty string, and a terminal line - break does not result in an extra line:: - - >>> b"".split(b'\n'), b"Two lines\n".split(b'\n') - ([b''], [b'Two lines', b'']) - >>> b"".splitlines(), b"One line\n".splitlines() - ([], [b'One line']) - - -.. method:: bytes.swapcase() - bytearray.swapcase() - - Return a copy of the sequence with all the lowercase ASCII characters - converted to their corresponding uppercase counterpart and vice-versa. - - For example:: - - >>> b'Hello World'.swapcase() - b'hELLO wORLD' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - Unlike :func:`str.swapcase()`, it is always the case that - ``bin.swapcase().swapcase() == bin`` for the binary versions. Case - conversions are symmetrical in ASCII, even though that is not generally - true for arbitrary Unicode code points. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.title() - bytearray.title() - - Return a titlecased version of the binary sequence where words start with - an uppercase ASCII character and the remaining characters are lowercase. - Uncased byte values are left unmodified. - - For example:: - - >>> b'Hello world'.title() - b'Hello World' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - All other byte values are uncased. - - The algorithm uses a simple language-independent definition of a word as - groups of consecutive letters. The definition works in many contexts but - it means that apostrophes in contractions and possessives form word - boundaries, which may not be the desired result:: - - >>> b"they're bill's friends from the UK".title() - b"They'Re Bill'S Friends From The Uk" - - A workaround for apostrophes can be constructed using regular expressions:: - - >>> import re - >>> def titlecase(s): - ... return re.sub(rb"[A-Za-z]+('[A-Za-z]+)?", - ... lambda mo: mo.group(0)[0:1].upper() + - ... mo.group(0)[1:].lower(), - ... s) - ... - >>> titlecase(b"they're bill's friends.") - b"They're Bill's Friends." - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.upper() - bytearray.upper() - - Return a copy of the sequence with all the lowercase ASCII characters - converted to their corresponding uppercase counterpart. - - For example:: - - >>> b'Hello World'.upper() - b'HELLO WORLD' - - Lowercase ASCII characters are those byte values in the sequence - ``b'abcdefghijklmnopqrstuvwxyz'``. Uppercase ASCII characters - are those byte values in the sequence ``b'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. method:: bytes.zfill(width) - bytearray.zfill(width) - - Return a copy of the sequence left filled with ASCII ``b'0'`` digits to - make a sequence of length *width*. A leading sign prefix (``b'+'``/ - ``b'-'``) is handled by inserting the padding *after* the sign character - rather than before. For :class:`bytes` objects, the original sequence is - returned if *width* is less than or equal to ``len(seq)``. - - For example:: - - >>> b"42".zfill(5) - b'00042' - >>> b"-42".zfill(5) - b'-0042' - - .. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - - -.. _bytes-formatting: - -``printf``-style Bytes Formatting ----------------------------------- - -.. index:: - single: formatting; bytes (%) - single: formatting; bytearray (%) - single: interpolation; bytes (%) - single: interpolation; bytearray (%) - single: bytes; formatting - single: bytearray; formatting - single: bytes; interpolation - single: bytearray; interpolation - single: printf-style formatting - single: sprintf-style formatting - single: % (percent); printf-style formatting - -.. note:: - - The formatting operations described here exhibit a variety of quirks that - lead to a number of common errors (such as failing to display tuples and - dictionaries correctly). If the value being printed may be a tuple or - dictionary, wrap it in a tuple. - -Bytes objects (``bytes``/``bytearray``) have one unique built-in operation: -the ``%`` operator (modulo). -This is also known as the bytes *formatting* or *interpolation* operator. -Given ``format % values`` (where *format* is a bytes object), ``%`` conversion -specifications in *format* are replaced with zero or more elements of *values*. -The effect is similar to using the :c:func:`sprintf` in the C language. - -If *format* requires a single argument, *values* may be a single non-tuple -object. [5]_ Otherwise, *values* must be a tuple with exactly the number of -items specified by the format bytes object, or a single mapping object (for -example, a dictionary). - -.. index:: - single: () (parentheses); in printf-style formatting - single: * (asterisk); in printf-style formatting - single: . (dot); in printf-style formatting - -A conversion specifier contains two or more characters and has the following -components, which must occur in this order: - -#. The ``'%'`` character, which marks the start of the specifier. - -#. Mapping key (optional), consisting of a parenthesised sequence of characters - (for example, ``(somename)``). - -#. Conversion flags (optional), which affect the result of some conversion - types. - -#. Minimum field width (optional). If specified as an ``'*'`` (asterisk), the - actual width is read from the next element of the tuple in *values*, and the - object to convert comes after the minimum field width and optional precision. - -#. Precision (optional), given as a ``'.'`` (dot) followed by the precision. If - specified as ``'*'`` (an asterisk), the actual precision is read from the next - element of the tuple in *values*, and the value to convert comes after the - precision. - -#. Length modifier (optional). - -#. Conversion type. - -When the right argument is a dictionary (or other mapping type), then the -formats in the bytes object *must* include a parenthesised mapping key into that -dictionary inserted immediately after the ``'%'`` character. The mapping key -selects the value to be formatted from the mapping. For example: - - >>> print(b'%(language)s has %(number)03d quote types.' % - ... {b'language': b"Python", b"number": 2}) - b'Python has 002 quote types.' - -In this case no ``*`` specifiers may occur in a format (since they require a -sequential parameter list). - -The conversion flag characters are: - -.. index:: - single: # (hash); in printf-style formatting - single: - (minus); in printf-style formatting - single: + (plus); in printf-style formatting - single: space; in printf-style formatting - -+---------+---------------------------------------------------------------------+ -| Flag | Meaning | -+=========+=====================================================================+ -| ``'#'`` | The value conversion will use the "alternate form" (where defined | -| | below). | -+---------+---------------------------------------------------------------------+ -| ``'0'`` | The conversion will be zero padded for numeric values. | -+---------+---------------------------------------------------------------------+ -| ``'-'`` | The converted value is left adjusted (overrides the ``'0'`` | -| | conversion if both are given). | -+---------+---------------------------------------------------------------------+ -| ``' '`` | (a space) A blank should be left before a positive number (or empty | -| | string) produced by a signed conversion. | -+---------+---------------------------------------------------------------------+ -| ``'+'`` | A sign character (``'+'`` or ``'-'``) will precede the conversion | -| | (overrides a "space" flag). | -+---------+---------------------------------------------------------------------+ - -A length modifier (``h``, ``l``, or ``L``) may be present, but is ignored as it -is not necessary for Python -- so e.g. ``%ld`` is identical to ``%d``. - -The conversion types are: - -+------------+-----------------------------------------------------+-------+ -| Conversion | Meaning | Notes | -+============+=====================================================+=======+ -| ``'d'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'i'`` | Signed integer decimal. | | -+------------+-----------------------------------------------------+-------+ -| ``'o'`` | Signed octal value. | \(1) | -+------------+-----------------------------------------------------+-------+ -| ``'u'`` | Obsolete type -- it is identical to ``'d'``. | \(8) | -+------------+-----------------------------------------------------+-------+ -| ``'x'`` | Signed hexadecimal (lowercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'X'`` | Signed hexadecimal (uppercase). | \(2) | -+------------+-----------------------------------------------------+-------+ -| ``'e'`` | Floating point exponential format (lowercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'E'`` | Floating point exponential format (uppercase). | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'f'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'F'`` | Floating point decimal format. | \(3) | -+------------+-----------------------------------------------------+-------+ -| ``'g'`` | Floating point format. Uses lowercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'G'`` | Floating point format. Uses uppercase exponential | \(4) | -| | format if exponent is less than -4 or not less than | | -| | precision, decimal format otherwise. | | -+------------+-----------------------------------------------------+-------+ -| ``'c'`` | Single byte (accepts integer or single | | -| | byte objects). | | -+------------+-----------------------------------------------------+-------+ -| ``'b'`` | Bytes (any object that follows the | \(5) | -| | :ref:`buffer protocol ` or has | | -| | :meth:`__bytes__`). | | -+------------+-----------------------------------------------------+-------+ -| ``'s'`` | ``'s'`` is an alias for ``'b'`` and should only | \(6) | -| | be used for Python2/3 code bases. | | -+------------+-----------------------------------------------------+-------+ -| ``'a'`` | Bytes (converts any Python object using | \(5) | -| | ``repr(obj).encode('ascii', 'backslashreplace')``). | | -+------------+-----------------------------------------------------+-------+ -| ``'r'`` | ``'r'`` is an alias for ``'a'`` and should only | \(7) | -| | be used for Python2/3 code bases. | | -+------------+-----------------------------------------------------+-------+ -| ``'%'`` | No argument is converted, results in a ``'%'`` | | -| | character in the result. | | -+------------+-----------------------------------------------------+-------+ - -Notes: - -(1) - The alternate form causes a leading octal specifier (``'0o'``) to be - inserted before the first digit. - -(2) - The alternate form causes a leading ``'0x'`` or ``'0X'`` (depending on whether - the ``'x'`` or ``'X'`` format was used) to be inserted before the first digit. - -(3) - The alternate form causes the result to always contain a decimal point, even if - no digits follow it. - - The precision determines the number of digits after the decimal point and - defaults to 6. - -(4) - The alternate form causes the result to always contain a decimal point, and - trailing zeroes are not removed as they would otherwise be. - - The precision determines the number of significant digits before and after the - decimal point and defaults to 6. - -(5) - If precision is ``N``, the output is truncated to ``N`` characters. - -(6) - ``b'%s'`` is deprecated, but will not be removed during the 3.x series. - -(7) - ``b'%r'`` is deprecated, but will not be removed during the 3.x series. - -(8) - See :pep:`237`. - -.. note:: - - The bytearray version of this method does *not* operate in place - it - always produces a new object, even if no changes were made. - -.. seealso:: - - :pep:`461` - Adding % formatting to bytes and bytearray - -.. versionadded:: 3.5 - -.. _typememoryview: - -Memory Views ------------- - -:class:`memoryview` objects allow Python code to access the internal data -of an object that supports the :ref:`buffer protocol ` without -copying. - -.. class:: memoryview(object) - - Create a :class:`memoryview` that references *object*. *object* must - support the buffer protocol. Built-in objects that support the buffer - protocol include :class:`bytes` and :class:`bytearray`. - - A :class:`memoryview` has the notion of an *element*, which is the - atomic memory unit handled by the originating *object*. For many simple - types such as :class:`bytes` and :class:`bytearray`, an element is a single - byte, but other types such as :class:`array.array` may have bigger elements. - - ``len(view)`` is equal to the length of :class:`~memoryview.tolist`. - If ``view.ndim = 0``, the length is 1. If ``view.ndim = 1``, the length - is equal to the number of elements in the view. For higher dimensions, - the length is equal to the length of the nested list representation of - the view. The :class:`~memoryview.itemsize` attribute will give you the - number of bytes in a single element. - - A :class:`memoryview` supports slicing and indexing to expose its data. - One-dimensional slicing will result in a subview:: - - >>> v = memoryview(b'abcefg') - >>> v[1] - 98 - >>> v[-1] - 103 - >>> v[1:4] - - >>> bytes(v[1:4]) - b'bce' - - If :class:`~memoryview.format` is one of the native format specifiers - from the :mod:`struct` module, indexing with an integer or a tuple of - integers is also supported and returns a single *element* with - the correct type. One-dimensional memoryviews can be indexed - with an integer or a one-integer tuple. Multi-dimensional memoryviews - can be indexed with tuples of exactly *ndim* integers where *ndim* is - the number of dimensions. Zero-dimensional memoryviews can be indexed - with the empty tuple. - - Here is an example with a non-byte format:: - - >>> import array - >>> a = array.array('l', [-11111111, 22222222, -33333333, 44444444]) - >>> m = memoryview(a) - >>> m[0] - -11111111 - >>> m[-1] - 44444444 - >>> m[::2].tolist() - [-11111111, -33333333] - - If the underlying object is writable, the memoryview supports - one-dimensional slice assignment. Resizing is not allowed:: - - >>> data = bytearray(b'abcefg') - >>> v = memoryview(data) - >>> v.readonly - False - >>> v[0] = ord(b'z') - >>> data - bytearray(b'zbcefg') - >>> v[1:4] = b'123' - >>> data - bytearray(b'z123fg') - >>> v[2:3] = b'spam' - Traceback (most recent call last): - File "", line 1, in - ValueError: memoryview assignment: lvalue and rvalue have different structures - >>> v[2:6] = b'spam' - >>> data - bytearray(b'z1spam') - - One-dimensional memoryviews of :term:`hashable` (read-only) types with formats - 'B', 'b' or 'c' are also hashable. The hash is defined as - ``hash(m) == hash(m.tobytes())``:: - - >>> v = memoryview(b'abcefg') - >>> hash(v) == hash(b'abcefg') - True - >>> hash(v[2:4]) == hash(b'ce') - True - >>> hash(v[::-2]) == hash(b'abcefg'[::-2]) - True - - .. versionchanged:: 3.3 - One-dimensional memoryviews can now be sliced. - One-dimensional memoryviews with formats 'B', 'b' or 'c' are now :term:`hashable`. - - .. versionchanged:: 3.4 - memoryview is now registered automatically with - :class:`collections.abc.Sequence` - - .. versionchanged:: 3.5 - memoryviews can now be indexed with tuple of integers. - - :class:`memoryview` has several methods: - - .. method:: __eq__(exporter) - - A memoryview and a :pep:`3118` exporter are equal if their shapes are - equivalent and if all corresponding values are equal when the operands' - respective format codes are interpreted using :mod:`struct` syntax. - - For the subset of :mod:`struct` format strings currently supported by - :meth:`tolist`, ``v`` and ``w`` are equal if ``v.tolist() == w.tolist()``:: - - >>> import array - >>> a = array.array('I', [1, 2, 3, 4, 5]) - >>> b = array.array('d', [1.0, 2.0, 3.0, 4.0, 5.0]) - >>> c = array.array('b', [5, 3, 1]) - >>> x = memoryview(a) - >>> y = memoryview(b) - >>> x == a == y == b - True - >>> x.tolist() == a.tolist() == y.tolist() == b.tolist() - True - >>> z = y[::-2] - >>> z == c - True - >>> z.tolist() == c.tolist() - True - - If either format string is not supported by the :mod:`struct` module, - then the objects will always compare as unequal (even if the format - strings and buffer contents are identical):: - - >>> from ctypes import BigEndianStructure, c_long - >>> class BEPoint(BigEndianStructure): - ... _fields_ = [("x", c_long), ("y", c_long)] - ... - >>> point = BEPoint(100, 200) - >>> a = memoryview(point) - >>> b = memoryview(point) - >>> a == point - False - >>> a == b - False - - Note that, as with floating point numbers, ``v is w`` does *not* imply - ``v == w`` for memoryview objects. - - .. versionchanged:: 3.3 - Previous versions compared the raw memory disregarding the item format - and the logical array structure. - - .. method:: tobytes(order='C') - - Return the data in the buffer as a bytestring. This is equivalent to - calling the :class:`bytes` constructor on the memoryview. :: - - >>> m = memoryview(b"abc") - >>> m.tobytes() - b'abc' - >>> bytes(m) - b'abc' - - For non-contiguous arrays the result is equal to the flattened list - representation with all elements converted to bytes. :meth:`tobytes` - supports all format strings, including those that are not in - :mod:`struct` module syntax. - - .. versionadded:: 3.8 - *order* can be {'C', 'F', 'A'}. When *order* is 'C' or 'F', the data - of the original array is converted to C or Fortran order. For contiguous - views, 'A' returns an exact copy of the physical memory. In particular, - in-memory Fortran order is preserved. For non-contiguous views, the - data is converted to C first. *order=None* is the same as *order='C'*. - - .. method:: hex([sep[, bytes_per_sep]]) - - Return a string object containing two hexadecimal digits for each - byte in the buffer. :: - - >>> m = memoryview(b"abc") - >>> m.hex() - '616263' - - .. versionadded:: 3.5 - - .. versionchanged:: 3.8 - Similar to :meth:`bytes.hex`, :meth:`memoryview.hex` now supports - optional *sep* and *bytes_per_sep* parameters to insert separators - between bytes in the hex output. - - .. method:: tolist() - - Return the data in the buffer as a list of elements. :: - - >>> memoryview(b'abc').tolist() - [97, 98, 99] - >>> import array - >>> a = array.array('d', [1.1, 2.2, 3.3]) - >>> m = memoryview(a) - >>> m.tolist() - [1.1, 2.2, 3.3] - - .. versionchanged:: 3.3 - :meth:`tolist` now supports all single character native formats in - :mod:`struct` module syntax as well as multi-dimensional - representations. - - .. method:: toreadonly() - - Return a readonly version of the memoryview object. The original - memoryview object is unchanged. :: - - >>> m = memoryview(bytearray(b'abc')) - >>> mm = m.toreadonly() - >>> mm.tolist() - [97, 98, 99] - >>> mm[0] = 42 - Traceback (most recent call last): - File "", line 1, in - TypeError: cannot modify read-only memory - >>> m[0] = 43 - >>> mm.tolist() - [43, 98, 99] - - .. versionadded:: 3.8 - - .. method:: release() - - Release the underlying buffer exposed by the memoryview object. Many - objects take special actions when a view is held on them (for example, - a :class:`bytearray` would temporarily forbid resizing); therefore, - calling release() is handy to remove these restrictions (and free any - dangling resources) as soon as possible. - - After this method has been called, any further operation on the view - raises a :class:`ValueError` (except :meth:`release()` itself which can - be called multiple times):: - - >>> m = memoryview(b'abc') - >>> m.release() - >>> m[0] - Traceback (most recent call last): - File "", line 1, in - ValueError: operation forbidden on released memoryview object - - The context management protocol can be used for a similar effect, - using the ``with`` statement:: - - >>> with memoryview(b'abc') as m: - ... m[0] - ... - 97 - >>> m[0] - Traceback (most recent call last): - File "", line 1, in - ValueError: operation forbidden on released memoryview object - - .. versionadded:: 3.2 - - .. method:: cast(format[, shape]) - - Cast a memoryview to a new format or shape. *shape* defaults to - ``[byte_length//new_itemsize]``, which means that the result view - will be one-dimensional. The return value is a new memoryview, but - the buffer itself is not copied. Supported casts are 1D -> C-:term:`contiguous` - and C-contiguous -> 1D. - - The destination format is restricted to a single element native format in - :mod:`struct` syntax. One of the formats must be a byte format - ('B', 'b' or 'c'). The byte length of the result must be the same - as the original length. - Note that all byte lengths may depend on the operating system. - - Cast 1D/long to 1D/unsigned bytes:: - - >>> import array - >>> a = array.array('l', [1,2,3]) - >>> x = memoryview(a) - >>> x.format - 'l' - >>> x.itemsize - 8 - >>> len(x) - 3 - >>> x.nbytes - 24 - >>> y = x.cast('B') - >>> y.format - 'B' - >>> y.itemsize - 1 - >>> len(y) - 24 - >>> y.nbytes - 24 - - Cast 1D/unsigned bytes to 1D/char:: - - >>> b = bytearray(b'zyz') - >>> x = memoryview(b) - >>> x[0] = b'a' - Traceback (most recent call last): - ... - TypeError: memoryview: invalid type for format 'B' - >>> y = x.cast('c') - >>> y[0] = b'a' - >>> b - bytearray(b'ayz') - - Cast 1D/bytes to 3D/ints to 1D/signed char:: - - >>> import struct - >>> buf = struct.pack("i"*12, *list(range(12))) - >>> x = memoryview(buf) - >>> y = x.cast('i', shape=[2,2,3]) - >>> y.tolist() - [[[0, 1, 2], [3, 4, 5]], [[6, 7, 8], [9, 10, 11]]] - >>> y.format - 'i' - >>> y.itemsize - 4 - >>> len(y) - 2 - >>> y.nbytes - 48 - >>> z = y.cast('b') - >>> z.format - 'b' - >>> z.itemsize - 1 - >>> len(z) - 48 - >>> z.nbytes - 48 - - Cast 1D/unsigned long to 2D/unsigned long:: - - >>> buf = struct.pack("L"*6, *list(range(6))) - >>> x = memoryview(buf) - >>> y = x.cast('L', shape=[2,3]) - >>> len(y) - 2 - >>> y.nbytes - 48 - >>> y.tolist() - [[0, 1, 2], [3, 4, 5]] - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - The source format is no longer restricted when casting to a byte view. - - There are also several readonly attributes available: - - .. attribute:: obj - - The underlying object of the memoryview:: - - >>> b = bytearray(b'xyz') - >>> m = memoryview(b) - >>> m.obj is b - True - - .. versionadded:: 3.3 - - .. attribute:: nbytes - - ``nbytes == product(shape) * itemsize == len(m.tobytes())``. This is - the amount of space in bytes that the array would use in a contiguous - representation. It is not necessarily equal to ``len(m)``:: - - >>> import array - >>> a = array.array('i', [1,2,3,4,5]) - >>> m = memoryview(a) - >>> len(m) - 5 - >>> m.nbytes - 20 - >>> y = m[::2] - >>> len(y) - 3 - >>> y.nbytes - 12 - >>> len(y.tobytes()) - 12 - - Multi-dimensional arrays:: - - >>> import struct - >>> buf = struct.pack("d"*12, *[1.5*x for x in range(12)]) - >>> x = memoryview(buf) - >>> y = x.cast('d', shape=[3,4]) - >>> y.tolist() - [[0.0, 1.5, 3.0, 4.5], [6.0, 7.5, 9.0, 10.5], [12.0, 13.5, 15.0, 16.5]] - >>> len(y) - 3 - >>> y.nbytes - 96 - - .. versionadded:: 3.3 - - .. attribute:: readonly - - A bool indicating whether the memory is read only. - - .. attribute:: format - - A string containing the format (in :mod:`struct` module style) for each - element in the view. A memoryview can be created from exporters with - arbitrary format strings, but some methods (e.g. :meth:`tolist`) are - restricted to native single element formats. - - .. versionchanged:: 3.3 - format ``'B'`` is now handled according to the struct module syntax. - This means that ``memoryview(b'abc')[0] == b'abc'[0] == 97``. - - .. attribute:: itemsize - - The size in bytes of each element of the memoryview:: - - >>> import array, struct - >>> m = memoryview(array.array('H', [32000, 32001, 32002])) - >>> m.itemsize - 2 - >>> m[0] - 32000 - >>> struct.calcsize('H') == m.itemsize - True - - .. attribute:: ndim - - An integer indicating how many dimensions of a multi-dimensional array the - memory represents. - - .. attribute:: shape - - A tuple of integers the length of :attr:`ndim` giving the shape of the - memory as an N-dimensional array. - - .. versionchanged:: 3.3 - An empty tuple instead of ``None`` when ndim = 0. - - .. attribute:: strides - - A tuple of integers the length of :attr:`ndim` giving the size in bytes to - access each element for each dimension of the array. - - .. versionchanged:: 3.3 - An empty tuple instead of ``None`` when ndim = 0. - - .. attribute:: suboffsets - - Used internally for PIL-style arrays. The value is informational only. - - .. attribute:: c_contiguous - - A bool indicating whether the memory is C-:term:`contiguous`. - - .. versionadded:: 3.3 - - .. attribute:: f_contiguous - - A bool indicating whether the memory is Fortran :term:`contiguous`. - - .. versionadded:: 3.3 - - .. attribute:: contiguous - - A bool indicating whether the memory is :term:`contiguous`. - - .. versionadded:: 3.3 - - -.. _types-set: - -Set Types --- :class:`set`, :class:`frozenset` -============================================== - -.. index:: pair: object; set - -A :dfn:`set` object is an unordered collection of distinct :term:`hashable` objects. -Common uses include membership testing, removing duplicates from a sequence, and -computing mathematical operations such as intersection, union, difference, and -symmetric difference. -(For other containers see the built-in :class:`dict`, :class:`list`, -and :class:`tuple` classes, and the :mod:`collections` module.) - -Like other collections, sets support ``x in set``, ``len(set)``, and ``for x in -set``. Being an unordered collection, sets do not record element position or -order of insertion. Accordingly, sets do not support indexing, slicing, or -other sequence-like behavior. - -There are currently two built-in set types, :class:`set` and :class:`frozenset`. -The :class:`set` type is mutable --- the contents can be changed using methods -like :meth:`~set.add` and :meth:`~set.remove`. Since it is mutable, it has no -hash value and cannot be used as either a dictionary key or as an element of -another set. The :class:`frozenset` type is immutable and :term:`hashable` --- -its contents cannot be altered after it is created; it can therefore be used as -a dictionary key or as an element of another set. - -Non-empty sets (not frozensets) can be created by placing a comma-separated list -of elements within braces, for example: ``{'jack', 'sjoerd'}``, in addition to the -:class:`set` constructor. - -The constructors for both classes work the same: - -.. class:: set([iterable]) - frozenset([iterable]) - - Return a new set or frozenset object whose elements are taken from - *iterable*. The elements of a set must be :term:`hashable`. To - represent sets of sets, the inner sets must be :class:`frozenset` - objects. If *iterable* is not specified, a new empty set is - returned. - - Sets can be created by several means: - - * Use a comma-separated list of elements within braces: ``{'jack', 'sjoerd'}`` - * Use a set comprehension: ``{c for c in 'abracadabra' if c not in 'abc'}`` - * Use the type constructor: ``set()``, ``set('foobar')``, ``set(['a', 'b', 'foo'])`` - - Instances of :class:`set` and :class:`frozenset` provide the following - operations: - - .. describe:: len(s) - - Return the number of elements in set *s* (cardinality of *s*). - - .. describe:: x in s - - Test *x* for membership in *s*. - - .. describe:: x not in s - - Test *x* for non-membership in *s*. - - .. method:: isdisjoint(other) - - Return ``True`` if the set has no elements in common with *other*. Sets are - disjoint if and only if their intersection is the empty set. - - .. method:: issubset(other) - set <= other - - Test whether every element in the set is in *other*. - - .. method:: set < other - - Test whether the set is a proper subset of *other*, that is, - ``set <= other and set != other``. - - .. method:: issuperset(other) - set >= other - - Test whether every element in *other* is in the set. - - .. method:: set > other - - Test whether the set is a proper superset of *other*, that is, ``set >= - other and set != other``. - - .. method:: union(*others) - set | other | ... - - Return a new set with elements from the set and all others. - - .. method:: intersection(*others) - set & other & ... - - Return a new set with elements common to the set and all others. - - .. method:: difference(*others) - set - other - ... - - Return a new set with elements in the set that are not in the others. - - .. method:: symmetric_difference(other) - set ^ other - - Return a new set with elements in either the set or *other* but not both. - - .. method:: copy() - - Return a shallow copy of the set. - - - Note, the non-operator versions of :meth:`union`, :meth:`intersection`, - :meth:`difference`, :meth:`symmetric_difference`, :meth:`issubset`, and - :meth:`issuperset` methods will accept any iterable as an argument. In - contrast, their operator based counterparts require their arguments to be - sets. This precludes error-prone constructions like ``set('abc') & 'cbs'`` - in favor of the more readable ``set('abc').intersection('cbs')``. - - Both :class:`set` and :class:`frozenset` support set to set comparisons. Two - sets are equal if and only if every element of each set is contained in the - other (each is a subset of the other). A set is less than another set if and - only if the first set is a proper subset of the second set (is a subset, but - is not equal). A set is greater than another set if and only if the first set - is a proper superset of the second set (is a superset, but is not equal). - - Instances of :class:`set` are compared to instances of :class:`frozenset` - based on their members. For example, ``set('abc') == frozenset('abc')`` - returns ``True`` and so does ``set('abc') in set([frozenset('abc')])``. - - The subset and equality comparisons do not generalize to a total ordering - function. For example, any two nonempty disjoint sets are not equal and are not - subsets of each other, so *all* of the following return ``False``: ``ab``. - - Since sets only define partial ordering (subset relationships), the output of - the :meth:`list.sort` method is undefined for lists of sets. - - Set elements, like dictionary keys, must be :term:`hashable`. - - Binary operations that mix :class:`set` instances with :class:`frozenset` - return the type of the first operand. For example: ``frozenset('ab') | - set('bc')`` returns an instance of :class:`frozenset`. - - The following table lists operations available for :class:`set` that do not - apply to immutable instances of :class:`frozenset`: - - .. method:: update(*others) - set |= other | ... - - Update the set, adding elements from all others. - - .. method:: intersection_update(*others) - set &= other & ... - - Update the set, keeping only elements found in it and all others. - - .. method:: difference_update(*others) - set -= other | ... - - Update the set, removing elements found in others. - - .. method:: symmetric_difference_update(other) - set ^= other - - Update the set, keeping only elements found in either set, but not in both. - - .. method:: add(elem) - - Add element *elem* to the set. - - .. method:: remove(elem) - - Remove element *elem* from the set. Raises :exc:`KeyError` if *elem* is - not contained in the set. - - .. method:: discard(elem) - - Remove element *elem* from the set if it is present. - - .. method:: pop() - - Remove and return an arbitrary element from the set. Raises - :exc:`KeyError` if the set is empty. - - .. method:: clear() - - Remove all elements from the set. - - - Note, the non-operator versions of the :meth:`update`, - :meth:`intersection_update`, :meth:`difference_update`, and - :meth:`symmetric_difference_update` methods will accept any iterable as an - argument. - - Note, the *elem* argument to the :meth:`__contains__`, :meth:`remove`, and - :meth:`discard` methods may be a set. To support searching for an equivalent - frozenset, a temporary one is created from *elem*. - - -.. _typesmapping: - -Mapping Types --- :class:`dict` -=============================== - -.. index:: - pair: object; mapping - pair: object; dictionary - triple: operations on; mapping; types - triple: operations on; dictionary; type - pair: statement; del - pair: built-in function; len - -A :term:`mapping` object maps :term:`hashable` values to arbitrary objects. -Mappings are mutable objects. There is currently only one standard mapping -type, the :dfn:`dictionary`. (For other containers see the built-in -:class:`list`, :class:`set`, and :class:`tuple` classes, and the -:mod:`collections` module.) - -A dictionary's keys are *almost* arbitrary values. Values that are not -:term:`hashable`, that is, values containing lists, dictionaries or other -mutable types (that are compared by value rather than by object identity) may -not be used as keys. -Values that compare equal (such as ``1``, ``1.0``, and ``True``) -can be used interchangeably to index the same dictionary entry. - -.. class:: dict(**kwargs) - dict(mapping, **kwargs) - dict(iterable, **kwargs) - - Return a new dictionary initialized from an optional positional argument - and a possibly empty set of keyword arguments. - - Dictionaries can be created by several means: - - * Use a comma-separated list of ``key: value`` pairs within braces: - ``{'jack': 4098, 'sjoerd': 4127}`` or ``{4098: 'jack', 4127: 'sjoerd'}`` - * Use a dict comprehension: ``{}``, ``{x: x ** 2 for x in range(10)}`` - * Use the type constructor: ``dict()``, - ``dict([('foo', 100), ('bar', 200)])``, ``dict(foo=100, bar=200)`` - - If no positional argument is given, an empty dictionary is created. - If a positional argument is given and it is a mapping object, a dictionary - is created with the same key-value pairs as the mapping object. Otherwise, - the positional argument must be an :term:`iterable` object. Each item in - the iterable must itself be an iterable with exactly two objects. The - first object of each item becomes a key in the new dictionary, and the - second object the corresponding value. If a key occurs more than once, the - last value for that key becomes the corresponding value in the new - dictionary. - - If keyword arguments are given, the keyword arguments and their values are - added to the dictionary created from the positional argument. If a key - being added is already present, the value from the keyword argument - replaces the value from the positional argument. - - To illustrate, the following examples all return a dictionary equal to - ``{"one": 1, "two": 2, "three": 3}``:: - - >>> a = dict(one=1, two=2, three=3) - >>> b = {'one': 1, 'two': 2, 'three': 3} - >>> c = dict(zip(['one', 'two', 'three'], [1, 2, 3])) - >>> d = dict([('two', 2), ('one', 1), ('three', 3)]) - >>> e = dict({'three': 3, 'one': 1, 'two': 2}) - >>> f = dict({'one': 1, 'three': 3}, two=2) - >>> a == b == c == d == e == f - True - - Providing keyword arguments as in the first example only works for keys that - are valid Python identifiers. Otherwise, any valid keys can be used. - - - These are the operations that dictionaries support (and therefore, custom - mapping types should support too): - - .. describe:: list(d) - - Return a list of all the keys used in the dictionary *d*. - - .. describe:: len(d) - - Return the number of items in the dictionary *d*. - - .. describe:: d[key] - - Return the item of *d* with key *key*. Raises a :exc:`KeyError` if *key* is - not in the map. - - .. index:: __missing__() - - If a subclass of dict defines a method :meth:`__missing__` and *key* - is not present, the ``d[key]`` operation calls that method with the key *key* - as argument. The ``d[key]`` operation then returns or raises whatever is - returned or raised by the ``__missing__(key)`` call. - No other operations or methods invoke :meth:`__missing__`. If - :meth:`__missing__` is not defined, :exc:`KeyError` is raised. - :meth:`__missing__` must be a method; it cannot be an instance variable:: - - >>> class Counter(dict): - ... def __missing__(self, key): - ... return 0 - >>> c = Counter() - >>> c['red'] - 0 - >>> c['red'] += 1 - >>> c['red'] - 1 - - The example above shows part of the implementation of - :class:`collections.Counter`. A different ``__missing__`` method is used - by :class:`collections.defaultdict`. - - .. describe:: d[key] = value - - Set ``d[key]`` to *value*. - - .. describe:: del d[key] - - Remove ``d[key]`` from *d*. Raises a :exc:`KeyError` if *key* is not in the - map. - - .. describe:: key in d - - Return ``True`` if *d* has a key *key*, else ``False``. - - .. describe:: key not in d - - Equivalent to ``not key in d``. - - .. describe:: iter(d) - - Return an iterator over the keys of the dictionary. This is a shortcut - for ``iter(d.keys())``. - - .. method:: clear() - - Remove all items from the dictionary. - - .. method:: copy() - - Return a shallow copy of the dictionary. - - .. classmethod:: fromkeys(iterable[, value]) - - Create a new dictionary with keys from *iterable* and values set to *value*. - - :meth:`fromkeys` is a class method that returns a new dictionary. *value* - defaults to ``None``. All of the values refer to just a single instance, - so it generally doesn't make sense for *value* to be a mutable object - such as an empty list. To get distinct values, use a :ref:`dict - comprehension ` instead. - - .. method:: get(key[, default]) - - Return the value for *key* if *key* is in the dictionary, else *default*. - If *default* is not given, it defaults to ``None``, so that this method - never raises a :exc:`KeyError`. - - .. method:: items() - - Return a new view of the dictionary's items (``(key, value)`` pairs). - See the :ref:`documentation of view objects `. - - .. method:: keys() - - Return a new view of the dictionary's keys. See the :ref:`documentation - of view objects `. - - .. method:: pop(key[, default]) - - If *key* is in the dictionary, remove it and return its value, else return - *default*. If *default* is not given and *key* is not in the dictionary, - a :exc:`KeyError` is raised. - - .. method:: popitem() - - Remove and return a ``(key, value)`` pair from the dictionary. - Pairs are returned in :abbr:`LIFO (last-in, first-out)` order. - - :meth:`popitem` is useful to destructively iterate over a dictionary, as - often used in set algorithms. If the dictionary is empty, calling - :meth:`popitem` raises a :exc:`KeyError`. - - .. versionchanged:: 3.7 - LIFO order is now guaranteed. In prior versions, :meth:`popitem` would - return an arbitrary key/value pair. - - .. describe:: reversed(d) - - Return a reverse iterator over the keys of the dictionary. This is a - shortcut for ``reversed(d.keys())``. - - .. versionadded:: 3.8 - - .. method:: setdefault(key[, default]) - - If *key* is in the dictionary, return its value. If not, insert *key* - with a value of *default* and return *default*. *default* defaults to - ``None``. - - .. method:: update([other]) - - Update the dictionary with the key/value pairs from *other*, overwriting - existing keys. Return ``None``. - - :meth:`update` accepts either another dictionary object or an iterable of - key/value pairs (as tuples or other iterables of length two). If keyword - arguments are specified, the dictionary is then updated with those - key/value pairs: ``d.update(red=1, blue=2)``. - - .. method:: values() - - Return a new view of the dictionary's values. See the - :ref:`documentation of view objects `. - - An equality comparison between one ``dict.values()`` view and another - will always return ``False``. This also applies when comparing - ``dict.values()`` to itself:: - - >>> d = {'a': 1} - >>> d.values() == d.values() - False - - .. describe:: d | other - - Create a new dictionary with the merged keys and values of *d* and - *other*, which must both be dictionaries. The values of *other* take - priority when *d* and *other* share keys. - - .. versionadded:: 3.9 - - .. describe:: d |= other - - Update the dictionary *d* with keys and values from *other*, which may be - either a :term:`mapping` or an :term:`iterable` of key/value pairs. The - values of *other* take priority when *d* and *other* share keys. - - .. versionadded:: 3.9 - - Dictionaries compare equal if and only if they have the same ``(key, - value)`` pairs (regardless of ordering). Order comparisons ('<', '<=', '>=', '>') raise - :exc:`TypeError`. - - Dictionaries preserve insertion order. Note that updating a key does not - affect the order. Keys added after deletion are inserted at the end. :: - - >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} - >>> d - {'one': 1, 'two': 2, 'three': 3, 'four': 4} - >>> list(d) - ['one', 'two', 'three', 'four'] - >>> list(d.values()) - [1, 2, 3, 4] - >>> d["one"] = 42 - >>> d - {'one': 42, 'two': 2, 'three': 3, 'four': 4} - >>> del d["two"] - >>> d["two"] = None - >>> d - {'one': 42, 'three': 3, 'four': 4, 'two': None} - - .. versionchanged:: 3.7 - Dictionary order is guaranteed to be insertion order. This behavior was - an implementation detail of CPython from 3.6. - - Dictionaries and dictionary views are reversible. :: - - >>> d = {"one": 1, "two": 2, "three": 3, "four": 4} - >>> d - {'one': 1, 'two': 2, 'three': 3, 'four': 4} - >>> list(reversed(d)) - ['four', 'three', 'two', 'one'] - >>> list(reversed(d.values())) - [4, 3, 2, 1] - >>> list(reversed(d.items())) - [('four', 4), ('three', 3), ('two', 2), ('one', 1)] - - .. versionchanged:: 3.8 - Dictionaries are now reversible. - - -.. seealso:: - :class:`types.MappingProxyType` can be used to create a read-only view - of a :class:`dict`. - - -.. _dict-views: - -Dictionary view objects ------------------------ - -The objects returned by :meth:`dict.keys`, :meth:`dict.values` and -:meth:`dict.items` are *view objects*. They provide a dynamic view on the -dictionary's entries, which means that when the dictionary changes, the view -reflects these changes. - -Dictionary views can be iterated over to yield their respective data, and -support membership tests: - -.. describe:: len(dictview) - - Return the number of entries in the dictionary. - -.. describe:: iter(dictview) - - Return an iterator over the keys, values or items (represented as tuples of - ``(key, value)``) in the dictionary. - - Keys and values are iterated over in insertion order. - This allows the creation of ``(value, key)`` pairs - using :func:`zip`: ``pairs = zip(d.values(), d.keys())``. Another way to - create the same list is ``pairs = [(v, k) for (k, v) in d.items()]``. - - Iterating views while adding or deleting entries in the dictionary may raise - a :exc:`RuntimeError` or fail to iterate over all entries. - - .. versionchanged:: 3.7 - Dictionary order is guaranteed to be insertion order. - -.. describe:: x in dictview - - Return ``True`` if *x* is in the underlying dictionary's keys, values or - items (in the latter case, *x* should be a ``(key, value)`` tuple). - -.. describe:: reversed(dictview) - - Return a reverse iterator over the keys, values or items of the dictionary. - The view will be iterated in reverse order of the insertion. - - .. versionchanged:: 3.8 - Dictionary views are now reversible. - -.. describe:: dictview.mapping - - Return a :class:`types.MappingProxyType` that wraps the original - dictionary to which the view refers. - - .. versionadded:: 3.10 - -Keys views are set-like since their entries are unique and :term:`hashable`. If all -values are hashable, so that ``(key, value)`` pairs are unique and hashable, -then the items view is also set-like. (Values views are not treated as set-like -since the entries are generally not unique.) For set-like views, all of the -operations defined for the abstract base class :class:`collections.abc.Set` are -available (for example, ``==``, ``<``, or ``^``). - -An example of dictionary view usage:: - - >>> dishes = {'eggs': 2, 'sausage': 1, 'bacon': 1, 'spam': 500} - >>> keys = dishes.keys() - >>> values = dishes.values() - - >>> # iteration - >>> n = 0 - >>> for val in values: - ... n += val - >>> print(n) - 504 - - >>> # keys and values are iterated over in the same order (insertion order) - >>> list(keys) - ['eggs', 'sausage', 'bacon', 'spam'] - >>> list(values) - [2, 1, 1, 500] - - >>> # view objects are dynamic and reflect dict changes - >>> del dishes['eggs'] - >>> del dishes['sausage'] - >>> list(keys) - ['bacon', 'spam'] - - >>> # set operations - >>> keys & {'eggs', 'bacon', 'salad'} - {'bacon'} - >>> keys ^ {'sausage', 'juice'} == {'juice', 'sausage', 'bacon', 'spam'} - True - >>> keys | ['juice', 'juice', 'juice'] == {'bacon', 'spam', 'juice'} - True - - >>> # get back a read-only proxy for the original dictionary - >>> values.mapping - mappingproxy({'bacon': 1, 'spam': 500}) - >>> values.mapping['spam'] - 500 - - -.. _typecontextmanager: - -Context Manager Types -===================== - -.. index:: - single: context manager - single: context management protocol - single: protocol; context management - -Python's :keyword:`with` statement supports the concept of a runtime context -defined by a context manager. This is implemented using a pair of methods -that allow user-defined classes to define a runtime context that is entered -before the statement body is executed and exited when the statement ends: - - -.. method:: contextmanager.__enter__() - - Enter the runtime context and return either this object or another object - related to the runtime context. The value returned by this method is bound to - the identifier in the :keyword:`!as` clause of :keyword:`with` statements using - this context manager. - - An example of a context manager that returns itself is a :term:`file object`. - File objects return themselves from __enter__() to allow :func:`open` to be - used as the context expression in a :keyword:`with` statement. - - An example of a context manager that returns a related object is the one - returned by :func:`decimal.localcontext`. These managers set the active - decimal context to a copy of the original decimal context and then return the - copy. This allows changes to be made to the current decimal context in the body - of the :keyword:`with` statement without affecting code outside the - :keyword:`!with` statement. - - -.. method:: contextmanager.__exit__(exc_type, exc_val, exc_tb) - - Exit the runtime context and return a Boolean flag indicating if any exception - that occurred should be suppressed. If an exception occurred while executing the - body of the :keyword:`with` statement, the arguments contain the exception type, - value and traceback information. Otherwise, all three arguments are ``None``. - - Returning a true value from this method will cause the :keyword:`with` statement - to suppress the exception and continue execution with the statement immediately - following the :keyword:`!with` statement. Otherwise the exception continues - propagating after this method has finished executing. Exceptions that occur - during execution of this method will replace any exception that occurred in the - body of the :keyword:`!with` statement. - - The exception passed in should never be reraised explicitly - instead, this - method should return a false value to indicate that the method completed - successfully and does not want to suppress the raised exception. This allows - context management code to easily detect whether or not an :meth:`~object.__exit__` - method has actually failed. - -Python defines several context managers to support easy thread synchronisation, -prompt closure of files or other objects, and simpler manipulation of the active -decimal arithmetic context. The specific types are not treated specially beyond -their implementation of the context management protocol. See the -:mod:`contextlib` module for some examples. - -Python's :term:`generator`\s and the :class:`contextlib.contextmanager` decorator -provide a convenient way to implement these protocols. If a generator function is -decorated with the :class:`contextlib.contextmanager` decorator, it will return a -context manager implementing the necessary :meth:`~contextmanager.__enter__` and -:meth:`~contextmanager.__exit__` methods, rather than the iterator produced by an -undecorated generator function. - -Note that there is no specific slot for any of these methods in the type -structure for Python objects in the Python/C API. Extension types wanting to -define these methods must provide them as a normal Python accessible method. -Compared to the overhead of setting up the runtime context, the overhead of a -single class dictionary lookup is negligible. - - -Type Annotation Types --- :ref:`Generic Alias `, :ref:`Union ` -=============================================================================================== - -.. index:: - single: annotation; type annotation; type hint - -The core built-in types for :term:`type annotations ` are -:ref:`Generic Alias ` and :ref:`Union `. - - -.. _types-genericalias: - -Generic Alias Type ------------------- - -.. index:: - pair: object; GenericAlias - pair: Generic; Alias - -``GenericAlias`` objects are generally created by -:ref:`subscripting ` a class. They are most often used with -:ref:`container classes `, such as :class:`list` or -:class:`dict`. For example, ``list[int]`` is a ``GenericAlias`` object created -by subscripting the ``list`` class with the argument :class:`int`. -``GenericAlias`` objects are intended primarily for use with -:term:`type annotations `. - -.. note:: - - It is generally only possible to subscript a class if the class implements - the special method :meth:`~object.__class_getitem__`. - -A ``GenericAlias`` object acts as a proxy for a :term:`generic type`, -implementing *parameterized generics*. - -For a container class, the -argument(s) supplied to a :ref:`subscription ` of the class may -indicate the type(s) of the elements an object contains. For example, -``set[bytes]`` can be used in type annotations to signify a :class:`set` in -which all the elements are of type :class:`bytes`. - -For a class which defines :meth:`~object.__class_getitem__` but is not a -container, the argument(s) supplied to a subscription of the class will often -indicate the return type(s) of one or more methods defined on an object. For -example, :mod:`regular expressions ` can be used on both the :class:`str` data -type and the :class:`bytes` data type: - -* If ``x = re.search('foo', 'foo')``, ``x`` will be a - :ref:`re.Match ` object where the return values of - ``x.group(0)`` and ``x[0]`` will both be of type :class:`str`. We can - represent this kind of object in type annotations with the ``GenericAlias`` - ``re.Match[str]``. - -* If ``y = re.search(b'bar', b'bar')``, (note the ``b`` for :class:`bytes`), - ``y`` will also be an instance of ``re.Match``, but the return - values of ``y.group(0)`` and ``y[0]`` will both be of type - :class:`bytes`. In type annotations, we would represent this - variety of :ref:`re.Match ` objects with ``re.Match[bytes]``. - -``GenericAlias`` objects are instances of the class -:class:`types.GenericAlias`, which can also be used to create ``GenericAlias`` -objects directly. - -.. describe:: T[X, Y, ...] - - Creates a ``GenericAlias`` representing a type ``T`` parameterized by types - *X*, *Y*, and more depending on the ``T`` used. - For example, a function expecting a :class:`list` containing - :class:`float` elements:: - - def average(values: list[float]) -> float: - return sum(values) / len(values) - - Another example for :term:`mapping` objects, using a :class:`dict`, which - is a generic type expecting two type parameters representing the key type - and the value type. In this example, the function expects a ``dict`` with - keys of type :class:`str` and values of type :class:`int`:: - - def send_post_request(url: str, body: dict[str, int]) -> None: - ... - -The builtin functions :func:`isinstance` and :func:`issubclass` do not accept -``GenericAlias`` types for their second argument:: - - >>> isinstance([1, 2], list[str]) - Traceback (most recent call last): - File "", line 1, in - TypeError: isinstance() argument 2 cannot be a parameterized generic - -The Python runtime does not enforce :term:`type annotations `. -This extends to generic types and their type parameters. When creating -a container object from a ``GenericAlias``, the elements in the container are not checked -against their type. For example, the following code is discouraged, but will -run without errors:: - - >>> t = list[str] - >>> t([1, 2, 3]) - [1, 2, 3] - -Furthermore, parameterized generics erase type parameters during object -creation:: - - >>> t = list[str] - >>> type(t) - - - >>> l = t() - >>> type(l) - - -Calling :func:`repr` or :func:`str` on a generic shows the parameterized type:: - - >>> repr(list[int]) - 'list[int]' - - >>> str(list[int]) - 'list[int]' - -The :meth:`~object.__getitem__` method of generic containers will raise an -exception to disallow mistakes like ``dict[str][str]``:: - - >>> dict[str][str] - Traceback (most recent call last): - ... - TypeError: dict[str] is not a generic class - -However, such expressions are valid when :ref:`type variables ` are -used. The index must have as many elements as there are type variable items -in the ``GenericAlias`` object's :attr:`~genericalias.__args__`. :: - - >>> from typing import TypeVar - >>> Y = TypeVar('Y') - >>> dict[str, Y][int] - dict[str, int] - - -Standard Generic Classes -^^^^^^^^^^^^^^^^^^^^^^^^ - -The following standard library classes support parameterized generics. This -list is non-exhaustive. - -* :class:`tuple` -* :class:`list` -* :class:`dict` -* :class:`set` -* :class:`frozenset` -* :class:`type` -* :class:`collections.deque` -* :class:`collections.defaultdict` -* :class:`collections.OrderedDict` -* :class:`collections.Counter` -* :class:`collections.ChainMap` -* :class:`collections.abc.Awaitable` -* :class:`collections.abc.Coroutine` -* :class:`collections.abc.AsyncIterable` -* :class:`collections.abc.AsyncIterator` -* :class:`collections.abc.AsyncGenerator` -* :class:`collections.abc.Iterable` -* :class:`collections.abc.Iterator` -* :class:`collections.abc.Generator` -* :class:`collections.abc.Reversible` -* :class:`collections.abc.Container` -* :class:`collections.abc.Collection` -* :class:`collections.abc.Callable` -* :class:`collections.abc.Set` -* :class:`collections.abc.MutableSet` -* :class:`collections.abc.Mapping` -* :class:`collections.abc.MutableMapping` -* :class:`collections.abc.Sequence` -* :class:`collections.abc.MutableSequence` -* :class:`collections.abc.ByteString` -* :class:`collections.abc.MappingView` -* :class:`collections.abc.KeysView` -* :class:`collections.abc.ItemsView` -* :class:`collections.abc.ValuesView` -* :class:`contextlib.AbstractContextManager` -* :class:`contextlib.AbstractAsyncContextManager` -* :class:`dataclasses.Field` -* :class:`functools.cached_property` -* :class:`functools.partialmethod` -* :class:`os.PathLike` -* :class:`queue.LifoQueue` -* :class:`queue.Queue` -* :class:`queue.PriorityQueue` -* :class:`queue.SimpleQueue` -* :ref:`re.Pattern ` -* :ref:`re.Match ` -* :class:`shelve.BsdDbShelf` -* :class:`shelve.DbfilenameShelf` -* :class:`shelve.Shelf` -* :class:`types.MappingProxyType` -* :class:`weakref.WeakKeyDictionary` -* :class:`weakref.WeakMethod` -* :class:`weakref.WeakSet` -* :class:`weakref.WeakValueDictionary` - - - -Special Attributes of ``GenericAlias`` objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -All parameterized generics implement special read-only attributes. - -.. attribute:: genericalias.__origin__ - - This attribute points at the non-parameterized generic class:: - - >>> list[int].__origin__ - - - -.. attribute:: genericalias.__args__ - - This attribute is a :class:`tuple` (possibly of length 1) of generic - types passed to the original :meth:`~object.__class_getitem__` of the - generic class:: - - >>> dict[str, list[int]].__args__ - (, list[int]) - - -.. attribute:: genericalias.__parameters__ - - This attribute is a lazily computed tuple (possibly empty) of unique type - variables found in ``__args__``:: - - >>> from typing import TypeVar - - >>> T = TypeVar('T') - >>> list[T].__parameters__ - (~T,) - - - .. note:: - A ``GenericAlias`` object with :class:`typing.ParamSpec` parameters may not - have correct ``__parameters__`` after substitution because - :class:`typing.ParamSpec` is intended primarily for static type checking. - - -.. attribute:: genericalias.__unpacked__ - - A boolean that is true if the alias has been unpacked using the - ``*`` operator (see :data:`~typing.TypeVarTuple`). - - .. versionadded:: 3.11 - - -.. seealso:: - - :pep:`484` - Type Hints - Introducing Python's framework for type annotations. - - :pep:`585` - Type Hinting Generics In Standard Collections - Introducing the ability to natively parameterize standard-library - classes, provided they implement the special class method - :meth:`~object.__class_getitem__`. - - :ref:`Generics`, :ref:`user-defined generics ` and :class:`typing.Generic` - Documentation on how to implement generic classes that can be - parameterized at runtime and understood by static type-checkers. - -.. versionadded:: 3.9 - - -.. _types-union: - -Union Type ----------- - -.. index:: - pair: object; Union - pair: union; type - -A union object holds the value of the ``|`` (bitwise or) operation on -multiple :ref:`type objects `. These types are intended -primarily for :term:`type annotations `. The union type expression -enables cleaner type hinting syntax compared to :data:`typing.Union`. - -.. describe:: X | Y | ... - - Defines a union object which holds types *X*, *Y*, and so forth. ``X | Y`` - means either X or Y. It is equivalent to ``typing.Union[X, Y]``. - For example, the following function expects an argument of type - :class:`int` or :class:`float`:: - - def square(number: int | float) -> int | float: - return number ** 2 - - .. note:: - - The ``|`` operand cannot be used at runtime to define unions where one or - more members is a forward reference. For example, ``int | "Foo"``, where - ``"Foo"`` is a reference to a class not yet defined, will fail at - runtime. For unions which include forward references, present the - whole expression as a string, e.g. ``"int | Foo"``. - -.. describe:: union_object == other - - Union objects can be tested for equality with other union objects. Details: - - * Unions of unions are flattened:: - - (int | str) | float == int | str | float - - * Redundant types are removed:: - - int | str | int == int | str - - * When comparing unions, the order is ignored:: - - int | str == str | int - - * It is compatible with :data:`typing.Union`:: - - int | str == typing.Union[int, str] - - * Optional types can be spelled as a union with ``None``:: - - str | None == typing.Optional[str] - -.. describe:: isinstance(obj, union_object) -.. describe:: issubclass(obj, union_object) - - Calls to :func:`isinstance` and :func:`issubclass` are also supported with a - union object:: - - >>> isinstance("", int | str) - True - - However, :ref:`parameterized generics ` in - union objects cannot be checked:: - - >>> isinstance(1, int | list[int]) # short-circuit evaluation - True - >>> isinstance([1], int | list[int]) - Traceback (most recent call last): - ... - TypeError: isinstance() argument 2 cannot be a parameterized generic - -The user-exposed type for the union object can be accessed from -:data:`types.UnionType` and used for :func:`isinstance` checks. An object cannot be -instantiated from the type:: - - >>> import types - >>> isinstance(int | str, types.UnionType) - True - >>> types.UnionType() - Traceback (most recent call last): - File "", line 1, in - TypeError: cannot create 'types.UnionType' instances - -.. note:: - The :meth:`__or__` method for type objects was added to support the syntax - ``X | Y``. If a metaclass implements :meth:`__or__`, the Union may - override it:: - - >>> class M(type): - ... def __or__(self, other): - ... return "Hello" - ... - >>> class C(metaclass=M): - ... pass - ... - >>> C | int - 'Hello' - >>> int | C - int | __main__.C - -.. seealso:: - - :pep:`604` -- PEP proposing the ``X | Y`` syntax and the Union type. - -.. versionadded:: 3.10 - - -.. _typesother: - -Other Built-in Types -==================== - -The interpreter supports several other kinds of objects. Most of these support -only one or two operations. - - -.. _typesmodules: - -Modules -------- - -The only special operation on a module is attribute access: ``m.name``, where -*m* is a module and *name* accesses a name defined in *m*'s symbol table. -Module attributes can be assigned to. (Note that the :keyword:`import` -statement is not, strictly speaking, an operation on a module object; ``import -foo`` does not require a module object named *foo* to exist, rather it requires -an (external) *definition* for a module named *foo* somewhere.) - -A special attribute of every module is :attr:`~object.__dict__`. This is the -dictionary containing the module's symbol table. Modifying this dictionary will -actually change the module's symbol table, but direct assignment to the -:attr:`~object.__dict__` attribute is not possible (you can write -``m.__dict__['a'] = 1``, which defines ``m.a`` to be ``1``, but you can't write -``m.__dict__ = {}``). Modifying :attr:`~object.__dict__` directly is -not recommended. - -Modules built into the interpreter are written like this: ````. If loaded from a file, they are written as ````. - - -.. _typesobjects: - -Classes and Class Instances ---------------------------- - -See :ref:`objects` and :ref:`class` for these. - - -.. _typesfunctions: - -Functions ---------- - -Function objects are created by function definitions. The only operation on a -function object is to call it: ``func(argument-list)``. - -There are really two flavors of function objects: built-in functions and -user-defined functions. Both support the same operation (to call the function), -but the implementation is different, hence the different object types. - -See :ref:`function` for more information. - - -.. _typesmethods: - -Methods -------- - -.. index:: pair: object; method - -Methods are functions that are called using the attribute notation. There are -two flavors: built-in methods (such as :meth:`append` on lists) and class -instance methods. Built-in methods are described with the types that support -them. - -If you access a method (a function defined in a class namespace) through an -instance, you get a special object: a :dfn:`bound method` (also called -:dfn:`instance method`) object. When called, it will add the ``self`` argument -to the argument list. Bound methods have two special read-only attributes: -``m.__self__`` is the object on which the method operates, and ``m.__func__`` is -the function implementing the method. Calling ``m(arg-1, arg-2, ..., arg-n)`` -is completely equivalent to calling ``m.__func__(m.__self__, arg-1, arg-2, ..., -arg-n)``. - -Like function objects, bound method objects support getting arbitrary -attributes. However, since method attributes are actually stored on the -underlying function object (``meth.__func__``), setting method attributes on -bound methods is disallowed. Attempting to set an attribute on a method -results in an :exc:`AttributeError` being raised. In order to set a method -attribute, you need to explicitly set it on the underlying function object:: - - >>> class C: - ... def method(self): - ... pass - ... - >>> c = C() - >>> c.method.whoami = 'my name is method' # can't set on the method - Traceback (most recent call last): - File "", line 1, in - AttributeError: 'method' object has no attribute 'whoami' - >>> c.method.__func__.whoami = 'my name is method' - >>> c.method.whoami - 'my name is method' - -See :ref:`types` for more information. - - -.. index:: object; code, code object - -.. _bltin-code-objects: - -Code Objects ------------- - -.. index:: - pair: built-in function; compile - single: __code__ (function object attribute) - -Code objects are used by the implementation to represent "pseudo-compiled" -executable Python code such as a function body. They differ from function -objects because they don't contain a reference to their global execution -environment. Code objects are returned by the built-in :func:`compile` function -and can be extracted from function objects through their :attr:`__code__` -attribute. See also the :mod:`code` module. - -Accessing ``__code__`` raises an :ref:`auditing event ` -``object.__getattr__`` with arguments ``obj`` and ``"__code__"``. - -.. index:: - pair: built-in function; exec - pair: built-in function; eval - -A code object can be executed or evaluated by passing it (instead of a source -string) to the :func:`exec` or :func:`eval` built-in functions. - -See :ref:`types` for more information. - - -.. _bltin-type-objects: - -Type Objects ------------- - -.. index:: - pair: built-in function; type - pair: module; types - -Type objects represent the various object types. An object's type is accessed -by the built-in function :func:`type`. There are no special operations on -types. The standard module :mod:`types` defines names for all standard built-in -types. - -Types are written like this: ````. - - -.. _bltin-null-object: - -The Null Object ---------------- - -This object is returned by functions that don't explicitly return a value. It -supports no special operations. There is exactly one null object, named -``None`` (a built-in name). ``type(None)()`` produces the same singleton. - -It is written as ``None``. - - -.. index:: single: ...; ellipsis literal -.. _bltin-ellipsis-object: - -The Ellipsis Object -------------------- - -This object is commonly used by slicing (see :ref:`slicings`). It supports no -special operations. There is exactly one ellipsis object, named -:const:`Ellipsis` (a built-in name). ``type(Ellipsis)()`` produces the -:const:`Ellipsis` singleton. - -It is written as ``Ellipsis`` or ``...``. - - -.. _bltin-notimplemented-object: - -The NotImplemented Object -------------------------- - -This object is returned from comparisons and binary operations when they are -asked to operate on types they don't support. See :ref:`comparisons` for more -information. There is exactly one ``NotImplemented`` object. -``type(NotImplemented)()`` produces the singleton instance. - -It is written as ``NotImplemented``. - - -.. _bltin-boolean-values: - -Boolean Values --------------- - -Boolean values are the two constant objects ``False`` and ``True``. They are -used to represent truth values (although other values can also be considered -false or true). In numeric contexts (for example when used as the argument to -an arithmetic operator), they behave like the integers 0 and 1, respectively. -The built-in function :func:`bool` can be used to convert any value to a -Boolean, if the value can be interpreted as a truth value (see section -:ref:`truth` above). - -.. index:: - single: False - single: True - pair: Boolean; values - -They are written as ``False`` and ``True``, respectively. - - -.. _typesinternal: - -Internal Objects ----------------- - -See :ref:`types` for this information. It describes stack frame objects, -traceback objects, and slice objects. - - -.. _specialattrs: - -Special Attributes -================== - -The implementation adds a few special read-only attributes to several object -types, where they are relevant. Some of these are not reported by the -:func:`dir` built-in function. - - -.. attribute:: object.__dict__ - - A dictionary or other mapping object used to store an object's (writable) - attributes. - - -.. attribute:: instance.__class__ - - The class to which a class instance belongs. - - -.. attribute:: class.__bases__ - - The tuple of base classes of a class object. - - -.. attribute:: definition.__name__ - - The name of the class, function, method, descriptor, or - generator instance. - - -.. attribute:: definition.__qualname__ - - The :term:`qualified name` of the class, function, method, descriptor, - or generator instance. - - .. versionadded:: 3.3 - - -.. attribute:: class.__mro__ - - This attribute is a tuple of classes that are considered when looking for - base classes during method resolution. - - -.. method:: class.mro() - - This method can be overridden by a metaclass to customize the method - resolution order for its instances. It is called at class instantiation, and - its result is stored in :attr:`~class.__mro__`. - - -.. method:: class.__subclasses__ - - Each class keeps a list of weak references to its immediate subclasses. This - method returns a list of all those references still alive. The list is in - definition order. Example:: - - >>> int.__subclasses__() - [, , , ] - - -.. _int_max_str_digits: - -Integer string conversion length limitation -=========================================== - -CPython has a global limit for converting between :class:`int` and :class:`str` -to mitigate denial of service attacks. This limit *only* applies to decimal or -other non-power-of-two number bases. Hexadecimal, octal, and binary conversions -are unlimited. The limit can be configured. - -The :class:`int` type in CPython is an arbitrary length number stored in binary -form (commonly known as a "bignum"). There exists no algorithm that can convert -a string to a binary integer or a binary integer to a string in linear time, -*unless* the base is a power of 2. Even the best known algorithms for base 10 -have sub-quadratic complexity. Converting a large value such as ``int('1' * -500_000)`` can take over a second on a fast CPU. - -Limiting conversion size offers a practical way to avoid `CVE-2020-10735 -`_. - -The limit is applied to the number of digit characters in the input or output -string when a non-linear conversion algorithm would be involved. Underscores -and the sign are not counted towards the limit. - -When an operation would exceed the limit, a :exc:`ValueError` is raised: - -.. doctest:: - - >>> import sys - >>> sys.set_int_max_str_digits(4300) # Illustrative, this is the default. - >>> _ = int('2' * 5432) - Traceback (most recent call last): - ... - ValueError: Exceeds the limit (4300 digits) for integer string conversion: value has 5432 digits; use sys.set_int_max_str_digits() to increase the limit - >>> i = int('2' * 4300) - >>> len(str(i)) - 4300 - >>> i_squared = i*i - >>> len(str(i_squared)) - Traceback (most recent call last): - ... - ValueError: Exceeds the limit (4300 digits) for integer string conversion; use sys.set_int_max_str_digits() to increase the limit - >>> len(hex(i_squared)) - 7144 - >>> assert int(hex(i_squared), base=16) == i*i # Hexadecimal is unlimited. - -The default limit is 4300 digits as provided in -:data:`sys.int_info.default_max_str_digits `. -The lowest limit that can be configured is 640 digits as provided in -:data:`sys.int_info.str_digits_check_threshold `. - -Verification: - -.. doctest:: - - >>> import sys - >>> assert sys.int_info.default_max_str_digits == 4300, sys.int_info - >>> assert sys.int_info.str_digits_check_threshold == 640, sys.int_info - >>> msg = int('578966293710682886880994035146873798396722250538762761564' - ... '9252925514383915483333812743580549779436104706260696366600' - ... '571186405732').to_bytes(53, 'big') - ... - -.. versionadded:: 3.11 - -Affected APIs -------------- - -The limitation only applies to potentially slow conversions between :class:`int` -and :class:`str` or :class:`bytes`: - -* ``int(string)`` with default base 10. -* ``int(string, base)`` for all bases that are not a power of 2. -* ``str(integer)``. -* ``repr(integer)``. -* any other string conversion to base 10, for example ``f"{integer}"``, - ``"{}".format(integer)``, or ``b"%d" % integer``. - -The limitations do not apply to functions with a linear algorithm: - -* ``int(string, base)`` with base 2, 4, 8, 16, or 32. -* :func:`int.from_bytes` and :func:`int.to_bytes`. -* :func:`hex`, :func:`oct`, :func:`bin`. -* :ref:`formatspec` for hex, octal, and binary numbers. -* :class:`str` to :class:`float`. -* :class:`str` to :class:`decimal.Decimal`. - -Configuring the limit ---------------------- - -Before Python starts up you can use an environment variable or an interpreter -command line flag to configure the limit: - -* :envvar:`PYTHONINTMAXSTRDIGITS`, e.g. - ``PYTHONINTMAXSTRDIGITS=640 python3`` to set the limit to 640 or - ``PYTHONINTMAXSTRDIGITS=0 python3`` to disable the limitation. -* :option:`-X int_max_str_digits <-X>`, e.g. - ``python3 -X int_max_str_digits=640`` -* :data:`sys.flags.int_max_str_digits` contains the value of - :envvar:`PYTHONINTMAXSTRDIGITS` or :option:`-X int_max_str_digits <-X>`. - If both the env var and the ``-X`` option are set, the ``-X`` option takes - precedence. A value of *-1* indicates that both were unset, thus a value of - :data:`sys.int_info.default_max_str_digits` was used during initialization. - -From code, you can inspect the current limit and set a new one using these -:mod:`sys` APIs: - -* :func:`sys.get_int_max_str_digits` and :func:`sys.set_int_max_str_digits` are - a getter and setter for the interpreter-wide limit. Subinterpreters have - their own limit. - -Information about the default and minimum can be found in :data:`sys.int_info`: - -* :data:`sys.int_info.default_max_str_digits ` is the compiled-in - default limit. -* :data:`sys.int_info.str_digits_check_threshold ` is the lowest - accepted value for the limit (other than 0 which disables it). - -.. versionadded:: 3.11 - -.. caution:: - - Setting a low limit *can* lead to problems. While rare, code exists that - contains integer constants in decimal in their source that exceed the - minimum threshold. A consequence of setting the limit is that Python source - code containing decimal integer literals longer than the limit will - encounter an error during parsing, usually at startup time or import time or - even at installation time - anytime an up to date ``.pyc`` does not already - exist for the code. A workaround for source that contains such large - constants is to convert them to ``0x`` hexadecimal form as it has no limit. - - Test your application thoroughly if you use a low limit. Ensure your tests - run with the limit set early via the environment or flag so that it applies - during startup and even during any installation step that may invoke Python - to precompile ``.py`` sources to ``.pyc`` files. - -Recommended configuration -------------------------- - -The default :data:`sys.int_info.default_max_str_digits` is expected to be -reasonable for most applications. If your application requires a different -limit, set it from your main entry point using Python version agnostic code as -these APIs were added in security patch releases in versions before 3.11. - -Example:: - - >>> import sys - >>> if hasattr(sys, "set_int_max_str_digits"): - ... upper_bound = 68000 - ... lower_bound = 4004 - ... current_limit = sys.get_int_max_str_digits() - ... if current_limit == 0 or current_limit > upper_bound: - ... sys.set_int_max_str_digits(upper_bound) - ... elif current_limit < lower_bound: - ... sys.set_int_max_str_digits(lower_bound) - -If you need to disable it entirely, set it to ``0``. - - -.. rubric:: Footnotes - -.. [1] Additional information on these special methods may be found in the Python - Reference Manual (:ref:`customization`). - -.. [2] As a consequence, the list ``[1, 2]`` is considered equal to ``[1.0, 2.0]``, and - similarly for tuples. - -.. [3] They must have since the parser can't tell the type of the operands. - -.. [4] Cased characters are those with general category property being one of - "Lu" (Letter, uppercase), "Ll" (Letter, lowercase), or "Lt" (Letter, titlecase). - -.. [5] To format only a tuple you should therefore provide a singleton tuple whose only - element is the tuple to be formatted. diff --git a/Doc/library/string.rst b/Doc/library/string.rst deleted file mode 100644 index 5fbd82a5ec852e..00000000000000 --- a/Doc/library/string.rst +++ /dev/null @@ -1,915 +0,0 @@ -:mod:`string` --- Common string operations -========================================== - -.. module:: string - :synopsis: Common string operations. - -**Source code:** :source:`Lib/string.py` - --------------- - - -.. seealso:: - - :ref:`textseq` - - :ref:`string-methods` - -String constants ----------------- - -The constants defined in this module are: - - -.. data:: ascii_letters - - The concatenation of the :const:`ascii_lowercase` and :const:`ascii_uppercase` - constants described below. This value is not locale-dependent. - - -.. data:: ascii_lowercase - - The lowercase letters ``'abcdefghijklmnopqrstuvwxyz'``. This value is not - locale-dependent and will not change. - - -.. data:: ascii_uppercase - - The uppercase letters ``'ABCDEFGHIJKLMNOPQRSTUVWXYZ'``. This value is not - locale-dependent and will not change. - - -.. data:: digits - - The string ``'0123456789'``. - - -.. data:: hexdigits - - The string ``'0123456789abcdefABCDEF'``. - - -.. data:: octdigits - - The string ``'01234567'``. - - -.. data:: punctuation - - String of ASCII characters which are considered punctuation characters - in the ``C`` locale: ``!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~``. - -.. data:: printable - - String of ASCII characters which are considered printable. This is a - combination of :const:`digits`, :const:`ascii_letters`, :const:`punctuation`, - and :const:`whitespace`. - - -.. data:: whitespace - - A string containing all ASCII characters that are considered whitespace. - This includes the characters space, tab, linefeed, return, formfeed, and - vertical tab. - - -.. _string-formatting: - -Custom String Formatting ------------------------- - -The built-in string class provides the ability to do complex variable -substitutions and value formatting via the :meth:`~str.format` method described in -:pep:`3101`. The :class:`Formatter` class in the :mod:`string` module allows -you to create and customize your own string formatting behaviors using the same -implementation as the built-in :meth:`~str.format` method. - - -.. class:: Formatter - - The :class:`Formatter` class has the following public methods: - - .. method:: format(format_string, /, *args, **kwargs) - - The primary API method. It takes a format string and - an arbitrary set of positional and keyword arguments. - It is just a wrapper that calls :meth:`vformat`. - - .. versionchanged:: 3.7 - A format string argument is now :ref:`positional-only - `. - - .. method:: vformat(format_string, args, kwargs) - - This function does the actual work of formatting. It is exposed as a - separate function for cases where you want to pass in a predefined - dictionary of arguments, rather than unpacking and repacking the - dictionary as individual arguments using the ``*args`` and ``**kwargs`` - syntax. :meth:`vformat` does the work of breaking up the format string - into character data and replacement fields. It calls the various - methods described below. - - In addition, the :class:`Formatter` defines a number of methods that are - intended to be replaced by subclasses: - - .. method:: parse(format_string) - - Loop over the format_string and return an iterable of tuples - (*literal_text*, *field_name*, *format_spec*, *conversion*). This is used - by :meth:`vformat` to break the string into either literal text, or - replacement fields. - - The values in the tuple conceptually represent a span of literal text - followed by a single replacement field. If there is no literal text - (which can happen if two replacement fields occur consecutively), then - *literal_text* will be a zero-length string. If there is no replacement - field, then the values of *field_name*, *format_spec* and *conversion* - will be ``None``. - - .. method:: get_field(field_name, args, kwargs) - - Given *field_name* as returned by :meth:`parse` (see above), convert it to - an object to be formatted. Returns a tuple (obj, used_key). The default - version takes strings of the form defined in :pep:`3101`, such as - "0[name]" or "label.title". *args* and *kwargs* are as passed in to - :meth:`vformat`. The return value *used_key* has the same meaning as the - *key* parameter to :meth:`get_value`. - - .. method:: get_value(key, args, kwargs) - - Retrieve a given field value. The *key* argument will be either an - integer or a string. If it is an integer, it represents the index of the - positional argument in *args*; if it is a string, then it represents a - named argument in *kwargs*. - - The *args* parameter is set to the list of positional arguments to - :meth:`vformat`, and the *kwargs* parameter is set to the dictionary of - keyword arguments. - - For compound field names, these functions are only called for the first - component of the field name; subsequent components are handled through - normal attribute and indexing operations. - - So for example, the field expression '0.name' would cause - :meth:`get_value` to be called with a *key* argument of 0. The ``name`` - attribute will be looked up after :meth:`get_value` returns by calling the - built-in :func:`getattr` function. - - If the index or keyword refers to an item that does not exist, then an - :exc:`IndexError` or :exc:`KeyError` should be raised. - - .. method:: check_unused_args(used_args, args, kwargs) - - Implement checking for unused arguments if desired. The arguments to this - function is the set of all argument keys that were actually referred to in - the format string (integers for positional arguments, and strings for - named arguments), and a reference to the *args* and *kwargs* that was - passed to vformat. The set of unused args can be calculated from these - parameters. :meth:`check_unused_args` is assumed to raise an exception if - the check fails. - - .. method:: format_field(value, format_spec) - - :meth:`format_field` simply calls the global :func:`format` built-in. The - method is provided so that subclasses can override it. - - .. method:: convert_field(value, conversion) - - Converts the value (returned by :meth:`get_field`) given a conversion type - (as in the tuple returned by the :meth:`parse` method). The default - version understands 's' (str), 'r' (repr) and 'a' (ascii) conversion - types. - - -.. _formatstrings: - -Format String Syntax --------------------- - -The :meth:`str.format` method and the :class:`Formatter` class share the same -syntax for format strings (although in the case of :class:`Formatter`, -subclasses can define their own format string syntax). The syntax is -related to that of :ref:`formatted string literals `, but it is -less sophisticated and, in particular, does not support arbitrary expressions. - -.. index:: - single: {} (curly brackets); in string formatting - single: . (dot); in string formatting - single: [] (square brackets); in string formatting - single: ! (exclamation); in string formatting - single: : (colon); in string formatting - -Format strings contain "replacement fields" surrounded by curly braces ``{}``. -Anything that is not contained in braces is considered literal text, which is -copied unchanged to the output. If you need to include a brace character in the -literal text, it can be escaped by doubling: ``{{`` and ``}}``. - -The grammar for a replacement field is as follows: - -.. productionlist:: format-string - replacement_field: "{" [`field_name`] ["!" `conversion`] [":" `format_spec`] "}" - field_name: arg_name ("." `attribute_name` | "[" `element_index` "]")* - arg_name: [`identifier` | `digit`+] - attribute_name: `identifier` - element_index: `digit`+ | `index_string` - index_string: + - conversion: "r" | "s" | "a" - format_spec: - -In less formal terms, the replacement field can start with a *field_name* that specifies -the object whose value is to be formatted and inserted -into the output instead of the replacement field. -The *field_name* is optionally followed by a *conversion* field, which is -preceded by an exclamation point ``'!'``, and a *format_spec*, which is preceded -by a colon ``':'``. These specify a non-default format for the replacement value. - -See also the :ref:`formatspec` section. - -The *field_name* itself begins with an *arg_name* that is either a number or a -keyword. If it's a number, it refers to a positional argument, and if it's a keyword, -it refers to a named keyword argument. An *arg_name* is treated as a number if -a call to :meth:`str.isdecimal` on the string would return true. -If the numerical arg_names in a format string -are 0, 1, 2, ... in sequence, they can all be omitted (not just some) -and the numbers 0, 1, 2, ... will be automatically inserted in that order. -Because *arg_name* is not quote-delimited, it is not possible to specify arbitrary -dictionary keys (e.g., the strings ``'10'`` or ``':-]'``) within a format string. -The *arg_name* can be followed by any number of index or -attribute expressions. An expression of the form ``'.name'`` selects the named -attribute using :func:`getattr`, while an expression of the form ``'[index]'`` -does an index lookup using :func:`__getitem__`. - -.. versionchanged:: 3.1 - The positional argument specifiers can be omitted for :meth:`str.format`, - so ``'{} {}'.format(a, b)`` is equivalent to ``'{0} {1}'.format(a, b)``. - -.. versionchanged:: 3.4 - The positional argument specifiers can be omitted for :class:`Formatter`. - -Some simple format string examples:: - - "First, thou shalt count to {0}" # References first positional argument - "Bring me a {}" # Implicitly references the first positional argument - "From {} to {}" # Same as "From {0} to {1}" - "My quest is {name}" # References keyword argument 'name' - "Weight in tons {0.weight}" # 'weight' attribute of first positional arg - "Units destroyed: {players[0]}" # First element of keyword argument 'players'. - -The *conversion* field causes a type coercion before formatting. Normally, the -job of formatting a value is done by the :meth:`__format__` method of the value -itself. However, in some cases it is desirable to force a type to be formatted -as a string, overriding its own definition of formatting. By converting the -value to a string before calling :meth:`__format__`, the normal formatting logic -is bypassed. - -Three conversion flags are currently supported: ``'!s'`` which calls :func:`str` -on the value, ``'!r'`` which calls :func:`repr` and ``'!a'`` which calls -:func:`ascii`. - -Some examples:: - - "Harold's a clever {0!s}" # Calls str() on the argument first - "Bring out the holy {name!r}" # Calls repr() on the argument first - "More {!a}" # Calls ascii() on the argument first - -The *format_spec* field contains a specification of how the value should be -presented, including such details as field width, alignment, padding, decimal -precision and so on. Each value type can define its own "formatting -mini-language" or interpretation of the *format_spec*. - -Most built-in types support a common formatting mini-language, which is -described in the next section. - -A *format_spec* field can also include nested replacement fields within it. -These nested replacement fields may contain a field name, conversion flag -and format specification, but deeper nesting is -not allowed. The replacement fields within the -format_spec are substituted before the *format_spec* string is interpreted. -This allows the formatting of a value to be dynamically specified. - -See the :ref:`formatexamples` section for some examples. - - -.. _formatspec: - -Format Specification Mini-Language -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -"Format specifications" are used within replacement fields contained within a -format string to define how individual values are presented (see -:ref:`formatstrings` and :ref:`f-strings`). -They can also be passed directly to the built-in -:func:`format` function. Each formattable type may define how the format -specification is to be interpreted. - -Most built-in types implement the following options for format specifications, -although some of the formatting options are only supported by the numeric types. - -A general convention is that an empty format specification produces -the same result as if you had called :func:`str` on the value. A -non-empty format specification typically modifies the result. - -The general form of a *standard format specifier* is: - -.. productionlist:: format-spec - format_spec: [[`fill`]`align`][`sign`]["z"]["#"]["0"][`width`][`grouping_option`]["." `precision`][`type`] - fill: - align: "<" | ">" | "=" | "^" - sign: "+" | "-" | " " - width: `digit`+ - grouping_option: "_" | "," - precision: `digit`+ - type: "b" | "c" | "d" | "e" | "E" | "f" | "F" | "g" | "G" | "n" | "o" | "s" | "x" | "X" | "%" - -If a valid *align* value is specified, it can be preceded by a *fill* -character that can be any character and defaults to a space if omitted. -It is not possible to use a literal curly brace ("``{``" or "``}``") as -the *fill* character in a :ref:`formatted string literal -` or when using the :meth:`str.format` -method. However, it is possible to insert a curly brace -with a nested replacement field. This limitation doesn't -affect the :func:`format` function. - -The meaning of the various alignment options is as follows: - -.. index:: - single: < (less); in string formatting - single: > (greater); in string formatting - single: = (equals); in string formatting - single: ^ (caret); in string formatting - -+---------+----------------------------------------------------------+ -| Option | Meaning | -+=========+==========================================================+ -| ``'<'`` | Forces the field to be left-aligned within the available | -| | space (this is the default for most objects). | -+---------+----------------------------------------------------------+ -| ``'>'`` | Forces the field to be right-aligned within the | -| | available space (this is the default for numbers). | -+---------+----------------------------------------------------------+ -| ``'='`` | Forces the padding to be placed after the sign (if any) | -| | but before the digits. This is used for printing fields | -| | in the form '+000000120'. This alignment option is only | -| | valid for numeric types. It becomes the default for | -| | numbers when '0' immediately precedes the field width. | -+---------+----------------------------------------------------------+ -| ``'^'`` | Forces the field to be centered within the available | -| | space. | -+---------+----------------------------------------------------------+ - -Note that unless a minimum field width is defined, the field width will always -be the same size as the data to fill it, so that the alignment option has no -meaning in this case. - -The *sign* option is only valid for number types, and can be one of the -following: - -.. index:: - single: + (plus); in string formatting - single: - (minus); in string formatting - single: space; in string formatting - -+---------+----------------------------------------------------------+ -| Option | Meaning | -+=========+==========================================================+ -| ``'+'`` | indicates that a sign should be used for both | -| | positive as well as negative numbers. | -+---------+----------------------------------------------------------+ -| ``'-'`` | indicates that a sign should be used only for negative | -| | numbers (this is the default behavior). | -+---------+----------------------------------------------------------+ -| space | indicates that a leading space should be used on | -| | positive numbers, and a minus sign on negative numbers. | -+---------+----------------------------------------------------------+ - - -.. index:: single: z; in string formatting - -The ``'z'`` option coerces negative zero floating-point values to positive -zero after rounding to the format precision. This option is only valid for -floating-point presentation types. - -.. versionchanged:: 3.11 - Added the ``'z'`` option (see also :pep:`682`). - -.. index:: single: # (hash); in string formatting - -The ``'#'`` option causes the "alternate form" to be used for the -conversion. The alternate form is defined differently for different -types. This option is only valid for integer, float and complex -types. For integers, when binary, octal, or hexadecimal output -is used, this option adds the respective prefix ``'0b'``, ``'0o'``, -``'0x'``, or ``'0X'`` to the output value. For float and complex the -alternate form causes the result of the conversion to always contain a -decimal-point character, even if no digits follow it. Normally, a -decimal-point character appears in the result of these conversions -only if a digit follows it. In addition, for ``'g'`` and ``'G'`` -conversions, trailing zeros are not removed from the result. - -.. index:: single: , (comma); in string formatting - -The ``','`` option signals the use of a comma for a thousands separator. -For a locale aware separator, use the ``'n'`` integer presentation type -instead. - -.. versionchanged:: 3.1 - Added the ``','`` option (see also :pep:`378`). - -.. index:: single: _ (underscore); in string formatting - -The ``'_'`` option signals the use of an underscore for a thousands -separator for floating point presentation types and for integer -presentation type ``'d'``. For integer presentation types ``'b'``, -``'o'``, ``'x'``, and ``'X'``, underscores will be inserted every 4 -digits. For other presentation types, specifying this option is an -error. - -.. versionchanged:: 3.6 - Added the ``'_'`` option (see also :pep:`515`). - -*width* is a decimal integer defining the minimum total field width, -including any prefixes, separators, and other formatting characters. -If not specified, then the field width will be determined by the content. - -When no explicit alignment is given, preceding the *width* field by a zero -(``'0'``) character enables -sign-aware zero-padding for numeric types. This is equivalent to a *fill* -character of ``'0'`` with an *alignment* type of ``'='``. - -.. versionchanged:: 3.10 - Preceding the *width* field by ``'0'`` no longer affects the default - alignment for strings. - -The *precision* is a decimal integer indicating how many digits should be -displayed after the decimal point for presentation types -``'f'`` and ``'F'``, or before and after the decimal point for presentation -types ``'g'`` or ``'G'``. For string presentation types the field -indicates the maximum field size - in other words, how many characters will be -used from the field content. The *precision* is not allowed for integer -presentation types. - -Finally, the *type* determines how the data should be presented. - -The available string presentation types are: - - +---------+----------------------------------------------------------+ - | Type | Meaning | - +=========+==========================================================+ - | ``'s'`` | String format. This is the default type for strings and | - | | may be omitted. | - +---------+----------------------------------------------------------+ - | None | The same as ``'s'``. | - +---------+----------------------------------------------------------+ - -The available integer presentation types are: - - +---------+----------------------------------------------------------+ - | Type | Meaning | - +=========+==========================================================+ - | ``'b'`` | Binary format. Outputs the number in base 2. | - +---------+----------------------------------------------------------+ - | ``'c'`` | Character. Converts the integer to the corresponding | - | | unicode character before printing. | - +---------+----------------------------------------------------------+ - | ``'d'`` | Decimal Integer. Outputs the number in base 10. | - +---------+----------------------------------------------------------+ - | ``'o'`` | Octal format. Outputs the number in base 8. | - +---------+----------------------------------------------------------+ - | ``'x'`` | Hex format. Outputs the number in base 16, using | - | | lower-case letters for the digits above 9. | - +---------+----------------------------------------------------------+ - | ``'X'`` | Hex format. Outputs the number in base 16, using | - | | upper-case letters for the digits above 9. | - | | In case ``'#'`` is specified, the prefix ``'0x'`` will | - | | be upper-cased to ``'0X'`` as well. | - +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'d'``, except that it uses | - | | the current locale setting to insert the appropriate | - | | number separator characters. | - +---------+----------------------------------------------------------+ - | None | The same as ``'d'``. | - +---------+----------------------------------------------------------+ - -In addition to the above presentation types, integers can be formatted -with the floating point presentation types listed below (except -``'n'`` and ``None``). When doing so, :func:`float` is used to convert the -integer to a floating point number before formatting. - -The available presentation types for :class:`float` and -:class:`~decimal.Decimal` values are: - - +---------+----------------------------------------------------------+ - | Type | Meaning | - +=========+==========================================================+ - | ``'e'`` | Scientific notation. For a given precision ``p``, | - | | formats the number in scientific notation with the | - | | letter 'e' separating the coefficient from the exponent. | - | | The coefficient has one digit before and ``p`` digits | - | | after the decimal point, for a total of ``p + 1`` | - | | significant digits. With no precision given, uses a | - | | precision of ``6`` digits after the decimal point for | - | | :class:`float`, and shows all coefficient digits | - | | for :class:`~decimal.Decimal`. If no digits follow the | - | | decimal point, the decimal point is also removed unless | - | | the ``#`` option is used. | - +---------+----------------------------------------------------------+ - | ``'E'`` | Scientific notation. Same as ``'e'`` except it uses | - | | an upper case 'E' as the separator character. | - +---------+----------------------------------------------------------+ - | ``'f'`` | Fixed-point notation. For a given precision ``p``, | - | | formats the number as a decimal number with exactly | - | | ``p`` digits following the decimal point. With no | - | | precision given, uses a precision of ``6`` digits after | - | | the decimal point for :class:`float`, and uses a | - | | precision large enough to show all coefficient digits | - | | for :class:`~decimal.Decimal`. If no digits follow the | - | | decimal point, the decimal point is also removed unless | - | | the ``#`` option is used. | - +---------+----------------------------------------------------------+ - | ``'F'`` | Fixed-point notation. Same as ``'f'``, but converts | - | | ``nan`` to ``NAN`` and ``inf`` to ``INF``. | - +---------+----------------------------------------------------------+ - | ``'g'`` | General format. For a given precision ``p >= 1``, | - | | this rounds the number to ``p`` significant digits and | - | | then formats the result in either fixed-point format | - | | or in scientific notation, depending on its magnitude. | - | | A precision of ``0`` is treated as equivalent to a | - | | precision of ``1``. | - | | | - | | The precise rules are as follows: suppose that the | - | | result formatted with presentation type ``'e'`` and | - | | precision ``p-1`` would have exponent ``exp``. Then, | - | | if ``m <= exp < p``, where ``m`` is -4 for floats and -6 | - | | for :class:`Decimals `, the number is | - | | formatted with presentation type ``'f'`` and precision | - | | ``p-1-exp``. Otherwise, the number is formatted | - | | with presentation type ``'e'`` and precision ``p-1``. | - | | In both cases insignificant trailing zeros are removed | - | | from the significand, and the decimal point is also | - | | removed if there are no remaining digits following it, | - | | unless the ``'#'`` option is used. | - | | | - | | With no precision given, uses a precision of ``6`` | - | | significant digits for :class:`float`. For | - | | :class:`~decimal.Decimal`, the coefficient of the result | - | | is formed from the coefficient digits of the value; | - | | scientific notation is used for values smaller than | - | | ``1e-6`` in absolute value and values where the place | - | | value of the least significant digit is larger than 1, | - | | and fixed-point notation is used otherwise. | - | | | - | | Positive and negative infinity, positive and negative | - | | zero, and nans, are formatted as ``inf``, ``-inf``, | - | | ``0``, ``-0`` and ``nan`` respectively, regardless of | - | | the precision. | - +---------+----------------------------------------------------------+ - | ``'G'`` | General format. Same as ``'g'`` except switches to | - | | ``'E'`` if the number gets too large. The | - | | representations of infinity and NaN are uppercased, too. | - +---------+----------------------------------------------------------+ - | ``'n'`` | Number. This is the same as ``'g'``, except that it uses | - | | the current locale setting to insert the appropriate | - | | number separator characters. | - +---------+----------------------------------------------------------+ - | ``'%'`` | Percentage. Multiplies the number by 100 and displays | - | | in fixed (``'f'``) format, followed by a percent sign. | - +---------+----------------------------------------------------------+ - | None | For :class:`float` this is the same as ``'g'``, except | - | | that when fixed-point notation is used to format the | - | | result, it always includes at least one digit past the | - | | decimal point. The precision used is as large as needed | - | | to represent the given value faithfully. | - | | | - | | For :class:`~decimal.Decimal`, this is the same as | - | | either ``'g'`` or ``'G'`` depending on the value of | - | | ``context.capitals`` for the current decimal context. | - | | | - | | The overall effect is to match the output of :func:`str` | - | | as altered by the other format modifiers. | - +---------+----------------------------------------------------------+ - - -.. _formatexamples: - -Format examples -^^^^^^^^^^^^^^^ - -This section contains examples of the :meth:`str.format` syntax and -comparison with the old ``%``-formatting. - -In most of the cases the syntax is similar to the old ``%``-formatting, with the -addition of the ``{}`` and with ``:`` used instead of ``%``. -For example, ``'%03.2f'`` can be translated to ``'{:03.2f}'``. - -The new format syntax also supports new and different options, shown in the -following examples. - -Accessing arguments by position:: - - >>> '{0}, {1}, {2}'.format('a', 'b', 'c') - 'a, b, c' - >>> '{}, {}, {}'.format('a', 'b', 'c') # 3.1+ only - 'a, b, c' - >>> '{2}, {1}, {0}'.format('a', 'b', 'c') - 'c, b, a' - >>> '{2}, {1}, {0}'.format(*'abc') # unpacking argument sequence - 'c, b, a' - >>> '{0}{1}{0}'.format('abra', 'cad') # arguments' indices can be repeated - 'abracadabra' - -Accessing arguments by name:: - - >>> 'Coordinates: {latitude}, {longitude}'.format(latitude='37.24N', longitude='-115.81W') - 'Coordinates: 37.24N, -115.81W' - >>> coord = {'latitude': '37.24N', 'longitude': '-115.81W'} - >>> 'Coordinates: {latitude}, {longitude}'.format(**coord) - 'Coordinates: 37.24N, -115.81W' - -Accessing arguments' attributes:: - - >>> c = 3-5j - >>> ('The complex number {0} is formed from the real part {0.real} ' - ... 'and the imaginary part {0.imag}.').format(c) - 'The complex number (3-5j) is formed from the real part 3.0 and the imaginary part -5.0.' - >>> class Point: - ... def __init__(self, x, y): - ... self.x, self.y = x, y - ... def __str__(self): - ... return 'Point({self.x}, {self.y})'.format(self=self) - ... - >>> str(Point(4, 2)) - 'Point(4, 2)' - -Accessing arguments' items:: - - >>> coord = (3, 5) - >>> 'X: {0[0]}; Y: {0[1]}'.format(coord) - 'X: 3; Y: 5' - -Replacing ``%s`` and ``%r``:: - - >>> "repr() shows quotes: {!r}; str() doesn't: {!s}".format('test1', 'test2') - "repr() shows quotes: 'test1'; str() doesn't: test2" - -Aligning the text and specifying a width:: - - >>> '{:<30}'.format('left aligned') - 'left aligned ' - >>> '{:>30}'.format('right aligned') - ' right aligned' - >>> '{:^30}'.format('centered') - ' centered ' - >>> '{:*^30}'.format('centered') # use '*' as a fill char - '***********centered***********' - -Replacing ``%+f``, ``%-f``, and ``% f`` and specifying a sign:: - - >>> '{:+f}; {:+f}'.format(3.14, -3.14) # show it always - '+3.140000; -3.140000' - >>> '{: f}; {: f}'.format(3.14, -3.14) # show a space for positive numbers - ' 3.140000; -3.140000' - >>> '{:-f}; {:-f}'.format(3.14, -3.14) # show only the minus -- same as '{:f}; {:f}' - '3.140000; -3.140000' - -Replacing ``%x`` and ``%o`` and converting the value to different bases:: - - >>> # format also supports binary numbers - >>> "int: {0:d}; hex: {0:x}; oct: {0:o}; bin: {0:b}".format(42) - 'int: 42; hex: 2a; oct: 52; bin: 101010' - >>> # with 0x, 0o, or 0b as prefix: - >>> "int: {0:d}; hex: {0:#x}; oct: {0:#o}; bin: {0:#b}".format(42) - 'int: 42; hex: 0x2a; oct: 0o52; bin: 0b101010' - -Using the comma as a thousands separator:: - - >>> '{:,}'.format(1234567890) - '1,234,567,890' - -Expressing a percentage:: - - >>> points = 19 - >>> total = 22 - >>> 'Correct answers: {:.2%}'.format(points/total) - 'Correct answers: 86.36%' - -Using type-specific formatting:: - - >>> import datetime - >>> d = datetime.datetime(2010, 7, 4, 12, 15, 58) - >>> '{:%Y-%m-%d %H:%M:%S}'.format(d) - '2010-07-04 12:15:58' - -Nesting arguments and more complex examples:: - - >>> for align, text in zip('<^>', ['left', 'center', 'right']): - ... '{0:{fill}{align}16}'.format(text, fill=align, align=align) - ... - 'left<<<<<<<<<<<<' - '^^^^^center^^^^^' - '>>>>>>>>>>>right' - >>> - >>> octets = [192, 168, 0, 1] - >>> '{:02X}{:02X}{:02X}{:02X}'.format(*octets) - 'C0A80001' - >>> int(_, 16) - 3232235521 - >>> - >>> width = 5 - >>> for num in range(5,12): #doctest: +NORMALIZE_WHITESPACE - ... for base in 'dXob': - ... print('{0:{width}{base}}'.format(num, base=base, width=width), end=' ') - ... print() - ... - 5 5 5 101 - 6 6 6 110 - 7 7 7 111 - 8 8 10 1000 - 9 9 11 1001 - 10 A 12 1010 - 11 B 13 1011 - - - -.. _template-strings: - -Template strings ----------------- - -Template strings provide simpler string substitutions as described in -:pep:`292`. A primary use case for template strings is for -internationalization (i18n) since in that context, the simpler syntax and -functionality makes it easier to translate than other built-in string -formatting facilities in Python. As an example of a library built on template -strings for i18n, see the -`flufl.i18n `_ package. - -.. index:: single: $ (dollar); in template strings - -Template strings support ``$``-based substitutions, using the following rules: - -* ``$$`` is an escape; it is replaced with a single ``$``. - -* ``$identifier`` names a substitution placeholder matching a mapping key of - ``"identifier"``. By default, ``"identifier"`` is restricted to any - case-insensitive ASCII alphanumeric string (including underscores) that - starts with an underscore or ASCII letter. The first non-identifier - character after the ``$`` character terminates this placeholder - specification. - -* ``${identifier}`` is equivalent to ``$identifier``. It is required when - valid identifier characters follow the placeholder but are not part of the - placeholder, such as ``"${noun}ification"``. - -Any other appearance of ``$`` in the string will result in a :exc:`ValueError` -being raised. - -The :mod:`string` module provides a :class:`Template` class that implements -these rules. The methods of :class:`Template` are: - - -.. class:: Template(template) - - The constructor takes a single argument which is the template string. - - - .. method:: substitute(mapping={}, /, **kwds) - - Performs the template substitution, returning a new string. *mapping* is - any dictionary-like object with keys that match the placeholders in the - template. Alternatively, you can provide keyword arguments, where the - keywords are the placeholders. When both *mapping* and *kwds* are given - and there are duplicates, the placeholders from *kwds* take precedence. - - - .. method:: safe_substitute(mapping={}, /, **kwds) - - Like :meth:`substitute`, except that if placeholders are missing from - *mapping* and *kwds*, instead of raising a :exc:`KeyError` exception, the - original placeholder will appear in the resulting string intact. Also, - unlike with :meth:`substitute`, any other appearances of the ``$`` will - simply return ``$`` instead of raising :exc:`ValueError`. - - While other exceptions may still occur, this method is called "safe" - because it always tries to return a usable string instead of - raising an exception. In another sense, :meth:`safe_substitute` may be - anything other than safe, since it will silently ignore malformed - templates containing dangling delimiters, unmatched braces, or - placeholders that are not valid Python identifiers. - - - .. method:: is_valid() - - Returns false if the template has invalid placeholders that will cause - :meth:`substitute` to raise :exc:`ValueError`. - - .. versionadded:: 3.11 - - - .. method:: get_identifiers() - - Returns a list of the valid identifiers in the template, in the order - they first appear, ignoring any invalid identifiers. - - .. versionadded:: 3.11 - - :class:`Template` instances also provide one public data attribute: - - .. attribute:: template - - This is the object passed to the constructor's *template* argument. In - general, you shouldn't change it, but read-only access is not enforced. - -Here is an example of how to use a Template:: - - >>> from string import Template - >>> s = Template('$who likes $what') - >>> s.substitute(who='tim', what='kung pao') - 'tim likes kung pao' - >>> d = dict(who='tim') - >>> Template('Give $who $100').substitute(d) - Traceback (most recent call last): - ... - ValueError: Invalid placeholder in string: line 1, col 11 - >>> Template('$who likes $what').substitute(d) - Traceback (most recent call last): - ... - KeyError: 'what' - >>> Template('$who likes $what').safe_substitute(d) - 'tim likes $what' - -Advanced usage: you can derive subclasses of :class:`Template` to customize -the placeholder syntax, delimiter character, or the entire regular expression -used to parse template strings. To do this, you can override these class -attributes: - -* *delimiter* -- This is the literal string describing a placeholder - introducing delimiter. The default value is ``$``. Note that this should - *not* be a regular expression, as the implementation will call - :meth:`re.escape` on this string as needed. Note further that you cannot - change the delimiter after class creation (i.e. a different delimiter must - be set in the subclass's class namespace). - -* *idpattern* -- This is the regular expression describing the pattern for - non-braced placeholders. The default value is the regular expression - ``(?a:[_a-z][_a-z0-9]*)``. If this is given and *braceidpattern* is - ``None`` this pattern will also apply to braced placeholders. - - .. note:: - - Since default *flags* is ``re.IGNORECASE``, pattern ``[a-z]`` can match - with some non-ASCII characters. That's why we use the local ``a`` flag - here. - - .. versionchanged:: 3.7 - *braceidpattern* can be used to define separate patterns used inside and - outside the braces. - -* *braceidpattern* -- This is like *idpattern* but describes the pattern for - braced placeholders. Defaults to ``None`` which means to fall back to - *idpattern* (i.e. the same pattern is used both inside and outside braces). - If given, this allows you to define different patterns for braced and - unbraced placeholders. - - .. versionadded:: 3.7 - -* *flags* -- The regular expression flags that will be applied when compiling - the regular expression used for recognizing substitutions. The default value - is ``re.IGNORECASE``. Note that ``re.VERBOSE`` will always be added to the - flags, so custom *idpattern*\ s must follow conventions for verbose regular - expressions. - - .. versionadded:: 3.2 - -Alternatively, you can provide the entire regular expression pattern by -overriding the class attribute *pattern*. If you do this, the value must be a -regular expression object with four named capturing groups. The capturing -groups correspond to the rules given above, along with the invalid placeholder -rule: - -* *escaped* -- This group matches the escape sequence, e.g. ``$$``, in the - default pattern. - -* *named* -- This group matches the unbraced placeholder name; it should not - include the delimiter in capturing group. - -* *braced* -- This group matches the brace enclosed placeholder name; it should - not include either the delimiter or braces in the capturing group. - -* *invalid* -- This group matches any other delimiter pattern (usually a single - delimiter), and it should appear last in the regular expression. - -The methods on this class will raise :exc:`ValueError` if the pattern matches -the template without one of these named groups matching. - - -Helper functions ----------------- - -.. function:: capwords(s, sep=None) - - Split the argument into words using :meth:`str.split`, capitalize each word - using :meth:`str.capitalize`, and join the capitalized words using - :meth:`str.join`. If the optional second argument *sep* is absent - or ``None``, runs of whitespace characters are replaced by a single space - and leading and trailing whitespace are removed, otherwise *sep* is used to - split and join the words. diff --git a/Doc/library/stringprep.rst b/Doc/library/stringprep.rst deleted file mode 100644 index c6d78a356d97bc..00000000000000 --- a/Doc/library/stringprep.rst +++ /dev/null @@ -1,143 +0,0 @@ -:mod:`stringprep` --- Internet String Preparation -================================================= - -.. module:: stringprep - :synopsis: String preparation, as per RFC 3453 - -.. moduleauthor:: Martin v. Löwis -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/stringprep.py` - --------------- - -When identifying things (such as host names) in the internet, it is often -necessary to compare such identifications for "equality". Exactly how this -comparison is executed may depend on the application domain, e.g. whether it -should be case-insensitive or not. It may be also necessary to restrict the -possible identifications, to allow only identifications consisting of -"printable" characters. - -:rfc:`3454` defines a procedure for "preparing" Unicode strings in internet -protocols. Before passing strings onto the wire, they are processed with the -preparation procedure, after which they have a certain normalized form. The RFC -defines a set of tables, which can be combined into profiles. Each profile must -define which tables it uses, and what other optional parts of the ``stringprep`` -procedure are part of the profile. One example of a ``stringprep`` profile is -``nameprep``, which is used for internationalized domain names. - -The module :mod:`stringprep` only exposes the tables from :rfc:`3454`. As these -tables would be very large to represent as dictionaries or lists, the -module uses the Unicode character database internally. The module source code -itself was generated using the ``mkstringprep.py`` utility. - -As a result, these tables are exposed as functions, not as data structures. -There are two kinds of tables in the RFC: sets and mappings. For a set, -:mod:`stringprep` provides the "characteristic function", i.e. a function that -returns ``True`` if the parameter is part of the set. For mappings, it provides the -mapping function: given the key, it returns the associated value. Below is a -list of all functions available in the module. - - -.. function:: in_table_a1(code) - - Determine whether *code* is in tableA.1 (Unassigned code points in Unicode 3.2). - - -.. function:: in_table_b1(code) - - Determine whether *code* is in tableB.1 (Commonly mapped to nothing). - - -.. function:: map_table_b2(code) - - Return the mapped value for *code* according to tableB.2 (Mapping for - case-folding used with NFKC). - - -.. function:: map_table_b3(code) - - Return the mapped value for *code* according to tableB.3 (Mapping for - case-folding used with no normalization). - - -.. function:: in_table_c11(code) - - Determine whether *code* is in tableC.1.1 (ASCII space characters). - - -.. function:: in_table_c12(code) - - Determine whether *code* is in tableC.1.2 (Non-ASCII space characters). - - -.. function:: in_table_c11_c12(code) - - Determine whether *code* is in tableC.1 (Space characters, union of C.1.1 and - C.1.2). - - -.. function:: in_table_c21(code) - - Determine whether *code* is in tableC.2.1 (ASCII control characters). - - -.. function:: in_table_c22(code) - - Determine whether *code* is in tableC.2.2 (Non-ASCII control characters). - - -.. function:: in_table_c21_c22(code) - - Determine whether *code* is in tableC.2 (Control characters, union of C.2.1 and - C.2.2). - - -.. function:: in_table_c3(code) - - Determine whether *code* is in tableC.3 (Private use). - - -.. function:: in_table_c4(code) - - Determine whether *code* is in tableC.4 (Non-character code points). - - -.. function:: in_table_c5(code) - - Determine whether *code* is in tableC.5 (Surrogate codes). - - -.. function:: in_table_c6(code) - - Determine whether *code* is in tableC.6 (Inappropriate for plain text). - - -.. function:: in_table_c7(code) - - Determine whether *code* is in tableC.7 (Inappropriate for canonical - representation). - - -.. function:: in_table_c8(code) - - Determine whether *code* is in tableC.8 (Change display properties or are - deprecated). - - -.. function:: in_table_c9(code) - - Determine whether *code* is in tableC.9 (Tagging characters). - - -.. function:: in_table_d1(code) - - Determine whether *code* is in tableD.1 (Characters with bidirectional property - "R" or "AL"). - - -.. function:: in_table_d2(code) - - Determine whether *code* is in tableD.2 (Characters with bidirectional property - "L"). - diff --git a/Doc/library/struct.rst b/Doc/library/struct.rst deleted file mode 100644 index 416b01db615fa0..00000000000000 --- a/Doc/library/struct.rst +++ /dev/null @@ -1,605 +0,0 @@ -:mod:`struct` --- Interpret bytes as packed binary data -======================================================= - -.. module:: struct - :synopsis: Interpret bytes as packed binary data. - -**Source code:** :source:`Lib/struct.py` - -.. index:: - pair: C; structures - triple: packing; binary; data - --------------- - -This module converts between Python values and C structs represented -as Python :class:`bytes` objects. Compact :ref:`format strings ` -describe the intended conversions to/from Python values. -The module's functions and objects can be used for two largely -distinct applications, data exchange with external sources (files or -network connections), or data transfer between the Python application -and the C layer. - -.. note:: - - When no prefix character is given, native mode is the default. It - packs or unpacks data based on the platform and compiler on which - the Python interpreter was built. - The result of packing a given C struct includes pad bytes which - maintain proper alignment for the C types involved; similarly, - alignment is taken into account when unpacking. In contrast, when - communicating data between external sources, the programmer is - responsible for defining byte ordering and padding between elements. - See :ref:`struct-alignment` for details. - -Several :mod:`struct` functions (and methods of :class:`Struct`) take a *buffer* -argument. This refers to objects that implement the :ref:`bufferobjects` and -provide either a readable or read-writable buffer. The most common types used -for that purpose are :class:`bytes` and :class:`bytearray`, but many other types -that can be viewed as an array of bytes implement the buffer protocol, so that -they can be read/filled without additional copying from a :class:`bytes` object. - - -Functions and Exceptions ------------------------- - -The module defines the following exception and functions: - - -.. exception:: error - - Exception raised on various occasions; argument is a string describing what - is wrong. - - -.. function:: pack(format, v1, v2, ...) - - Return a bytes object containing the values *v1*, *v2*, ... packed according - to the format string *format*. The arguments must match the values required by - the format exactly. - - -.. function:: pack_into(format, buffer, offset, v1, v2, ...) - - Pack the values *v1*, *v2*, ... according to the format string *format* and - write the packed bytes into the writable buffer *buffer* starting at - position *offset*. Note that *offset* is a required argument. - - -.. function:: unpack(format, buffer) - - Unpack from the buffer *buffer* (presumably packed by ``pack(format, ...)``) - according to the format string *format*. The result is a tuple even if it - contains exactly one item. The buffer's size in bytes must match the - size required by the format, as reflected by :func:`calcsize`. - - -.. function:: unpack_from(format, /, buffer, offset=0) - - Unpack from *buffer* starting at position *offset*, according to the format - string *format*. The result is a tuple even if it contains exactly one - item. The buffer's size in bytes, starting at position *offset*, must be at - least the size required by the format, as reflected by :func:`calcsize`. - - -.. function:: iter_unpack(format, buffer) - - Iteratively unpack from the buffer *buffer* according to the format - string *format*. This function returns an iterator which will read - equally sized chunks from the buffer until all its contents have been - consumed. The buffer's size in bytes must be a multiple of the size - required by the format, as reflected by :func:`calcsize`. - - Each iteration yields a tuple as specified by the format string. - - .. versionadded:: 3.4 - - -.. function:: calcsize(format) - - Return the size of the struct (and hence of the bytes object produced by - ``pack(format, ...)``) corresponding to the format string *format*. - - -.. _struct-format-strings: - -Format Strings --------------- - -Format strings describe the data layout when -packing and unpacking data. They are built up from :ref:`format characters`, -which specify the type of data being packed/unpacked. In addition, -special characters control the :ref:`byte order, size and alignment`. -Each format string consists of an optional prefix character which -describes the overall properties of the data and one or more format -characters which describe the actual data values and padding. - - -.. _struct-alignment: - -Byte Order, Size, and Alignment -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -By default, C types are represented in the machine's native format and byte -order, and properly aligned by skipping pad bytes if necessary (according to the -rules used by the C compiler). -This behavior is chosen so -that the bytes of a packed struct correspond exactly to the memory layout -of the corresponding C struct. -Whether to use native byte ordering -and padding or standard formats depends on the application. - -.. index:: - single: @ (at); in struct format strings - single: = (equals); in struct format strings - single: < (less); in struct format strings - single: > (greater); in struct format strings - single: ! (exclamation); in struct format strings - -Alternatively, the first character of the format string can be used to indicate -the byte order, size and alignment of the packed data, according to the -following table: - -+-----------+------------------------+----------+-----------+ -| Character | Byte order | Size | Alignment | -+===========+========================+==========+===========+ -| ``@`` | native | native | native | -+-----------+------------------------+----------+-----------+ -| ``=`` | native | standard | none | -+-----------+------------------------+----------+-----------+ -| ``<`` | little-endian | standard | none | -+-----------+------------------------+----------+-----------+ -| ``>`` | big-endian | standard | none | -+-----------+------------------------+----------+-----------+ -| ``!`` | network (= big-endian) | standard | none | -+-----------+------------------------+----------+-----------+ - -If the first character is not one of these, ``'@'`` is assumed. - -Native byte order is big-endian or little-endian, depending on the -host system. For example, Intel x86, AMD64 (x86-64), and Apple M1 are -little-endian; IBM z and many legacy architectures are big-endian. -Use :data:`sys.byteorder` to check the endianness of your system. - -Native size and alignment are determined using the C compiler's -``sizeof`` expression. This is always combined with native byte order. - -Standard size depends only on the format character; see the table in -the :ref:`format-characters` section. - -Note the difference between ``'@'`` and ``'='``: both use native byte order, but -the size and alignment of the latter is standardized. - -The form ``'!'`` represents the network byte order which is always big-endian -as defined in `IETF RFC 1700 `_. - -There is no way to indicate non-native byte order (force byte-swapping); use the -appropriate choice of ``'<'`` or ``'>'``. - -Notes: - -(1) Padding is only automatically added between successive structure members. - No padding is added at the beginning or the end of the encoded struct. - -(2) No padding is added when using non-native size and alignment, e.g. - with '<', '>', '=', and '!'. - -(3) To align the end of a structure to the alignment requirement of a - particular type, end the format with the code for that type with a repeat - count of zero. See :ref:`struct-examples`. - - -.. _format-characters: - -Format Characters -^^^^^^^^^^^^^^^^^ - -Format characters have the following meaning; the conversion between C and -Python values should be obvious given their types. The 'Standard size' column -refers to the size of the packed value in bytes when using standard size; that -is, when the format string starts with one of ``'<'``, ``'>'``, ``'!'`` or -``'='``. When using native size, the size of the packed value is -platform-dependent. - -+--------+--------------------------+--------------------+----------------+------------+ -| Format | C Type | Python type | Standard size | Notes | -+========+==========================+====================+================+============+ -| ``x`` | pad byte | no value | | \(7) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``c`` | :c:expr:`char` | bytes of length 1 | 1 | | -+--------+--------------------------+--------------------+----------------+------------+ -| ``b`` | :c:expr:`signed char` | integer | 1 | \(1), \(2) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``B`` | :c:expr:`unsigned char` | integer | 1 | \(2) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``?`` | :c:expr:`_Bool` | bool | 1 | \(1) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``h`` | :c:expr:`short` | integer | 2 | \(2) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``H`` | :c:expr:`unsigned short` | integer | 2 | \(2) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``i`` | :c:expr:`int` | integer | 4 | \(2) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``I`` | :c:expr:`unsigned int` | integer | 4 | \(2) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``l`` | :c:expr:`long` | integer | 4 | \(2) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``L`` | :c:expr:`unsigned long` | integer | 4 | \(2) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``q`` | :c:expr:`long long` | integer | 8 | \(2) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``Q`` | :c:expr:`unsigned long | integer | 8 | \(2) | -| | long` | | | | -+--------+--------------------------+--------------------+----------------+------------+ -| ``n`` | :c:type:`ssize_t` | integer | | \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``N`` | :c:type:`size_t` | integer | | \(3) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``e`` | \(6) | float | 2 | \(4) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``f`` | :c:expr:`float` | float | 4 | \(4) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``d`` | :c:expr:`double` | float | 8 | \(4) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``s`` | :c:expr:`char[]` | bytes | | \(9) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``p`` | :c:expr:`char[]` | bytes | | \(8) | -+--------+--------------------------+--------------------+----------------+------------+ -| ``P`` | :c:expr:`void \*` | integer | | \(5) | -+--------+--------------------------+--------------------+----------------+------------+ - -.. versionchanged:: 3.3 - Added support for the ``'n'`` and ``'N'`` formats. - -.. versionchanged:: 3.6 - Added support for the ``'e'`` format. - - -Notes: - -(1) - .. index:: single: ? (question mark); in struct format strings - - The ``'?'`` conversion code corresponds to the :c:expr:`_Bool` type defined by - C99. If this type is not available, it is simulated using a :c:expr:`char`. In - standard mode, it is always represented by one byte. - -(2) - When attempting to pack a non-integer using any of the integer conversion - codes, if the non-integer has a :meth:`~object.__index__` method then that method is - called to convert the argument to an integer before packing. - - .. versionchanged:: 3.2 - Added use of the :meth:`~object.__index__` method for non-integers. - -(3) - The ``'n'`` and ``'N'`` conversion codes are only available for the native - size (selected as the default or with the ``'@'`` byte order character). - For the standard size, you can use whichever of the other integer formats - fits your application. - -(4) - For the ``'f'``, ``'d'`` and ``'e'`` conversion codes, the packed - representation uses the IEEE 754 binary32, binary64 or binary16 format (for - ``'f'``, ``'d'`` or ``'e'`` respectively), regardless of the floating-point - format used by the platform. - -(5) - The ``'P'`` format character is only available for the native byte ordering - (selected as the default or with the ``'@'`` byte order character). The byte - order character ``'='`` chooses to use little- or big-endian ordering based - on the host system. The struct module does not interpret this as native - ordering, so the ``'P'`` format is not available. - -(6) - The IEEE 754 binary16 "half precision" type was introduced in the 2008 - revision of the `IEEE 754 standard `_. It has a sign - bit, a 5-bit exponent and 11-bit precision (with 10 bits explicitly stored), - and can represent numbers between approximately ``6.1e-05`` and ``6.5e+04`` - at full precision. This type is not widely supported by C compilers: on a - typical machine, an unsigned short can be used for storage, but not for math - operations. See the Wikipedia page on the `half-precision floating-point - format `_ for more information. - -(7) - When packing, ``'x'`` inserts one NUL byte. - -(8) - The ``'p'`` format character encodes a "Pascal string", meaning a short - variable-length string stored in a *fixed number of bytes*, given by the count. - The first byte stored is the length of the string, or 255, whichever is - smaller. The bytes of the string follow. If the string passed in to - :func:`pack` is too long (longer than the count minus 1), only the leading - ``count-1`` bytes of the string are stored. If the string is shorter than - ``count-1``, it is padded with null bytes so that exactly count bytes in all - are used. Note that for :func:`unpack`, the ``'p'`` format character consumes - ``count`` bytes, but that the string returned can never contain more than 255 - bytes. - -(9) - For the ``'s'`` format character, the count is interpreted as the length of the - bytes, not a repeat count like for the other format characters; for example, - ``'10s'`` means a single 10-byte string mapping to or from a single - Python byte string, while ``'10c'`` means 10 - separate one byte character elements (e.g., ``cccccccccc``) mapping - to or from ten different Python byte objects. (See :ref:`struct-examples` - for a concrete demonstration of the difference.) - If a count is not given, it defaults to 1. For packing, the string is - truncated or padded with null bytes as appropriate to make it fit. For - unpacking, the resulting bytes object always has exactly the specified number - of bytes. As a special case, ``'0s'`` means a single, empty string (while - ``'0c'`` means 0 characters). - -A format character may be preceded by an integral repeat count. For example, -the format string ``'4h'`` means exactly the same as ``'hhhh'``. - -Whitespace characters between formats are ignored; a count and its format must -not contain whitespace though. - -When packing a value ``x`` using one of the integer formats (``'b'``, -``'B'``, ``'h'``, ``'H'``, ``'i'``, ``'I'``, ``'l'``, ``'L'``, -``'q'``, ``'Q'``), if ``x`` is outside the valid range for that format -then :exc:`struct.error` is raised. - -.. versionchanged:: 3.1 - Previously, some of the integer formats wrapped out-of-range values and - raised :exc:`DeprecationWarning` instead of :exc:`struct.error`. - -.. index:: single: ? (question mark); in struct format strings - -For the ``'?'`` format character, the return value is either :const:`True` or -:const:`False`. When packing, the truth value of the argument object is used. -Either 0 or 1 in the native or standard bool representation will be packed, and -any non-zero value will be ``True`` when unpacking. - - - -.. _struct-examples: - -Examples -^^^^^^^^ - -.. note:: - Native byte order examples (designated by the ``'@'`` format prefix or - lack of any prefix character) may not match what the reader's - machine produces as - that depends on the platform and compiler. - -Pack and unpack integers of three different sizes, using big endian -ordering:: - - >>> from struct import * - >>> pack(">bhl", 1, 2, 3) - b'\x01\x00\x02\x00\x00\x00\x03' - >>> unpack('>bhl', b'\x01\x00\x02\x00\x00\x00\x03') - (1, 2, 3) - >>> calcsize('>bhl') - 7 - -Attempt to pack an integer which is too large for the defined field:: - - >>> pack(">h", 99999) - Traceback (most recent call last): - File "", line 1, in - struct.error: 'h' format requires -32768 <= number <= 32767 - -Demonstrate the difference between ``'s'`` and ``'c'`` format -characters:: - - >>> pack("@ccc", b'1', b'2', b'3') - b'123' - >>> pack("@3s", b'123') - b'123' - -Unpacked fields can be named by assigning them to variables or by wrapping -the result in a named tuple:: - - >>> record = b'raymond \x32\x12\x08\x01\x08' - >>> name, serialnum, school, gradelevel = unpack('<10sHHb', record) - - >>> from collections import namedtuple - >>> Student = namedtuple('Student', 'name serialnum school gradelevel') - >>> Student._make(unpack('<10sHHb', record)) - Student(name=b'raymond ', serialnum=4658, school=264, gradelevel=8) - -The ordering of format characters may have an impact on size in native -mode since padding is implicit. In standard mode, the user is -responsible for inserting any desired padding. -Note in -the first ``pack`` call below that three NUL bytes were added after the -packed ``'#'`` to align the following integer on a four-byte boundary. -In this example, the output was produced on a little endian machine:: - - >>> pack('@ci', b'#', 0x12131415) - b'#\x00\x00\x00\x15\x14\x13\x12' - >>> pack('@ic', 0x12131415, b'#') - b'\x15\x14\x13\x12#' - >>> calcsize('@ci') - 8 - >>> calcsize('@ic') - 5 - -The following format ``'llh0l'`` results in two pad bytes being added -at the end, assuming the platform's longs are aligned on 4-byte boundaries:: - - >>> pack('@llh0l', 1, 2, 3) - b'\x00\x00\x00\x01\x00\x00\x00\x02\x00\x03\x00\x00' - - -.. seealso:: - - Module :mod:`array` - Packed binary storage of homogeneous data. - - Module :mod:`json` - JSON encoder and decoder. - - Module :mod:`pickle` - Python object serialization. - - -.. _applications: - -Applications ------------- - -Two main applications for the :mod:`struct` module exist, data -interchange between Python and C code within an application or another -application compiled using the same compiler (:ref:`native formats`), and -data interchange between applications using agreed upon data layout -(:ref:`standard formats`). Generally speaking, the format strings -constructed for these two domains are distinct. - - -.. _struct-native-formats: - -Native Formats -^^^^^^^^^^^^^^ - -When constructing format strings which mimic native layouts, the -compiler and machine architecture determine byte ordering and padding. -In such cases, the ``@`` format character should be used to specify -native byte ordering and data sizes. Internal pad bytes are normally inserted -automatically. It is possible that a zero-repeat format code will be -needed at the end of a format string to round up to the correct -byte boundary for proper alignment of consective chunks of data. - -Consider these two simple examples (on a 64-bit, little-endian -machine):: - - >>> calcsize('@lhl') - 24 - >>> calcsize('@llh') - 18 - -Data is not padded to an 8-byte boundary at the end of the second -format string without the use of extra padding. A zero-repeat format -code solves that problem:: - - >>> calcsize('@llh0l') - 24 - -The ``'x'`` format code can be used to specify the repeat, but for -native formats it is better to use a zero-repeat format like ``'0l'``. - -By default, native byte ordering and alignment is used, but it is -better to be explicit and use the ``'@'`` prefix character. - - -.. _struct-standard-formats: - -Standard Formats -^^^^^^^^^^^^^^^^ - -When exchanging data beyond your process such as networking or storage, -be precise. Specify the exact byte order, size, and alignment. Do -not assume they match the native order of a particular machine. -For example, network byte order is big-endian, while many popular CPUs -are little-endian. By defining this explicitly, the user need not -care about the specifics of the platform their code is running on. -The first character should typically be ``<`` or ``>`` -(or ``!``). Padding is the responsibility of the programmer. The -zero-repeat format character won't work. Instead, the user must -explicitly add ``'x'`` pad bytes where needed. Revisiting the -examples from the previous section, we have:: - - >>> calcsize('>> pack('>> calcsize('@llh') - 18 - >>> pack('@llh', 1, 2, 3) == pack('>> calcsize('>> calcsize('@llh0l') - 24 - >>> pack('@llh0l', 1, 2, 3) == pack('>> calcsize('>> calcsize('@llh0l') - 12 - >>> pack('@llh0l', 1, 2, 3) == pack(' -.. sectionauthor:: Peter Åstrand - -**Source code:** :source:`Lib/subprocess.py` - --------------- - -The :mod:`subprocess` module allows you to spawn new processes, connect to their -input/output/error pipes, and obtain their return codes. This module intends to -replace several older modules and functions:: - - os.system - os.spawn* - -Information about how the :mod:`subprocess` module can be used to replace these -modules and functions can be found in the following sections. - -.. seealso:: - - :pep:`324` -- PEP proposing the subprocess module - -.. include:: ../includes/wasm-notavail.rst - -Using the :mod:`subprocess` Module ----------------------------------- - -The recommended approach to invoking subprocesses is to use the :func:`run` -function for all use cases it can handle. For more advanced use cases, the -underlying :class:`Popen` interface can be used directly. - - -.. function:: run(args, *, stdin=None, input=None, stdout=None, stderr=None,\ - capture_output=False, shell=False, cwd=None, timeout=None, \ - check=False, encoding=None, errors=None, text=None, env=None, \ - universal_newlines=None, **other_popen_kwargs) - - Run the command described by *args*. Wait for command to complete, then - return a :class:`CompletedProcess` instance. - - The arguments shown above are merely the most common ones, described below - in :ref:`frequently-used-arguments` (hence the use of keyword-only notation - in the abbreviated signature). The full function signature is largely the - same as that of the :class:`Popen` constructor - most of the arguments to - this function are passed through to that interface. (*timeout*, *input*, - *check*, and *capture_output* are not.) - - If *capture_output* is true, stdout and stderr will be captured. - When used, the internal :class:`Popen` object is automatically created with - ``stdout=PIPE`` and ``stderr=PIPE``. The *stdout* and *stderr* arguments may - not be supplied at the same time as *capture_output*. If you wish to capture - and combine both streams into one, use ``stdout=PIPE`` and ``stderr=STDOUT`` - instead of *capture_output*. - - A *timeout* may be specified in seconds, it is internally passed on to - :meth:`Popen.communicate`. If the timeout expires, the child process will be - killed and waited for. The :exc:`TimeoutExpired` exception will be - re-raised after the child process has terminated. The initial process - creation itself cannot be interrupted on many platform APIs so you are not - guaranteed to see a timeout exception until at least after however long - process creation takes. - - The *input* argument is passed to :meth:`Popen.communicate` and thus to the - subprocess's stdin. If used it must be a byte sequence, or a string if - *encoding* or *errors* is specified or *text* is true. When - used, the internal :class:`Popen` object is automatically created with - ``stdin=PIPE``, and the *stdin* argument may not be used as well. - - If *check* is true, and the process exits with a non-zero exit code, a - :exc:`CalledProcessError` exception will be raised. Attributes of that - exception hold the arguments, the exit code, and stdout and stderr if they - were captured. - - If *encoding* or *errors* are specified, or *text* is true, - file objects for stdin, stdout and stderr are opened in text mode using the - specified *encoding* and *errors* or the :class:`io.TextIOWrapper` default. - The *universal_newlines* argument is equivalent to *text* and is provided - for backwards compatibility. By default, file objects are opened in binary mode. - - If *env* is not ``None``, it must be a mapping that defines the environment - variables for the new process; these are used instead of the default - behavior of inheriting the current process' environment. It is passed - directly to :class:`Popen`. This mapping can be str to str on any platform - or bytes to bytes on POSIX platforms much like :data:`os.environ` or - :data:`os.environb`. - - Examples:: - - >>> subprocess.run(["ls", "-l"]) # doesn't capture output - CompletedProcess(args=['ls', '-l'], returncode=0) - - >>> subprocess.run("exit 1", shell=True, check=True) - Traceback (most recent call last): - ... - subprocess.CalledProcessError: Command 'exit 1' returned non-zero exit status 1 - - >>> subprocess.run(["ls", "-l", "/dev/null"], capture_output=True) - CompletedProcess(args=['ls', '-l', '/dev/null'], returncode=0, - stdout=b'crw-rw-rw- 1 root root 1, 3 Jan 23 16:23 /dev/null\n', stderr=b'') - - .. versionadded:: 3.5 - - .. versionchanged:: 3.6 - - Added *encoding* and *errors* parameters - - .. versionchanged:: 3.7 - - Added the *text* parameter, as a more understandable alias of *universal_newlines*. - Added the *capture_output* parameter. - - .. versionchanged:: 3.11.3 - - Changed Windows shell search order for ``shell=True``. The current - directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and - ``%SystemRoot%\System32\cmd.exe``. As a result, dropping a - malicious program named ``cmd.exe`` into a current directory no - longer works. - -.. class:: CompletedProcess - - The return value from :func:`run`, representing a process that has finished. - - .. attribute:: args - - The arguments used to launch the process. This may be a list or a string. - - .. attribute:: returncode - - Exit status of the child process. Typically, an exit status of 0 indicates - that it ran successfully. - - A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (POSIX only). - - .. attribute:: stdout - - Captured stdout from the child process. A bytes sequence, or a string if - :func:`run` was called with an encoding, errors, or text=True. - ``None`` if stdout was not captured. - - If you ran the process with ``stderr=subprocess.STDOUT``, stdout and - stderr will be combined in this attribute, and :attr:`stderr` will be - ``None``. - - .. attribute:: stderr - - Captured stderr from the child process. A bytes sequence, or a string if - :func:`run` was called with an encoding, errors, or text=True. - ``None`` if stderr was not captured. - - .. method:: check_returncode() - - If :attr:`returncode` is non-zero, raise a :exc:`CalledProcessError`. - - .. versionadded:: 3.5 - -.. data:: DEVNULL - - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :class:`Popen` and indicates that the special file :data:`os.devnull` - will be used. - - .. versionadded:: 3.3 - - -.. data:: PIPE - - Special value that can be used as the *stdin*, *stdout* or *stderr* argument - to :class:`Popen` and indicates that a pipe to the standard stream should be - opened. Most useful with :meth:`Popen.communicate`. - - -.. data:: STDOUT - - Special value that can be used as the *stderr* argument to :class:`Popen` and - indicates that standard error should go into the same handle as standard - output. - - -.. exception:: SubprocessError - - Base class for all other exceptions from this module. - - .. versionadded:: 3.3 - - -.. exception:: TimeoutExpired - - Subclass of :exc:`SubprocessError`, raised when a timeout expires - while waiting for a child process. - - .. attribute:: cmd - - Command that was used to spawn the child process. - - .. attribute:: timeout - - Timeout in seconds. - - .. attribute:: output - - Output of the child process if it was captured by :func:`run` or - :func:`check_output`. Otherwise, ``None``. This is always - :class:`bytes` when any output was captured regardless of the - ``text=True`` setting. It may remain ``None`` instead of ``b''`` - when no output was observed. - - .. attribute:: stdout - - Alias for output, for symmetry with :attr:`stderr`. - - .. attribute:: stderr - - Stderr output of the child process if it was captured by :func:`run`. - Otherwise, ``None``. This is always :class:`bytes` when stderr output - was captured regardless of the ``text=True`` setting. It may remain - ``None`` instead of ``b''`` when no stderr output was observed. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - *stdout* and *stderr* attributes added - -.. exception:: CalledProcessError - - Subclass of :exc:`SubprocessError`, raised when a process run by - :func:`check_call`, :func:`check_output`, or :func:`run` (with ``check=True``) - returns a non-zero exit status. - - - .. attribute:: returncode - - Exit status of the child process. If the process exited due to a - signal, this will be the negative signal number. - - .. attribute:: cmd - - Command that was used to spawn the child process. - - .. attribute:: output - - Output of the child process if it was captured by :func:`run` or - :func:`check_output`. Otherwise, ``None``. - - .. attribute:: stdout - - Alias for output, for symmetry with :attr:`stderr`. - - .. attribute:: stderr - - Stderr output of the child process if it was captured by :func:`run`. - Otherwise, ``None``. - - .. versionchanged:: 3.5 - *stdout* and *stderr* attributes added - - -.. _frequently-used-arguments: - -Frequently Used Arguments -^^^^^^^^^^^^^^^^^^^^^^^^^ - -To support a wide variety of use cases, the :class:`Popen` constructor (and -the convenience functions) accept a large number of optional arguments. For -most typical use cases, many of these arguments can be safely left at their -default values. The arguments that are most commonly needed are: - - *args* is required for all calls and should be a string, or a sequence of - program arguments. Providing a sequence of arguments is generally - preferred, as it allows the module to take care of any required escaping - and quoting of arguments (e.g. to permit spaces in file names). If passing - a single string, either *shell* must be :const:`True` (see below) or else - the string must simply name the program to be executed without specifying - any arguments. - - *stdin*, *stdout* and *stderr* specify the executed program's standard input, - standard output and standard error file handles, respectively. Valid values - are :data:`PIPE`, :data:`DEVNULL`, an existing file descriptor (a positive - integer), an existing file object with a valid file descriptor, and ``None``. - :data:`PIPE` indicates that a new pipe to the child should be created. - :data:`DEVNULL` indicates that the special file :data:`os.devnull` will - be used. With the default settings of ``None``, no redirection will occur; - the child's file handles will be inherited from the parent. - Additionally, *stderr* can be :data:`STDOUT`, which indicates that the - stderr data from the child process should be captured into the same file - handle as for *stdout*. - - .. index:: - single: universal newlines; subprocess module - - If *encoding* or *errors* are specified, or *text* (also known as - *universal_newlines*) is true, - the file objects *stdin*, *stdout* and *stderr* will be opened in text - mode using the *encoding* and *errors* specified in the call or the - defaults for :class:`io.TextIOWrapper`. - - For *stdin*, line ending characters ``'\n'`` in the input will be converted - to the default line separator :data:`os.linesep`. For *stdout* and *stderr*, - all line endings in the output will be converted to ``'\n'``. For more - information see the documentation of the :class:`io.TextIOWrapper` class - when the *newline* argument to its constructor is ``None``. - - If text mode is not used, *stdin*, *stdout* and *stderr* will be opened as - binary streams. No encoding or line ending conversion is performed. - - .. versionadded:: 3.6 - Added *encoding* and *errors* parameters. - - .. versionadded:: 3.7 - Added the *text* parameter as an alias for *universal_newlines*. - - .. note:: - - The newlines attribute of the file objects :attr:`Popen.stdin`, - :attr:`Popen.stdout` and :attr:`Popen.stderr` are not updated by - the :meth:`Popen.communicate` method. - - If *shell* is ``True``, the specified command will be executed through - the shell. This can be useful if you are using Python primarily for the - enhanced control flow it offers over most system shells and still want - convenient access to other shell features such as shell pipes, filename - wildcards, environment variable expansion, and expansion of ``~`` to a - user's home directory. However, note that Python itself offers - implementations of many shell-like features (in particular, :mod:`glob`, - :mod:`fnmatch`, :func:`os.walk`, :func:`os.path.expandvars`, - :func:`os.path.expanduser`, and :mod:`shutil`). - - .. versionchanged:: 3.3 - When *universal_newlines* is ``True``, the class uses the encoding - :func:`locale.getpreferredencoding(False) ` - instead of ``locale.getpreferredencoding()``. See the - :class:`io.TextIOWrapper` class for more information on this change. - - .. note:: - - Read the `Security Considerations`_ section before using ``shell=True``. - -These options, along with all of the other options, are described in more -detail in the :class:`Popen` constructor documentation. - - -Popen Constructor -^^^^^^^^^^^^^^^^^ - -The underlying process creation and management in this module is handled by -the :class:`Popen` class. It offers a lot of flexibility so that developers -are able to handle the less common cases not covered by the convenience -functions. - - -.. class:: Popen(args, bufsize=-1, executable=None, stdin=None, stdout=None, \ - stderr=None, preexec_fn=None, close_fds=True, shell=False, \ - cwd=None, env=None, universal_newlines=None, \ - startupinfo=None, creationflags=0, restore_signals=True, \ - start_new_session=False, pass_fds=(), *, group=None, \ - extra_groups=None, user=None, umask=-1, \ - encoding=None, errors=None, text=None, pipesize=-1, \ - process_group=None) - - Execute a child program in a new process. On POSIX, the class uses - :meth:`os.execvpe`-like behavior to execute the child program. On Windows, - the class uses the Windows ``CreateProcess()`` function. The arguments to - :class:`Popen` are as follows. - - *args* should be a sequence of program arguments or else a single string - or :term:`path-like object`. - By default, the program to execute is the first item in *args* if *args* is - a sequence. If *args* is a string, the interpretation is - platform-dependent and described below. See the *shell* and *executable* - arguments for additional differences from the default behavior. Unless - otherwise stated, it is recommended to pass *args* as a sequence. - - .. warning:: - - For maximum reliability, use a fully qualified path for the executable. - To search for an unqualified name on :envvar:`PATH`, use - :meth:`shutil.which`. On all platforms, passing :data:`sys.executable` - is the recommended way to launch the current Python interpreter again, - and use the ``-m`` command-line format to launch an installed module. - - Resolving the path of *executable* (or the first item of *args*) is - platform dependent. For POSIX, see :meth:`os.execvpe`, and note that - when resolving or searching for the executable path, *cwd* overrides the - current working directory and *env* can override the ``PATH`` - environment variable. For Windows, see the documentation of the - ``lpApplicationName`` and ``lpCommandLine`` parameters of WinAPI - ``CreateProcess``, and note that when resolving or searching for the - executable path with ``shell=False``, *cwd* does not override the - current working directory and *env* cannot override the ``PATH`` - environment variable. Using a full path avoids all of these variations. - - An example of passing some arguments to an external program - as a sequence is:: - - Popen(["/usr/bin/git", "commit", "-m", "Fixes a bug."]) - - On POSIX, if *args* is a string, the string is interpreted as the name or - path of the program to execute. However, this can only be done if not - passing arguments to the program. - - .. note:: - - It may not be obvious how to break a shell command into a sequence of arguments, - especially in complex cases. :meth:`shlex.split` can illustrate how to - determine the correct tokenization for *args*:: - - >>> import shlex, subprocess - >>> command_line = input() - /bin/vikings -input eggs.txt -output "spam spam.txt" -cmd "echo '$MONEY'" - >>> args = shlex.split(command_line) - >>> print(args) - ['/bin/vikings', '-input', 'eggs.txt', '-output', 'spam spam.txt', '-cmd', "echo '$MONEY'"] - >>> p = subprocess.Popen(args) # Success! - - Note in particular that options (such as *-input*) and arguments (such - as *eggs.txt*) that are separated by whitespace in the shell go in separate - list elements, while arguments that need quoting or backslash escaping when - used in the shell (such as filenames containing spaces or the *echo* command - shown above) are single list elements. - - On Windows, if *args* is a sequence, it will be converted to a string in a - manner described in :ref:`converting-argument-sequence`. This is because - the underlying ``CreateProcess()`` operates on strings. - - .. versionchanged:: 3.6 - *args* parameter accepts a :term:`path-like object` if *shell* is - ``False`` and a sequence containing path-like objects on POSIX. - - .. versionchanged:: 3.8 - *args* parameter accepts a :term:`path-like object` if *shell* is - ``False`` and a sequence containing bytes and path-like objects - on Windows. - - The *shell* argument (which defaults to ``False``) specifies whether to use - the shell as the program to execute. If *shell* is ``True``, it is - recommended to pass *args* as a string rather than as a sequence. - - On POSIX with ``shell=True``, the shell defaults to :file:`/bin/sh`. If - *args* is a string, the string specifies the command - to execute through the shell. This means that the string must be - formatted exactly as it would be when typed at the shell prompt. This - includes, for example, quoting or backslash escaping filenames with spaces in - them. If *args* is a sequence, the first item specifies the command string, and - any additional items will be treated as additional arguments to the shell - itself. That is to say, :class:`Popen` does the equivalent of:: - - Popen(['/bin/sh', '-c', args[0], args[1], ...]) - - On Windows with ``shell=True``, the :envvar:`COMSPEC` environment variable - specifies the default shell. The only time you need to specify - ``shell=True`` on Windows is when the command you wish to execute is built - into the shell (e.g. :command:`dir` or :command:`copy`). You do not need - ``shell=True`` to run a batch file or console-based executable. - - .. note:: - - Read the `Security Considerations`_ section before using ``shell=True``. - - *bufsize* will be supplied as the corresponding argument to the - :func:`open` function when creating the stdin/stdout/stderr pipe - file objects: - - - ``0`` means unbuffered (read and write are one - system call and can return short) - - ``1`` means line buffered - (only usable if ``text=True`` or ``universal_newlines=True``) - - any other positive value means use a buffer of approximately that - size - - negative bufsize (the default) means the system default of - io.DEFAULT_BUFFER_SIZE will be used. - - .. versionchanged:: 3.3.1 - *bufsize* now defaults to -1 to enable buffering by default to match the - behavior that most code expects. In versions prior to Python 3.2.4 and - 3.3.1 it incorrectly defaulted to ``0`` which was unbuffered - and allowed short reads. This was unintentional and did not match the - behavior of Python 2 as most code expected. - - The *executable* argument specifies a replacement program to execute. It - is very seldom needed. When ``shell=False``, *executable* replaces the - program to execute specified by *args*. However, the original *args* is - still passed to the program. Most programs treat the program specified - by *args* as the command name, which can then be different from the program - actually executed. On POSIX, the *args* name - becomes the display name for the executable in utilities such as - :program:`ps`. If ``shell=True``, on POSIX the *executable* argument - specifies a replacement shell for the default :file:`/bin/sh`. - - .. versionchanged:: 3.6 - *executable* parameter accepts a :term:`path-like object` on POSIX. - - .. versionchanged:: 3.8 - *executable* parameter accepts a bytes and :term:`path-like object` - on Windows. - - .. versionchanged:: 3.11.3 - - Changed Windows shell search order for ``shell=True``. The current - directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and - ``%SystemRoot%\System32\cmd.exe``. As a result, dropping a - malicious program named ``cmd.exe`` into a current directory no - longer works. - - *stdin*, *stdout* and *stderr* specify the executed program's standard input, - standard output and standard error file handles, respectively. Valid values - are :data:`PIPE`, :data:`DEVNULL`, an existing file descriptor (a positive - integer), an existing :term:`file object` with a valid file descriptor, - and ``None``. :data:`PIPE` indicates that a new pipe to the child should - be created. :data:`DEVNULL` indicates that the special file - :data:`os.devnull` will be used. With the default settings of ``None``, - no redirection will occur; the child's file handles will be inherited from - the parent. Additionally, *stderr* can be :data:`STDOUT`, which indicates - that the stderr data from the applications should be captured into the same - file handle as for stdout. - - If *preexec_fn* is set to a callable object, this object will be called in the - child process just before the child is executed. - (POSIX only) - - .. warning:: - - The *preexec_fn* parameter is NOT SAFE to use in the presence of threads - in your application. The child process could deadlock before exec is - called. - - .. note:: - - If you need to modify the environment for the child use the *env* - parameter rather than doing it in a *preexec_fn*. - The *start_new_session* and *process_group* parameters should take the place of - code using *preexec_fn* to call :func:`os.setsid` or :func:`os.setpgid` in the child. - - .. versionchanged:: 3.8 - - The *preexec_fn* parameter is no longer supported in subinterpreters. - The use of the parameter in a subinterpreter raises - :exc:`RuntimeError`. The new restriction may affect applications that - are deployed in mod_wsgi, uWSGI, and other embedded environments. - - If *close_fds* is true, all file descriptors except ``0``, ``1`` and - ``2`` will be closed before the child process is executed. Otherwise - when *close_fds* is false, file descriptors obey their inheritable flag - as described in :ref:`fd_inheritance`. - - On Windows, if *close_fds* is true then no handles will be inherited by the - child process unless explicitly passed in the ``handle_list`` element of - :attr:`STARTUPINFO.lpAttributeList`, or by standard handle redirection. - - .. versionchanged:: 3.2 - The default for *close_fds* was changed from :const:`False` to - what is described above. - - .. versionchanged:: 3.7 - On Windows the default for *close_fds* was changed from :const:`False` to - :const:`True` when redirecting the standard handles. It's now possible to - set *close_fds* to :const:`True` when redirecting the standard handles. - - *pass_fds* is an optional sequence of file descriptors to keep open - between the parent and child. Providing any *pass_fds* forces - *close_fds* to be :const:`True`. (POSIX only) - - .. versionchanged:: 3.2 - The *pass_fds* parameter was added. - - If *cwd* is not ``None``, the function changes the working directory to - *cwd* before executing the child. *cwd* can be a string, bytes or - :term:`path-like ` object. On POSIX, the function - looks for *executable* (or for the first item in *args*) relative to *cwd* - if the executable path is a relative path. - - .. versionchanged:: 3.6 - *cwd* parameter accepts a :term:`path-like object` on POSIX. - - .. versionchanged:: 3.7 - *cwd* parameter accepts a :term:`path-like object` on Windows. - - .. versionchanged:: 3.8 - *cwd* parameter accepts a bytes object on Windows. - - If *restore_signals* is true (the default) all signals that Python has set to - SIG_IGN are restored to SIG_DFL in the child process before the exec. - Currently this includes the SIGPIPE, SIGXFZ and SIGXFSZ signals. - (POSIX only) - - .. versionchanged:: 3.2 - *restore_signals* was added. - - If *start_new_session* is true the ``setsid()`` system call will be made in the - child process prior to the execution of the subprocess. - - .. availability:: POSIX - .. versionchanged:: 3.2 - *start_new_session* was added. - - If *process_group* is a non-negative integer, the ``setpgid(0, value)`` system call will - be made in the child process prior to the execution of the subprocess. - - .. availability:: POSIX - .. versionchanged:: 3.11 - *process_group* was added. - - If *group* is not ``None``, the setregid() system call will be made in the - child process prior to the execution of the subprocess. If the provided - value is a string, it will be looked up via :func:`grp.getgrnam()` and - the value in ``gr_gid`` will be used. If the value is an integer, it - will be passed verbatim. (POSIX only) - - .. availability:: POSIX - .. versionadded:: 3.9 - - If *extra_groups* is not ``None``, the setgroups() system call will be - made in the child process prior to the execution of the subprocess. - Strings provided in *extra_groups* will be looked up via - :func:`grp.getgrnam()` and the values in ``gr_gid`` will be used. - Integer values will be passed verbatim. (POSIX only) - - .. availability:: POSIX - .. versionadded:: 3.9 - - If *user* is not ``None``, the setreuid() system call will be made in the - child process prior to the execution of the subprocess. If the provided - value is a string, it will be looked up via :func:`pwd.getpwnam()` and - the value in ``pw_uid`` will be used. If the value is an integer, it will - be passed verbatim. (POSIX only) - - .. availability:: POSIX - .. versionadded:: 3.9 - - If *umask* is not negative, the umask() system call will be made in the - child process prior to the execution of the subprocess. - - .. availability:: POSIX - .. versionadded:: 3.9 - - If *env* is not ``None``, it must be a mapping that defines the environment - variables for the new process; these are used instead of the default - behavior of inheriting the current process' environment. This mapping can be - str to str on any platform or bytes to bytes on POSIX platforms much like - :data:`os.environ` or :data:`os.environb`. - - .. note:: - - If specified, *env* must provide any variables required for the program to - execute. On Windows, in order to run a `side-by-side assembly`_ the - specified *env* **must** include a valid :envvar:`SystemRoot`. - - .. _side-by-side assembly: https://en.wikipedia.org/wiki/Side-by-Side_Assembly - - If *encoding* or *errors* are specified, or *text* is true, the file objects - *stdin*, *stdout* and *stderr* are opened in text mode with the specified - *encoding* and *errors*, as described above in :ref:`frequently-used-arguments`. - The *universal_newlines* argument is equivalent to *text* and is provided - for backwards compatibility. By default, file objects are opened in binary mode. - - .. versionadded:: 3.6 - *encoding* and *errors* were added. - - .. versionadded:: 3.7 - *text* was added as a more readable alias for *universal_newlines*. - - If given, *startupinfo* will be a :class:`STARTUPINFO` object, which is - passed to the underlying ``CreateProcess`` function. - *creationflags*, if given, can be one or more of the following flags: - - * :data:`CREATE_NEW_CONSOLE` - * :data:`CREATE_NEW_PROCESS_GROUP` - * :data:`ABOVE_NORMAL_PRIORITY_CLASS` - * :data:`BELOW_NORMAL_PRIORITY_CLASS` - * :data:`HIGH_PRIORITY_CLASS` - * :data:`IDLE_PRIORITY_CLASS` - * :data:`NORMAL_PRIORITY_CLASS` - * :data:`REALTIME_PRIORITY_CLASS` - * :data:`CREATE_NO_WINDOW` - * :data:`DETACHED_PROCESS` - * :data:`CREATE_DEFAULT_ERROR_MODE` - * :data:`CREATE_BREAKAWAY_FROM_JOB` - - *pipesize* can be used to change the size of the pipe when - :data:`PIPE` is used for *stdin*, *stdout* or *stderr*. The size of the pipe - is only changed on platforms that support this (only Linux at this time of - writing). Other platforms will ignore this parameter. - - .. versionadded:: 3.10 - The ``pipesize`` parameter was added. - - Popen objects are supported as context managers via the :keyword:`with` statement: - on exit, standard file descriptors are closed, and the process is waited for. - :: - - with Popen(["ifconfig"], stdout=PIPE) as proc: - log.write(proc.stdout.read()) - - .. audit-event:: subprocess.Popen executable,args,cwd,env subprocess.Popen - - Popen and the other functions in this module that use it raise an - :ref:`auditing event ` ``subprocess.Popen`` with arguments - ``executable``, ``args``, ``cwd``, and ``env``. The value for ``args`` - may be a single string or a list of strings, depending on platform. - - .. versionchanged:: 3.2 - Added context manager support. - - .. versionchanged:: 3.6 - Popen destructor now emits a :exc:`ResourceWarning` warning if the child - process is still running. - - .. versionchanged:: 3.8 - Popen can use :func:`os.posix_spawn` in some cases for better - performance. On Windows Subsystem for Linux and QEMU User Emulation, - Popen constructor using :func:`os.posix_spawn` no longer raise an - exception on errors like missing program, but the child process fails - with a non-zero :attr:`~Popen.returncode`. - - -Exceptions -^^^^^^^^^^ - -Exceptions raised in the child process, before the new program has started to -execute, will be re-raised in the parent. - -The most common exception raised is :exc:`OSError`. This occurs, for example, -when trying to execute a non-existent file. Applications should prepare for -:exc:`OSError` exceptions. Note that, when ``shell=True``, :exc:`OSError` -will be raised by the child only if the selected shell itself was not found. -To determine if the shell failed to find the requested application, it is -necessary to check the return code or output from the subprocess. - -A :exc:`ValueError` will be raised if :class:`Popen` is called with invalid -arguments. - -:func:`check_call` and :func:`check_output` will raise -:exc:`CalledProcessError` if the called process returns a non-zero return -code. - -All of the functions and methods that accept a *timeout* parameter, such as -:func:`run` and :meth:`Popen.communicate` will raise :exc:`TimeoutExpired` if -the timeout expires before the process exits. - -Exceptions defined in this module all inherit from :exc:`SubprocessError`. - -.. versionadded:: 3.3 - The :exc:`SubprocessError` base class was added. - -.. _subprocess-security: - -Security Considerations ------------------------ - -Unlike some other popen functions, this implementation will never -implicitly call a system shell. This means that all characters, -including shell metacharacters, can safely be passed to child processes. -If the shell is invoked explicitly, via ``shell=True``, it is the application's -responsibility to ensure that all whitespace and metacharacters are -quoted appropriately to avoid -`shell injection `_ -vulnerabilities. On :ref:`some platforms `, it is possible -to use :func:`shlex.quote` for this escaping. - - -Popen Objects -------------- - -Instances of the :class:`Popen` class have the following methods: - - -.. method:: Popen.poll() - - Check if child process has terminated. Set and return - :attr:`~Popen.returncode` attribute. Otherwise, returns ``None``. - - -.. method:: Popen.wait(timeout=None) - - Wait for child process to terminate. Set and return - :attr:`~Popen.returncode` attribute. - - If the process does not terminate after *timeout* seconds, raise a - :exc:`TimeoutExpired` exception. It is safe to catch this exception and - retry the wait. - - .. note:: - - This will deadlock when using ``stdout=PIPE`` or ``stderr=PIPE`` - and the child process generates enough output to a pipe such that - it blocks waiting for the OS pipe buffer to accept more data. - Use :meth:`Popen.communicate` when using pipes to avoid that. - - .. note:: - - The function is implemented using a busy loop (non-blocking call and - short sleeps). Use the :mod:`asyncio` module for an asynchronous wait: - see :class:`asyncio.create_subprocess_exec`. - - .. versionchanged:: 3.3 - *timeout* was added. - -.. method:: Popen.communicate(input=None, timeout=None) - - Interact with process: Send data to stdin. Read data from stdout and stderr, - until end-of-file is reached. Wait for process to terminate and set the - :attr:`~Popen.returncode` attribute. The optional *input* argument should be - data to be sent to the child process, or ``None``, if no data should be sent - to the child. If streams were opened in text mode, *input* must be a string. - Otherwise, it must be bytes. - - :meth:`communicate` returns a tuple ``(stdout_data, stderr_data)``. - The data will be strings if streams were opened in text mode; otherwise, - bytes. - - Note that if you want to send data to the process's stdin, you need to create - the Popen object with ``stdin=PIPE``. Similarly, to get anything other than - ``None`` in the result tuple, you need to give ``stdout=PIPE`` and/or - ``stderr=PIPE`` too. - - If the process does not terminate after *timeout* seconds, a - :exc:`TimeoutExpired` exception will be raised. Catching this exception and - retrying communication will not lose any output. - - The child process is not killed if the timeout expires, so in order to - cleanup properly a well-behaved application should kill the child process and - finish communication:: - - proc = subprocess.Popen(...) - try: - outs, errs = proc.communicate(timeout=15) - except TimeoutExpired: - proc.kill() - outs, errs = proc.communicate() - - .. note:: - - The data read is buffered in memory, so do not use this method if the data - size is large or unlimited. - - .. versionchanged:: 3.3 - *timeout* was added. - - -.. method:: Popen.send_signal(signal) - - Sends the signal *signal* to the child. - - Do nothing if the process completed. - - .. note:: - - On Windows, SIGTERM is an alias for :meth:`terminate`. CTRL_C_EVENT and - CTRL_BREAK_EVENT can be sent to processes started with a *creationflags* - parameter which includes ``CREATE_NEW_PROCESS_GROUP``. - - -.. method:: Popen.terminate() - - Stop the child. On POSIX OSs the method sends SIGTERM to the - child. On Windows the Win32 API function :c:func:`TerminateProcess` is called - to stop the child. - - -.. method:: Popen.kill() - - Kills the child. On POSIX OSs the function sends SIGKILL to the child. - On Windows :meth:`kill` is an alias for :meth:`terminate`. - - -The following attributes are also set by the class for you to access. -Reassigning them to new values is unsupported: - -.. attribute:: Popen.args - - The *args* argument as it was passed to :class:`Popen` -- a - sequence of program arguments or else a single string. - - .. versionadded:: 3.3 - -.. attribute:: Popen.stdin - - If the *stdin* argument was :data:`PIPE`, this attribute is a writeable - stream object as returned by :func:`open`. If the *encoding* or *errors* - arguments were specified or the *text* or *universal_newlines* argument - was ``True``, the stream is a text stream, otherwise it is a byte stream. - If the *stdin* argument was not :data:`PIPE`, this attribute is ``None``. - - -.. attribute:: Popen.stdout - - If the *stdout* argument was :data:`PIPE`, this attribute is a readable - stream object as returned by :func:`open`. Reading from the stream provides - output from the child process. If the *encoding* or *errors* arguments were - specified or the *text* or *universal_newlines* argument was ``True``, the - stream is a text stream, otherwise it is a byte stream. If the *stdout* - argument was not :data:`PIPE`, this attribute is ``None``. - - -.. attribute:: Popen.stderr - - If the *stderr* argument was :data:`PIPE`, this attribute is a readable - stream object as returned by :func:`open`. Reading from the stream provides - error output from the child process. If the *encoding* or *errors* arguments - were specified or the *text* or *universal_newlines* argument was ``True``, the - stream is a text stream, otherwise it is a byte stream. If the *stderr* argument - was not :data:`PIPE`, this attribute is ``None``. - -.. warning:: - - Use :meth:`~Popen.communicate` rather than :attr:`.stdin.write `, - :attr:`.stdout.read ` or :attr:`.stderr.read ` to avoid - deadlocks due to any of the other OS pipe buffers filling up and blocking the - child process. - - -.. attribute:: Popen.pid - - The process ID of the child process. - - Note that if you set the *shell* argument to ``True``, this is the process ID - of the spawned shell. - - -.. attribute:: Popen.returncode - - The child return code. Initially ``None``, :attr:`returncode` is set by - a call to the :meth:`poll`, :meth:`wait`, or :meth:`communicate` methods - if they detect that the process has terminated. - - A ``None`` value indicates that the process hadn't yet terminated at the - time of the last method call. - - A negative value ``-N`` indicates that the child was terminated by signal - ``N`` (POSIX only). - - -Windows Popen Helpers ---------------------- - -The :class:`STARTUPINFO` class and following constants are only available -on Windows. - -.. class:: STARTUPINFO(*, dwFlags=0, hStdInput=None, hStdOutput=None, \ - hStdError=None, wShowWindow=0, lpAttributeList=None) - - Partial support of the Windows - `STARTUPINFO `__ - structure is used for :class:`Popen` creation. The following attributes can - be set by passing them as keyword-only arguments. - - .. versionchanged:: 3.7 - Keyword-only argument support was added. - - .. attribute:: dwFlags - - A bit field that determines whether certain :class:`STARTUPINFO` - attributes are used when the process creates a window. :: - - si = subprocess.STARTUPINFO() - si.dwFlags = subprocess.STARTF_USESTDHANDLES | subprocess.STARTF_USESHOWWINDOW - - .. attribute:: hStdInput - - If :attr:`dwFlags` specifies :data:`STARTF_USESTDHANDLES`, this attribute - is the standard input handle for the process. If - :data:`STARTF_USESTDHANDLES` is not specified, the default for standard - input is the keyboard buffer. - - .. attribute:: hStdOutput - - If :attr:`dwFlags` specifies :data:`STARTF_USESTDHANDLES`, this attribute - is the standard output handle for the process. Otherwise, this attribute - is ignored and the default for standard output is the console window's - buffer. - - .. attribute:: hStdError - - If :attr:`dwFlags` specifies :data:`STARTF_USESTDHANDLES`, this attribute - is the standard error handle for the process. Otherwise, this attribute is - ignored and the default for standard error is the console window's buffer. - - .. attribute:: wShowWindow - - If :attr:`dwFlags` specifies :data:`STARTF_USESHOWWINDOW`, this attribute - can be any of the values that can be specified in the ``nCmdShow`` - parameter for the - `ShowWindow `__ - function, except for ``SW_SHOWDEFAULT``. Otherwise, this attribute is - ignored. - - :data:`SW_HIDE` is provided for this attribute. It is used when - :class:`Popen` is called with ``shell=True``. - - .. attribute:: lpAttributeList - - A dictionary of additional attributes for process creation as given in - ``STARTUPINFOEX``, see - `UpdateProcThreadAttribute `__. - - Supported attributes: - - **handle_list** - Sequence of handles that will be inherited. *close_fds* must be true if - non-empty. - - The handles must be temporarily made inheritable by - :func:`os.set_handle_inheritable` when passed to the :class:`Popen` - constructor, else :class:`OSError` will be raised with Windows error - ``ERROR_INVALID_PARAMETER`` (87). - - .. warning:: - - In a multithreaded process, use caution to avoid leaking handles - that are marked inheritable when combining this feature with - concurrent calls to other process creation functions that inherit - all handles such as :func:`os.system`. This also applies to - standard handle redirection, which temporarily creates inheritable - handles. - - .. versionadded:: 3.7 - -Windows Constants -^^^^^^^^^^^^^^^^^ - -The :mod:`subprocess` module exposes the following constants. - -.. data:: STD_INPUT_HANDLE - - The standard input device. Initially, this is the console input buffer, - ``CONIN$``. - -.. data:: STD_OUTPUT_HANDLE - - The standard output device. Initially, this is the active console screen - buffer, ``CONOUT$``. - -.. data:: STD_ERROR_HANDLE - - The standard error device. Initially, this is the active console screen - buffer, ``CONOUT$``. - -.. data:: SW_HIDE - - Hides the window. Another window will be activated. - -.. data:: STARTF_USESTDHANDLES - - Specifies that the :attr:`STARTUPINFO.hStdInput`, - :attr:`STARTUPINFO.hStdOutput`, and :attr:`STARTUPINFO.hStdError` attributes - contain additional information. - -.. data:: STARTF_USESHOWWINDOW - - Specifies that the :attr:`STARTUPINFO.wShowWindow` attribute contains - additional information. - -.. data:: CREATE_NEW_CONSOLE - - The new process has a new console, instead of inheriting its parent's - console (the default). - -.. data:: CREATE_NEW_PROCESS_GROUP - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - group will be created. This flag is necessary for using :func:`os.kill` - on the subprocess. - - This flag is ignored if :data:`CREATE_NEW_CONSOLE` is specified. - -.. data:: ABOVE_NORMAL_PRIORITY_CLASS - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - will have an above average priority. - - .. versionadded:: 3.7 - -.. data:: BELOW_NORMAL_PRIORITY_CLASS - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - will have a below average priority. - - .. versionadded:: 3.7 - -.. data:: HIGH_PRIORITY_CLASS - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - will have a high priority. - - .. versionadded:: 3.7 - -.. data:: IDLE_PRIORITY_CLASS - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - will have an idle (lowest) priority. - - .. versionadded:: 3.7 - -.. data:: NORMAL_PRIORITY_CLASS - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - will have an normal priority. (default) - - .. versionadded:: 3.7 - -.. data:: REALTIME_PRIORITY_CLASS - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - will have realtime priority. - You should almost never use REALTIME_PRIORITY_CLASS, because this interrupts - system threads that manage mouse input, keyboard input, and background disk - flushing. This class can be appropriate for applications that "talk" directly - to hardware or that perform brief tasks that should have limited interruptions. - - .. versionadded:: 3.7 - -.. data:: CREATE_NO_WINDOW - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - will not create a window. - - .. versionadded:: 3.7 - -.. data:: DETACHED_PROCESS - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - will not inherit its parent's console. - This value cannot be used with CREATE_NEW_CONSOLE. - - .. versionadded:: 3.7 - -.. data:: CREATE_DEFAULT_ERROR_MODE - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - does not inherit the error mode of the calling process. Instead, the new - process gets the default error mode. - This feature is particularly useful for multithreaded shell applications - that run with hard errors disabled. - - .. versionadded:: 3.7 - -.. data:: CREATE_BREAKAWAY_FROM_JOB - - A :class:`Popen` ``creationflags`` parameter to specify that a new process - is not associated with the job. - - .. versionadded:: 3.7 - -.. _call-function-trio: - -Older high-level API --------------------- - -Prior to Python 3.5, these three functions comprised the high level API to -subprocess. You can now use :func:`run` in many cases, but lots of existing code -calls these functions. - -.. function:: call(args, *, stdin=None, stdout=None, stderr=None, \ - shell=False, cwd=None, timeout=None, **other_popen_kwargs) - - Run the command described by *args*. Wait for command to complete, then - return the :attr:`~Popen.returncode` attribute. - - Code needing to capture stdout or stderr should use :func:`run` instead:: - - run(...).returncode - - To suppress stdout or stderr, supply a value of :data:`DEVNULL`. - - The arguments shown above are merely some common ones. - The full function signature is the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionchanged:: 3.3 - *timeout* was added. - - .. versionchanged:: 3.11.3 - - Changed Windows shell search order for ``shell=True``. The current - directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and - ``%SystemRoot%\System32\cmd.exe``. As a result, dropping a - malicious program named ``cmd.exe`` into a current directory no - longer works. - -.. function:: check_call(args, *, stdin=None, stdout=None, stderr=None, \ - shell=False, cwd=None, timeout=None, \ - **other_popen_kwargs) - - Run command with arguments. Wait for command to complete. If the return - code was zero then return, otherwise raise :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute. - If :func:`check_call` was unable to start the process it will propagate the exception - that was raised. - - Code needing to capture stdout or stderr should use :func:`run` instead:: - - run(..., check=True) - - To suppress stdout or stderr, supply a value of :data:`DEVNULL`. - - The arguments shown above are merely some common ones. - The full function signature is the - same as that of the :class:`Popen` constructor - this function passes all - supplied arguments other than *timeout* directly through to that interface. - - .. note:: - - Do not use ``stdout=PIPE`` or ``stderr=PIPE`` with this - function. The child process will block if it generates enough - output to a pipe to fill up the OS pipe buffer as the pipes are - not being read from. - - .. versionchanged:: 3.3 - *timeout* was added. - - .. versionchanged:: 3.11.3 - - Changed Windows shell search order for ``shell=True``. The current - directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and - ``%SystemRoot%\System32\cmd.exe``. As a result, dropping a - malicious program named ``cmd.exe`` into a current directory no - longer works. - - -.. function:: check_output(args, *, stdin=None, stderr=None, shell=False, \ - cwd=None, encoding=None, errors=None, \ - universal_newlines=None, timeout=None, text=None, \ - **other_popen_kwargs) - - Run command with arguments and return its output. - - If the return code was non-zero it raises a :exc:`CalledProcessError`. The - :exc:`CalledProcessError` object will have the return code in the - :attr:`~CalledProcessError.returncode` attribute and any output in the - :attr:`~CalledProcessError.output` attribute. - - This is equivalent to:: - - run(..., check=True, stdout=PIPE).stdout - - The arguments shown above are merely some common ones. - The full function signature is largely the same as that of :func:`run` - - most arguments are passed directly through to that interface. - One API deviation from :func:`run` behavior exists: passing ``input=None`` - will behave the same as ``input=b''`` (or ``input=''``, depending on other - arguments) rather than using the parent's standard input file handle. - - By default, this function will return the data as encoded bytes. The actual - encoding of the output data may depend on the command being invoked, so the - decoding to text will often need to be handled at the application level. - - This behaviour may be overridden by setting *text*, *encoding*, *errors*, - or *universal_newlines* to ``True`` as described in - :ref:`frequently-used-arguments` and :func:`run`. - - To also capture standard error in the result, use - ``stderr=subprocess.STDOUT``:: - - >>> subprocess.check_output( - ... "ls non_existent_file; exit 0", - ... stderr=subprocess.STDOUT, - ... shell=True) - 'ls: non_existent_file: No such file or directory\n' - - .. versionadded:: 3.1 - - .. versionchanged:: 3.3 - *timeout* was added. - - .. versionchanged:: 3.4 - Support for the *input* keyword argument was added. - - .. versionchanged:: 3.6 - *encoding* and *errors* were added. See :func:`run` for details. - - .. versionadded:: 3.7 - *text* was added as a more readable alias for *universal_newlines*. - - .. versionchanged:: 3.11.3 - - Changed Windows shell search order for ``shell=True``. The current - directory and ``%PATH%`` are replaced with ``%COMSPEC%`` and - ``%SystemRoot%\System32\cmd.exe``. As a result, dropping a - malicious program named ``cmd.exe`` into a current directory no - longer works. - - -.. _subprocess-replacements: - -Replacing Older Functions with the :mod:`subprocess` Module ------------------------------------------------------------ - -In this section, "a becomes b" means that b can be used as a replacement for a. - -.. note:: - - All "a" functions in this section fail (more or less) silently if the - executed program cannot be found; the "b" replacements raise :exc:`OSError` - instead. - - In addition, the replacements using :func:`check_output` will fail with a - :exc:`CalledProcessError` if the requested operation produces a non-zero - return code. The output is still available as the - :attr:`~CalledProcessError.output` attribute of the raised exception. - -In the following examples, we assume that the relevant functions have already -been imported from the :mod:`subprocess` module. - - -Replacing :program:`/bin/sh` shell command substitution -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: bash - - output=$(mycmd myarg) - -becomes:: - - output = check_output(["mycmd", "myarg"]) - -Replacing shell pipeline -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. code-block:: bash - - output=$(dmesg | grep hda) - -becomes:: - - p1 = Popen(["dmesg"], stdout=PIPE) - p2 = Popen(["grep", "hda"], stdin=p1.stdout, stdout=PIPE) - p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. - output = p2.communicate()[0] - -The ``p1.stdout.close()`` call after starting the p2 is important in order for -p1 to receive a SIGPIPE if p2 exits before p1. - -Alternatively, for trusted input, the shell's own pipeline support may still -be used directly: - -.. code-block:: bash - - output=$(dmesg | grep hda) - -becomes:: - - output = check_output("dmesg | grep hda", shell=True) - - -Replacing :func:`os.system` -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - sts = os.system("mycmd" + " myarg") - # becomes - retcode = call("mycmd" + " myarg", shell=True) - -Notes: - -* Calling the program through the shell is usually not required. -* The :func:`call` return value is encoded differently to that of - :func:`os.system`. - -* The :func:`os.system` function ignores SIGINT and SIGQUIT signals while - the command is running, but the caller must do this separately when - using the :mod:`subprocess` module. - -A more realistic example would look like this:: - - try: - retcode = call("mycmd" + " myarg", shell=True) - if retcode < 0: - print("Child was terminated by signal", -retcode, file=sys.stderr) - else: - print("Child returned", retcode, file=sys.stderr) - except OSError as e: - print("Execution failed:", e, file=sys.stderr) - - -Replacing the :func:`os.spawn ` family -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -P_NOWAIT example:: - - pid = os.spawnlp(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg") - ==> - pid = Popen(["/bin/mycmd", "myarg"]).pid - -P_WAIT example:: - - retcode = os.spawnlp(os.P_WAIT, "/bin/mycmd", "mycmd", "myarg") - ==> - retcode = call(["/bin/mycmd", "myarg"]) - -Vector example:: - - os.spawnvp(os.P_NOWAIT, path, args) - ==> - Popen([path] + args[1:]) - -Environment example:: - - os.spawnlpe(os.P_NOWAIT, "/bin/mycmd", "mycmd", "myarg", env) - ==> - Popen(["/bin/mycmd", "myarg"], env={"PATH": "/usr/bin"}) - - - -Replacing :func:`os.popen`, :func:`os.popen2`, :func:`os.popen3` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:: - - (child_stdin, child_stdout) = os.popen2(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdin, child_stdout) = (p.stdin, p.stdout) - -:: - - (child_stdin, - child_stdout, - child_stderr) = os.popen3(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=PIPE, close_fds=True) - (child_stdin, - child_stdout, - child_stderr) = (p.stdin, p.stdout, p.stderr) - -:: - - (child_stdin, child_stdout_and_stderr) = os.popen4(cmd, mode, bufsize) - ==> - p = Popen(cmd, shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, stderr=STDOUT, close_fds=True) - (child_stdin, child_stdout_and_stderr) = (p.stdin, p.stdout) - -Return code handling translates as follows:: - - pipe = os.popen(cmd, 'w') - ... - rc = pipe.close() - if rc is not None and rc >> 8: - print("There were some errors") - ==> - process = Popen(cmd, stdin=PIPE) - ... - process.stdin.close() - if process.wait() != 0: - print("There were some errors") - - -Replacing functions from the :mod:`popen2` module -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. note:: - - If the cmd argument to popen2 functions is a string, the command is executed - through /bin/sh. If it is a list, the command is directly executed. - -:: - - (child_stdout, child_stdin) = popen2.popen2("somestring", bufsize, mode) - ==> - p = Popen("somestring", shell=True, bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:: - - (child_stdout, child_stdin) = popen2.popen2(["mycmd", "myarg"], bufsize, mode) - ==> - p = Popen(["mycmd", "myarg"], bufsize=bufsize, - stdin=PIPE, stdout=PIPE, close_fds=True) - (child_stdout, child_stdin) = (p.stdout, p.stdin) - -:class:`popen2.Popen3` and :class:`popen2.Popen4` basically work as -:class:`subprocess.Popen`, except that: - -* :class:`Popen` raises an exception if the execution fails. - -* The *capturestderr* argument is replaced with the *stderr* argument. - -* ``stdin=PIPE`` and ``stdout=PIPE`` must be specified. - -* popen2 closes all file descriptors by default, but you have to specify - ``close_fds=True`` with :class:`Popen` to guarantee this behavior on - all platforms or past Python versions. - - -Legacy Shell Invocation Functions ---------------------------------- - -This module also provides the following legacy functions from the 2.x -``commands`` module. These operations implicitly invoke the system shell and -none of the guarantees described above regarding security and exception -handling consistency are valid for these functions. - -.. function:: getstatusoutput(cmd, *, encoding=None, errors=None) - - Return ``(exitcode, output)`` of executing *cmd* in a shell. - - Execute the string *cmd* in a shell with :meth:`Popen.check_output` and - return a 2-tuple ``(exitcode, output)``. - *encoding* and *errors* are used to decode output; - see the notes on :ref:`frequently-used-arguments` for more details. - - A trailing newline is stripped from the output. - The exit code for the command can be interpreted as the return code - of subprocess. Example:: - - >>> subprocess.getstatusoutput('ls /bin/ls') - (0, '/bin/ls') - >>> subprocess.getstatusoutput('cat /bin/junk') - (1, 'cat: /bin/junk: No such file or directory') - >>> subprocess.getstatusoutput('/bin/junk') - (127, 'sh: /bin/junk: not found') - >>> subprocess.getstatusoutput('/bin/kill $$') - (-15, '') - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.3.4 - Windows support was added. - - The function now returns (exitcode, output) instead of (status, output) - as it did in Python 3.3.3 and earlier. exitcode has the same value as - :attr:`~Popen.returncode`. - - .. versionadded:: 3.11 - Added *encoding* and *errors* arguments. - -.. function:: getoutput(cmd, *, encoding=None, errors=None) - - Return output (stdout and stderr) of executing *cmd* in a shell. - - Like :func:`getstatusoutput`, except the exit code is ignored and the return - value is a string containing the command's output. Example:: - - >>> subprocess.getoutput('ls /bin/ls') - '/bin/ls' - - .. availability:: Unix, Windows. - - .. versionchanged:: 3.3.4 - Windows support added - - .. versionadded:: 3.11 - Added *encoding* and *errors* arguments. - - -Notes ------ - -.. _converting-argument-sequence: - -Converting an argument sequence to a string on Windows -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -On Windows, an *args* sequence is converted to a string that can be parsed -using the following rules (which correspond to the rules used by the MS C -runtime): - -1. Arguments are delimited by white space, which is either a - space or a tab. - -2. A string surrounded by double quotation marks is - interpreted as a single argument, regardless of white space - contained within. A quoted string can be embedded in an - argument. - -3. A double quotation mark preceded by a backslash is - interpreted as a literal double quotation mark. - -4. Backslashes are interpreted literally, unless they - immediately precede a double quotation mark. - -5. If backslashes immediately precede a double quotation mark, - every pair of backslashes is interpreted as a literal - backslash. If the number of backslashes is odd, the last - backslash escapes the next double quotation mark as - described in rule 3. - - -.. seealso:: - - :mod:`shlex` - Module which provides function to parse and escape command lines. - - -.. _disable_vfork: -.. _disable_posix_spawn: - -Disabling use of ``vfork()`` or ``posix_spawn()`` -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -On Linux, :mod:`subprocess` defaults to using the ``vfork()`` system call -internally when it is safe to do so rather than ``fork()``. This greatly -improves performance. - -If you ever encounter a presumed highly unusual situation where you need to -prevent ``vfork()`` from being used by Python, you can set the -:const:`subprocess._USE_VFORK` attribute to a false value. - -:: - - subprocess._USE_VFORK = False # See CPython issue gh-NNNNNN. - -Setting this has no impact on use of ``posix_spawn()`` which could use -``vfork()`` internally within its libc implementation. There is a similar -:const:`subprocess._USE_POSIX_SPAWN` attribute if you need to prevent use of -that. - -:: - - subprocess._USE_POSIX_SPAWN = False # See CPython issue gh-NNNNNN. - -It is safe to set these to false on any Python version. They will have no -effect on older versions when unsupported. Do not assume the attributes are -available to read. Despite their names, a true value does not indicate that the -corresponding function will be used, only that it may be. - -Please file issues any time you have to use these private knobs with a way to -reproduce the issue you were seeing. Link to that issue from a comment in your -code. - -.. versionadded:: 3.8 ``_USE_POSIX_SPAWN`` -.. versionadded:: 3.11 ``_USE_VFORK`` diff --git a/Doc/library/sunau.rst b/Doc/library/sunau.rst deleted file mode 100644 index c7a38d96ade131..00000000000000 --- a/Doc/library/sunau.rst +++ /dev/null @@ -1,274 +0,0 @@ -:mod:`sunau` --- Read and write Sun AU files -============================================ - -.. module:: sunau - :synopsis: Provide an interface to the Sun AU sound format. - :deprecated: - -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/sunau.py` - -.. deprecated-removed:: 3.11 3.13 - The :mod:`sunau` module is deprecated - (see :pep:`PEP 594 <594#sunau>` for details). - --------------- - -The :mod:`sunau` module provides a convenient interface to the Sun AU sound -format. Note that this module is interface-compatible with the modules -:mod:`aifc` and :mod:`wave`. - -An audio file consists of a header followed by the data. The fields of the -header are: - -+---------------+-----------------------------------------------+ -| Field | Contents | -+===============+===============================================+ -| magic word | The four bytes ``.snd``. | -+---------------+-----------------------------------------------+ -| header size | Size of the header, including info, in bytes. | -+---------------+-----------------------------------------------+ -| data size | Physical size of the data, in bytes. | -+---------------+-----------------------------------------------+ -| encoding | Indicates how the audio samples are encoded. | -+---------------+-----------------------------------------------+ -| sample rate | The sampling rate. | -+---------------+-----------------------------------------------+ -| # of channels | The number of channels in the samples. | -+---------------+-----------------------------------------------+ -| info | ASCII string giving a description of the | -| | audio file (padded with null bytes). | -+---------------+-----------------------------------------------+ - -Apart from the info field, all header fields are 4 bytes in size. They are all -32-bit unsigned integers encoded in big-endian byte order. - -The :mod:`sunau` module defines the following functions: - - -.. function:: open(file, mode) - - If *file* is a string, open the file by that name, otherwise treat it as a - seekable file-like object. *mode* can be any of - - ``'r'`` - Read only mode. - - ``'w'`` - Write only mode. - - Note that it does not allow read/write files. - - A *mode* of ``'r'`` returns an :class:`AU_read` object, while a *mode* of ``'w'`` - or ``'wb'`` returns an :class:`AU_write` object. - - -The :mod:`sunau` module defines the following exception: - -.. exception:: Error - - An error raised when something is impossible because of Sun AU specs or - implementation deficiency. - - -The :mod:`sunau` module defines the following data items: - -.. data:: AUDIO_FILE_MAGIC - - An integer every valid Sun AU file begins with, stored in big-endian form. This - is the string ``.snd`` interpreted as an integer. - - -.. data:: AUDIO_FILE_ENCODING_MULAW_8 - AUDIO_FILE_ENCODING_LINEAR_8 - AUDIO_FILE_ENCODING_LINEAR_16 - AUDIO_FILE_ENCODING_LINEAR_24 - AUDIO_FILE_ENCODING_LINEAR_32 - AUDIO_FILE_ENCODING_ALAW_8 - - Values of the encoding field from the AU header which are supported by this - module. - - -.. data:: AUDIO_FILE_ENCODING_FLOAT - AUDIO_FILE_ENCODING_DOUBLE - AUDIO_FILE_ENCODING_ADPCM_G721 - AUDIO_FILE_ENCODING_ADPCM_G722 - AUDIO_FILE_ENCODING_ADPCM_G723_3 - AUDIO_FILE_ENCODING_ADPCM_G723_5 - - Additional known values of the encoding field from the AU header, but which are - not supported by this module. - - -.. _au-read-objects: - -AU_read Objects ---------------- - -AU_read objects, as returned by :func:`.open` above, have the following methods: - - -.. method:: AU_read.close() - - Close the stream, and make the instance unusable. (This is called automatically - on deletion.) - - -.. method:: AU_read.getnchannels() - - Returns number of audio channels (1 for mono, 2 for stereo). - - -.. method:: AU_read.getsampwidth() - - Returns sample width in bytes. - - -.. method:: AU_read.getframerate() - - Returns sampling frequency. - - -.. method:: AU_read.getnframes() - - Returns number of audio frames. - - -.. method:: AU_read.getcomptype() - - Returns compression type. Supported compression types are ``'ULAW'``, ``'ALAW'`` - and ``'NONE'``. - - -.. method:: AU_read.getcompname() - - Human-readable version of :meth:`getcomptype`. The supported types have the - respective names ``'CCITT G.711 u-law'``, ``'CCITT G.711 A-law'`` and ``'not - compressed'``. - - -.. method:: AU_read.getparams() - - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - :meth:`get\*` methods. - - -.. method:: AU_read.readframes(n) - - Reads and returns at most *n* frames of audio, as a :class:`bytes` object. The data - will be returned in linear format. If the original data is in u-LAW format, it - will be converted. - - -.. method:: AU_read.rewind() - - Rewind the file pointer to the beginning of the audio stream. - -The following two methods define a term "position" which is compatible between -them, and is otherwise implementation dependent. - - -.. method:: AU_read.setpos(pos) - - Set the file pointer to the specified position. Only values returned from - :meth:`tell` should be used for *pos*. - - -.. method:: AU_read.tell() - - Return current file pointer position. Note that the returned value has nothing - to do with the actual position in the file. - -The following two functions are defined for compatibility with the :mod:`aifc`, -and don't do anything interesting. - - -.. method:: AU_read.getmarkers() - - Returns ``None``. - - -.. method:: AU_read.getmark(id) - - Raise an error. - - -.. _au-write-objects: - -AU_write Objects ----------------- - -AU_write objects, as returned by :func:`.open` above, have the following methods: - - -.. method:: AU_write.setnchannels(n) - - Set the number of channels. - - -.. method:: AU_write.setsampwidth(n) - - Set the sample width (in bytes.) - - .. versionchanged:: 3.4 - Added support for 24-bit samples. - - -.. method:: AU_write.setframerate(n) - - Set the frame rate. - - -.. method:: AU_write.setnframes(n) - - Set the number of frames. This can be later changed, when and if more frames - are written. - - -.. method:: AU_write.setcomptype(type, name) - - Set the compression type and description. Only ``'NONE'`` and ``'ULAW'`` are - supported on output. - - -.. method:: AU_write.setparams(tuple) - - The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, with values valid for the :meth:`set\*` methods. Set all - parameters. - - -.. method:: AU_write.tell() - - Return current position in the file, with the same disclaimer for the - :meth:`AU_read.tell` and :meth:`AU_read.setpos` methods. - - -.. method:: AU_write.writeframesraw(data) - - Write audio frames, without correcting *nframes*. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: AU_write.writeframes(data) - - Write audio frames and make sure *nframes* is correct. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - -.. method:: AU_write.close() - - Make sure *nframes* is correct, and close the file. - - This method is called upon deletion. - -Note that it is invalid to set any parameters after calling :meth:`writeframes` -or :meth:`writeframesraw`. - diff --git a/Doc/library/superseded.rst b/Doc/library/superseded.rst deleted file mode 100644 index 25cbe6c94e8ec5..00000000000000 --- a/Doc/library/superseded.rst +++ /dev/null @@ -1,37 +0,0 @@ -.. _superseded: - -****************** -Superseded Modules -****************** - -The modules described in this chapter are deprecated and only kept for -backwards compatibility. They have been superseded by other modules. - - -.. toctree:: - :maxdepth: 1 - - aifc.rst - asynchat.rst - asyncore.rst - audioop.rst - cgi.rst - cgitb.rst - chunk.rst - crypt.rst - imghdr.rst - imp.rst - mailcap.rst - msilib.rst - nis.rst - nntplib.rst - optparse.rst - ossaudiodev.rst - pipes.rst - smtpd.rst - sndhdr.rst - spwd.rst - sunau.rst - telnetlib.rst - uu.rst - xdrlib.rst diff --git a/Doc/library/symtable.rst b/Doc/library/symtable.rst deleted file mode 100644 index 65ff5bfe7abd61..00000000000000 --- a/Doc/library/symtable.rst +++ /dev/null @@ -1,199 +0,0 @@ -:mod:`symtable` --- Access to the compiler's symbol tables -========================================================== - -.. module:: symtable - :synopsis: Interface to the compiler's internal symbol tables. - -**Source code:** :source:`Lib/symtable.py` - --------------- - -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Benjamin Peterson - - -Symbol tables are generated by the compiler from AST just before bytecode is -generated. The symbol table is responsible for calculating the scope of every -identifier in the code. :mod:`symtable` provides an interface to examine these -tables. - - -Generating Symbol Tables ------------------------- - -.. function:: symtable(code, filename, compile_type) - - Return the toplevel :class:`SymbolTable` for the Python source *code*. - *filename* is the name of the file containing the code. *compile_type* is - like the *mode* argument to :func:`compile`. - - -Examining Symbol Tables ------------------------ - -.. class:: SymbolTable - - A namespace table for a block. The constructor is not public. - - .. method:: get_type() - - Return the type of the symbol table. Possible values are ``'class'``, - ``'module'``, and ``'function'``. - - .. method:: get_id() - - Return the table's identifier. - - .. method:: get_name() - - Return the table's name. This is the name of the class if the table is - for a class, the name of the function if the table is for a function, or - ``'top'`` if the table is global (:meth:`get_type` returns ``'module'``). - - .. method:: get_lineno() - - Return the number of the first line in the block this table represents. - - .. method:: is_optimized() - - Return ``True`` if the locals in this table can be optimized. - - .. method:: is_nested() - - Return ``True`` if the block is a nested class or function. - - .. method:: has_children() - - Return ``True`` if the block has nested namespaces within it. These can - be obtained with :meth:`get_children`. - - .. method:: get_identifiers() - - Return a view object containing the names of symbols in the table. - See the :ref:`documentation of view objects `. - - .. method:: lookup(name) - - Lookup *name* in the table and return a :class:`Symbol` instance. - - .. method:: get_symbols() - - Return a list of :class:`Symbol` instances for names in the table. - - .. method:: get_children() - - Return a list of the nested symbol tables. - - -.. class:: Function - - A namespace for a function or method. This class inherits - :class:`SymbolTable`. - - .. method:: get_parameters() - - Return a tuple containing names of parameters to this function. - - .. method:: get_locals() - - Return a tuple containing names of locals in this function. - - .. method:: get_globals() - - Return a tuple containing names of globals in this function. - - .. method:: get_nonlocals() - - Return a tuple containing names of nonlocals in this function. - - .. method:: get_frees() - - Return a tuple containing names of free variables in this function. - - -.. class:: Class - - A namespace of a class. This class inherits :class:`SymbolTable`. - - .. method:: get_methods() - - Return a tuple containing the names of methods declared in the class. - - -.. class:: Symbol - - An entry in a :class:`SymbolTable` corresponding to an identifier in the - source. The constructor is not public. - - .. method:: get_name() - - Return the symbol's name. - - .. method:: is_referenced() - - Return ``True`` if the symbol is used in its block. - - .. method:: is_imported() - - Return ``True`` if the symbol is created from an import statement. - - .. method:: is_parameter() - - Return ``True`` if the symbol is a parameter. - - .. method:: is_global() - - Return ``True`` if the symbol is global. - - .. method:: is_nonlocal() - - Return ``True`` if the symbol is nonlocal. - - .. method:: is_declared_global() - - Return ``True`` if the symbol is declared global with a global statement. - - .. method:: is_local() - - Return ``True`` if the symbol is local to its block. - - .. method:: is_annotated() - - Return ``True`` if the symbol is annotated. - - .. versionadded:: 3.6 - - .. method:: is_free() - - Return ``True`` if the symbol is referenced in its block, but not assigned - to. - - .. method:: is_assigned() - - Return ``True`` if the symbol is assigned to in its block. - - .. method:: is_namespace() - - Return ``True`` if name binding introduces new namespace. - - If the name is used as the target of a function or class statement, this - will be true. - - For example:: - - >>> table = symtable.symtable("def some_func(): pass", "string", "exec") - >>> table.lookup("some_func").is_namespace() - True - - Note that a single name can be bound to multiple objects. If the result - is ``True``, the name may also be bound to other objects, like an int or - list, that does not introduce a new namespace. - - .. method:: get_namespaces() - - Return a list of namespaces bound to this name. - - .. method:: get_namespace() - - Return the namespace bound to this name. If more than one or no namespace - is bound to this name, a :exc:`ValueError` is raised. diff --git a/Doc/library/sys.rst b/Doc/library/sys.rst deleted file mode 100644 index f5b94f567ff749..00000000000000 --- a/Doc/library/sys.rst +++ /dev/null @@ -1,1932 +0,0 @@ -:mod:`sys` --- System-specific parameters and functions -======================================================= - -.. module:: sys - :synopsis: Access system-specific parameters and functions. - --------------- - -This module provides access to some variables used or maintained by the -interpreter and to functions that interact strongly with the interpreter. It is -always available. - - -.. data:: abiflags - - On POSIX systems where Python was built with the standard ``configure`` - script, this contains the ABI flags as specified by :pep:`3149`. - - .. versionchanged:: 3.8 - Default flags became an empty string (``m`` flag for pymalloc has been - removed). - - .. versionadded:: 3.2 - - -.. function:: addaudithook(hook) - - Append the callable *hook* to the list of active auditing hooks for the - current (sub)interpreter. - - When an auditing event is raised through the :func:`sys.audit` function, each - hook will be called in the order it was added with the event name and the - tuple of arguments. Native hooks added by :c:func:`PySys_AddAuditHook` are - called first, followed by hooks added in the current (sub)interpreter. Hooks - can then log the event, raise an exception to abort the operation, - or terminate the process entirely. - - Note that audit hooks are primarily for collecting information about internal - or otherwise unobservable actions, whether by Python or libraries written in - Python. They are not suitable for implementing a "sandbox". In particular, - malicious code can trivially disable or bypass hooks added using this - function. At a minimum, any security-sensitive hooks must be added using the - C API :c:func:`PySys_AddAuditHook` before initialising the runtime, and any - modules allowing arbitrary memory modification (such as :mod:`ctypes`) should - be completely removed or closely monitored. - - .. audit-event:: sys.addaudithook "" sys.addaudithook - - Calling :func:`sys.addaudithook` will itself raise an auditing event - named ``sys.addaudithook`` with no arguments. If any - existing hooks raise an exception derived from :class:`RuntimeError`, the - new hook will not be added and the exception suppressed. As a result, - callers cannot assume that their hook has been added unless they control - all existing hooks. - - See the :ref:`audit events table ` for all events raised by - CPython, and :pep:`578` for the original design discussion. - - .. versionadded:: 3.8 - - .. versionchanged:: 3.8.1 - - Exceptions derived from :class:`Exception` but not :class:`RuntimeError` - are no longer suppressed. - - .. impl-detail:: - - When tracing is enabled (see :func:`settrace`), Python hooks are only - traced if the callable has a ``__cantrace__`` member that is set to a - true value. Otherwise, trace functions will skip the hook. - - -.. data:: argv - - The list of command line arguments passed to a Python script. ``argv[0]`` is the - script name (it is operating system dependent whether this is a full pathname or - not). If the command was executed using the :option:`-c` command line option to - the interpreter, ``argv[0]`` is set to the string ``'-c'``. If no script name - was passed to the Python interpreter, ``argv[0]`` is the empty string. - - To loop over the standard input, or the list of files given on the - command line, see the :mod:`fileinput` module. - - See also :data:`sys.orig_argv`. - - .. note:: - On Unix, command line arguments are passed by bytes from OS. Python decodes - them with filesystem encoding and "surrogateescape" error handler. - When you need original bytes, you can get it by - ``[os.fsencode(arg) for arg in sys.argv]``. - - -.. _auditing: - -.. function:: audit(event, *args) - - .. index:: single: auditing - - Raise an auditing event and trigger any active auditing hooks. - *event* is a string identifying the event, and *args* may contain - optional arguments with more information about the event. The - number and types of arguments for a given event are considered a - public and stable API and should not be modified between releases. - - For example, one auditing event is named ``os.chdir``. This event has - one argument called *path* that will contain the requested new - working directory. - - :func:`sys.audit` will call the existing auditing hooks, passing - the event name and arguments, and will re-raise the first exception - from any hook. In general, if an exception is raised, it should not - be handled and the process should be terminated as quickly as - possible. This allows hook implementations to decide how to respond - to particular events: they can merely log the event or abort the - operation by raising an exception. - - Hooks are added using the :func:`sys.addaudithook` or - :c:func:`PySys_AddAuditHook` functions. - - The native equivalent of this function is :c:func:`PySys_Audit`. Using the - native function is preferred when possible. - - See the :ref:`audit events table ` for all events raised by - CPython. - - .. versionadded:: 3.8 - - -.. data:: base_exec_prefix - - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`exec_prefix`. If not running in a - :ref:`virtual environment `, the values will stay the same; if - ``site.py`` finds that a virtual environment is in use, the values of - :data:`prefix` and :data:`exec_prefix` will be changed to point to the - virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). - - .. versionadded:: 3.3 - - -.. data:: base_prefix - - Set during Python startup, before ``site.py`` is run, to the same value as - :data:`prefix`. If not running in a :ref:`virtual environment `, the values - will stay the same; if ``site.py`` finds that a virtual environment is in - use, the values of :data:`prefix` and :data:`exec_prefix` will be changed to - point to the virtual environment, whereas :data:`base_prefix` and - :data:`base_exec_prefix` will remain pointing to the base Python - installation (the one which the virtual environment was created from). - - .. versionadded:: 3.3 - - -.. data:: byteorder - - An indicator of the native byte order. This will have the value ``'big'`` on - big-endian (most-significant byte first) platforms, and ``'little'`` on - little-endian (least-significant byte first) platforms. - - -.. data:: builtin_module_names - - A tuple of strings containing the names of all modules that are compiled into this - Python interpreter. (This information is not available in any other way --- - ``modules.keys()`` only lists the imported modules.) - - See also the :data:`sys.stdlib_module_names` list. - - -.. function:: call_tracing(func, args) - - Call ``func(*args)``, while tracing is enabled. The tracing state is saved, - and restored afterwards. This is intended to be called from a debugger from - a checkpoint, to recursively debug or profile some other code. - - Tracing is suspended while calling a tracing function set by - :func:`settrace` or :func:`setprofile` to avoid infinite recursion. - :func:`!call_tracing` enables explicit recursion of the tracing function. - - -.. data:: copyright - - A string containing the copyright pertaining to the Python interpreter. - - -.. function:: _clear_type_cache() - - Clear the internal type cache. The type cache is used to speed up attribute - and method lookups. Use the function *only* to drop unnecessary references - during reference leak debugging. - - This function should be used for internal and specialized purposes only. - - -.. function:: _current_frames() - - Return a dictionary mapping each thread's identifier to the topmost stack frame - currently active in that thread at the time the function is called. Note that - functions in the :mod:`traceback` module can build the call stack given such a - frame. - - This is most useful for debugging deadlock: this function does not require the - deadlocked threads' cooperation, and such threads' call stacks are frozen for as - long as they remain deadlocked. The frame returned for a non-deadlocked thread - may bear no relationship to that thread's current activity by the time calling - code examines the frame. - - This function should be used for internal and specialized purposes only. - - .. audit-event:: sys._current_frames "" sys._current_frames - -.. function:: _current_exceptions() - - Return a dictionary mapping each thread's identifier to the topmost exception - currently active in that thread at the time the function is called. - If a thread is not currently handling an exception, it is not included in - the result dictionary. - - This is most useful for statistical profiling. - - This function should be used for internal and specialized purposes only. - - .. audit-event:: sys._current_exceptions "" sys._current_exceptions - -.. function:: breakpointhook() - - This hook function is called by built-in :func:`breakpoint`. By default, - it drops you into the :mod:`pdb` debugger, but it can be set to any other - function so that you can choose which debugger gets used. - - The signature of this function is dependent on what it calls. For example, - the default binding (e.g. ``pdb.set_trace()``) expects no arguments, but - you might bind it to a function that expects additional arguments - (positional and/or keyword). The built-in ``breakpoint()`` function passes - its ``*args`` and ``**kws`` straight through. Whatever - ``breakpointhooks()`` returns is returned from ``breakpoint()``. - - The default implementation first consults the environment variable - :envvar:`PYTHONBREAKPOINT`. If that is set to ``"0"`` then this function - returns immediately; i.e. it is a no-op. If the environment variable is - not set, or is set to the empty string, ``pdb.set_trace()`` is called. - Otherwise this variable should name a function to run, using Python's - dotted-import nomenclature, e.g. ``package.subpackage.module.function``. - In this case, ``package.subpackage.module`` would be imported and the - resulting module must have a callable named ``function()``. This is run, - passing in ``*args`` and ``**kws``, and whatever ``function()`` returns, - ``sys.breakpointhook()`` returns to the built-in :func:`breakpoint` - function. - - Note that if anything goes wrong while importing the callable named by - :envvar:`PYTHONBREAKPOINT`, a :exc:`RuntimeWarning` is reported and the - breakpoint is ignored. - - Also note that if ``sys.breakpointhook()`` is overridden programmatically, - :envvar:`PYTHONBREAKPOINT` is *not* consulted. - - .. versionadded:: 3.7 - -.. function:: _debugmallocstats() - - Print low-level information to stderr about the state of CPython's memory - allocator. - - If Python is :ref:`built in debug mode ` (:option:`configure - --with-pydebug option <--with-pydebug>`), it also performs some expensive - internal consistency checks. - - .. versionadded:: 3.3 - - .. impl-detail:: - - This function is specific to CPython. The exact output format is not - defined here, and may change. - - -.. data:: dllhandle - - Integer specifying the handle of the Python DLL. - - .. availability:: Windows. - - -.. function:: displayhook(value) - - If *value* is not ``None``, this function prints ``repr(value)`` to - ``sys.stdout``, and saves *value* in ``builtins._``. If ``repr(value)`` is - not encodable to ``sys.stdout.encoding`` with ``sys.stdout.errors`` error - handler (which is probably ``'strict'``), encode it to - ``sys.stdout.encoding`` with ``'backslashreplace'`` error handler. - - ``sys.displayhook`` is called on the result of evaluating an :term:`expression` - entered in an interactive Python session. The display of these values can be - customized by assigning another one-argument function to ``sys.displayhook``. - - Pseudo-code:: - - def displayhook(value): - if value is None: - return - # Set '_' to None to avoid recursion - builtins._ = None - text = repr(value) - try: - sys.stdout.write(text) - except UnicodeEncodeError: - bytes = text.encode(sys.stdout.encoding, 'backslashreplace') - if hasattr(sys.stdout, 'buffer'): - sys.stdout.buffer.write(bytes) - else: - text = bytes.decode(sys.stdout.encoding, 'strict') - sys.stdout.write(text) - sys.stdout.write("\n") - builtins._ = value - - .. versionchanged:: 3.2 - Use ``'backslashreplace'`` error handler on :exc:`UnicodeEncodeError`. - - -.. data:: dont_write_bytecode - - If this is true, Python won't try to write ``.pyc`` files on the - import of source modules. This value is initially set to ``True`` or - ``False`` depending on the :option:`-B` command line option and the - :envvar:`PYTHONDONTWRITEBYTECODE` environment variable, but you can set it - yourself to control bytecode file generation. - - -.. data:: _emscripten_info - - A :term:`named tuple` holding information about the environment on the - *wasm32-emscripten* platform. The named tuple is provisional and may change - in the future. - - .. attribute:: _emscripten_info.emscripten_version - - Emscripten version as tuple of ints (major, minor, micro), e.g. ``(3, 1, 8)``. - - .. attribute:: _emscripten_info.runtime - - Runtime string, e.g. browser user agent, ``'Node.js v14.18.2'``, or ``'UNKNOWN'``. - - .. attribute:: _emscripten_info.pthreads - - ``True`` if Python is compiled with Emscripten pthreads support. - - .. attribute:: _emscripten_info.shared_memory - - ``True`` if Python is compiled with shared memory support. - - .. availability:: Emscripten. - - .. versionadded:: 3.11 - - -.. data:: pycache_prefix - - If this is set (not ``None``), Python will write bytecode-cache ``.pyc`` - files to (and read them from) a parallel directory tree rooted at this - directory, rather than from ``__pycache__`` directories in the source code - tree. Any ``__pycache__`` directories in the source code tree will be ignored - and new ``.pyc`` files written within the pycache prefix. Thus if you use - :mod:`compileall` as a pre-build step, you must ensure you run it with the - same pycache prefix (if any) that you will use at runtime. - - A relative path is interpreted relative to the current working directory. - - This value is initially set based on the value of the :option:`-X` - ``pycache_prefix=PATH`` command-line option or the - :envvar:`PYTHONPYCACHEPREFIX` environment variable (command-line takes - precedence). If neither are set, it is ``None``. - - .. versionadded:: 3.8 - - -.. function:: excepthook(type, value, traceback) - - This function prints out a given traceback and exception to ``sys.stderr``. - - When an exception other than :exc:`SystemExit` is raised and uncaught, the interpreter calls - ``sys.excepthook`` with three arguments, the exception class, exception - instance, and a traceback object. In an interactive session this happens just - before control is returned to the prompt; in a Python program this happens just - before the program exits. The handling of such top-level exceptions can be - customized by assigning another three-argument function to ``sys.excepthook``. - - .. audit-event:: sys.excepthook hook,type,value,traceback sys.excepthook - - Raise an auditing event ``sys.excepthook`` with arguments ``hook``, - ``type``, ``value``, ``traceback`` when an uncaught exception occurs. - If no hook has been set, ``hook`` may be ``None``. If any hook raises - an exception derived from :class:`RuntimeError` the call to the hook will - be suppressed. Otherwise, the audit hook exception will be reported as - unraisable and ``sys.excepthook`` will be called. - - .. seealso:: - - The :func:`sys.unraisablehook` function handles unraisable exceptions - and the :func:`threading.excepthook` function handles exception raised - by :func:`threading.Thread.run`. - - -.. data:: __breakpointhook__ - __displayhook__ - __excepthook__ - __unraisablehook__ - - These objects contain the original values of ``breakpointhook``, - ``displayhook``, ``excepthook``, and ``unraisablehook`` at the start of the - program. They are saved so that ``breakpointhook``, ``displayhook`` and - ``excepthook``, ``unraisablehook`` can be restored in case they happen to - get replaced with broken or alternative objects. - - .. versionadded:: 3.7 - __breakpointhook__ - - .. versionadded:: 3.8 - __unraisablehook__ - - -.. function:: exception() - - This function, when called while an exception handler is executing (such as - an ``except`` or ``except*`` clause), returns the exception instance that - was caught by this handler. When exception handlers are nested within one - another, only the exception handled by the innermost handler is accessible. - - If no exception handler is executing, this function returns ``None``. - - .. versionadded:: 3.11 - - -.. function:: exc_info() - - This function returns the old-style representation of the handled - exception. If an exception ``e`` is currently handled (so - :func:`exception` would return ``e``), :func:`exc_info` returns the - tuple ``(type(e), e, e.__traceback__)``. - That is, a tuple containing the type of the exception (a subclass of - :exc:`BaseException`), the exception itself, and a :ref:`traceback - object ` which typically encapsulates the call - stack at the point where the exception last occurred. - - .. index:: pair: object; traceback - - If no exception is being handled anywhere on the stack, this function - return a tuple containing three ``None`` values. - - .. versionchanged:: 3.11 - The ``type`` and ``traceback`` fields are now derived from the ``value`` - (the exception instance), so when an exception is modified while it is - being handled, the changes are reflected in the results of subsequent - calls to :func:`exc_info`. - -.. data:: exec_prefix - - A string giving the site-specific directory prefix where the platform-dependent - Python files are installed; by default, this is also ``'/usr/local'``. This can - be set at build time with the ``--exec-prefix`` argument to the - :program:`configure` script. Specifically, all configuration files (e.g. the - :file:`pyconfig.h` header file) are installed in the directory - :file:`{exec_prefix}/lib/python{X.Y}/config`, and shared library modules are - installed in :file:`{exec_prefix}/lib/python{X.Y}/lib-dynload`, where *X.Y* - is the version number of Python, for example ``3.2``. - - .. note:: - - If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual environment. - The value for the Python installation will still be available, via - :data:`base_exec_prefix`. - - -.. data:: executable - - A string giving the absolute path of the executable binary for the Python - interpreter, on systems where this makes sense. If Python is unable to retrieve - the real path to its executable, :data:`sys.executable` will be an empty string - or ``None``. - - -.. function:: exit([arg]) - - Raise a :exc:`SystemExit` exception, signaling an intention to exit the interpreter. - - The optional argument *arg* can be an integer giving the exit status - (defaulting to zero), or another type of object. If it is an integer, zero - is considered "successful termination" and any nonzero value is considered - "abnormal termination" by shells and the like. Most systems require it to be - in the range 0--127, and produce undefined results otherwise. Some systems - have a convention for assigning specific meanings to specific exit codes, but - these are generally underdeveloped; Unix programs generally use 2 for command - line syntax errors and 1 for all other kind of errors. If another type of - object is passed, ``None`` is equivalent to passing zero, and any other - object is printed to :data:`stderr` and results in an exit code of 1. In - particular, ``sys.exit("some error message")`` is a quick way to exit a - program when an error occurs. - - Since :func:`exit` ultimately "only" raises an exception, it will only exit - the process when called from the main thread, and the exception is not - intercepted. Cleanup actions specified by finally clauses of :keyword:`try` statements - are honored, and it is possible to intercept the exit attempt at an outer level. - - .. versionchanged:: 3.6 - If an error occurs in the cleanup after the Python interpreter - has caught :exc:`SystemExit` (such as an error flushing buffered data - in the standard streams), the exit status is changed to 120. - - -.. data:: flags - - The :term:`named tuple` *flags* exposes the status of command line - flags. The attributes are read only. - - .. list-table:: - - * - .. attribute:: flags.debug - - :option:`-d` - - * - .. attribute:: flags.inspect - - :option:`-i` - - * - .. attribute:: flags.interactive - - :option:`-i` - - * - .. attribute:: flags.isolated - - :option:`-I` - - * - .. attribute:: flags.optimize - - :option:`-O` or :option:`-OO` - - * - .. attribute:: flags.dont_write_bytecode - - :option:`-B` - - * - .. attribute:: flags.no_user_site - - :option:`-s` - - * - .. attribute:: flags.no_site - - :option:`-S` - - * - .. attribute:: flags.ignore_environment - - :option:`-E` - - * - .. attribute:: flags.verbose - - :option:`-v` - - * - .. attribute:: flags.bytes_warning - - :option:`-b` - - * - .. attribute:: flags.quiet - - :option:`-q` - - * - .. attribute:: flags.hash_randomization - - :option:`-R` - - * - .. attribute:: flags.dev_mode - - :option:`-X dev <-X>` (:ref:`Python Development Mode `) - - * - .. attribute:: flags.utf8_mode - - :option:`-X utf8 <-X>` - - * - .. attribute:: flags.safe_path - - :option:`-P` - - * - .. attribute:: flags.int_max_str_digits - - :option:`-X int_max_str_digits <-X>` - (:ref:`integer string conversion length limitation `) - - * - .. attribute:: flags.warn_default_encoding - - :option:`-X warn_default_encoding <-X>` - - .. versionchanged:: 3.2 - Added ``quiet`` attribute for the new :option:`-q` flag. - - .. versionadded:: 3.2.3 - The ``hash_randomization`` attribute. - - .. versionchanged:: 3.3 - Removed obsolete ``division_warning`` attribute. - - .. versionchanged:: 3.4 - Added ``isolated`` attribute for :option:`-I` ``isolated`` flag. - - .. versionchanged:: 3.7 - Added the ``dev_mode`` attribute for the new :ref:`Python Development - Mode ` and the ``utf8_mode`` attribute for the new :option:`-X` - ``utf8`` flag. - - .. versionchanged:: 3.10 - Added ``warn_default_encoding`` attribute for :option:`-X` ``warn_default_encoding`` flag. - - .. versionchanged:: 3.11 - Added the ``safe_path`` attribute for :option:`-P` option. - - .. versionchanged:: 3.11 - Added the ``int_max_str_digits`` attribute. - - -.. data:: float_info - - A :term:`named tuple` holding information about the float type. It - contains low level information about the precision and internal - representation. The values correspond to the various floating-point - constants defined in the standard header file :file:`float.h` for the 'C' - programming language; see section 5.2.4.2.2 of the 1999 ISO/IEC C standard - [C99]_, 'Characteristics of floating types', for details. - - .. list-table:: Attributes of the :data:`!float_info` :term:`named tuple` - :header-rows: 1 - - * - attribute - - float.h macro - - explanation - - * - .. attribute:: float_info.epsilon - - :c:macro:`!DBL_EPSILON` - - difference between 1.0 and the least value greater than 1.0 that is - representable as a float. - - See also :func:`math.ulp`. - - * - .. attribute:: float_info.dig - - :c:macro:`!DBL_DIG` - - The maximum number of decimal digits that can be faithfully - represented in a float; see below. - - * - .. attribute:: float_info.mant_dig - - :c:macro:`!DBL_MANT_DIG` - - Float precision: the number of base-``radix`` digits in the - significand of a float. - - * - .. attribute:: float_info.max - - :c:macro:`!DBL_MAX` - - The maximum representable positive finite float. - - * - .. attribute:: float_info.max_exp - - :c:macro:`!DBL_MAX_EXP` - - The maximum integer *e* such that ``radix**(e-1)`` is a representable - finite float. - - * - .. attribute:: float_info.max_10_exp - - :c:macro:`!DBL_MAX_10_EXP` - - The maximum integer *e* such that ``10**e`` is in the range of - representable finite floats. - - * - .. attribute:: float_info.min - - :c:macro:`!DBL_MIN` - - The minimum representable positive *normalized* float. - - Use :func:`math.ulp(0.0) ` to get the smallest positive - *denormalized* representable float. - - * - .. attribute:: float_info.min_exp - - :c:macro:`!DBL_MIN_EXP` - - The minimum integer *e* such that ``radix**(e-1)`` is a normalized - float. - - * - .. attribute:: float_info.min_10_exp - - :c:macro:`!DBL_MIN_10_EXP` - - The minimum integer *e* such that ``10**e`` is a normalized float. - - * - .. attribute:: float_info.radix - - :c:macro:`!FLT_RADIX` - - The radix of exponent representation. - - * - .. attribute:: float_info.rounds - - :c:macro:`!FLT_ROUNDS` - - An integer representing the rounding mode for floating-point arithmetic. - This reflects the value of the system :c:macro:`!FLT_ROUNDS` macro - at interpreter startup time: - - * ``-1``: indeterminable - * ``0``: toward zero - * ``1``: to nearest - * ``2``: toward positive infinity - * ``3``: toward negative infinity - - All other values for :c:macro:`!FLT_ROUNDS` characterize - implementation-defined rounding behavior. - - The attribute :attr:`sys.float_info.dig` needs further explanation. If - ``s`` is any string representing a decimal number with at most - :attr:`!sys.float_info.dig` significant digits, then converting ``s`` to a - float and back again will recover a string representing the same decimal - value:: - - >>> import sys - >>> sys.float_info.dig - 15 - >>> s = '3.14159265358979' # decimal string with 15 significant digits - >>> format(float(s), '.15g') # convert to float and back -> same value - '3.14159265358979' - - But for strings with more than :attr:`sys.float_info.dig` significant digits, - this isn't always true:: - - >>> s = '9876543211234567' # 16 significant digits is too many! - >>> format(float(s), '.16g') # conversion changes value - '9876543211234568' - -.. data:: float_repr_style - - A string indicating how the :func:`repr` function behaves for - floats. If the string has value ``'short'`` then for a finite - float ``x``, ``repr(x)`` aims to produce a short string with the - property that ``float(repr(x)) == x``. This is the usual behaviour - in Python 3.1 and later. Otherwise, ``float_repr_style`` has value - ``'legacy'`` and ``repr(x)`` behaves in the same way as it did in - versions of Python prior to 3.1. - - .. versionadded:: 3.1 - - -.. function:: getallocatedblocks() - - Return the number of memory blocks currently allocated by the interpreter, - regardless of their size. This function is mainly useful for tracking - and debugging memory leaks. Because of the interpreter's internal - caches, the result can vary from call to call; you may have to call - :func:`_clear_type_cache()` and :func:`gc.collect()` to get more - predictable results. - - If a Python build or implementation cannot reasonably compute this - information, :func:`getallocatedblocks()` is allowed to return 0 instead. - - .. versionadded:: 3.4 - - -.. function:: getandroidapilevel() - - Return the build time API version of Android as an integer. - - .. availability:: Android. - - .. versionadded:: 3.7 - - -.. function:: getdefaultencoding() - - Return the name of the current default string encoding used by the Unicode - implementation. - - -.. function:: getdlopenflags() - - Return the current value of the flags that are used for - :c:func:`dlopen` calls. Symbolic names for the flag values can be - found in the :mod:`os` module (:samp:`RTLD_{xxx}` constants, e.g. - :const:`os.RTLD_LAZY`). - - .. availability:: Unix. - - -.. function:: getfilesystemencoding() - - Get the :term:`filesystem encoding `: - the encoding used with the :term:`filesystem error handler ` to convert between Unicode filenames and bytes - filenames. The filesystem error handler is returned from - :func:`getfilesystemencodeerrors`. - - For best compatibility, str should be used for filenames in all cases, - although representing filenames as bytes is also supported. Functions - accepting or returning filenames should support either str or bytes and - internally convert to the system's preferred representation. - - :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that - the correct encoding and errors mode are used. - - The :term:`filesystem encoding and error handler` are configured at Python - startup by the :c:func:`PyConfig_Read` function: see - :c:member:`~PyConfig.filesystem_encoding` and - :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. - - .. versionchanged:: 3.2 - :func:`getfilesystemencoding` result cannot be ``None`` anymore. - - .. versionchanged:: 3.6 - Windows is no longer guaranteed to return ``'mbcs'``. See :pep:`529` - and :func:`_enablelegacywindowsfsencoding` for more information. - - .. versionchanged:: 3.7 - Return ``'utf-8'`` if the :ref:`Python UTF-8 Mode ` is - enabled. - - -.. function:: getfilesystemencodeerrors() - - Get the :term:`filesystem error handler `: the error handler used with the :term:`filesystem encoding - ` to convert between Unicode - filenames and bytes filenames. The filesystem encoding is returned from - :func:`getfilesystemencoding`. - - :func:`os.fsencode` and :func:`os.fsdecode` should be used to ensure that - the correct encoding and errors mode are used. - - The :term:`filesystem encoding and error handler` are configured at Python - startup by the :c:func:`PyConfig_Read` function: see - :c:member:`~PyConfig.filesystem_encoding` and - :c:member:`~PyConfig.filesystem_errors` members of :c:type:`PyConfig`. - - .. versionadded:: 3.6 - -.. function:: get_int_max_str_digits() - - Returns the current value for the :ref:`integer string conversion length - limitation `. See also :func:`set_int_max_str_digits`. - - .. versionadded:: 3.11 - -.. function:: getrefcount(object) - - Return the reference count of the *object*. The count returned is generally one - higher than you might expect, because it includes the (temporary) reference as - an argument to :func:`getrefcount`. - - Note that the returned value may not actually reflect how many - references to the object are actually held. Consequently, do not rely - on the returned value to be accurate, other than a value of 0 or 1. - -.. function:: getrecursionlimit() - - Return the current value of the recursion limit, the maximum depth of the Python - interpreter stack. This limit prevents infinite recursion from causing an - overflow of the C stack and crashing Python. It can be set by - :func:`setrecursionlimit`. - - -.. function:: getsizeof(object[, default]) - - Return the size of an object in bytes. The object can be any type of - object. All built-in objects will return correct results, but this - does not have to hold true for third-party extensions as it is implementation - specific. - - Only the memory consumption directly attributed to the object is - accounted for, not the memory consumption of objects it refers to. - - If given, *default* will be returned if the object does not provide means to - retrieve the size. Otherwise a :exc:`TypeError` will be raised. - - :func:`getsizeof` calls the object's ``__sizeof__`` method and adds an - additional garbage collector overhead if the object is managed by the garbage - collector. - - See `recursive sizeof recipe `_ - for an example of using :func:`getsizeof` recursively to find the size of - containers and all their contents. - -.. function:: getswitchinterval() - - Return the interpreter's "thread switch interval"; see - :func:`setswitchinterval`. - - .. versionadded:: 3.2 - - -.. function:: _getframe([depth]) - - Return a frame object from the call stack. If optional integer *depth* is - given, return the frame object that many calls below the top of the stack. If - that is deeper than the call stack, :exc:`ValueError` is raised. The default - for *depth* is zero, returning the frame at the top of the call stack. - - .. audit-event:: sys._getframe frame sys._getframe - - .. impl-detail:: - - This function should be used for internal and specialized purposes only. - It is not guaranteed to exist in all implementations of Python. - - -.. function:: getprofile() - - .. index:: - single: profile function - single: profiler - - Get the profiler function as set by :func:`setprofile`. - - -.. function:: gettrace() - - .. index:: - single: trace function - single: debugger - - Get the trace function as set by :func:`settrace`. - - .. impl-detail:: - - The :func:`gettrace` function is intended only for implementing debuggers, - profilers, coverage tools and the like. Its behavior is part of the - implementation platform, rather than part of the language definition, and - thus may not be available in all Python implementations. - - -.. function:: getwindowsversion() - - Return a named tuple describing the Windows version - currently running. The named elements are *major*, *minor*, - *build*, *platform*, *service_pack*, *service_pack_minor*, - *service_pack_major*, *suite_mask*, *product_type* and - *platform_version*. *service_pack* contains a string, - *platform_version* a 3-tuple and all other values are - integers. The components can also be accessed by name, so - ``sys.getwindowsversion()[0]`` is equivalent to - ``sys.getwindowsversion().major``. For compatibility with prior - versions, only the first 5 elements are retrievable by indexing. - - *platform* will be ``2`` (VER_PLATFORM_WIN32_NT). - - *product_type* may be one of the following values: - - +---------------------------------------+---------------------------------+ - | Constant | Meaning | - +=======================================+=================================+ - | ``1`` (VER_NT_WORKSTATION) | The system is a workstation. | - +---------------------------------------+---------------------------------+ - | ``2`` (VER_NT_DOMAIN_CONTROLLER) | The system is a domain | - | | controller. | - +---------------------------------------+---------------------------------+ - | ``3`` (VER_NT_SERVER) | The system is a server, but not | - | | a domain controller. | - +---------------------------------------+---------------------------------+ - - This function wraps the Win32 :c:func:`!GetVersionEx` function; see the - Microsoft documentation on :c:func:`!OSVERSIONINFOEX` for more information - about these fields. - - *platform_version* returns the major version, minor version and - build number of the current operating system, rather than the version that - is being emulated for the process. It is intended for use in logging rather - than for feature detection. - - .. note:: - *platform_version* derives the version from kernel32.dll which can be of a different - version than the OS version. Please use :mod:`platform` module for achieving accurate - OS version. - - .. availability:: Windows. - - .. versionchanged:: 3.2 - Changed to a named tuple and added *service_pack_minor*, - *service_pack_major*, *suite_mask*, and *product_type*. - - .. versionchanged:: 3.6 - Added *platform_version* - - -.. function:: get_asyncgen_hooks() - - Returns an *asyncgen_hooks* object, which is similar to a - :class:`~collections.namedtuple` of the form ``(firstiter, finalizer)``, - where *firstiter* and *finalizer* are expected to be either ``None`` or - functions which take an :term:`asynchronous generator iterator` as an - argument, and are used to schedule finalization of an asynchronous - generator by an event loop. - - .. versionadded:: 3.6 - See :pep:`525` for more details. - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) - - -.. function:: get_coroutine_origin_tracking_depth() - - Get the current coroutine origin tracking depth, as set by - :func:`set_coroutine_origin_tracking_depth`. - - .. versionadded:: 3.7 - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - - -.. data:: hash_info - - A :term:`named tuple` giving parameters of the numeric hash - implementation. For more details about hashing of numeric types, see - :ref:`numeric-hash`. - - .. attribute:: hash_info.width - - The width in bits used for hash values - - .. attribute:: hash_info.modulus - - The prime modulus P used for numeric hash scheme - - .. attribute:: hash_info.inf - - The hash value returned for a positive infinity - - .. attribute:: hash_info.nan - - (This attribute is no longer used) - - .. attribute:: hash_info.imag - - The multiplier used for the imaginary part of a complex number - - .. attribute:: hash_info.algorithm - - The name of the algorithm for hashing of str, bytes, and memoryview - - .. attribute:: hash_info.hash_bits - - The internal output size of the hash algorithm - - .. attribute:: hash_info.seed_bits - - The size of the seed key of the hash algorithm - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Added *algorithm*, *hash_bits* and *seed_bits* - - -.. data:: hexversion - - The version number encoded as a single integer. This is guaranteed to increase - with each version, including proper support for non-production releases. For - example, to test that the Python interpreter is at least version 1.5.2, use:: - - if sys.hexversion >= 0x010502F0: - # use some advanced feature - ... - else: - # use an alternative implementation or warn the user - ... - - This is called ``hexversion`` since it only really looks meaningful when viewed - as the result of passing it to the built-in :func:`hex` function. The - :term:`named tuple` :data:`sys.version_info` may be used for a more - human-friendly encoding of the same information. - - More details of ``hexversion`` can be found at :ref:`apiabiversion`. - - -.. data:: implementation - - An object containing information about the implementation of the - currently running Python interpreter. The following attributes are - required to exist in all Python implementations. - - *name* is the implementation's identifier, e.g. ``'cpython'``. The actual - string is defined by the Python implementation, but it is guaranteed to be - lower case. - - *version* is a named tuple, in the same format as - :data:`sys.version_info`. It represents the version of the Python - *implementation*. This has a distinct meaning from the specific - version of the Python *language* to which the currently running - interpreter conforms, which ``sys.version_info`` represents. For - example, for PyPy 1.8 ``sys.implementation.version`` might be - ``sys.version_info(1, 8, 0, 'final', 0)``, whereas ``sys.version_info`` - would be ``sys.version_info(2, 7, 2, 'final', 0)``. For CPython they - are the same value, since it is the reference implementation. - - *hexversion* is the implementation version in hexadecimal format, like - :data:`sys.hexversion`. - - *cache_tag* is the tag used by the import machinery in the filenames of - cached modules. By convention, it would be a composite of the - implementation's name and version, like ``'cpython-33'``. However, a - Python implementation may use some other value if appropriate. If - ``cache_tag`` is set to ``None``, it indicates that module caching should - be disabled. - - :data:`sys.implementation` may contain additional attributes specific to - the Python implementation. These non-standard attributes must start with - an underscore, and are not described here. Regardless of its contents, - :data:`sys.implementation` will not change during a run of the interpreter, - nor between implementation versions. (It may change between Python - language versions, however.) See :pep:`421` for more information. - - .. versionadded:: 3.3 - - .. note:: - - The addition of new required attributes must go through the normal PEP - process. See :pep:`421` for more information. - -.. data:: int_info - - A :term:`named tuple` that holds information about Python's internal - representation of integers. The attributes are read only. - - .. attribute:: int_info.bits_per_digit - - The number of bits held in each digit. - Python integers are stored internally in base ``2**int_info.bits_per_digit``. - - .. attribute:: int_info.sizeof_digit - - The size in bytes of the C type used to represent a digit. - - .. attribute:: int_info.default_max_str_digits - - The default value for :func:`sys.get_int_max_str_digits` - when it is not otherwise explicitly configured. - - .. attribute:: int_info.str_digits_check_threshold - - The minimum non-zero value for :func:`sys.set_int_max_str_digits`, - :envvar:`PYTHONINTMAXSTRDIGITS`, or :option:`-X int_max_str_digits <-X>`. - - .. versionadded:: 3.1 - - .. versionchanged:: 3.11 - - Added :attr:`~int_info.default_max_str_digits` and - :attr:`~int_info.str_digits_check_threshold`. - - -.. data:: __interactivehook__ - - When this attribute exists, its value is automatically called (with no - arguments) when the interpreter is launched in :ref:`interactive mode - `. This is done after the :envvar:`PYTHONSTARTUP` file is - read, so that you can set this hook there. The :mod:`site` module - :ref:`sets this `. - - .. audit-event:: cpython.run_interactivehook hook sys.__interactivehook__ - - Raises an :ref:`auditing event ` - ``cpython.run_interactivehook`` with the hook object as the argument when - the hook is called on startup. - - .. versionadded:: 3.4 - - -.. function:: intern(string) - - Enter *string* in the table of "interned" strings and return the interned string - -- which is *string* itself or a copy. Interning strings is useful to gain a - little performance on dictionary lookup -- if the keys in a dictionary are - interned, and the lookup key is interned, the key comparisons (after hashing) - can be done by a pointer compare instead of a string compare. Normally, the - names used in Python programs are automatically interned, and the dictionaries - used to hold module, class or instance attributes have interned keys. - - Interned strings are not immortal; you must keep a reference to the return - value of :func:`intern` around to benefit from it. - - -.. function:: is_finalizing() - - Return :const:`True` if the Python interpreter is - :term:`shutting down `, :const:`False` otherwise. - - .. versionadded:: 3.5 - - -.. data:: last_type - last_value - last_traceback - - These three variables are not always defined; they are set when an exception is - not handled and the interpreter prints an error message and a stack traceback. - Their intended use is to allow an interactive user to import a debugger module - and engage in post-mortem debugging without having to re-execute the command - that caused the error. (Typical use is ``import pdb; pdb.pm()`` to enter the - post-mortem debugger; see :mod:`pdb` module for - more information.) - - The meaning of the variables is the same as that of the return values from - :func:`exc_info` above. - - -.. data:: maxsize - - An integer giving the maximum value a variable of type :c:type:`Py_ssize_t` can - take. It's usually ``2**31 - 1`` on a 32-bit platform and ``2**63 - 1`` on a - 64-bit platform. - - -.. data:: maxunicode - - An integer giving the value of the largest Unicode code point, - i.e. ``1114111`` (``0x10FFFF`` in hexadecimal). - - .. versionchanged:: 3.3 - Before :pep:`393`, ``sys.maxunicode`` used to be either ``0xFFFF`` - or ``0x10FFFF``, depending on the configuration option that specified - whether Unicode characters were stored as UCS-2 or UCS-4. - - -.. data:: meta_path - - A list of :term:`meta path finder` objects that have their - :meth:`~importlib.abc.MetaPathFinder.find_spec` methods called to see if one - of the objects can find the module to be imported. By default, it holds entries - that implement Python's default import semantics. The - :meth:`~importlib.abc.MetaPathFinder.find_spec` method is called with at - least the absolute name of the module being imported. If the module to be - imported is contained in a package, then the parent package's :attr:`__path__` - attribute is passed in as a second argument. The method returns a - :term:`module spec`, or ``None`` if the module cannot be found. - - .. seealso:: - - :class:`importlib.abc.MetaPathFinder` - The abstract base class defining the interface of finder objects on - :data:`meta_path`. - :class:`importlib.machinery.ModuleSpec` - The concrete class which - :meth:`~importlib.abc.MetaPathFinder.find_spec` should return - instances of. - - .. versionchanged:: 3.4 - - :term:`Module specs ` were introduced in Python 3.4, by - :pep:`451`. Earlier versions of Python looked for a method called - :meth:`~importlib.abc.MetaPathFinder.find_module`. - This is still called as a fallback if a :data:`meta_path` entry doesn't - have a :meth:`~importlib.abc.MetaPathFinder.find_spec` method. - -.. data:: modules - - This is a dictionary that maps module names to modules which have already been - loaded. This can be manipulated to force reloading of modules and other tricks. - However, replacing the dictionary will not necessarily work as expected and - deleting essential items from the dictionary may cause Python to fail. If - you want to iterate over this global dictionary always use - ``sys.modules.copy()`` or ``tuple(sys.modules)`` to avoid exceptions as its - size may change during iteration as a side effect of code or activity in - other threads. - - -.. data:: orig_argv - - The list of the original command line arguments passed to the Python - executable. - - See also :data:`sys.argv`. - - .. versionadded:: 3.10 - - -.. data:: path - - .. index:: triple: module; search; path - - A list of strings that specifies the search path for modules. Initialized from - the environment variable :envvar:`PYTHONPATH`, plus an installation-dependent - default. - - By default, as initialized upon program startup, a potentially unsafe path - is prepended to :data:`sys.path` (*before* the entries inserted as a result - of :envvar:`PYTHONPATH`): - - * ``python -m module`` command line: prepend the current working - directory. - * ``python script.py`` command line: prepend the script's directory. - If it's a symbolic link, resolve symbolic links. - * ``python -c code`` and ``python`` (REPL) command lines: prepend an empty - string, which means the current working directory. - - To not prepend this potentially unsafe path, use the :option:`-P` command - line option or the :envvar:`PYTHONSAFEPATH` environment variable. - - A program is free to modify this list for its own purposes. Only strings - should be added to :data:`sys.path`; all other data types are - ignored during import. - - - .. seealso:: - * Module :mod:`site` This describes how to use .pth files to - extend :data:`sys.path`. - -.. data:: path_hooks - - A list of callables that take a path argument to try to create a - :term:`finder` for the path. If a finder can be created, it is to be - returned by the callable, else raise :exc:`ImportError`. - - Originally specified in :pep:`302`. - - -.. data:: path_importer_cache - - A dictionary acting as a cache for :term:`finder` objects. The keys are - paths that have been passed to :data:`sys.path_hooks` and the values are - the finders that are found. If a path is a valid file system path but no - finder is found on :data:`sys.path_hooks` then ``None`` is - stored. - - Originally specified in :pep:`302`. - - .. versionchanged:: 3.3 - ``None`` is stored instead of :class:`imp.NullImporter` when no finder - is found. - - -.. data:: platform - - This string contains a platform identifier that can be used to append - platform-specific components to :data:`sys.path`, for instance. - - For Unix systems, except on Linux and AIX, this is the lowercased OS name as - returned by ``uname -s`` with the first part of the version as returned by - ``uname -r`` appended, e.g. ``'sunos5'`` or ``'freebsd8'``, *at the time - when Python was built*. Unless you want to test for a specific system - version, it is therefore recommended to use the following idiom:: - - if sys.platform.startswith('freebsd'): - # FreeBSD-specific code here... - elif sys.platform.startswith('linux'): - # Linux-specific code here... - elif sys.platform.startswith('aix'): - # AIX-specific code here... - - For other systems, the values are: - - ================ =========================== - System ``platform`` value - ================ =========================== - AIX ``'aix'`` - Emscripten ``'emscripten'`` - Linux ``'linux'`` - WASI ``'wasi'`` - Windows ``'win32'`` - Windows/Cygwin ``'cygwin'`` - macOS ``'darwin'`` - ================ =========================== - - .. versionchanged:: 3.3 - On Linux, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'linux'``, instead of ``'linux2'`` or ``'linux3'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. - - .. versionchanged:: 3.8 - On AIX, :data:`sys.platform` doesn't contain the major version anymore. - It is always ``'aix'``, instead of ``'aix5'`` or ``'aix7'``. Since - older Python versions include the version number, it is recommended to - always use the ``startswith`` idiom presented above. - - .. seealso:: - - :data:`os.name` has a coarser granularity. :func:`os.uname` gives - system-dependent version information. - - The :mod:`platform` module provides detailed checks for the - system's identity. - - -.. data:: platlibdir - - Name of the platform-specific library directory. It is used to build the - path of standard library and the paths of installed extension modules. - - It is equal to ``"lib"`` on most platforms. On Fedora and SuSE, it is equal - to ``"lib64"`` on 64-bit platforms which gives the following ``sys.path`` - paths (where ``X.Y`` is the Python ``major.minor`` version): - - * ``/usr/lib64/pythonX.Y/``: - Standard library (like ``os.py`` of the :mod:`os` module) - * ``/usr/lib64/pythonX.Y/lib-dynload/``: - C extension modules of the standard library (like the :mod:`errno` module, - the exact filename is platform specific) - * ``/usr/lib/pythonX.Y/site-packages/`` (always use ``lib``, not - :data:`sys.platlibdir`): Third-party modules - * ``/usr/lib64/pythonX.Y/site-packages/``: - C extension modules of third-party packages - - .. versionadded:: 3.9 - - -.. data:: prefix - - A string giving the site-specific directory prefix where the platform - independent Python files are installed; on Unix, the default is - :file:`/usr/local`. This can be set at build time with the :option:`--prefix` - argument to the :program:`configure` script. See - :ref:`installation_paths` for derived paths. - - .. note:: If a :ref:`virtual environment ` is in effect, this - value will be changed in ``site.py`` to point to the virtual - environment. The value for the Python installation will still be - available, via :data:`base_prefix`. - - -.. data:: ps1 - ps2 - - .. index:: - single: interpreter prompts - single: prompts, interpreter - single: >>>; interpreter prompt - single: ...; interpreter prompt - - Strings specifying the primary and secondary prompt of the interpreter. These - are only defined if the interpreter is in interactive mode. Their initial - values in this case are ``'>>> '`` and ``'... '``. If a non-string object is - assigned to either variable, its :func:`str` is re-evaluated each time the - interpreter prepares to read a new interactive command; this can be used to - implement a dynamic prompt. - - -.. function:: setdlopenflags(n) - - Set the flags used by the interpreter for :c:func:`dlopen` calls, such as when - the interpreter loads extension modules. Among other things, this will enable a - lazy resolving of symbols when importing a module, if called as - ``sys.setdlopenflags(0)``. To share symbols across extension modules, call as - ``sys.setdlopenflags(os.RTLD_GLOBAL)``. Symbolic names for the flag values - can be found in the :mod:`os` module (:samp:`RTLD_{xxx}` constants, e.g. - :const:`os.RTLD_LAZY`). - - .. availability:: Unix. - -.. function:: set_int_max_str_digits(maxdigits) - - Set the :ref:`integer string conversion length limitation - ` used by this interpreter. See also - :func:`get_int_max_str_digits`. - - .. versionadded:: 3.11 - -.. function:: setprofile(profilefunc) - - .. index:: - single: profile function - single: profiler - - Set the system's profile function, which allows you to implement a Python source - code profiler in Python. See chapter :ref:`profile` for more information on the - Python profiler. The system's profile function is called similarly to the - system's trace function (see :func:`settrace`), but it is called with different events, - for example it isn't called for each executed line of code (only on call and return, - but the return event is reported even when an exception has been set). The function is - thread-specific, but there is no way for the profiler to know about context switches between - threads, so it does not make sense to use this in the presence of multiple threads. Also, - its return value is not used, so it can simply return ``None``. Error in the profile - function will cause itself unset. - - .. note:: - The same tracing mechanism is used for :func:`!setprofile` as :func:`settrace`. - To trace calls with :func:`!setprofile` inside a tracing function - (e.g. in a debugger breakpoint), see :func:`call_tracing`. - - Profile functions should have three arguments: *frame*, *event*, and - *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, - ``'return'``, ``'c_call'``, ``'c_return'``, or ``'c_exception'``. *arg* depends - on the event type. - - The events have the following meaning: - - ``'call'`` - A function is called (or some other code block entered). The - profile function is called; *arg* is ``None``. - - ``'return'`` - A function (or other code block) is about to return. The profile - function is called; *arg* is the value that will be returned, or ``None`` - if the event is caused by an exception being raised. - - ``'c_call'`` - A C function is about to be called. This may be an extension function or - a built-in. *arg* is the C function object. - - ``'c_return'`` - A C function has returned. *arg* is the C function object. - - ``'c_exception'`` - A C function has raised an exception. *arg* is the C function object. - - .. audit-event:: sys.setprofile "" sys.setprofile - - -.. function:: setrecursionlimit(limit) - - Set the maximum depth of the Python interpreter stack to *limit*. This limit - prevents infinite recursion from causing an overflow of the C stack and crashing - Python. - - The highest possible limit is platform-dependent. A user may need to set the - limit higher when they have a program that requires deep recursion and a platform - that supports a higher limit. This should be done with care, because a too-high - limit can lead to a crash. - - If the new limit is too low at the current recursion depth, a - :exc:`RecursionError` exception is raised. - - .. versionchanged:: 3.5.1 - A :exc:`RecursionError` exception is now raised if the new limit is too - low at the current recursion depth. - - -.. function:: setswitchinterval(interval) - - Set the interpreter's thread switch interval (in seconds). This floating-point - value determines the ideal duration of the "timeslices" allocated to - concurrently running Python threads. Please note that the actual value - can be higher, especially if long-running internal functions or methods - are used. Also, which thread becomes scheduled at the end of the interval - is the operating system's decision. The interpreter doesn't have its - own scheduler. - - .. versionadded:: 3.2 - - -.. function:: settrace(tracefunc) - - .. index:: - single: trace function - single: debugger - - Set the system's trace function, which allows you to implement a Python - source code debugger in Python. The function is thread-specific; for a - debugger to support multiple threads, it must register a trace function using - :func:`settrace` for each thread being debugged or use :func:`threading.settrace`. - - Trace functions should have three arguments: *frame*, *event*, and - *arg*. *frame* is the current stack frame. *event* is a string: ``'call'``, - ``'line'``, ``'return'``, ``'exception'`` or ``'opcode'``. *arg* depends on - the event type. - - The trace function is invoked (with *event* set to ``'call'``) whenever a new - local scope is entered; it should return a reference to a local trace - function to be used for the new scope, or ``None`` if the scope shouldn't be - traced. - - The local trace function should return a reference to itself, or to another - function which would then be used as the local trace function for the scope. - - If there is any error occurred in the trace function, it will be unset, just - like ``settrace(None)`` is called. - - .. note:: - Tracing is disabled while calling the trace function (e.g. a function set by - :func:`!settrace`). For recursive tracing see :func:`call_tracing`. - - The events have the following meaning: - - ``'call'`` - A function is called (or some other code block entered). The - global trace function is called; *arg* is ``None``; the return value - specifies the local trace function. - - ``'line'`` - The interpreter is about to execute a new line of code or re-execute the - condition of a loop. The local trace function is called; *arg* is - ``None``; the return value specifies the new local trace function. See - :file:`Objects/lnotab_notes.txt` for a detailed explanation of how this - works. - Per-line events may be disabled for a frame by setting - :attr:`!f_trace_lines` to :const:`False` on that :ref:`frame `. - - ``'return'`` - A function (or other code block) is about to return. The local trace - function is called; *arg* is the value that will be returned, or ``None`` - if the event is caused by an exception being raised. The trace function's - return value is ignored. - - ``'exception'`` - An exception has occurred. The local trace function is called; *arg* is a - tuple ``(exception, value, traceback)``; the return value specifies the - new local trace function. - - ``'opcode'`` - The interpreter is about to execute a new opcode (see :mod:`dis` for - opcode details). The local trace function is called; *arg* is - ``None``; the return value specifies the new local trace function. - Per-opcode events are not emitted by default: they must be explicitly - requested by setting :attr:`!f_trace_opcodes` to :const:`True` on the - :ref:`frame `. - - Note that as an exception is propagated down the chain of callers, an - ``'exception'`` event is generated at each level. - - For more fine-grained usage, it's possible to set a trace function by - assigning ``frame.f_trace = tracefunc`` explicitly, rather than relying on - it being set indirectly via the return value from an already installed - trace function. This is also required for activating the trace function on - the current frame, which :func:`settrace` doesn't do. Note that in order - for this to work, a global tracing function must have been installed - with :func:`settrace` in order to enable the runtime tracing machinery, - but it doesn't need to be the same tracing function (e.g. it could be a - low overhead tracing function that simply returns ``None`` to disable - itself immediately on each frame). - - For more information on code and frame objects, refer to :ref:`types`. - - .. audit-event:: sys.settrace "" sys.settrace - - .. impl-detail:: - - The :func:`settrace` function is intended only for implementing debuggers, - profilers, coverage tools and the like. Its behavior is part of the - implementation platform, rather than part of the language definition, and - thus may not be available in all Python implementations. - - .. versionchanged:: 3.7 - - ``'opcode'`` event type added; :attr:`!f_trace_lines` and - :attr:`!f_trace_opcodes` attributes added to frames - -.. function:: set_asyncgen_hooks(firstiter, finalizer) - - Accepts two optional keyword arguments which are callables that accept an - :term:`asynchronous generator iterator` as an argument. The *firstiter* - callable will be called when an asynchronous generator is iterated for the - first time. The *finalizer* will be called when an asynchronous generator - is about to be garbage collected. - - .. audit-event:: sys.set_asyncgen_hooks_firstiter "" sys.set_asyncgen_hooks - - .. audit-event:: sys.set_asyncgen_hooks_finalizer "" sys.set_asyncgen_hooks - - Two auditing events are raised because the underlying API consists of two - calls, each of which must raise its own event. - - .. versionadded:: 3.6 - See :pep:`525` for more details, and for a reference example of a - *finalizer* method see the implementation of - ``asyncio.Loop.shutdown_asyncgens`` in - :source:`Lib/asyncio/base_events.py` - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) - -.. function:: set_coroutine_origin_tracking_depth(depth) - - Allows enabling or disabling coroutine origin tracking. When - enabled, the ``cr_origin`` attribute on coroutine objects will - contain a tuple of (filename, line number, function name) tuples - describing the traceback where the coroutine object was created, - with the most recent call first. When disabled, ``cr_origin`` will - be None. - - To enable, pass a *depth* value greater than zero; this sets the - number of frames whose information will be captured. To disable, - pass set *depth* to zero. - - This setting is thread-specific. - - .. versionadded:: 3.7 - - .. note:: - This function has been added on a provisional basis (see :pep:`411` - for details.) Use it only for debugging purposes. - -.. function:: _enablelegacywindowsfsencoding() - - Changes the :term:`filesystem encoding and error handler` to 'mbcs' and - 'replace' respectively, for consistency with versions of Python prior to - 3.6. - - This is equivalent to defining the :envvar:`PYTHONLEGACYWINDOWSFSENCODING` - environment variable before launching Python. - - See also :func:`sys.getfilesystemencoding` and - :func:`sys.getfilesystemencodeerrors`. - - .. availability:: Windows. - - .. versionadded:: 3.6 - See :pep:`529` for more details. - -.. data:: stdin - stdout - stderr - - :term:`File objects ` used by the interpreter for standard - input, output and errors: - - * ``stdin`` is used for all interactive input (including calls to - :func:`input`); - * ``stdout`` is used for the output of :func:`print` and :term:`expression` - statements and for the prompts of :func:`input`; - * The interpreter's own prompts and its error messages go to ``stderr``. - - These streams are regular :term:`text files ` like those - returned by the :func:`open` function. Their parameters are chosen as - follows: - - * The encoding and error handling are is initialized from - :c:member:`PyConfig.stdio_encoding` and :c:member:`PyConfig.stdio_errors`. - - On Windows, UTF-8 is used for the console device. Non-character - devices such as disk files and pipes use the system locale - encoding (i.e. the ANSI codepage). Non-console character - devices such as NUL (i.e. where ``isatty()`` returns ``True``) use the - value of the console input and output codepages at startup, - respectively for stdin and stdout/stderr. This defaults to the - system :term:`locale encoding` if the process is not initially attached - to a console. - - The special behaviour of the console can be overridden - by setting the environment variable PYTHONLEGACYWINDOWSSTDIO - before starting Python. In that case, the console codepages are - used as for any other character device. - - Under all platforms, you can override the character encoding by - setting the :envvar:`PYTHONIOENCODING` environment variable before - starting Python or by using the new :option:`-X` ``utf8`` command - line option and :envvar:`PYTHONUTF8` environment variable. However, - for the Windows console, this only applies when - :envvar:`PYTHONLEGACYWINDOWSSTDIO` is also set. - - * When interactive, the ``stdout`` stream is line-buffered. Otherwise, - it is block-buffered like regular text files. The ``stderr`` stream - is line-buffered in both cases. You can make both streams unbuffered - by passing the :option:`-u` command-line option or setting the - :envvar:`PYTHONUNBUFFERED` environment variable. - - .. versionchanged:: 3.9 - Non-interactive ``stderr`` is now line-buffered instead of fully - buffered. - - .. note:: - - To write or read binary data from/to the standard streams, use the - underlying binary :data:`~io.TextIOBase.buffer` object. For example, to - write bytes to :data:`stdout`, use ``sys.stdout.buffer.write(b'abc')``. - - However, if you are writing a library (and do not control in which - context its code will be executed), be aware that the standard streams - may be replaced with file-like objects like :class:`io.StringIO` which - do not support the :attr:`!buffer` attribute. - - -.. data:: __stdin__ - __stdout__ - __stderr__ - - These objects contain the original values of ``stdin``, ``stderr`` and - ``stdout`` at the start of the program. They are used during finalization, - and could be useful to print to the actual standard stream no matter if the - ``sys.std*`` object has been redirected. - - It can also be used to restore the actual files to known working file objects - in case they have been overwritten with a broken object. However, the - preferred way to do this is to explicitly save the previous stream before - replacing it, and restore the saved object. - - .. note:: - Under some conditions ``stdin``, ``stdout`` and ``stderr`` as well as the - original values ``__stdin__``, ``__stdout__`` and ``__stderr__`` can be - ``None``. It is usually the case for Windows GUI apps that aren't connected - to a console and Python apps started with :program:`pythonw`. - - -.. data:: stdlib_module_names - - A frozenset of strings containing the names of standard library modules. - - It is the same on all platforms. Modules which are not available on - some platforms and modules disabled at Python build are also listed. - All module kinds are listed: pure Python, built-in, frozen and extension - modules. Test modules are excluded. - - For packages, only the main package is listed: sub-packages and sub-modules - are not listed. For example, the ``email`` package is listed, but the - ``email.mime`` sub-package and the ``email.message`` sub-module are not - listed. - - See also the :data:`sys.builtin_module_names` list. - - .. versionadded:: 3.10 - - -.. data:: thread_info - - A :term:`named tuple` holding information about the thread - implementation. - - .. attribute:: thread_info.name - - The name of the thread implementation: - - * ``"nt"``: Windows threads - * ``"pthread"``: POSIX threads - * ``"pthread-stubs"``: stub POSIX threads - (on WebAssembly platforms without threading support) - * ``"solaris"``: Solaris threads - - .. attribute:: thread_info.lock - - The name of the lock implementation: - - * ``"semaphore"``: a lock uses a semaphore - * ``"mutex+cond"``: a lock uses a mutex and a condition variable - * ``None`` if this information is unknown - - .. attribute:: thread_info.version - - The name and version of the thread library. - It is a string, or ``None`` if this information is unknown. - - .. versionadded:: 3.3 - - -.. data:: tracebacklimit - - When this variable is set to an integer value, it determines the maximum number - of levels of traceback information printed when an unhandled exception occurs. - The default is ``1000``. When set to ``0`` or less, all traceback information - is suppressed and only the exception type and value are printed. - - -.. function:: unraisablehook(unraisable, /) - - Handle an unraisable exception. - - Called when an exception has occurred but there is no way for Python to - handle it. For example, when a destructor raises an exception or during - garbage collection (:func:`gc.collect`). - - The *unraisable* argument has the following attributes: - - * :attr:`!exc_type`: Exception type. - * :attr:`!exc_value`: Exception value, can be ``None``. - * :attr:`!exc_traceback`: Exception traceback, can be ``None``. - * :attr:`!err_msg`: Error message, can be ``None``. - * :attr:`!object`: Object causing the exception, can be ``None``. - - The default hook formats :attr:`!err_msg` and :attr:`!object` as: - ``f'{err_msg}: {object!r}'``; use "Exception ignored in" error message - if :attr:`!err_msg` is ``None``. - - :func:`sys.unraisablehook` can be overridden to control how unraisable - exceptions are handled. - - .. seealso:: - - :func:`excepthook` which handles uncaught exceptions. - - .. warning:: - - Storing :attr:`!exc_value` using a custom hook can create a reference cycle. - It should be cleared explicitly to break the reference cycle when the - exception is no longer needed. - - Storing :attr:`!object` using a custom hook can resurrect it if it is set to an - object which is being finalized. Avoid storing :attr:`!object` after the custom - hook completes to avoid resurrecting objects. - - .. audit-event:: sys.unraisablehook hook,unraisable sys.unraisablehook - - Raise an auditing event ``sys.unraisablehook`` with arguments - *hook*, *unraisable* when an exception that cannot be handled occurs. - The *unraisable* object is the same as what will be passed to the hook. - If no hook has been set, *hook* may be ``None``. - - .. versionadded:: 3.8 - -.. data:: version - - A string containing the version number of the Python interpreter plus additional - information on the build number and compiler used. This string is displayed - when the interactive interpreter is started. Do not extract version information - out of it, rather, use :data:`version_info` and the functions provided by the - :mod:`platform` module. - - -.. data:: api_version - - The C API version for this interpreter. Programmers may find this useful when - debugging version conflicts between Python and extension modules. - - -.. data:: version_info - - A tuple containing the five components of the version number: *major*, *minor*, - *micro*, *releaselevel*, and *serial*. All values except *releaselevel* are - integers; the release level is ``'alpha'``, ``'beta'``, ``'candidate'``, or - ``'final'``. The ``version_info`` value corresponding to the Python version 2.0 - is ``(2, 0, 0, 'final', 0)``. The components can also be accessed by name, - so ``sys.version_info[0]`` is equivalent to ``sys.version_info.major`` - and so on. - - .. versionchanged:: 3.1 - Added named component attributes. - -.. data:: warnoptions - - This is an implementation detail of the warnings framework; do not modify this - value. Refer to the :mod:`warnings` module for more information on the warnings - framework. - - -.. data:: winver - - The version number used to form registry keys on Windows platforms. This is - stored as string resource 1000 in the Python DLL. The value is normally the - major and minor versions of the running Python interpreter. It is provided in the :mod:`sys` - module for informational purposes; modifying this value has no effect on the - registry keys used by Python. - - .. availability:: Windows. - - -.. data:: _xoptions - - A dictionary of the various implementation-specific flags passed through - the :option:`-X` command-line option. Option names are either mapped to - their values, if given explicitly, or to :const:`True`. Example: - - .. code-block:: shell-session - - $ ./python -Xa=b -Xc - Python 3.2a3+ (py3k, Oct 16 2010, 20:14:50) - [GCC 4.4.3] on linux2 - Type "help", "copyright", "credits" or "license" for more information. - >>> import sys - >>> sys._xoptions - {'a': 'b', 'c': True} - - .. impl-detail:: - - This is a CPython-specific way of accessing options passed through - :option:`-X`. Other implementations may export them through other - means, or not at all. - - .. versionadded:: 3.2 - - -.. rubric:: Citations - -.. [C99] ISO/IEC 9899:1999. "Programming languages -- C." A public draft of this standard is available at https://www.open-std.org/jtc1/sc22/wg14/www/docs/n1256.pdf\ . diff --git a/Doc/library/sysconfig.rst b/Doc/library/sysconfig.rst deleted file mode 100644 index 2de47cfd36a0f7..00000000000000 --- a/Doc/library/sysconfig.rst +++ /dev/null @@ -1,314 +0,0 @@ -:mod:`sysconfig` --- Provide access to Python's configuration information -========================================================================= - -.. module:: sysconfig - :synopsis: Python's configuration information - -.. moduleauthor:: Tarek Ziadé -.. sectionauthor:: Tarek Ziadé - -.. versionadded:: 3.2 - -**Source code:** :source:`Lib/sysconfig.py` - -.. index:: - single: configuration information - --------------- - -The :mod:`sysconfig` module provides access to Python's configuration -information like the list of installation paths and the configuration variables -relevant for the current platform. - -Configuration variables ------------------------ - -A Python distribution contains a :file:`Makefile` and a :file:`pyconfig.h` -header file that are necessary to build both the Python binary itself and -third-party C extensions compiled using :mod:`distutils`. - -:mod:`sysconfig` puts all variables found in these files in a dictionary that -can be accessed using :func:`get_config_vars` or :func:`get_config_var`. - -Notice that on Windows, it's a much smaller set. - -.. function:: get_config_vars(*args) - - With no arguments, return a dictionary of all configuration variables - relevant for the current platform. - - With arguments, return a list of values that result from looking up each - argument in the configuration variable dictionary. - - For each argument, if the value is not found, return ``None``. - - -.. function:: get_config_var(name) - - Return the value of a single variable *name*. Equivalent to - ``get_config_vars().get(name)``. - - If *name* is not found, return ``None``. - -Example of usage:: - - >>> import sysconfig - >>> sysconfig.get_config_var('Py_ENABLE_SHARED') - 0 - >>> sysconfig.get_config_var('LIBDIR') - '/usr/local/lib' - >>> sysconfig.get_config_vars('AR', 'CXX') - ['ar', 'g++'] - -.. _installation_paths: - -Installation paths ------------------- - -Python uses an installation scheme that differs depending on the platform and on -the installation options. These schemes are stored in :mod:`sysconfig` under -unique identifiers based on the value returned by :const:`os.name`. - -Every new component that is installed using :mod:`!distutils` or a -Distutils-based system will follow the same scheme to copy its file in the right -places. - -Python currently supports nine schemes: - -- *posix_prefix*: scheme for POSIX platforms like Linux or macOS. This is - the default scheme used when Python or a component is installed. -- *posix_home*: scheme for POSIX platforms used when a *home* option is used - upon installation. This scheme is used when a component is installed through - Distutils with a specific home prefix. -- *posix_user*: scheme for POSIX platforms used when a component is installed - through Distutils and the *user* option is used. This scheme defines paths - located under the user home directory. -- *posix_venv*: scheme for :mod:`Python virtual environments ` on POSIX - platforms; by default it is the same as *posix_prefix*. -- *nt*: scheme for NT platforms like Windows. -- *nt_user*: scheme for NT platforms, when the *user* option is used. -- *nt_venv*: scheme for :mod:`Python virtual environments ` on NT - platforms; by default it is the same as *nt*. -- *venv*: a scheme with values from either *posix_venv* or *nt_venv* depending - on the platform Python runs on. -- *osx_framework_user*: scheme for macOS, when the *user* option is used. - -Each scheme is itself composed of a series of paths and each path has a unique -identifier. Python currently uses eight paths: - -- *stdlib*: directory containing the standard Python library files that are not - platform-specific. -- *platstdlib*: directory containing the standard Python library files that are - platform-specific. -- *platlib*: directory for site-specific, platform-specific files. -- *purelib*: directory for site-specific, non-platform-specific files. -- *include*: directory for non-platform-specific header files for - the Python C-API. -- *platinclude*: directory for platform-specific header files for - the Python C-API. -- *scripts*: directory for script files. -- *data*: directory for data files. - -:mod:`sysconfig` provides some functions to determine these paths. - -.. function:: get_scheme_names() - - Return a tuple containing all schemes currently supported in - :mod:`sysconfig`. - - -.. function:: get_default_scheme() - - Return the default scheme name for the current platform. - - .. versionadded:: 3.10 - This function was previously named ``_get_default_scheme()`` and - considered an implementation detail. - - .. versionchanged:: 3.11 - When Python runs from a virtual environment, - the *venv* scheme is returned. - -.. function:: get_preferred_scheme(key) - - Return a preferred scheme name for an installation layout specified by *key*. - - *key* must be either ``"prefix"``, ``"home"``, or ``"user"``. - - The return value is a scheme name listed in :func:`get_scheme_names`. It - can be passed to :mod:`sysconfig` functions that take a *scheme* argument, - such as :func:`get_paths`. - - .. versionadded:: 3.10 - - .. versionchanged:: 3.11 - When Python runs from a virtual environment and ``key="prefix"``, - the *venv* scheme is returned. - - -.. function:: _get_preferred_schemes() - - Return a dict containing preferred scheme names on the current platform. - Python implementers and redistributors may add their preferred schemes to - the ``_INSTALL_SCHEMES`` module-level global value, and modify this function - to return those scheme names, to e.g. provide different schemes for system - and language package managers to use, so packages installed by either do not - mix with those by the other. - - End users should not use this function, but :func:`get_default_scheme` and - :func:`get_preferred_scheme()` instead. - - .. versionadded:: 3.10 - - -.. function:: get_path_names() - - Return a tuple containing all path names currently supported in - :mod:`sysconfig`. - - -.. function:: get_path(name, [scheme, [vars, [expand]]]) - - Return an installation path corresponding to the path *name*, from the - install scheme named *scheme*. - - *name* has to be a value from the list returned by :func:`get_path_names`. - - :mod:`sysconfig` stores installation paths corresponding to each path name, - for each platform, with variables to be expanded. For instance the *stdlib* - path for the *nt* scheme is: ``{base}/Lib``. - - :func:`get_path` will use the variables returned by :func:`get_config_vars` - to expand the path. All variables have default values for each platform so - one may call this function and get the default value. - - If *scheme* is provided, it must be a value from the list returned by - :func:`get_scheme_names`. Otherwise, the default scheme for the current - platform is used. - - If *vars* is provided, it must be a dictionary of variables that will update - the dictionary returned by :func:`get_config_vars`. - - If *expand* is set to ``False``, the path will not be expanded using the - variables. - - If *name* is not found, raise a :exc:`KeyError`. - - -.. function:: get_paths([scheme, [vars, [expand]]]) - - Return a dictionary containing all installation paths corresponding to an - installation scheme. See :func:`get_path` for more information. - - If *scheme* is not provided, will use the default scheme for the current - platform. - - If *vars* is provided, it must be a dictionary of variables that will - update the dictionary used to expand the paths. - - If *expand* is set to false, the paths will not be expanded. - - If *scheme* is not an existing scheme, :func:`get_paths` will raise a - :exc:`KeyError`. - - -Other functions ---------------- - -.. function:: get_python_version() - - Return the ``MAJOR.MINOR`` Python version number as a string. Similar to - ``'%d.%d' % sys.version_info[:2]``. - - -.. function:: get_platform() - - Return a string that identifies the current platform. - - This is used mainly to distinguish platform-specific build directories and - platform-specific built distributions. Typically includes the OS name and - version and the architecture (as supplied by 'os.uname()'), although the - exact information included depends on the OS; e.g., on Linux, the kernel - version isn't particularly important. - - Examples of returned values: - - - linux-i586 - - linux-alpha (?) - - solaris-2.6-sun4u - - Windows will return one of: - - - win-amd64 (64bit Windows on AMD64, aka x86_64, Intel64, and EM64T) - - win32 (all others - specifically, sys.platform is returned) - - macOS can return: - - - macosx-10.6-ppc - - macosx-10.4-ppc64 - - macosx-10.3-i386 - - macosx-10.4-fat - - For other non-POSIX platforms, currently just returns :data:`sys.platform`. - - -.. function:: is_python_build() - - Return ``True`` if the running Python interpreter was built from source and - is being run from its built location, and not from a location resulting from - e.g. running ``make install`` or installing via a binary installer. - - -.. function:: parse_config_h(fp[, vars]) - - Parse a :file:`config.h`\-style file. - - *fp* is a file-like object pointing to the :file:`config.h`\-like file. - - A dictionary containing name/value pairs is returned. If an optional - dictionary is passed in as the second argument, it is used instead of a new - dictionary, and updated with the values read in the file. - - -.. function:: get_config_h_filename() - - Return the path of :file:`pyconfig.h`. - -.. function:: get_makefile_filename() - - Return the path of :file:`Makefile`. - -.. _sysconfig-cli: - -Using :mod:`sysconfig` as a script ----------------------------------- - -You can use :mod:`sysconfig` as a script with Python's *-m* option: - -.. code-block:: shell-session - - $ python -m sysconfig - Platform: "macosx-10.4-i386" - Python version: "3.2" - Current installation scheme: "posix_prefix" - - Paths: - data = "/usr/local" - include = "/Users/tarek/Dev/svn.python.org/py3k/Include" - platinclude = "." - platlib = "/usr/local/lib/python3.2/site-packages" - platstdlib = "/usr/local/lib/python3.2" - purelib = "/usr/local/lib/python3.2/site-packages" - scripts = "/usr/local/bin" - stdlib = "/usr/local/lib/python3.2" - - Variables: - AC_APPLE_UNIVERSAL_BUILD = "0" - AIX_GENUINE_CPLUSPLUS = "0" - AR = "ar" - ARFLAGS = "rc" - ... - -This call will print in the standard output the information returned by -:func:`get_platform`, :func:`get_python_version`, :func:`get_path` and -:func:`get_config_vars`. diff --git a/Doc/library/syslog.rst b/Doc/library/syslog.rst deleted file mode 100644 index 889bbb39d58232..00000000000000 --- a/Doc/library/syslog.rst +++ /dev/null @@ -1,126 +0,0 @@ -:mod:`syslog` --- Unix syslog library routines -============================================== - -.. module:: syslog - :platform: Unix - :synopsis: An interface to the Unix syslog library routines. - --------------- - -This module provides an interface to the Unix ``syslog`` library routines. -Refer to the Unix manual pages for a detailed description of the ``syslog`` -facility. - -.. availability:: Unix, not Emscripten, not WASI. - -This module wraps the system ``syslog`` family of routines. A pure Python -library that can speak to a syslog server is available in the -:mod:`logging.handlers` module as :class:`SysLogHandler`. - -The module defines the following functions: - - -.. function:: syslog(message) - syslog(priority, message) - - Send the string *message* to the system logger. A trailing newline is added - if necessary. Each message is tagged with a priority composed of a - *facility* and a *level*. The optional *priority* argument, which defaults - to :const:`LOG_INFO`, determines the message priority. If the facility is - not encoded in *priority* using logical-or (``LOG_INFO | LOG_USER``), the - value given in the :func:`openlog` call is used. - - If :func:`openlog` has not been called prior to the call to :func:`syslog`, - :func:`openlog` will be called with no arguments. - - .. audit-event:: syslog.syslog priority,message syslog.syslog - - .. versionchanged:: 3.2 - In previous versions, :func:`openlog` would not be called automatically if - it wasn't called prior to the call to :func:`syslog`, deferring to the syslog - implementation to call ``openlog()``. - - -.. function:: openlog([ident[, logoption[, facility]]]) - - Logging options of subsequent :func:`syslog` calls can be set by calling - :func:`openlog`. :func:`syslog` will call :func:`openlog` with no arguments - if the log is not currently open. - - The optional *ident* keyword argument is a string which is prepended to every - message, and defaults to ``sys.argv[0]`` with leading path components - stripped. The optional *logoption* keyword argument (default is 0) is a bit - field -- see below for possible values to combine. The optional *facility* - keyword argument (default is :const:`LOG_USER`) sets the default facility for - messages which do not have a facility explicitly encoded. - - .. audit-event:: syslog.openlog ident,logoption,facility syslog.openlog - - .. versionchanged:: 3.2 - In previous versions, keyword arguments were not allowed, and *ident* was - required. - - -.. function:: closelog() - - Reset the syslog module values and call the system library ``closelog()``. - - This causes the module to behave as it does when initially imported. For - example, :func:`openlog` will be called on the first :func:`syslog` call (if - :func:`openlog` hasn't already been called), and *ident* and other - :func:`openlog` parameters are reset to defaults. - - .. audit-event:: syslog.closelog "" syslog.closelog - - -.. function:: setlogmask(maskpri) - - Set the priority mask to *maskpri* and return the previous mask value. Calls - to :func:`syslog` with a priority level not set in *maskpri* are ignored. - The default is to log all priorities. The function ``LOG_MASK(pri)`` - calculates the mask for the individual priority *pri*. The function - ``LOG_UPTO(pri)`` calculates the mask for all priorities up to and including - *pri*. - - .. audit-event:: syslog.setlogmask maskpri syslog.setlogmask - -The module defines the following constants: - -Priority levels (high to low): - :const:`LOG_EMERG`, :const:`LOG_ALERT`, :const:`LOG_CRIT`, :const:`LOG_ERR`, - :const:`LOG_WARNING`, :const:`LOG_NOTICE`, :const:`LOG_INFO`, - :const:`LOG_DEBUG`. - -Facilities: - :const:`LOG_KERN`, :const:`LOG_USER`, :const:`LOG_MAIL`, :const:`LOG_DAEMON`, - :const:`LOG_AUTH`, :const:`LOG_LPR`, :const:`LOG_NEWS`, :const:`LOG_UUCP`, - :const:`LOG_CRON`, :const:`LOG_SYSLOG`, :const:`LOG_LOCAL0` to - :const:`LOG_LOCAL7`, and, if defined in ````, - :const:`LOG_AUTHPRIV`. - -Log options: - :const:`LOG_PID`, :const:`LOG_CONS`, :const:`LOG_NDELAY`, and, if defined - in ````, :const:`LOG_ODELAY`, :const:`LOG_NOWAIT`, and - :const:`LOG_PERROR`. - - -Examples --------- - -Simple example -~~~~~~~~~~~~~~ - -A simple set of examples:: - - import syslog - - syslog.syslog('Processing started') - if error: - syslog.syslog(syslog.LOG_ERR, 'Processing started') - -An example of setting some log options, these would include the process ID in -logged messages, and write the messages to the destination facility used for -mail logging:: - - syslog.openlog(logoption=syslog.LOG_PID, facility=syslog.LOG_MAIL) - syslog.syslog('E-mail processing initiated...') diff --git a/Doc/library/tabnanny.rst b/Doc/library/tabnanny.rst deleted file mode 100644 index dfe688a2f93e0c..00000000000000 --- a/Doc/library/tabnanny.rst +++ /dev/null @@ -1,67 +0,0 @@ -:mod:`tabnanny` --- Detection of ambiguous indentation -====================================================== - -.. module:: tabnanny - :synopsis: Tool for detecting white space related problems in Python - source files in a directory tree. - -.. moduleauthor:: Tim Peters -.. sectionauthor:: Peter Funk - -.. rudimentary documentation based on module comments - -**Source code:** :source:`Lib/tabnanny.py` - --------------- - -For the time being this module is intended to be called as a script. However it -is possible to import it into an IDE and use the function :func:`check` -described below. - -.. note:: - - The API provided by this module is likely to change in future releases; such - changes may not be backward compatible. - - -.. function:: check(file_or_dir) - - If *file_or_dir* is a directory and not a symbolic link, then recursively - descend the directory tree named by *file_or_dir*, checking all :file:`.py` - files along the way. If *file_or_dir* is an ordinary Python source file, it - is checked for whitespace related problems. The diagnostic messages are - written to standard output using the :func:`print` function. - - -.. data:: verbose - - Flag indicating whether to print verbose messages. This is incremented by the - ``-v`` option if called as a script. - - -.. data:: filename_only - - Flag indicating whether to print only the filenames of files containing - whitespace related problems. This is set to true by the ``-q`` option if called - as a script. - - -.. exception:: NannyNag - - Raised by :func:`process_tokens` if detecting an ambiguous indent. Captured and - handled in :func:`check`. - - -.. function:: process_tokens(tokens) - - This function is used by :func:`check` to process tokens generated by the - :mod:`tokenize` module. - -.. XXX document errprint, format_witnesses, Whitespace, check_equal, indents, - reset_globals - - -.. seealso:: - - Module :mod:`tokenize` - Lexical scanner for Python source code. diff --git a/Doc/library/tarfile.rst b/Doc/library/tarfile.rst deleted file mode 100644 index 76c8602b7537b7..00000000000000 --- a/Doc/library/tarfile.rst +++ /dev/null @@ -1,1330 +0,0 @@ -:mod:`tarfile` --- Read and write tar archive files -=================================================== - -.. module:: tarfile - :synopsis: Read and write tar-format archive files. - -.. moduleauthor:: Lars Gustäbel -.. sectionauthor:: Lars Gustäbel - -**Source code:** :source:`Lib/tarfile.py` - --------------- - -The :mod:`tarfile` module makes it possible to read and write tar -archives, including those using gzip, bz2 and lzma compression. -Use the :mod:`zipfile` module to read or write :file:`.zip` files, or the -higher-level functions in :ref:`shutil `. - -Some facts and figures: - -* reads and writes :mod:`gzip`, :mod:`bz2` and :mod:`lzma` compressed archives - if the respective modules are available. - -* read/write support for the POSIX.1-1988 (ustar) format. - -* read/write support for the GNU tar format including *longname* and *longlink* - extensions, read-only support for all variants of the *sparse* extension - including restoration of sparse files. - -* read/write support for the POSIX.1-2001 (pax) format. - -* handles directories, regular files, hardlinks, symbolic links, fifos, - character devices and block devices and is able to acquire and restore file - information like timestamp, access permissions and owner. - -.. versionchanged:: 3.3 - Added support for :mod:`lzma` compression. - - -.. function:: open(name=None, mode='r', fileobj=None, bufsize=10240, **kwargs) - - Return a :class:`TarFile` object for the pathname *name*. For detailed - information on :class:`TarFile` objects and the keyword arguments that are - allowed, see :ref:`tarfile-objects`. - - *mode* has to be a string of the form ``'filemode[:compression]'``, it defaults - to ``'r'``. Here is a full list of mode combinations: - - +------------------+---------------------------------------------+ - | mode | action | - +==================+=============================================+ - | ``'r' or 'r:*'`` | Open for reading with transparent | - | | compression (recommended). | - +------------------+---------------------------------------------+ - | ``'r:'`` | Open for reading exclusively without | - | | compression. | - +------------------+---------------------------------------------+ - | ``'r:gz'`` | Open for reading with gzip compression. | - +------------------+---------------------------------------------+ - | ``'r:bz2'`` | Open for reading with bzip2 compression. | - +------------------+---------------------------------------------+ - | ``'r:xz'`` | Open for reading with lzma compression. | - +------------------+---------------------------------------------+ - | ``'x'`` or | Create a tarfile exclusively without | - | ``'x:'`` | compression. | - | | Raise a :exc:`FileExistsError` exception | - | | if it already exists. | - +------------------+---------------------------------------------+ - | ``'x:gz'`` | Create a tarfile with gzip compression. | - | | Raise a :exc:`FileExistsError` exception | - | | if it already exists. | - +------------------+---------------------------------------------+ - | ``'x:bz2'`` | Create a tarfile with bzip2 compression. | - | | Raise a :exc:`FileExistsError` exception | - | | if it already exists. | - +------------------+---------------------------------------------+ - | ``'x:xz'`` | Create a tarfile with lzma compression. | - | | Raise a :exc:`FileExistsError` exception | - | | if it already exists. | - +------------------+---------------------------------------------+ - | ``'a' or 'a:'`` | Open for appending with no compression. The | - | | file is created if it does not exist. | - +------------------+---------------------------------------------+ - | ``'w' or 'w:'`` | Open for uncompressed writing. | - +------------------+---------------------------------------------+ - | ``'w:gz'`` | Open for gzip compressed writing. | - +------------------+---------------------------------------------+ - | ``'w:bz2'`` | Open for bzip2 compressed writing. | - +------------------+---------------------------------------------+ - | ``'w:xz'`` | Open for lzma compressed writing. | - +------------------+---------------------------------------------+ - - Note that ``'a:gz'``, ``'a:bz2'`` or ``'a:xz'`` is not possible. If *mode* - is not suitable to open a certain (compressed) file for reading, - :exc:`ReadError` is raised. Use *mode* ``'r'`` to avoid this. If a - compression method is not supported, :exc:`CompressionError` is raised. - - If *fileobj* is specified, it is used as an alternative to a :term:`file object` - opened in binary mode for *name*. It is supposed to be at position 0. - - For modes ``'w:gz'``, ``'r:gz'``, ``'w:bz2'``, ``'r:bz2'``, ``'x:gz'``, - ``'x:bz2'``, :func:`tarfile.open` accepts the keyword argument - *compresslevel* (default ``9``) to specify the compression level of the file. - - For modes ``'w:xz'`` and ``'x:xz'``, :func:`tarfile.open` accepts the - keyword argument *preset* to specify the compression level of the file. - - For special purposes, there is a second format for *mode*: - ``'filemode|[compression]'``. :func:`tarfile.open` will return a :class:`TarFile` - object that processes its data as a stream of blocks. No random seeking will - be done on the file. If given, *fileobj* may be any object that has a - :meth:`read` or :meth:`write` method (depending on the *mode*). *bufsize* - specifies the blocksize and defaults to ``20 * 512`` bytes. Use this variant - in combination with e.g. ``sys.stdin``, a socket :term:`file object` or a tape - device. However, such a :class:`TarFile` object is limited in that it does - not allow random access, see :ref:`tar-examples`. The currently - possible modes: - - +-------------+--------------------------------------------+ - | Mode | Action | - +=============+============================================+ - | ``'r|*'`` | Open a *stream* of tar blocks for reading | - | | with transparent compression. | - +-------------+--------------------------------------------+ - | ``'r|'`` | Open a *stream* of uncompressed tar blocks | - | | for reading. | - +-------------+--------------------------------------------+ - | ``'r|gz'`` | Open a gzip compressed *stream* for | - | | reading. | - +-------------+--------------------------------------------+ - | ``'r|bz2'`` | Open a bzip2 compressed *stream* for | - | | reading. | - +-------------+--------------------------------------------+ - | ``'r|xz'`` | Open an lzma compressed *stream* for | - | | reading. | - +-------------+--------------------------------------------+ - | ``'w|'`` | Open an uncompressed *stream* for writing. | - +-------------+--------------------------------------------+ - | ``'w|gz'`` | Open a gzip compressed *stream* for | - | | writing. | - +-------------+--------------------------------------------+ - | ``'w|bz2'`` | Open a bzip2 compressed *stream* for | - | | writing. | - +-------------+--------------------------------------------+ - | ``'w|xz'`` | Open an lzma compressed *stream* for | - | | writing. | - +-------------+--------------------------------------------+ - - .. versionchanged:: 3.5 - The ``'x'`` (exclusive creation) mode was added. - - .. versionchanged:: 3.6 - The *name* parameter accepts a :term:`path-like object`. - - -.. class:: TarFile - :noindex: - - Class for reading and writing tar archives. Do not use this class directly: - use :func:`tarfile.open` instead. See :ref:`tarfile-objects`. - - -.. function:: is_tarfile(name) - - Return :const:`True` if *name* is a tar archive file, that the :mod:`tarfile` - module can read. *name* may be a :class:`str`, file, or file-like object. - - .. versionchanged:: 3.9 - Support for file and file-like objects. - - -The :mod:`tarfile` module defines the following exceptions: - - -.. exception:: TarError - - Base class for all :mod:`tarfile` exceptions. - - -.. exception:: ReadError - - Is raised when a tar archive is opened, that either cannot be handled by the - :mod:`tarfile` module or is somehow invalid. - - -.. exception:: CompressionError - - Is raised when a compression method is not supported or when the data cannot be - decoded properly. - - -.. exception:: StreamError - - Is raised for the limitations that are typical for stream-like :class:`TarFile` - objects. - - -.. exception:: ExtractError - - Is raised for *non-fatal* errors when using :meth:`TarFile.extract`, but only if - :attr:`TarFile.errorlevel`\ ``== 2``. - - -.. exception:: HeaderError - - Is raised by :meth:`TarInfo.frombuf` if the buffer it gets is invalid. - - -.. exception:: FilterError - - Base class for members :ref:`refused ` by - filters. - - .. attribute:: tarinfo - - Information about the member that the filter refused to extract, - as :ref:`TarInfo `. - -.. exception:: AbsolutePathError - - Raised to refuse extracting a member with an absolute path. - -.. exception:: OutsideDestinationError - - Raised to refuse extracting a member outside the destination directory. - -.. exception:: SpecialFileError - - Raised to refuse extracting a special file (e.g. a device or pipe). - -.. exception:: AbsoluteLinkError - - Raised to refuse extracting a symbolic link with an absolute path. - -.. exception:: LinkOutsideDestinationError - - Raised to refuse extracting a symbolic link pointing outside the destination - directory. - - -The following constants are available at the module level: - -.. data:: ENCODING - - The default character encoding: ``'utf-8'`` on Windows, the value returned by - :func:`sys.getfilesystemencoding` otherwise. - - -Each of the following constants defines a tar archive format that the -:mod:`tarfile` module is able to create. See section :ref:`tar-formats` for -details. - - -.. data:: USTAR_FORMAT - - POSIX.1-1988 (ustar) format. - - -.. data:: GNU_FORMAT - - GNU tar format. - - -.. data:: PAX_FORMAT - - POSIX.1-2001 (pax) format. - - -.. data:: DEFAULT_FORMAT - - The default format for creating archives. This is currently :const:`PAX_FORMAT`. - - .. versionchanged:: 3.8 - The default format for new archives was changed to - :const:`PAX_FORMAT` from :const:`GNU_FORMAT`. - - -.. seealso:: - - Module :mod:`zipfile` - Documentation of the :mod:`zipfile` standard module. - - :ref:`archiving-operations` - Documentation of the higher-level archiving facilities provided by the - standard :mod:`shutil` module. - - `GNU tar manual, Basic Tar Format `_ - Documentation for tar archive files, including GNU tar extensions. - - -.. _tarfile-objects: - -TarFile Objects ---------------- - -The :class:`TarFile` object provides an interface to a tar archive. A tar -archive is a sequence of blocks. An archive member (a stored file) is made up of -a header block followed by data blocks. It is possible to store a file in a tar -archive several times. Each archive member is represented by a :class:`TarInfo` -object, see :ref:`tarinfo-objects` for details. - -A :class:`TarFile` object can be used as a context manager in a :keyword:`with` -statement. It will automatically be closed when the block is completed. Please -note that in the event of an exception an archive opened for writing will not -be finalized; only the internally used file object will be closed. See the -:ref:`tar-examples` section for a use case. - -.. versionadded:: 3.2 - Added support for the context management protocol. - -.. class:: TarFile(name=None, mode='r', fileobj=None, format=DEFAULT_FORMAT, tarinfo=TarInfo, dereference=False, ignore_zeros=False, encoding=ENCODING, errors='surrogateescape', pax_headers=None, debug=0, errorlevel=1) - - All following arguments are optional and can be accessed as instance attributes - as well. - - *name* is the pathname of the archive. *name* may be a :term:`path-like object`. - It can be omitted if *fileobj* is given. - In this case, the file object's :attr:`name` attribute is used if it exists. - - *mode* is either ``'r'`` to read from an existing archive, ``'a'`` to append - data to an existing file, ``'w'`` to create a new file overwriting an existing - one, or ``'x'`` to create a new file only if it does not already exist. - - If *fileobj* is given, it is used for reading or writing data. If it can be - determined, *mode* is overridden by *fileobj*'s mode. *fileobj* will be used - from position 0. - - .. note:: - - *fileobj* is not closed, when :class:`TarFile` is closed. - - *format* controls the archive format for writing. It must be one of the constants - :const:`USTAR_FORMAT`, :const:`GNU_FORMAT` or :const:`PAX_FORMAT` that are - defined at module level. When reading, format will be automatically detected, even - if different formats are present in a single archive. - - The *tarinfo* argument can be used to replace the default :class:`TarInfo` class - with a different one. - - If *dereference* is :const:`False`, add symbolic and hard links to the archive. If it - is :const:`True`, add the content of the target files to the archive. This has no - effect on systems that do not support symbolic links. - - If *ignore_zeros* is :const:`False`, treat an empty block as the end of the archive. - If it is :const:`True`, skip empty (and invalid) blocks and try to get as many members - as possible. This is only useful for reading concatenated or damaged archives. - - *debug* can be set from ``0`` (no debug messages) up to ``3`` (all debug - messages). The messages are written to ``sys.stderr``. - - *errorlevel* controls how extraction errors are handled, - see :attr:`the corresponding attribute <~TarFile.errorlevel>`. - - The *encoding* and *errors* arguments define the character encoding to be - used for reading or writing the archive and how conversion errors are going - to be handled. The default settings will work for most users. - See section :ref:`tar-unicode` for in-depth information. - - The *pax_headers* argument is an optional dictionary of strings which - will be added as a pax global header if *format* is :const:`PAX_FORMAT`. - - .. versionchanged:: 3.2 - Use ``'surrogateescape'`` as the default for the *errors* argument. - - .. versionchanged:: 3.5 - The ``'x'`` (exclusive creation) mode was added. - - .. versionchanged:: 3.6 - The *name* parameter accepts a :term:`path-like object`. - - -.. classmethod:: TarFile.open(...) - - Alternative constructor. The :func:`tarfile.open` function is actually a - shortcut to this classmethod. - - -.. method:: TarFile.getmember(name) - - Return a :class:`TarInfo` object for member *name*. If *name* can not be found - in the archive, :exc:`KeyError` is raised. - - .. note:: - - If a member occurs more than once in the archive, its last occurrence is assumed - to be the most up-to-date version. - - -.. method:: TarFile.getmembers() - - Return the members of the archive as a list of :class:`TarInfo` objects. The - list has the same order as the members in the archive. - - -.. method:: TarFile.getnames() - - Return the members as a list of their names. It has the same order as the list - returned by :meth:`getmembers`. - - -.. method:: TarFile.list(verbose=True, *, members=None) - - Print a table of contents to ``sys.stdout``. If *verbose* is :const:`False`, - only the names of the members are printed. If it is :const:`True`, output - similar to that of :program:`ls -l` is produced. If optional *members* is - given, it must be a subset of the list returned by :meth:`getmembers`. - - .. versionchanged:: 3.5 - Added the *members* parameter. - - -.. method:: TarFile.next() - - Return the next member of the archive as a :class:`TarInfo` object, when - :class:`TarFile` is opened for reading. Return :const:`None` if there is no more - available. - - -.. method:: TarFile.extractall(path=".", members=None, *, numeric_owner=False, filter=None) - - Extract all members from the archive to the current working directory or - directory *path*. If optional *members* is given, it must be a subset of the - list returned by :meth:`getmembers`. Directory information like owner, - modification time and permissions are set after all members have been extracted. - This is done to work around two problems: A directory's modification time is - reset each time a file is created in it. And, if a directory's permissions do - not allow writing, extracting files to it will fail. - - If *numeric_owner* is :const:`True`, the uid and gid numbers from the tarfile - are used to set the owner/group for the extracted files. Otherwise, the named - values from the tarfile are used. - - The *filter* argument, which was added in Python 3.11.4, specifies how - ``members`` are modified or rejected before extraction. - See :ref:`tarfile-extraction-filter` for details. - It is recommended to set this explicitly depending on which *tar* features - you need to support. - - .. warning:: - - Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of *path*, e.g. members - that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. - - Set ``filter='data'`` to prevent the most dangerous security issues, - and read the :ref:`tarfile-extraction-filter` section for details. - - .. versionchanged:: 3.5 - Added the *numeric_owner* parameter. - - .. versionchanged:: 3.6 - The *path* parameter accepts a :term:`path-like object`. - - .. versionchanged:: 3.11.4 - Added the *filter* parameter. - - -.. method:: TarFile.extract(member, path="", set_attrs=True, *, numeric_owner=False, filter=None) - - Extract a member from the archive to the current working directory, using its - full name. Its file information is extracted as accurately as possible. *member* - may be a filename or a :class:`TarInfo` object. You can specify a different - directory using *path*. *path* may be a :term:`path-like object`. - File attributes (owner, mtime, mode) are set unless *set_attrs* is false. - - The *numeric_owner* and *filter* arguments are the same as - for :meth:`extractall`. - - .. note:: - - The :meth:`extract` method does not take care of several extraction issues. - In most cases you should consider using the :meth:`extractall` method. - - .. warning:: - - See the warning for :meth:`extractall`. - - Set ``filter='data'`` to prevent the most dangerous security issues, - and read the :ref:`tarfile-extraction-filter` section for details. - - .. versionchanged:: 3.2 - Added the *set_attrs* parameter. - - .. versionchanged:: 3.5 - Added the *numeric_owner* parameter. - - .. versionchanged:: 3.6 - The *path* parameter accepts a :term:`path-like object`. - - .. versionchanged:: 3.11.4 - Added the *filter* parameter. - - -.. method:: TarFile.extractfile(member) - - Extract a member from the archive as a file object. *member* may be - a filename or a :class:`TarInfo` object. If *member* is a regular file or - a link, an :class:`io.BufferedReader` object is returned. For all other - existing members, :const:`None` is returned. If *member* does not appear - in the archive, :exc:`KeyError` is raised. - - .. versionchanged:: 3.3 - Return an :class:`io.BufferedReader` object. - -.. attribute:: TarFile.errorlevel - :type: int - - If *errorlevel* is ``0``, errors are ignored when using :meth:`TarFile.extract` - and :meth:`TarFile.extractall`. - Nevertheless, they appear as error messages in the debug output when - *debug* is greater than 0. - If ``1`` (the default), all *fatal* errors are raised as :exc:`OSError` or - :exc:`FilterError` exceptions. If ``2``, all *non-fatal* errors are raised - as :exc:`TarError` exceptions as well. - - Some exceptions, e.g. ones caused by wrong argument types or data - corruption, are always raised. - - Custom :ref:`extraction filters ` - should raise :exc:`FilterError` for *fatal* errors - and :exc:`ExtractError` for *non-fatal* ones. - - Note that when an exception is raised, the archive may be partially - extracted. It is the user’s responsibility to clean up. - -.. attribute:: TarFile.extraction_filter - - .. versionadded:: 3.11.4 - - The :ref:`extraction filter ` used - as a default for the *filter* argument of :meth:`~TarFile.extract` - and :meth:`~TarFile.extractall`. - - The attribute may be ``None`` or a callable. - String names are not allowed for this attribute, unlike the *filter* - argument to :meth:`~TarFile.extract`. - - If ``extraction_filter`` is ``None`` (the default), - calling an extraction method without a *filter* argument will - use the :func:`fully_trusted ` filter for - compatibility with previous Python versions. - - In Python 3.12+, leaving ``extraction_filter=None`` will emit a - ``DeprecationWarning``. - - In Python 3.14+, leaving ``extraction_filter=None`` will cause - extraction methods to use the :func:`data ` filter by default. - - The attribute may be set on instances or overridden in subclasses. - It also is possible to set it on the ``TarFile`` class itself to set a - global default, although, since it affects all uses of *tarfile*, - it is best practice to only do so in top-level applications or - :mod:`site configuration `. - To set a global default this way, a filter function needs to be wrapped in - :func:`staticmethod()` to prevent injection of a ``self`` argument. - -.. method:: TarFile.add(name, arcname=None, recursive=True, *, filter=None) - - Add the file *name* to the archive. *name* may be any type of file - (directory, fifo, symbolic link, etc.). If given, *arcname* specifies an - alternative name for the file in the archive. Directories are added - recursively by default. This can be avoided by setting *recursive* to - :const:`False`. Recursion adds entries in sorted order. - If *filter* is given, it - should be a function that takes a :class:`TarInfo` object argument and - returns the changed :class:`TarInfo` object. If it instead returns - :const:`None` the :class:`TarInfo` object will be excluded from the - archive. See :ref:`tar-examples` for an example. - - .. versionchanged:: 3.2 - Added the *filter* parameter. - - .. versionchanged:: 3.7 - Recursion adds entries in sorted order. - - -.. method:: TarFile.addfile(tarinfo, fileobj=None) - - Add the :class:`TarInfo` object *tarinfo* to the archive. If *fileobj* is given, - it should be a :term:`binary file`, and - ``tarinfo.size`` bytes are read from it and added to the archive. You can - create :class:`TarInfo` objects directly, or by using :meth:`gettarinfo`. - - -.. method:: TarFile.gettarinfo(name=None, arcname=None, fileobj=None) - - Create a :class:`TarInfo` object from the result of :func:`os.stat` or - equivalent on an existing file. The file is either named by *name*, or - specified as a :term:`file object` *fileobj* with a file descriptor. - *name* may be a :term:`path-like object`. If - given, *arcname* specifies an alternative name for the file in the - archive, otherwise, the name is taken from *fileobj*’s - :attr:`~io.FileIO.name` attribute, or the *name* argument. The name - should be a text string. - - You can modify - some of the :class:`TarInfo`’s attributes before you add it using :meth:`addfile`. - If the file object is not an ordinary file object positioned at the - beginning of the file, attributes such as :attr:`~TarInfo.size` may need - modifying. This is the case for objects such as :class:`~gzip.GzipFile`. - The :attr:`~TarInfo.name` may also be modified, in which case *arcname* - could be a dummy string. - - .. versionchanged:: 3.6 - The *name* parameter accepts a :term:`path-like object`. - - -.. method:: TarFile.close() - - Close the :class:`TarFile`. In write mode, two finishing zero blocks are - appended to the archive. - - -.. attribute:: TarFile.pax_headers - - A dictionary containing key-value pairs of pax global headers. - - - -.. _tarinfo-objects: - -TarInfo Objects ---------------- - -A :class:`TarInfo` object represents one member in a :class:`TarFile`. Aside -from storing all required attributes of a file (like file type, size, time, -permissions, owner etc.), it provides some useful methods to determine its type. -It does *not* contain the file's data itself. - -:class:`TarInfo` objects are returned by :class:`TarFile`'s methods -:meth:`~TarFile.getmember`, :meth:`~TarFile.getmembers` and -:meth:`~TarFile.gettarinfo`. - -Modifying the objects returned by :meth:`~!TarFile.getmember` or -:meth:`~!TarFile.getmembers` will affect all subsequent -operations on the archive. -For cases where this is unwanted, you can use :mod:`copy.copy() ` or -call the :meth:`~TarInfo.replace` method to create a modified copy in one step. - -Several attributes can be set to ``None`` to indicate that a piece of metadata -is unused or unknown. -Different :class:`TarInfo` methods handle ``None`` differently: - -- The :meth:`~TarFile.extract` or :meth:`~TarFile.extractall` methods will - ignore the corresponding metadata, leaving it set to a default. -- :meth:`~TarFile.addfile` will fail. -- :meth:`~TarFile.list` will print a placeholder string. - - -.. versionchanged:: 3.11.4 - Added :meth:`~TarInfo.replace` and handling of ``None``. - - -.. class:: TarInfo(name="") - - Create a :class:`TarInfo` object. - - -.. classmethod:: TarInfo.frombuf(buf, encoding, errors) - - Create and return a :class:`TarInfo` object from string buffer *buf*. - - Raises :exc:`HeaderError` if the buffer is invalid. - - -.. classmethod:: TarInfo.fromtarfile(tarfile) - - Read the next member from the :class:`TarFile` object *tarfile* and return it as - a :class:`TarInfo` object. - - -.. method:: TarInfo.tobuf(format=DEFAULT_FORMAT, encoding=ENCODING, errors='surrogateescape') - - Create a string buffer from a :class:`TarInfo` object. For information on the - arguments see the constructor of the :class:`TarFile` class. - - .. versionchanged:: 3.2 - Use ``'surrogateescape'`` as the default for the *errors* argument. - - -A ``TarInfo`` object has the following public data attributes: - - -.. attribute:: TarInfo.name - :type: str - - Name of the archive member. - - -.. attribute:: TarInfo.size - :type: int - - Size in bytes. - - -.. attribute:: TarInfo.mtime - :type: int | float - - Time of last modification in seconds since the :ref:`epoch `, - as in :attr:`os.stat_result.st_mtime`. - - .. versionchanged:: 3.11.4 - - Can be set to ``None`` for :meth:`~TarFile.extract` and - :meth:`~TarFile.extractall`, causing extraction to skip applying this - attribute. - -.. attribute:: TarInfo.mode - :type: int - - Permission bits, as for :func:`os.chmod`. - - .. versionchanged:: 3.11.4 - - Can be set to ``None`` for :meth:`~TarFile.extract` and - :meth:`~TarFile.extractall`, causing extraction to skip applying this - attribute. - -.. attribute:: TarInfo.type - - File type. *type* is usually one of these constants: :const:`REGTYPE`, - :const:`AREGTYPE`, :const:`LNKTYPE`, :const:`SYMTYPE`, :const:`DIRTYPE`, - :const:`FIFOTYPE`, :const:`CONTTYPE`, :const:`CHRTYPE`, :const:`BLKTYPE`, - :const:`GNUTYPE_SPARSE`. To determine the type of a :class:`TarInfo` object - more conveniently, use the ``is*()`` methods below. - - -.. attribute:: TarInfo.linkname - :type: str - - Name of the target file name, which is only present in :class:`TarInfo` objects - of type :const:`LNKTYPE` and :const:`SYMTYPE`. - - For symbolic links (``SYMTYPE``), the *linkname* is relative to the directory - that contains the link. - For hard links (``LNKTYPE``), the *linkname* is relative to the root of - the archive. - - -.. attribute:: TarInfo.uid - :type: int - - User ID of the user who originally stored this member. - - .. versionchanged:: 3.11.4 - - Can be set to ``None`` for :meth:`~TarFile.extract` and - :meth:`~TarFile.extractall`, causing extraction to skip applying this - attribute. - -.. attribute:: TarInfo.gid - :type: int - - Group ID of the user who originally stored this member. - - .. versionchanged:: 3.11.4 - - Can be set to ``None`` for :meth:`~TarFile.extract` and - :meth:`~TarFile.extractall`, causing extraction to skip applying this - attribute. - -.. attribute:: TarInfo.uname - :type: str - - User name. - - .. versionchanged:: 3.11.4 - - Can be set to ``None`` for :meth:`~TarFile.extract` and - :meth:`~TarFile.extractall`, causing extraction to skip applying this - attribute. - -.. attribute:: TarInfo.gname - :type: str - - Group name. - - .. versionchanged:: 3.11.4 - - Can be set to ``None`` for :meth:`~TarFile.extract` and - :meth:`~TarFile.extractall`, causing extraction to skip applying this - attribute. - -.. attribute:: TarInfo.pax_headers - :type: dict - - A dictionary containing key-value pairs of an associated pax extended header. - -.. method:: TarInfo.replace(name=..., mtime=..., mode=..., linkname=..., - uid=..., gid=..., uname=..., gname=..., - deep=True) - - .. versionadded:: 3.11.4 - - Return a *new* copy of the :class:`!TarInfo` object with the given attributes - changed. For example, to return a ``TarInfo`` with the group name set to - ``'staff'``, use:: - - new_tarinfo = old_tarinfo.replace(gname='staff') - - By default, a deep copy is made. - If *deep* is false, the copy is shallow, i.e. ``pax_headers`` - and any custom attributes are shared with the original ``TarInfo`` object. - -A :class:`TarInfo` object also provides some convenient query methods: - - -.. method:: TarInfo.isfile() - - Return :const:`True` if the :class:`Tarinfo` object is a regular file. - - -.. method:: TarInfo.isreg() - - Same as :meth:`isfile`. - - -.. method:: TarInfo.isdir() - - Return :const:`True` if it is a directory. - - -.. method:: TarInfo.issym() - - Return :const:`True` if it is a symbolic link. - - -.. method:: TarInfo.islnk() - - Return :const:`True` if it is a hard link. - - -.. method:: TarInfo.ischr() - - Return :const:`True` if it is a character device. - - -.. method:: TarInfo.isblk() - - Return :const:`True` if it is a block device. - - -.. method:: TarInfo.isfifo() - - Return :const:`True` if it is a FIFO. - - -.. method:: TarInfo.isdev() - - Return :const:`True` if it is one of character device, block device or FIFO. - - -.. _tarfile-extraction-filter: - -Extraction filters ------------------- - -.. versionadded:: 3.11.4 - -The *tar* format is designed to capture all details of a UNIX-like filesystem, -which makes it very powerful. -Unfortunately, the features make it easy to create tar files that have -unintended -- and possibly malicious -- effects when extracted. -For example, extracting a tar file can overwrite arbitrary files in various -ways (e.g. by using absolute paths, ``..`` path components, or symlinks that -affect later members). - -In most cases, the full functionality is not needed. -Therefore, *tarfile* supports extraction filters: a mechanism to limit -functionality, and thus mitigate some of the security issues. - -.. seealso:: - - :pep:`706` - Contains further motivation and rationale behind the design. - -The *filter* argument to :meth:`TarFile.extract` or :meth:`~TarFile.extractall` -can be: - -* the string ``'fully_trusted'``: Honor all metadata as specified in the - archive. - Should be used if the user trusts the archive completely, or implements - their own complex verification. - -* the string ``'tar'``: Honor most *tar*-specific features (i.e. features of - UNIX-like filesystems), but block features that are very likely to be - surprising or malicious. See :func:`tar_filter` for details. - -* the string ``'data'``: Ignore or block most features specific to UNIX-like - filesystems. Intended for extracting cross-platform data archives. - See :func:`data_filter` for details. - -* ``None`` (default): Use :attr:`TarFile.extraction_filter`. - - If that is also ``None`` (the default), the ``'fully_trusted'`` - filter will be used (for compatibility with earlier versions of Python). - - In Python 3.12, the default will emit a ``DeprecationWarning``. - - In Python 3.14, the ``'data'`` filter will become the default instead. - It's possible to switch earlier; see :attr:`TarFile.extraction_filter`. - -* A callable which will be called for each extracted member with a - :ref:`TarInfo ` describing the member and the destination - path to where the archive is extracted (i.e. the same path is used for all - members):: - - filter(member: TarInfo, path: str, /) -> TarInfo | None - - The callable is called just before each member is extracted, so it can - take the current state of the disk into account. - It can: - - - return a :class:`TarInfo` object which will be used instead of the metadata - in the archive, or - - return ``None``, in which case the member will be skipped, or - - raise an exception to abort the operation or skip the member, - depending on :attr:`~TarFile.errorlevel`. - Note that when extraction is aborted, :meth:`~TarFile.extractall` may leave - the archive partially extracted. It does not attempt to clean up. - -Default named filters -~~~~~~~~~~~~~~~~~~~~~ - -The pre-defined, named filters are available as functions, so they can be -reused in custom filters: - -.. function:: fully_trusted_filter(member, path) - - Return *member* unchanged. - - This implements the ``'fully_trusted'`` filter. - -.. function:: tar_filter(member, path) - - Implements the ``'tar'`` filter. - - - Strip leading slashes (``/`` and :data:`os.sep`) from filenames. - - :ref:`Refuse ` to extract files with absolute - paths (in case the name is absolute - even after stripping slashes, e.g. ``C:/foo`` on Windows). - This raises :class:`~tarfile.AbsolutePathError`. - - :ref:`Refuse ` to extract files whose absolute - path (after following symlinks) would end up outside the destination. - This raises :class:`~tarfile.OutsideDestinationError`. - - Clear high mode bits (setuid, setgid, sticky) and group/other write bits - (:const:`~stat.S_IWGRP`|:const:`~stat.S_IWOTH`). - - Return the modified ``TarInfo`` member. - -.. function:: data_filter(member, path) - - Implements the ``'data'`` filter. - In addition to what ``tar_filter`` does: - - - :ref:`Refuse ` to extract links (hard or soft) - that link to absolute paths, or ones that link outside the destination. - - This raises :class:`~tarfile.AbsoluteLinkError` or - :class:`~tarfile.LinkOutsideDestinationError`. - - Note that such files are refused even on platforms that do not support - symbolic links. - - - :ref:`Refuse ` to extract device files - (including pipes). - This raises :class:`~tarfile.SpecialFileError`. - - - For regular files, including hard links: - - - Set the owner read and write permissions - (:const:`~stat.S_IRUSR`|:const:`~stat.S_IWUSR`). - - Remove the group & other executable permission - (:const:`~stat.S_IXGRP`|:const:`~stat.S_IXOTH`) - if the owner doesn’t have it (:const:`~stat.S_IXUSR`). - - - For other files (directories), set ``mode`` to ``None``, so - that extraction methods skip applying permission bits. - - Set user and group info (``uid``, ``gid``, ``uname``, ``gname``) - to ``None``, so that extraction methods skip setting it. - - Return the modified ``TarInfo`` member. - - -.. _tarfile-extraction-refuse: - -Filter errors -~~~~~~~~~~~~~ - -When a filter refuses to extract a file, it will raise an appropriate exception, -a subclass of :class:`~tarfile.FilterError`. -This will abort the extraction if :attr:`TarFile.errorlevel` is 1 or more. -With ``errorlevel=0`` the error will be logged and the member will be skipped, -but extraction will continue. - - -Hints for further verification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Even with ``filter='data'``, *tarfile* is not suited for extracting untrusted -files without prior inspection. -Among other issues, the pre-defined filters do not prevent denial-of-service -attacks. Users should do additional checks. - -Here is an incomplete list of things to consider: - -* Extract to a :func:`new temporary directory ` - to prevent e.g. exploiting pre-existing links, and to make it easier to - clean up after a failed extraction. -* When working with untrusted data, use external (e.g. OS-level) limits on - disk, memory and CPU usage. -* Check filenames against an allow-list of characters - (to filter out control characters, confusables, foreign path separators, - etc.). -* Check that filenames have expected extensions (discouraging files that - execute when you “click on them”, or extension-less files like Windows special device names). -* Limit the number of extracted files, total size of extracted data, - filename length (including symlink length), and size of individual files. -* Check for files that would be shadowed on case-insensitive filesystems. - -Also note that: - -* Tar files may contain multiple versions of the same file. - Later ones are expected to overwrite any earlier ones. - This feature is crucial to allow updating tape archives, but can be abused - maliciously. -* *tarfile* does not protect against issues with “live” data, - e.g. an attacker tinkering with the destination (or source) directory while - extraction (or archiving) is in progress. - - -Supporting older Python versions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Extraction filters were added to Python 3.12, and are backported to older -versions as security updates. -To check whether the feature is available, use e.g. -``hasattr(tarfile, 'data_filter')`` rather than checking the Python version. - -The following examples show how to support Python versions with and without -the feature. -Note that setting ``extraction_filter`` will affect any subsequent operations. - -* Fully trusted archive:: - - my_tarfile.extraction_filter = (lambda member, path: member) - my_tarfile.extractall() - -* Use the ``'data'`` filter if available, but revert to Python 3.11 behavior - (``'fully_trusted'``) if this feature is not available:: - - my_tarfile.extraction_filter = getattr(tarfile, 'data_filter', - (lambda member, path: member)) - my_tarfile.extractall() - -* Use the ``'data'`` filter; *fail* if it is not available:: - - my_tarfile.extractall(filter=tarfile.data_filter) - - or:: - - my_tarfile.extraction_filter = tarfile.data_filter - my_tarfile.extractall() - -* Use the ``'data'`` filter; *warn* if it is not available:: - - if hasattr(tarfile, 'data_filter'): - my_tarfile.extractall(filter='data') - else: - # remove this when no longer needed - warn_the_user('Extracting may be unsafe; consider updating Python') - my_tarfile.extractall() - - -Stateful extraction filter example -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -While *tarfile*'s extraction methods take a simple *filter* callable, -custom filters may be more complex objects with an internal state. -It may be useful to write these as context managers, to be used like this:: - - with StatefulFilter() as filter_func: - tar.extractall(path, filter=filter_func) - -Such a filter can be written as, for example:: - - class StatefulFilter: - def __init__(self): - self.file_count = 0 - - def __enter__(self): - return self - - def __call__(self, member, path): - self.file_count += 1 - return member - - def __exit__(self, *exc_info): - print(f'{self.file_count} files extracted') - - -.. _tarfile-commandline: -.. program:: tarfile - - -Command-Line Interface ----------------------- - -.. versionadded:: 3.4 - -The :mod:`tarfile` module provides a simple command-line interface to interact -with tar archives. - -If you want to create a new tar archive, specify its name after the :option:`-c` -option and then list the filename(s) that should be included: - -.. code-block:: shell-session - - $ python -m tarfile -c monty.tar spam.txt eggs.txt - -Passing a directory is also acceptable: - -.. code-block:: shell-session - - $ python -m tarfile -c monty.tar life-of-brian_1979/ - -If you want to extract a tar archive into the current directory, use -the :option:`-e` option: - -.. code-block:: shell-session - - $ python -m tarfile -e monty.tar - -You can also extract a tar archive into a different directory by passing the -directory's name: - -.. code-block:: shell-session - - $ python -m tarfile -e monty.tar other-dir/ - -For a list of the files in a tar archive, use the :option:`-l` option: - -.. code-block:: shell-session - - $ python -m tarfile -l monty.tar - - -Command-line options -~~~~~~~~~~~~~~~~~~~~ - -.. option:: -l - --list - - List files in a tarfile. - -.. option:: -c ... - --create ... - - Create tarfile from source files. - -.. option:: -e [] - --extract [] - - Extract tarfile into the current directory if *output_dir* is not specified. - -.. option:: -t - --test - - Test whether the tarfile is valid or not. - -.. option:: -v, --verbose - - Verbose output. - -.. option:: --filter - - Specifies the *filter* for ``--extract``. - See :ref:`tarfile-extraction-filter` for details. - Only string names are accepted (that is, ``fully_trusted``, ``tar``, - and ``data``). - - .. versionadded:: 3.11.4 - -.. _tar-examples: - -Examples --------- - -How to extract an entire tar archive to the current working directory:: - - import tarfile - tar = tarfile.open("sample.tar.gz") - tar.extractall() - tar.close() - -How to extract a subset of a tar archive with :meth:`TarFile.extractall` using -a generator function instead of a list:: - - import os - import tarfile - - def py_files(members): - for tarinfo in members: - if os.path.splitext(tarinfo.name)[1] == ".py": - yield tarinfo - - tar = tarfile.open("sample.tar.gz") - tar.extractall(members=py_files(tar)) - tar.close() - -How to create an uncompressed tar archive from a list of filenames:: - - import tarfile - tar = tarfile.open("sample.tar", "w") - for name in ["foo", "bar", "quux"]: - tar.add(name) - tar.close() - -The same example using the :keyword:`with` statement:: - - import tarfile - with tarfile.open("sample.tar", "w") as tar: - for name in ["foo", "bar", "quux"]: - tar.add(name) - -How to read a gzip compressed tar archive and display some member information:: - - import tarfile - tar = tarfile.open("sample.tar.gz", "r:gz") - for tarinfo in tar: - print(tarinfo.name, "is", tarinfo.size, "bytes in size and is ", end="") - if tarinfo.isreg(): - print("a regular file.") - elif tarinfo.isdir(): - print("a directory.") - else: - print("something else.") - tar.close() - -How to create an archive and reset the user information using the *filter* -parameter in :meth:`TarFile.add`:: - - import tarfile - def reset(tarinfo): - tarinfo.uid = tarinfo.gid = 0 - tarinfo.uname = tarinfo.gname = "root" - return tarinfo - tar = tarfile.open("sample.tar.gz", "w:gz") - tar.add("foo", filter=reset) - tar.close() - - -.. _tar-formats: - -Supported tar formats ---------------------- - -There are three tar formats that can be created with the :mod:`tarfile` module: - -* The POSIX.1-1988 ustar format (:const:`USTAR_FORMAT`). It supports filenames - up to a length of at best 256 characters and linknames up to 100 characters. - The maximum file size is 8 GiB. This is an old and limited but widely - supported format. - -* The GNU tar format (:const:`GNU_FORMAT`). It supports long filenames and - linknames, files bigger than 8 GiB and sparse files. It is the de facto - standard on GNU/Linux systems. :mod:`tarfile` fully supports the GNU tar - extensions for long names, sparse file support is read-only. - -* The POSIX.1-2001 pax format (:const:`PAX_FORMAT`). It is the most flexible - format with virtually no limits. It supports long filenames and linknames, large - files and stores pathnames in a portable way. Modern tar implementations, - including GNU tar, bsdtar/libarchive and star, fully support extended *pax* - features; some old or unmaintained libraries may not, but should treat - *pax* archives as if they were in the universally supported *ustar* format. - It is the current default format for new archives. - - It extends the existing *ustar* format with extra headers for information - that cannot be stored otherwise. There are two flavours of pax headers: - Extended headers only affect the subsequent file header, global - headers are valid for the complete archive and affect all following files. - All the data in a pax header is encoded in *UTF-8* for portability reasons. - -There are some more variants of the tar format which can be read, but not -created: - -* The ancient V7 format. This is the first tar format from Unix Seventh Edition, - storing only regular files and directories. Names must not be longer than 100 - characters, there is no user/group name information. Some archives have - miscalculated header checksums in case of fields with non-ASCII characters. - -* The SunOS tar extended format. This format is a variant of the POSIX.1-2001 - pax format, but is not compatible. - -.. _tar-unicode: - -Unicode issues --------------- - -The tar format was originally conceived to make backups on tape drives with the -main focus on preserving file system information. Nowadays tar archives are -commonly used for file distribution and exchanging archives over networks. One -problem of the original format (which is the basis of all other formats) is -that there is no concept of supporting different character encodings. For -example, an ordinary tar archive created on a *UTF-8* system cannot be read -correctly on a *Latin-1* system if it contains non-*ASCII* characters. Textual -metadata (like filenames, linknames, user/group names) will appear damaged. -Unfortunately, there is no way to autodetect the encoding of an archive. The -pax format was designed to solve this problem. It stores non-ASCII metadata -using the universal character encoding *UTF-8*. - -The details of character conversion in :mod:`tarfile` are controlled by the -*encoding* and *errors* keyword arguments of the :class:`TarFile` class. - -*encoding* defines the character encoding to use for the metadata in the -archive. The default value is :func:`sys.getfilesystemencoding` or ``'ascii'`` -as a fallback. Depending on whether the archive is read or written, the -metadata must be either decoded or encoded. If *encoding* is not set -appropriately, this conversion may fail. - -The *errors* argument defines how characters are treated that cannot be -converted. Possible values are listed in section :ref:`error-handlers`. -The default scheme is ``'surrogateescape'`` which Python also uses for its -file system calls, see :ref:`os-filenames`. - -For :const:`PAX_FORMAT` archives (the default), *encoding* is generally not needed -because all the metadata is stored using *UTF-8*. *encoding* is only used in -the rare cases when binary pax headers are decoded or when strings with -surrogate characters are stored. diff --git a/Doc/library/telnetlib.rst b/Doc/library/telnetlib.rst deleted file mode 100644 index 5a993dc42a5ab2..00000000000000 --- a/Doc/library/telnetlib.rst +++ /dev/null @@ -1,262 +0,0 @@ -:mod:`telnetlib` --- Telnet client -================================== - -.. module:: telnetlib - :synopsis: Telnet client class. - :deprecated: - -.. sectionauthor:: Skip Montanaro - -**Source code:** :source:`Lib/telnetlib.py` - -.. index:: single: protocol; Telnet - -.. deprecated-removed:: 3.11 3.13 - The :mod:`telnetlib` module is deprecated - (see :pep:`PEP 594 <594#telnetlib>` for details and alternatives). - --------------- - -The :mod:`telnetlib` module provides a :class:`Telnet` class that implements the -Telnet protocol. See :rfc:`854` for details about the protocol. In addition, it -provides symbolic constants for the protocol characters (see below), and for the -telnet options. The symbolic names of the telnet options follow the definitions -in ``arpa/telnet.h``, with the leading ``TELOPT_`` removed. For symbolic names -of options which are traditionally not included in ``arpa/telnet.h``, see the -module source itself. - -The symbolic constants for the telnet commands are: IAC, DONT, DO, WONT, WILL, -SE (Subnegotiation End), NOP (No Operation), DM (Data Mark), BRK (Break), IP -(Interrupt process), AO (Abort output), AYT (Are You There), EC (Erase -Character), EL (Erase Line), GA (Go Ahead), SB (Subnegotiation Begin). - -.. include:: ../includes/wasm-notavail.rst - -.. class:: Telnet(host=None, port=0[, timeout]) - - :class:`Telnet` represents a connection to a Telnet server. The instance is - initially not connected by default; the :meth:`~Telnet.open` method must be used to - establish a connection. Alternatively, the host name and optional port - number can be passed to the constructor too, in which case the connection to - the server will be established before the constructor returns. The optional - *timeout* parameter specifies a timeout in seconds for blocking operations - like the connection attempt (if not specified, the global default timeout - setting will be used). - - Do not reopen an already connected instance. - - This class has many :meth:`read_\*` methods. Note that some of them raise - :exc:`EOFError` when the end of the connection is read, because they can return - an empty string for other reasons. See the individual descriptions below. - - A :class:`Telnet` object is a context manager and can be used in a - :keyword:`with` statement. When the :keyword:`!with` block ends, the - :meth:`close` method is called:: - - >>> from telnetlib import Telnet - >>> with Telnet('localhost', 23) as tn: - ... tn.interact() - ... - - .. versionchanged:: 3.6 Context manager support added - - -.. seealso:: - - :rfc:`854` - Telnet Protocol Specification - Definition of the Telnet protocol. - - -.. _telnet-objects: - -Telnet Objects --------------- - -:class:`Telnet` instances have the following methods: - - -.. method:: Telnet.read_until(expected, timeout=None) - - Read until a given byte string, *expected*, is encountered or until *timeout* - seconds have passed. - - When no match is found, return whatever is available instead, possibly empty - bytes. Raise :exc:`EOFError` if the connection is closed and no cooked data - is available. - - -.. method:: Telnet.read_all() - - Read all data until EOF as bytes; block until connection closed. - - -.. method:: Telnet.read_some() - - Read at least one byte of cooked data unless EOF is hit. Return ``b''`` if - EOF is hit. Block if no data is immediately available. - - -.. method:: Telnet.read_very_eager() - - Read everything that can be without blocking in I/O (eager). - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_eager() - - Read readily available data. - - Raise :exc:`EOFError` if connection closed and no cooked data available. - Return ``b''`` if no cooked data available otherwise. Do not block unless in - the midst of an IAC sequence. - - -.. method:: Telnet.read_lazy() - - Process and return data already in the queues (lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. Do not block unless in the - midst of an IAC sequence. - - -.. method:: Telnet.read_very_lazy() - - Return any data available in the cooked queue (very lazy). - - Raise :exc:`EOFError` if connection closed and no data available. Return - ``b''`` if no cooked data available otherwise. This method never blocks. - - -.. method:: Telnet.read_sb_data() - - Return the data collected between a SB/SE pair (suboption begin/end). The - callback should access these data when it was invoked with a ``SE`` command. - This method never blocks. - - -.. method:: Telnet.open(host, port=0[, timeout]) - - Connect to a host. The optional second argument is the port number, which - defaults to the standard Telnet port (23). The optional *timeout* parameter - specifies a timeout in seconds for blocking operations like the connection - attempt (if not specified, the global default timeout setting will be used). - - Do not try to reopen an already connected instance. - - .. audit-event:: telnetlib.Telnet.open self,host,port telnetlib.Telnet.open - - -.. method:: Telnet.msg(msg, *args) - - Print a debug message when the debug level is ``>`` 0. If extra arguments are - present, they are substituted in the message using the standard string - formatting operator. - - -.. method:: Telnet.set_debuglevel(debuglevel) - - Set the debug level. The higher the value of *debuglevel*, the more debug - output you get (on ``sys.stdout``). - - -.. method:: Telnet.close() - - Close the connection. - - -.. method:: Telnet.get_socket() - - Return the socket object used internally. - - -.. method:: Telnet.fileno() - - Return the file descriptor of the socket object used internally. - - -.. method:: Telnet.write(buffer) - - Write a byte string to the socket, doubling any IAC characters. This can - block if the connection is blocked. May raise :exc:`OSError` if the - connection is closed. - - .. audit-event:: telnetlib.Telnet.write self,buffer telnetlib.Telnet.write - - .. versionchanged:: 3.3 - This method used to raise :exc:`socket.error`, which is now an alias - of :exc:`OSError`. - - -.. method:: Telnet.interact() - - Interaction function, emulates a very dumb Telnet client. - - -.. method:: Telnet.mt_interact() - - Multithreaded version of :meth:`interact`. - - -.. method:: Telnet.expect(list, timeout=None) - - Read until one from a list of a regular expressions matches. - - The first argument is a list of regular expressions, either compiled - (:ref:`regex objects `) or uncompiled (byte strings). The - optional second argument is a timeout, in seconds; the default is to block - indefinitely. - - Return a tuple of three items: the index in the list of the first regular - expression that matches; the match object returned; and the bytes read up - till and including the match. - - If end of file is found and no bytes were read, raise :exc:`EOFError`. - Otherwise, when nothing matches, return ``(-1, None, data)`` where *data* is - the bytes received so far (may be empty bytes if a timeout happened). - - If a regular expression ends with a greedy match (such as ``.*``) or if more - than one expression can match the same input, the results are - non-deterministic, and may depend on the I/O timing. - - -.. method:: Telnet.set_option_negotiation_callback(callback) - - Each time a telnet option is read on the input flow, this *callback* (if set) is - called with the following parameters: callback(telnet socket, command - (DO/DONT/WILL/WONT), option). No other action is done afterwards by telnetlib. - - -.. _telnet-example: - -Telnet Example --------------- - -.. sectionauthor:: Peter Funk - - -A simple example illustrating typical use:: - - import getpass - import telnetlib - - HOST = "localhost" - user = input("Enter your remote account: ") - password = getpass.getpass() - - tn = telnetlib.Telnet(HOST) - - tn.read_until(b"login: ") - tn.write(user.encode('ascii') + b"\n") - if password: - tn.read_until(b"Password: ") - tn.write(password.encode('ascii') + b"\n") - - tn.write(b"ls\n") - tn.write(b"exit\n") - - print(tn.read_all().decode('ascii')) - diff --git a/Doc/library/tempfile.rst b/Doc/library/tempfile.rst deleted file mode 100644 index 54096f083fd98b..00000000000000 --- a/Doc/library/tempfile.rst +++ /dev/null @@ -1,398 +0,0 @@ -:mod:`tempfile` --- Generate temporary files and directories -============================================================ - -.. module:: tempfile - :synopsis: Generate temporary files and directories. - -.. sectionauthor:: Zack Weinberg - -**Source code:** :source:`Lib/tempfile.py` - -.. index:: - pair: temporary; file name - pair: temporary; file - --------------- - -This module creates temporary files and directories. It works on all -supported platforms. :class:`TemporaryFile`, :class:`NamedTemporaryFile`, -:class:`TemporaryDirectory`, and :class:`SpooledTemporaryFile` are high-level -interfaces which provide automatic cleanup and can be used as -context managers. :func:`mkstemp` and -:func:`mkdtemp` are lower-level functions which require manual cleanup. - -All the user-callable functions and constructors take additional arguments which -allow direct control over the location and name of temporary files and -directories. Files names used by this module include a string of -random characters which allows those files to be securely created in -shared temporary directories. -To maintain backward compatibility, the argument order is somewhat odd; it -is recommended to use keyword arguments for clarity. - -The module defines the following user-callable items: - -.. function:: TemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) - - Return a :term:`file-like object` that can be used as a temporary storage area. - The file is created securely, using the same rules as :func:`mkstemp`. It will be destroyed as soon - as it is closed (including an implicit close when the object is garbage - collected). Under Unix, the directory entry for the file is either not created at all or is removed - immediately after the file is created. Other platforms do not support - this; your code should not rely on a temporary file created using this - function having or not having a visible name in the file system. - - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or - destruction of the file object the temporary file will be removed - from the filesystem. - - The *mode* parameter defaults to ``'w+b'`` so that the file created can - be read and written without being closed. Binary mode is used so that it - behaves consistently on all platforms without regard for the data that is - stored. *buffering*, *encoding*, *errors* and *newline* are interpreted as for - :func:`open`. - - The *dir*, *prefix* and *suffix* parameters have the same meaning and - defaults as with :func:`mkstemp`. - - The returned object is a true file object on POSIX platforms. On other - platforms, it is a file-like object whose :attr:`!file` attribute is the - underlying true file object. - - The :py:const:`os.O_TMPFILE` flag is used if it is available and works - (Linux-specific, requires Linux kernel 3.11 or later). - - On platforms that are neither Posix nor Cygwin, TemporaryFile is an alias - for NamedTemporaryFile. - - .. audit-event:: tempfile.mkstemp fullpath tempfile.TemporaryFile - - .. versionchanged:: 3.5 - - The :py:const:`os.O_TMPFILE` flag is now used if available. - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. function:: NamedTemporaryFile(mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, delete=True, *, errors=None) - - This function operates exactly as :func:`TemporaryFile` does, except that - the file is guaranteed to have a visible name in the file system (on - Unix, the directory entry is not unlinked). That name can be retrieved - from the :attr:`name` attribute of the returned - file-like object. Whether the name can be - used to open the file a second time, while the named temporary file is - still open, varies across platforms (it can be so used on Unix; it cannot - on Windows). If *delete* is true (the default), the file is - deleted as soon as it is closed. - The returned object is always a file-like object whose :attr:`!file` - attribute is the underlying true file object. This file-like object can - be used in a :keyword:`with` statement, just like a normal file. - - On POSIX (only), a process that is terminated abruptly with SIGKILL - cannot automatically delete any NamedTemporaryFiles it created. - - .. audit-event:: tempfile.mkstemp fullpath tempfile.NamedTemporaryFile - - .. versionchanged:: 3.8 - Added *errors* parameter. - - -.. class:: SpooledTemporaryFile(max_size=0, mode='w+b', buffering=-1, encoding=None, newline=None, suffix=None, prefix=None, dir=None, *, errors=None) - - This class operates exactly as :func:`TemporaryFile` does, except that - data is spooled in memory until the file size exceeds *max_size*, or - until the file's :func:`~io.IOBase.fileno` method is called, at which point the - contents are written to disk and operation proceeds as with - :func:`TemporaryFile`. - - The resulting file has one additional method, :func:`rollover`, which - causes the file to roll over to an on-disk file regardless of its size. - - The returned object is a file-like object whose :attr:`_file` attribute - is either an :class:`io.BytesIO` or :class:`io.TextIOWrapper` object - (depending on whether binary or text *mode* was specified) or a true file - object, depending on whether :func:`rollover` has been called. This - file-like object can be used in a :keyword:`with` statement, just like - a normal file. - - .. versionchanged:: 3.3 - the truncate method now accepts a ``size`` argument. - - .. versionchanged:: 3.8 - Added *errors* parameter. - - .. versionchanged:: 3.11 - Fully implements the :class:`io.BufferedIOBase` and - :class:`io.TextIOBase` abstract base classes (depending on whether binary - or text *mode* was specified). - - -.. class:: TemporaryDirectory(suffix=None, prefix=None, dir=None, ignore_cleanup_errors=False) - - This class securely creates a temporary directory using the same rules as :func:`mkdtemp`. - The resulting object can be used as a context manager (see - :ref:`tempfile-examples`). On completion of the context or destruction - of the temporary directory object, the newly created temporary directory - and all its contents are removed from the filesystem. - - The directory name can be retrieved from the :attr:`name` attribute of the - returned object. When the returned object is used as a context manager, the - :attr:`name` will be assigned to the target of the :keyword:`!as` clause in - the :keyword:`with` statement, if there is one. - - The directory can be explicitly cleaned up by calling the - :func:`cleanup` method. If *ignore_cleanup_errors* is true, any unhandled - exceptions during explicit or implicit cleanup (such as a - :exc:`PermissionError` removing open files on Windows) will be ignored, - and the remaining removable items deleted on a "best-effort" basis. - Otherwise, errors will be raised in whatever context cleanup occurs - (the :func:`cleanup` call, exiting the context manager, when the object - is garbage-collected or during interpreter shutdown). - - .. audit-event:: tempfile.mkdtemp fullpath tempfile.TemporaryDirectory - - .. versionadded:: 3.2 - - .. versionchanged:: 3.10 - Added *ignore_cleanup_errors* parameter. - - -.. function:: mkstemp(suffix=None, prefix=None, dir=None, text=False) - - Creates a temporary file in the most secure manner possible. There are - no race conditions in the file's creation, assuming that the platform - properly implements the :const:`os.O_EXCL` flag for :func:`os.open`. The - file is readable and writable only by the creating user ID. If the - platform uses permission bits to indicate whether a file is executable, - the file is executable by no one. The file descriptor is not inherited - by child processes. - - Unlike :func:`TemporaryFile`, the user of :func:`mkstemp` is responsible - for deleting the temporary file when done with it. - - If *suffix* is not ``None``, the file name will end with that suffix, - otherwise there will be no suffix. :func:`mkstemp` does not put a dot - between the file name and the suffix; if you need one, put it at the - beginning of *suffix*. - - If *prefix* is not ``None``, the file name will begin with that prefix; - otherwise, a default prefix is used. The default is the return value of - :func:`gettempprefix` or :func:`gettempprefixb`, as appropriate. - - If *dir* is not ``None``, the file will be created in that directory; - otherwise, a default directory is used. The default directory is chosen - from a platform-dependent list, but the user of the application can - control the directory location by setting the *TMPDIR*, *TEMP* or *TMP* - environment variables. There is thus no guarantee that the generated - filename will have any nice properties, such as not requiring quoting - when passed to external commands via ``os.popen()``. - - If any of *suffix*, *prefix*, and *dir* are not - ``None``, they must be the same type. - If they are bytes, the returned name will be bytes instead of str. - If you want to force a bytes return value with otherwise default behavior, - pass ``suffix=b''``. - - If *text* is specified and true, the file is opened in text mode. - Otherwise, (the default) the file is opened in binary mode. - - :func:`mkstemp` returns a tuple containing an OS-level handle to an open - file (as would be returned by :func:`os.open`) and the absolute pathname - of that file, in that order. - - .. audit-event:: tempfile.mkstemp fullpath tempfile.mkstemp - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - .. versionchanged:: 3.6 - The *dir* parameter now accepts a :term:`path-like object`. - - -.. function:: mkdtemp(suffix=None, prefix=None, dir=None) - - Creates a temporary directory in the most secure manner possible. There - are no race conditions in the directory's creation. The directory is - readable, writable, and searchable only by the creating user ID. - - The user of :func:`mkdtemp` is responsible for deleting the temporary - directory and its contents when done with it. - - The *prefix*, *suffix*, and *dir* arguments are the same as for - :func:`mkstemp`. - - :func:`mkdtemp` returns the absolute pathname of the new directory if *dir* - is ``None`` or is an absolute path. If *dir* is a relative path, - :func:`mkdtemp` returns a relative path on Python 3.11 and lower. However, - on 3.12 it will return an absolute path in all situations. - - .. audit-event:: tempfile.mkdtemp fullpath tempfile.mkdtemp - - .. versionchanged:: 3.5 - *suffix*, *prefix*, and *dir* may now be supplied in bytes in order to - obtain a bytes return value. Prior to this, only str was allowed. - *suffix* and *prefix* now accept and default to ``None`` to cause - an appropriate default value to be used. - - .. versionchanged:: 3.6 - The *dir* parameter now accepts a :term:`path-like object`. - - -.. function:: gettempdir() - - Return the name of the directory used for temporary files. This - defines the default value for the *dir* argument to all functions - in this module. - - Python searches a standard list of directories to find one which - the calling user can create files in. The list is: - - #. The directory named by the :envvar:`TMPDIR` environment variable. - - #. The directory named by the :envvar:`TEMP` environment variable. - - #. The directory named by the :envvar:`TMP` environment variable. - - #. A platform-specific location: - - * On Windows, the directories :file:`C:\\TEMP`, :file:`C:\\TMP`, - :file:`\\TEMP`, and :file:`\\TMP`, in that order. - - * On all other platforms, the directories :file:`/tmp`, :file:`/var/tmp`, and - :file:`/usr/tmp`, in that order. - - #. As a last resort, the current working directory. - - The result of this search is cached, see the description of - :data:`tempdir` below. - - .. versionchanged:: 3.10 - - Always returns a str. Previously it would return any :data:`tempdir` - value regardless of type so long as it was not ``None``. - -.. function:: gettempdirb() - - Same as :func:`gettempdir` but the return value is in bytes. - - .. versionadded:: 3.5 - -.. function:: gettempprefix() - - Return the filename prefix used to create temporary files. This does not - contain the directory component. - -.. function:: gettempprefixb() - - Same as :func:`gettempprefix` but the return value is in bytes. - - .. versionadded:: 3.5 - -The module uses a global variable to store the name of the directory -used for temporary files returned by :func:`gettempdir`. It can be -set directly to override the selection process, but this is discouraged. -All functions in this module take a *dir* argument which can be used -to specify the directory. This is the recommended approach that does -not surprise other unsuspecting code by changing global API behavior. - -.. data:: tempdir - - When set to a value other than ``None``, this variable defines the - default value for the *dir* argument to the functions defined in this - module, including its type, bytes or str. It cannot be a - :term:`path-like object`. - - If ``tempdir`` is ``None`` (the default) at any call to any of the above - functions except :func:`gettempprefix` it is initialized following the - algorithm described in :func:`gettempdir`. - - .. note:: - - Beware that if you set ``tempdir`` to a bytes value, there is a - nasty side effect: The global default return type of - :func:`mkstemp` and :func:`mkdtemp` changes to bytes when no - explicit ``prefix``, ``suffix``, or ``dir`` arguments of type - str are supplied. Please do not write code expecting or - depending on this. This awkward behavior is maintained for - compatibility with the historical implementation. - -.. _tempfile-examples: - -Examples --------- - -Here are some examples of typical usage of the :mod:`tempfile` module:: - - >>> import tempfile - - # create a temporary file and write some data to it - >>> fp = tempfile.TemporaryFile() - >>> fp.write(b'Hello world!') - # read data from file - >>> fp.seek(0) - >>> fp.read() - b'Hello world!' - # close the file, it will be removed - >>> fp.close() - - # create a temporary file using a context manager - >>> with tempfile.TemporaryFile() as fp: - ... fp.write(b'Hello world!') - ... fp.seek(0) - ... fp.read() - b'Hello world!' - >>> - # file is now closed and removed - - # create a temporary directory using the context manager - >>> with tempfile.TemporaryDirectory() as tmpdirname: - ... print('created temporary directory', tmpdirname) - >>> - # directory and contents have been removed - -.. _tempfile-mktemp-deprecated: - -Deprecated functions and variables ----------------------------------- - -A historical way to create temporary files was to first generate a -file name with the :func:`mktemp` function and then create a file -using this name. Unfortunately this is not secure, because a different -process may create a file with this name in the time between the call -to :func:`mktemp` and the subsequent attempt to create the file by the -first process. The solution is to combine the two steps and create the -file immediately. This approach is used by :func:`mkstemp` and the -other functions described above. - -.. function:: mktemp(suffix='', prefix='tmp', dir=None) - - .. deprecated:: 2.3 - Use :func:`mkstemp` instead. - - Return an absolute pathname of a file that did not exist at the time the - call is made. The *prefix*, *suffix*, and *dir* arguments are similar - to those of :func:`mkstemp`, except that bytes file names, ``suffix=None`` - and ``prefix=None`` are not supported. - - .. warning:: - - Use of this function may introduce a security hole in your program. By - the time you get around to doing anything with the file name it returns, - someone else may have beaten you to the punch. :func:`mktemp` usage can - be replaced easily with :func:`NamedTemporaryFile`, passing it the - ``delete=False`` parameter:: - - >>> f = NamedTemporaryFile(delete=False) - >>> f.name - '/tmp/tmptjujjt' - >>> f.write(b"Hello World!\n") - 13 - >>> f.close() - >>> os.unlink(f.name) - >>> os.path.exists(f.name) - False diff --git a/Doc/library/termios.rst b/Doc/library/termios.rst deleted file mode 100644 index 57705ddc4e6470..00000000000000 --- a/Doc/library/termios.rst +++ /dev/null @@ -1,137 +0,0 @@ -:mod:`termios` --- POSIX style tty control -========================================== - -.. module:: termios - :platform: Unix - :synopsis: POSIX style tty control. - -.. index:: - pair: POSIX; I/O control - pair: tty; I/O control - --------------- - -This module provides an interface to the POSIX calls for tty I/O control. For a -complete description of these calls, see :manpage:`termios(3)` Unix manual -page. It is only available for those Unix versions that support POSIX -*termios* style tty I/O control configured during installation. - -.. availability:: Unix. - -All functions in this module take a file descriptor *fd* as their first -argument. This can be an integer file descriptor, such as returned by -``sys.stdin.fileno()``, or a :term:`file object`, such as ``sys.stdin`` itself. - -This module also defines all the constants needed to work with the functions -provided here; these have the same name as their counterparts in C. Please -refer to your system documentation for more information on using these terminal -control interfaces. - -The module defines the following functions: - - -.. function:: tcgetattr(fd) - - Return a list containing the tty attributes for file descriptor *fd*, as - follows: ``[iflag, oflag, cflag, lflag, ispeed, ospeed, cc]`` where *cc* is a - list of the tty special characters (each a string of length 1, except the - items with indices :const:`VMIN` and :const:`VTIME`, which are integers when - these fields are defined). The interpretation of the flags and the speeds as - well as the indexing in the *cc* array must be done using the symbolic - constants defined in the :mod:`termios` module. - - -.. function:: tcsetattr(fd, when, attributes) - - Set the tty attributes for file descriptor *fd* from the *attributes*, which is - a list like the one returned by :func:`tcgetattr`. The *when* argument - determines when the attributes are changed: - - .. data:: TCSANOW - - Change attributes immediately. - - .. data:: TCSADRAIN - - Change attributes after transmitting all queued output. - - .. data:: TCSAFLUSH - - Change attributes after transmitting all queued output and - discarding all queued input. - - -.. function:: tcsendbreak(fd, duration) - - Send a break on file descriptor *fd*. A zero *duration* sends a break for - 0.25--0.5 seconds; a nonzero *duration* has a system dependent meaning. - - -.. function:: tcdrain(fd) - - Wait until all output written to file descriptor *fd* has been transmitted. - - -.. function:: tcflush(fd, queue) - - Discard queued data on file descriptor *fd*. The *queue* selector specifies - which queue: :const:`TCIFLUSH` for the input queue, :const:`TCOFLUSH` for the - output queue, or :const:`TCIOFLUSH` for both queues. - - -.. function:: tcflow(fd, action) - - Suspend or resume input or output on file descriptor *fd*. The *action* - argument can be :const:`TCOOFF` to suspend output, :const:`TCOON` to restart - output, :const:`TCIOFF` to suspend input, or :const:`TCION` to restart input. - - -.. function:: tcgetwinsize(fd) - - Return a tuple ``(ws_row, ws_col)`` containing the tty window size for file - descriptor *fd*. Requires :const:`termios.TIOCGWINSZ` or - :const:`termios.TIOCGSIZE`. - - .. versionadded:: 3.11 - - -.. function:: tcsetwinsize(fd, winsize) - - Set the tty window size for file descriptor *fd* from *winsize*, which is - a two-item tuple ``(ws_row, ws_col)`` like the one returned by - :func:`tcgetwinsize`. Requires at least one of the pairs - (:const:`termios.TIOCGWINSZ`, :const:`termios.TIOCSWINSZ`); - (:const:`termios.TIOCGSIZE`, :const:`termios.TIOCSSIZE`) to be defined. - - .. versionadded:: 3.11 - - -.. seealso:: - - Module :mod:`tty` - Convenience functions for common terminal control operations. - - -.. _termios-example: - -Example -------- - -Here's a function that prompts for a password with echoing turned off. Note the -technique using a separate :func:`tcgetattr` call and a :keyword:`try` ... -:keyword:`finally` statement to ensure that the old tty attributes are restored -exactly no matter what happens:: - - def getpass(prompt="Password: "): - import termios, sys - fd = sys.stdin.fileno() - old = termios.tcgetattr(fd) - new = termios.tcgetattr(fd) - new[3] = new[3] & ~termios.ECHO # lflags - try: - termios.tcsetattr(fd, termios.TCSADRAIN, new) - passwd = input(prompt) - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old) - return passwd - diff --git a/Doc/library/test.rst b/Doc/library/test.rst deleted file mode 100644 index 7fc2a7c8466f66..00000000000000 --- a/Doc/library/test.rst +++ /dev/null @@ -1,1739 +0,0 @@ -:mod:`test` --- Regression tests package for Python -=================================================== - -.. module:: test - :synopsis: Regression tests package containing the testing suite for Python. - -.. sectionauthor:: Brett Cannon - -.. note:: - The :mod:`test` package is meant for internal use by Python only. It is - documented for the benefit of the core developers of Python. Any use of - this package outside of Python's standard library is discouraged as code - mentioned here can change or be removed without notice between releases of - Python. - --------------- - -The :mod:`test` package contains all regression tests for Python as well as the -modules :mod:`test.support` and :mod:`test.regrtest`. -:mod:`test.support` is used to enhance your tests while -:mod:`test.regrtest` drives the testing suite. - -Each module in the :mod:`test` package whose name starts with ``test_`` is a -testing suite for a specific module or feature. All new tests should be written -using the :mod:`unittest` or :mod:`doctest` module. Some older tests are -written using a "traditional" testing style that compares output printed to -``sys.stdout``; this style of test is considered deprecated. - - -.. seealso:: - - Module :mod:`unittest` - Writing PyUnit regression tests. - - Module :mod:`doctest` - Tests embedded in documentation strings. - - -.. _writing-tests: - -Writing Unit Tests for the :mod:`test` package ----------------------------------------------- - -It is preferred that tests that use the :mod:`unittest` module follow a few -guidelines. One is to name the test module by starting it with ``test_`` and end -it with the name of the module being tested. The test methods in the test module -should start with ``test_`` and end with a description of what the method is -testing. This is needed so that the methods are recognized by the test driver as -test methods. Also, no documentation string for the method should be included. A -comment (such as ``# Tests function returns only True or False``) should be used -to provide documentation for test methods. This is done because documentation -strings get printed out if they exist and thus what test is being run is not -stated. - -A basic boilerplate is often used:: - - import unittest - from test import support - - class MyTestCase1(unittest.TestCase): - - # Only use setUp() and tearDown() if necessary - - def setUp(self): - ... code to execute in preparation for tests ... - - def tearDown(self): - ... code to execute to clean up after tests ... - - def test_feature_one(self): - # Test feature one. - ... testing code ... - - def test_feature_two(self): - # Test feature two. - ... testing code ... - - ... more test methods ... - - class MyTestCase2(unittest.TestCase): - ... same structure as MyTestCase1 ... - - ... more test classes ... - - if __name__ == '__main__': - unittest.main() - -This code pattern allows the testing suite to be run by :mod:`test.regrtest`, -on its own as a script that supports the :mod:`unittest` CLI, or via the -``python -m unittest`` CLI. - -The goal for regression testing is to try to break code. This leads to a few -guidelines to be followed: - -* The testing suite should exercise all classes, functions, and constants. This - includes not just the external API that is to be presented to the outside - world but also "private" code. - -* Whitebox testing (examining the code being tested when the tests are being - written) is preferred. Blackbox testing (testing only the published user - interface) is not complete enough to make sure all boundary and edge cases - are tested. - -* Make sure all possible values are tested including invalid ones. This makes - sure that not only all valid values are acceptable but also that improper - values are handled correctly. - -* Exhaust as many code paths as possible. Test where branching occurs and thus - tailor input to make sure as many different paths through the code are taken. - -* Add an explicit test for any bugs discovered for the tested code. This will - make sure that the error does not crop up again if the code is changed in the - future. - -* Make sure to clean up after your tests (such as close and remove all temporary - files). - -* If a test is dependent on a specific condition of the operating system then - verify the condition already exists before attempting the test. - -* Import as few modules as possible and do it as soon as possible. This - minimizes external dependencies of tests and also minimizes possible anomalous - behavior from side-effects of importing a module. - -* Try to maximize code reuse. On occasion, tests will vary by something as small - as what type of input is used. Minimize code duplication by subclassing a - basic test class with a class that specifies the input:: - - class TestFuncAcceptsSequencesMixin: - - func = mySuperWhammyFunction - - def test_func(self): - self.func(self.arg) - - class AcceptLists(TestFuncAcceptsSequencesMixin, unittest.TestCase): - arg = [1, 2, 3] - - class AcceptStrings(TestFuncAcceptsSequencesMixin, unittest.TestCase): - arg = 'abc' - - class AcceptTuples(TestFuncAcceptsSequencesMixin, unittest.TestCase): - arg = (1, 2, 3) - - When using this pattern, remember that all classes that inherit from - :class:`unittest.TestCase` are run as tests. The :class:`Mixin` class in the example above - does not have any data and so can't be run by itself, thus it does not - inherit from :class:`unittest.TestCase`. - - -.. seealso:: - - Test Driven Development - A book by Kent Beck on writing tests before code. - - -.. _regrtest: - -Running tests using the command-line interface ----------------------------------------------- - -The :mod:`test` package can be run as a script to drive Python's regression -test suite, thanks to the :option:`-m` option: :program:`python -m test`. Under -the hood, it uses :mod:`test.regrtest`; the call :program:`python -m -test.regrtest` used in previous Python versions still works. Running the -script by itself automatically starts running all regression tests in the -:mod:`test` package. It does this by finding all modules in the package whose -name starts with ``test_``, importing them, and executing the function -:func:`test_main` if present or loading the tests via -unittest.TestLoader.loadTestsFromModule if ``test_main`` does not exist. The -names of tests to execute may also be passed to the script. Specifying a single -regression test (:program:`python -m test test_spam`) will minimize output and -only print whether the test passed or failed. - -Running :mod:`test` directly allows what resources are available for -tests to use to be set. You do this by using the ``-u`` command-line -option. Specifying ``all`` as the value for the ``-u`` option enables all -possible resources: :program:`python -m test -uall`. -If all but one resource is desired (a more common case), a -comma-separated list of resources that are not desired may be listed after -``all``. The command :program:`python -m test -uall,-audio,-largefile` -will run :mod:`test` with all resources except the ``audio`` and -``largefile`` resources. For a list of all resources and more command-line -options, run :program:`python -m test -h`. - -Some other ways to execute the regression tests depend on what platform the -tests are being executed on. On Unix, you can run :program:`make test` at the -top-level directory where Python was built. On Windows, -executing :program:`rt.bat` from your :file:`PCbuild` directory will run all -regression tests. - - -:mod:`test.support` --- Utilities for the Python test suite -=========================================================== - -.. module:: test.support - :synopsis: Support for Python's regression test suite. - - -The :mod:`test.support` module provides support for Python's regression -test suite. - -.. note:: - - :mod:`test.support` is not a public module. It is documented here to help - Python developers write tests. The API of this module is subject to change - without backwards compatibility concerns between releases. - - -This module defines the following exceptions: - -.. exception:: TestFailed - - Exception to be raised when a test fails. This is deprecated in favor of - :mod:`unittest`\ -based tests and :class:`unittest.TestCase`'s assertion - methods. - - -.. exception:: ResourceDenied - - Subclass of :exc:`unittest.SkipTest`. Raised when a resource (such as a - network connection) is not available. Raised by the :func:`requires` - function. - - -The :mod:`test.support` module defines the following constants: - -.. data:: verbose - - ``True`` when verbose output is enabled. Should be checked when more - detailed information is desired about a running test. *verbose* is set by - :mod:`test.regrtest`. - - -.. data:: is_jython - - ``True`` if the running interpreter is Jython. - - -.. data:: is_android - - ``True`` if the system is Android. - - -.. data:: unix_shell - - Path for shell if not on Windows; otherwise ``None``. - - -.. data:: LOOPBACK_TIMEOUT - - Timeout in seconds for tests using a network server listening on the network - local loopback interface like ``127.0.0.1``. - - The timeout is long enough to prevent test failure: it takes into account - that the client and the server can run in different threads or even - different processes. - - The timeout should be long enough for :meth:`~socket.socket.connect`, - :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` methods of - :class:`socket.socket`. - - Its default value is 5 seconds. - - See also :data:`INTERNET_TIMEOUT`. - - -.. data:: INTERNET_TIMEOUT - - Timeout in seconds for network requests going to the internet. - - The timeout is short enough to prevent a test to wait for too long if the - internet request is blocked for whatever reason. - - Usually, a timeout using :data:`INTERNET_TIMEOUT` should not mark a test as - failed, but skip the test instead: see - :func:`~test.support.socket_helper.transient_internet`. - - Its default value is 1 minute. - - See also :data:`LOOPBACK_TIMEOUT`. - - -.. data:: SHORT_TIMEOUT - - Timeout in seconds to mark a test as failed if the test takes "too long". - - The timeout value depends on the regrtest ``--timeout`` command line option. - - If a test using :data:`SHORT_TIMEOUT` starts to fail randomly on slow - buildbots, use :data:`LONG_TIMEOUT` instead. - - Its default value is 30 seconds. - - -.. data:: LONG_TIMEOUT - - Timeout in seconds to detect when a test hangs. - - It is long enough to reduce the risk of test failure on the slowest Python - buildbots. It should not be used to mark a test as failed if the test takes - "too long". The timeout value depends on the regrtest ``--timeout`` command - line option. - - Its default value is 5 minutes. - - See also :data:`LOOPBACK_TIMEOUT`, :data:`INTERNET_TIMEOUT` and - :data:`SHORT_TIMEOUT`. - - -.. data:: PGO - - Set when tests can be skipped when they are not useful for PGO. - - -.. data:: PIPE_MAX_SIZE - - A constant that is likely larger than the underlying OS pipe buffer size, - to make writes blocking. - - -.. data:: SOCK_MAX_SIZE - - A constant that is likely larger than the underlying OS socket buffer size, - to make writes blocking. - - -.. data:: TEST_SUPPORT_DIR - - Set to the top level directory that contains :mod:`test.support`. - - -.. data:: TEST_HOME_DIR - - Set to the top level directory for the test package. - - -.. data:: TEST_DATA_DIR - - Set to the ``data`` directory within the test package. - - -.. data:: MAX_Py_ssize_t - - Set to :data:`sys.maxsize` for big memory tests. - - -.. data:: max_memuse - - Set by :func:`set_memlimit` as the memory limit for big memory tests. - Limited by :data:`MAX_Py_ssize_t`. - - -.. data:: real_max_memuse - - Set by :func:`set_memlimit` as the memory limit for big memory tests. Not - limited by :data:`MAX_Py_ssize_t`. - - -.. data:: MISSING_C_DOCSTRINGS - - Set to ``True`` if Python is built without docstrings (the - :c:macro:`WITH_DOC_STRINGS` macro is not defined). - See the :option:`configure --without-doc-strings <--without-doc-strings>` option. - - See also the :data:`HAVE_DOCSTRINGS` variable. - - -.. data:: HAVE_DOCSTRINGS - - Set to ``True`` if function docstrings are available. - See the :option:`python -OO <-O>` option, which strips docstrings of functions implemented in Python. - - See also the :data:`MISSING_C_DOCSTRINGS` variable. - - -.. data:: TEST_HTTP_URL - - Define the URL of a dedicated HTTP server for the network tests. - - -.. data:: ALWAYS_EQ - - Object that is equal to anything. Used to test mixed type comparison. - - -.. data:: NEVER_EQ - - Object that is not equal to anything (even to :data:`ALWAYS_EQ`). - Used to test mixed type comparison. - - -.. data:: LARGEST - - Object that is greater than anything (except itself). - Used to test mixed type comparison. - - -.. data:: SMALLEST - - Object that is less than anything (except itself). - Used to test mixed type comparison. - - -The :mod:`test.support` module defines the following functions: - -.. function:: busy_retry(timeout, err_msg=None, /, *, error=True) - - Run the loop body until ``break`` stops the loop. - - After *timeout* seconds, raise an :exc:`AssertionError` if *error* is true, - or just stop the loop if *error* is false. - - Example:: - - for _ in support.busy_retry(support.SHORT_TIMEOUT): - if check(): - break - - Example of error=False usage:: - - for _ in support.busy_retry(support.SHORT_TIMEOUT, error=False): - if check(): - break - else: - raise RuntimeError('my custom error') - -.. function:: sleeping_retry(timeout, err_msg=None, /, *, init_delay=0.010, max_delay=1.0, error=True) - - Wait strategy that applies exponential backoff. - - Run the loop body until ``break`` stops the loop. Sleep at each loop - iteration, but not at the first iteration. The sleep delay is doubled at - each iteration (up to *max_delay* seconds). - - See :func:`busy_retry` documentation for the parameters usage. - - Example raising an exception after SHORT_TIMEOUT seconds:: - - for _ in support.sleeping_retry(support.SHORT_TIMEOUT): - if check(): - break - - Example of error=False usage:: - - for _ in support.sleeping_retry(support.SHORT_TIMEOUT, error=False): - if check(): - break - else: - raise RuntimeError('my custom error') - -.. function:: is_resource_enabled(resource) - - Return ``True`` if *resource* is enabled and available. The list of - available resources is only set when :mod:`test.regrtest` is executing the - tests. - - -.. function:: python_is_optimized() - - Return ``True`` if Python was not built with ``-O0`` or ``-Og``. - - -.. function:: with_pymalloc() - - Return :const:`_testcapi.WITH_PYMALLOC`. - - -.. function:: requires(resource, msg=None) - - Raise :exc:`ResourceDenied` if *resource* is not available. *msg* is the - argument to :exc:`ResourceDenied` if it is raised. Always returns - ``True`` if called by a function whose ``__name__`` is ``'__main__'``. - Used when tests are executed by :mod:`test.regrtest`. - - -.. function:: sortdict(dict) - - Return a repr of *dict* with keys sorted. - - -.. function:: findfile(filename, subdir=None) - - Return the path to the file named *filename*. If no match is found - *filename* is returned. This does not equal a failure since it could be the - path to the file. - - Setting *subdir* indicates a relative path to use to find the file - rather than looking directly in the path directories. - - -.. function:: setswitchinterval(interval) - - Set the :func:`sys.setswitchinterval` to the given *interval*. Defines - a minimum interval for Android systems to prevent the system from hanging. - - -.. function:: check_impl_detail(**guards) - - Use this check to guard CPython's implementation-specific tests or to - run them only on the implementations guarded by the arguments. This - function returns ``True`` or ``False`` depending on the host platform. - Example usage:: - - check_impl_detail() # Only on CPython (default). - check_impl_detail(jython=True) # Only on Jython. - check_impl_detail(cpython=False) # Everywhere except CPython. - - -.. function:: set_memlimit(limit) - - Set the values for :data:`max_memuse` and :data:`real_max_memuse` for big - memory tests. - - -.. function:: record_original_stdout(stdout) - - Store the value from *stdout*. It is meant to hold the stdout at the - time the regrtest began. - - -.. function:: get_original_stdout() - - Return the original stdout set by :func:`record_original_stdout` or - ``sys.stdout`` if it's not set. - - -.. function:: args_from_interpreter_flags() - - Return a list of command line arguments reproducing the current settings - in ``sys.flags`` and ``sys.warnoptions``. - - -.. function:: optim_args_from_interpreter_flags() - - Return a list of command line arguments reproducing the current - optimization settings in ``sys.flags``. - - -.. function:: captured_stdin() - captured_stdout() - captured_stderr() - - A context managers that temporarily replaces the named stream with - :class:`io.StringIO` object. - - Example use with output streams:: - - with captured_stdout() as stdout, captured_stderr() as stderr: - print("hello") - print("error", file=sys.stderr) - assert stdout.getvalue() == "hello\n" - assert stderr.getvalue() == "error\n" - - Example use with input stream:: - - with captured_stdin() as stdin: - stdin.write('hello\n') - stdin.seek(0) - # call test code that consumes from sys.stdin - captured = input() - self.assertEqual(captured, "hello") - - -.. function:: disable_faulthandler() - - A context manager that temporary disables :mod:`faulthandler`. - - -.. function:: gc_collect() - - Force as many objects as possible to be collected. This is needed because - timely deallocation is not guaranteed by the garbage collector. This means - that ``__del__`` methods may be called later than expected and weakrefs - may remain alive for longer than expected. - - -.. function:: disable_gc() - - A context manager that disables the garbage collector on entry. On - exit, the garbage collector is restored to its prior state. - - -.. function:: swap_attr(obj, attr, new_val) - - Context manager to swap out an attribute with a new object. - - Usage:: - - with swap_attr(obj, "attr", 5): - ... - - This will set ``obj.attr`` to 5 for the duration of the ``with`` block, - restoring the old value at the end of the block. If ``attr`` doesn't - exist on ``obj``, it will be created and then deleted at the end of the - block. - - The old value (or ``None`` if it doesn't exist) will be assigned to the - target of the "as" clause, if there is one. - - -.. function:: swap_item(obj, attr, new_val) - - Context manager to swap out an item with a new object. - - Usage:: - - with swap_item(obj, "item", 5): - ... - - This will set ``obj["item"]`` to 5 for the duration of the ``with`` block, - restoring the old value at the end of the block. If ``item`` doesn't - exist on ``obj``, it will be created and then deleted at the end of the - block. - - The old value (or ``None`` if it doesn't exist) will be assigned to the - target of the "as" clause, if there is one. - - -.. function:: flush_std_streams() - - Call the ``flush()`` method on :data:`sys.stdout` and then on - :data:`sys.stderr`. It can be used to make sure that the logs order is - consistent before writing into stderr. - - .. versionadded:: 3.11 - - -.. function:: print_warning(msg) - - Print a warning into :data:`sys.__stderr__`. Format the message as: - ``f"Warning -- {msg}"``. If *msg* is made of multiple lines, add - ``"Warning -- "`` prefix to each line. - - .. versionadded:: 3.9 - - -.. function:: wait_process(pid, *, exitcode, timeout=None) - - Wait until process *pid* completes and check that the process exit code is - *exitcode*. - - Raise an :exc:`AssertionError` if the process exit code is not equal to - *exitcode*. - - If the process runs longer than *timeout* seconds (:data:`SHORT_TIMEOUT` by - default), kill the process and raise an :exc:`AssertionError`. The timeout - feature is not available on Windows. - - .. versionadded:: 3.9 - - -.. function:: calcobjsize(fmt) - - Return the size of the :c:type:`PyObject` whose structure members are - defined by *fmt*. The returned value includes the size of the Python object header and alignment. - - -.. function:: calcvobjsize(fmt) - - Return the size of the :c:type:`PyVarObject` whose structure members are - defined by *fmt*. The returned value includes the size of the Python object header and alignment. - - -.. function:: checksizeof(test, o, size) - - For testcase *test*, assert that the ``sys.getsizeof`` for *o* plus the GC - header size equals *size*. - - -.. decorator:: anticipate_failure(condition) - - A decorator to conditionally mark tests with - :func:`unittest.expectedFailure`. Any use of this decorator should - have an associated comment identifying the relevant tracker issue. - - -.. function:: system_must_validate_cert(f) - - A decorator that skips the decorated test on TLS certification validation failures. - - -.. decorator:: run_with_locale(catstr, *locales) - - A decorator for running a function in a different locale, correctly - resetting it after it has finished. *catstr* is the locale category as - a string (for example ``"LC_ALL"``). The *locales* passed will be tried - sequentially, and the first valid locale will be used. - - -.. decorator:: run_with_tz(tz) - - A decorator for running a function in a specific timezone, correctly - resetting it after it has finished. - - -.. decorator:: requires_freebsd_version(*min_version) - - Decorator for the minimum version when running test on FreeBSD. If the - FreeBSD version is less than the minimum, the test is skipped. - - -.. decorator:: requires_linux_version(*min_version) - - Decorator for the minimum version when running test on Linux. If the - Linux version is less than the minimum, the test is skipped. - - -.. decorator:: requires_mac_version(*min_version) - - Decorator for the minimum version when running test on macOS. If the - macOS version is less than the minimum, the test is skipped. - - -.. decorator:: requires_IEEE_754 - - Decorator for skipping tests on non-IEEE 754 platforms. - - -.. decorator:: requires_zlib - - Decorator for skipping tests if :mod:`zlib` doesn't exist. - - -.. decorator:: requires_gzip - - Decorator for skipping tests if :mod:`gzip` doesn't exist. - - -.. decorator:: requires_bz2 - - Decorator for skipping tests if :mod:`bz2` doesn't exist. - - -.. decorator:: requires_lzma - - Decorator for skipping tests if :mod:`lzma` doesn't exist. - - -.. decorator:: requires_resource(resource) - - Decorator for skipping tests if *resource* is not available. - - -.. decorator:: requires_docstrings - - Decorator for only running the test if :data:`HAVE_DOCSTRINGS`. - - -.. decorator:: cpython_only - - Decorator for tests only applicable to CPython. - - -.. decorator:: impl_detail(msg=None, **guards) - - Decorator for invoking :func:`check_impl_detail` on *guards*. If that - returns ``False``, then uses *msg* as the reason for skipping the test. - - -.. decorator:: no_tracing - - Decorator to temporarily turn off tracing for the duration of the test. - - -.. decorator:: refcount_test - - Decorator for tests which involve reference counting. The decorator does - not run the test if it is not run by CPython. Any trace function is unset - for the duration of the test to prevent unexpected refcounts caused by - the trace function. - - -.. decorator:: bigmemtest(size, memuse, dry_run=True) - - Decorator for bigmem tests. - - *size* is a requested size for the test (in arbitrary, test-interpreted - units.) *memuse* is the number of bytes per unit for the test, or a good - estimate of it. For example, a test that needs two byte buffers, of 4 GiB - each, could be decorated with ``@bigmemtest(size=_4G, memuse=2)``. - - The *size* argument is normally passed to the decorated test method as an - extra argument. If *dry_run* is ``True``, the value passed to the test - method may be less than the requested value. If *dry_run* is ``False``, it - means the test doesn't support dummy runs when ``-M`` is not specified. - - -.. decorator:: bigaddrspacetest - - Decorator for tests that fill the address space. - - -.. function:: check_syntax_error(testcase, statement, errtext='', *, lineno=None, offset=None) - - Test for syntax errors in *statement* by attempting to compile *statement*. - *testcase* is the :mod:`unittest` instance for the test. *errtext* is the - regular expression which should match the string representation of the - raised :exc:`SyntaxError`. If *lineno* is not ``None``, compares to - the line of the exception. If *offset* is not ``None``, compares to - the offset of the exception. - - -.. function:: open_urlresource(url, *args, **kw) - - Open *url*. If open fails, raises :exc:`TestFailed`. - - -.. function:: reap_children() - - Use this at the end of ``test_main`` whenever sub-processes are started. - This will help ensure that no extra children (zombies) stick around to - hog resources and create problems when looking for refleaks. - - -.. function:: get_attribute(obj, name) - - Get an attribute, raising :exc:`unittest.SkipTest` if :exc:`AttributeError` - is raised. - - -.. function:: catch_unraisable_exception() - - Context manager catching unraisable exception using - :func:`sys.unraisablehook`. - - Storing the exception value (``cm.unraisable.exc_value``) creates a - reference cycle. The reference cycle is broken explicitly when the context - manager exits. - - Storing the object (``cm.unraisable.object``) can resurrect it if it is set - to an object which is being finalized. Exiting the context manager clears - the stored object. - - Usage:: - - with support.catch_unraisable_exception() as cm: - # code creating an "unraisable exception" - ... - - # check the unraisable exception: use cm.unraisable - ... - - # cm.unraisable attribute no longer exists at this point - # (to break a reference cycle) - - .. versionadded:: 3.8 - - -.. function:: load_package_tests(pkg_dir, loader, standard_tests, pattern) - - Generic implementation of the :mod:`unittest` ``load_tests`` protocol for - use in test packages. *pkg_dir* is the root directory of the package; - *loader*, *standard_tests*, and *pattern* are the arguments expected by - ``load_tests``. In simple cases, the test package's ``__init__.py`` - can be the following:: - - import os - from test.support import load_package_tests - - def load_tests(*args): - return load_package_tests(os.path.dirname(__file__), *args) - - -.. function:: detect_api_mismatch(ref_api, other_api, *, ignore=()) - - Returns the set of attributes, functions or methods of *ref_api* not - found on *other_api*, except for a defined list of items to be - ignored in this check specified in *ignore*. - - By default this skips private attributes beginning with '_' but - includes all magic methods, i.e. those starting and ending in '__'. - - .. versionadded:: 3.5 - - -.. function:: patch(test_instance, object_to_patch, attr_name, new_value) - - Override *object_to_patch.attr_name* with *new_value*. Also add - cleanup procedure to *test_instance* to restore *object_to_patch* for - *attr_name*. The *attr_name* should be a valid attribute for - *object_to_patch*. - - -.. function:: run_in_subinterp(code) - - Run *code* in subinterpreter. Raise :exc:`unittest.SkipTest` if - :mod:`tracemalloc` is enabled. - - -.. function:: check_free_after_iterating(test, iter, cls, args=()) - - Assert instances of *cls* are deallocated after iterating. - - -.. function:: missing_compiler_executable(cmd_names=[]) - - Check for the existence of the compiler executables whose names are listed - in *cmd_names* or all the compiler executables when *cmd_names* is empty - and return the first missing executable or ``None`` when none is found - missing. - - -.. function:: check__all__(test_case, module, name_of_module=None, extra=(), not_exported=()) - - Assert that the ``__all__`` variable of *module* contains all public names. - - The module's public names (its API) are detected automatically - based on whether they match the public name convention and were defined in - *module*. - - The *name_of_module* argument can specify (as a string or tuple thereof) what - module(s) an API could be defined in order to be detected as a public - API. One case for this is when *module* imports part of its public API from - other modules, possibly a C backend (like ``csv`` and its ``_csv``). - - The *extra* argument can be a set of names that wouldn't otherwise be automatically - detected as "public", like objects without a proper ``__module__`` - attribute. If provided, it will be added to the automatically detected ones. - - The *not_exported* argument can be a set of names that must not be treated - as part of the public API even though their names indicate otherwise. - - Example use:: - - import bar - import foo - import unittest - from test import support - - class MiscTestCase(unittest.TestCase): - def test__all__(self): - support.check__all__(self, foo) - - class OtherTestCase(unittest.TestCase): - def test__all__(self): - extra = {'BAR_CONST', 'FOO_CONST'} - not_exported = {'baz'} # Undocumented name. - # bar imports part of its API from _bar. - support.check__all__(self, bar, ('bar', '_bar'), - extra=extra, not_exported=not_exported) - - .. versionadded:: 3.6 - -.. function:: skip_if_broken_multiprocessing_synchronize() - - Skip tests if the :mod:`multiprocessing.synchronize` module is missing, if - there is no available semaphore implementation, or if creating a lock raises - an :exc:`OSError`. - - .. versionadded:: 3.10 - - -.. function:: check_disallow_instantiation(test_case, tp, *args, **kwds) - - Assert that type *tp* cannot be instantiated using *args* and *kwds*. - - .. versionadded:: 3.10 - - -.. function:: adjust_int_max_str_digits(max_digits) - - This function returns a context manager that will change the global - :func:`sys.set_int_max_str_digits` setting for the duration of the - context to allow execution of test code that needs a different limit - on the number of digits when converting between an integer and string. - - .. versionadded:: 3.11 - - -The :mod:`test.support` module defines the following classes: - - -.. class:: SuppressCrashReport() - - A context manager used to try to prevent crash dialog popups on tests that - are expected to crash a subprocess. - - On Windows, it disables Windows Error Reporting dialogs using - `SetErrorMode `_. - - On UNIX, :func:`resource.setrlimit` is used to set - :const:`resource.RLIMIT_CORE`'s soft limit to 0 to prevent coredump file - creation. - - On both platforms, the old value is restored by :meth:`~object.__exit__`. - - -.. class:: SaveSignals() - - Class to save and restore signal handlers registered by the Python signal - handler. - - .. method:: save(self) - - Save the signal handlers to a dictionary mapping signal numbers to the - current signal handler. - - .. method:: restore(self) - - Set the signal numbers from the :meth:`save` dictionary to the saved - handler. - - -.. class:: Matcher() - - .. method:: matches(self, d, **kwargs) - - Try to match a single dict with the supplied arguments. - - - .. method:: match_value(self, k, dv, v) - - Try to match a single stored value (*dv*) with a supplied value (*v*). - - -:mod:`test.support.socket_helper` --- Utilities for socket tests -================================================================ - -.. module:: test.support.socket_helper - :synopsis: Support for socket tests. - - -The :mod:`test.support.socket_helper` module provides support for socket tests. - -.. versionadded:: 3.9 - - -.. data:: IPV6_ENABLED - - Set to ``True`` if IPv6 is enabled on this host, ``False`` otherwise. - - -.. function:: find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM) - - Returns an unused port that should be suitable for binding. This is - achieved by creating a temporary socket with the same family and type as - the ``sock`` parameter (default is :const:`~socket.AF_INET`, - :const:`~socket.SOCK_STREAM`), - and binding it to the specified host address (defaults to ``0.0.0.0``) - with the port set to 0, eliciting an unused ephemeral port from the OS. - The temporary socket is then closed and deleted, and the ephemeral port is - returned. - - Either this method or :func:`bind_port` should be used for any tests - where a server socket needs to be bound to a particular port for the - duration of the test. - Which one to use depends on whether the calling code is creating a Python - socket, or if an unused port needs to be provided in a constructor - or passed to an external program (i.e. the ``-accept`` argument to - openssl's s_server mode). Always prefer :func:`bind_port` over - :func:`find_unused_port` where possible. Using a hard coded port is - discouraged since it can make multiple instances of the test impossible to - run simultaneously, which is a problem for buildbots. - - -.. function:: bind_port(sock, host=HOST) - - Bind the socket to a free port and return the port number. Relies on - ephemeral ports in order to ensure we are using an unbound port. This is - important as many tests may be running simultaneously, especially in a - buildbot environment. This method raises an exception if the - ``sock.family`` is :const:`~socket.AF_INET` and ``sock.type`` is - :const:`~socket.SOCK_STREAM`, and the socket has - :const:`~socket.SO_REUSEADDR` or :const:`~socket.SO_REUSEPORT` set on it. - Tests should never set these socket options for TCP/IP sockets. - The only case for setting these options is testing multicasting via - multiple UDP sockets. - - Additionally, if the :const:`~socket.SO_EXCLUSIVEADDRUSE` socket option is - available (i.e. on Windows), it will be set on the socket. This will - prevent anyone else from binding to our host/port for the duration of the - test. - - -.. function:: bind_unix_socket(sock, addr) - - Bind a Unix socket, raising :exc:`unittest.SkipTest` if - :exc:`PermissionError` is raised. - - -.. decorator:: skip_unless_bind_unix_socket - - A decorator for running tests that require a functional ``bind()`` for Unix - sockets. - - -.. function:: transient_internet(resource_name, *, timeout=30.0, errnos=()) - - A context manager that raises :exc:`~test.support.ResourceDenied` when - various issues with the internet connection manifest themselves as - exceptions. - - -:mod:`test.support.script_helper` --- Utilities for the Python execution tests -============================================================================== - -.. module:: test.support.script_helper - :synopsis: Support for Python's script execution tests. - - -The :mod:`test.support.script_helper` module provides support for Python's -script execution tests. - -.. function:: interpreter_requires_environment() - - Return ``True`` if ``sys.executable interpreter`` requires environment - variables in order to be able to run at all. - - This is designed to be used with ``@unittest.skipIf()`` to annotate tests - that need to use an ``assert_python*()`` function to launch an isolated - mode (``-I``) or no environment mode (``-E``) sub-interpreter process. - - A normal build & test does not run into this situation but it can happen - when trying to run the standard library test suite from an interpreter that - doesn't have an obvious home with Python's current home finding logic. - - Setting :envvar:`PYTHONHOME` is one way to get most of the testsuite to run - in that situation. :envvar:`PYTHONPATH` or :envvar:`PYTHONUSERSITE` are - other common environment variables that might impact whether or not the - interpreter can start. - - -.. function:: run_python_until_end(*args, **env_vars) - - Set up the environment based on *env_vars* for running the interpreter - in a subprocess. The values can include ``__isolated``, ``__cleanenv``, - ``__cwd``, and ``TERM``. - - .. versionchanged:: 3.9 - The function no longer strips whitespaces from *stderr*. - - -.. function:: assert_python_ok(*args, **env_vars) - - Assert that running the interpreter with *args* and optional environment - variables *env_vars* succeeds (``rc == 0``) and return a ``(return code, - stdout, stderr)`` tuple. - - If the *__cleanenv* keyword-only parameter is set, *env_vars* is used as a fresh - environment. - - Python is started in isolated mode (command line option ``-I``), - except if the *__isolated* keyword-only parameter is set to ``False``. - - .. versionchanged:: 3.9 - The function no longer strips whitespaces from *stderr*. - - -.. function:: assert_python_failure(*args, **env_vars) - - Assert that running the interpreter with *args* and optional environment - variables *env_vars* fails (``rc != 0``) and return a ``(return code, - stdout, stderr)`` tuple. - - See :func:`assert_python_ok` for more options. - - .. versionchanged:: 3.9 - The function no longer strips whitespaces from *stderr*. - - -.. function:: spawn_python(*args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw) - - Run a Python subprocess with the given arguments. - - *kw* is extra keyword args to pass to :func:`subprocess.Popen`. Returns a - :class:`subprocess.Popen` object. - - -.. function:: kill_python(p) - - Run the given :class:`subprocess.Popen` process until completion and return - stdout. - - -.. function:: make_script(script_dir, script_basename, source, omit_suffix=False) - - Create script containing *source* in path *script_dir* and *script_basename*. - If *omit_suffix* is ``False``, append ``.py`` to the name. Return the full - script path. - - -.. function:: make_zip_script(zip_dir, zip_basename, script_name, name_in_zip=None) - - Create zip file at *zip_dir* and *zip_basename* with extension ``zip`` which - contains the files in *script_name*. *name_in_zip* is the archive name. - Return a tuple containing ``(full path, full path of archive name)``. - - -.. function:: make_pkg(pkg_dir, init_source='') - - Create a directory named *pkg_dir* containing an ``__init__`` file with - *init_source* as its contents. - - -.. function:: make_zip_pkg(zip_dir, zip_basename, pkg_name, script_basename, \ - source, depth=1, compiled=False) - - Create a zip package directory with a path of *zip_dir* and *zip_basename* - containing an empty ``__init__`` file and a file *script_basename* - containing the *source*. If *compiled* is ``True``, both source files will - be compiled and added to the zip package. Return a tuple of the full zip - path and the archive name for the zip file. - - -:mod:`test.support.bytecode_helper` --- Support tools for testing correct bytecode generation -============================================================================================= - -.. module:: test.support.bytecode_helper - :synopsis: Support tools for testing correct bytecode generation. - -The :mod:`test.support.bytecode_helper` module provides support for testing -and inspecting bytecode generation. - -.. versionadded:: 3.9 - -The module defines the following class: - -.. class:: BytecodeTestCase(unittest.TestCase) - - This class has custom assertion methods for inspecting bytecode. - -.. method:: BytecodeTestCase.get_disassembly_as_string(co) - - Return the disassembly of *co* as string. - - -.. method:: BytecodeTestCase.assertInBytecode(x, opname, argval=_UNSPECIFIED) - - Return instr if *opname* is found, otherwise throws :exc:`AssertionError`. - - -.. method:: BytecodeTestCase.assertNotInBytecode(x, opname, argval=_UNSPECIFIED) - - Throws :exc:`AssertionError` if *opname* is found. - - -:mod:`test.support.threading_helper` --- Utilities for threading tests -====================================================================== - -.. module:: test.support.threading_helper - :synopsis: Support for threading tests. - -The :mod:`test.support.threading_helper` module provides support for threading tests. - -.. versionadded:: 3.10 - - -.. function:: join_thread(thread, timeout=None) - - Join a *thread* within *timeout*. Raise an :exc:`AssertionError` if thread - is still alive after *timeout* seconds. - - -.. decorator:: reap_threads - - Decorator to ensure the threads are cleaned up even if the test fails. - - -.. function:: start_threads(threads, unlock=None) - - Context manager to start *threads*, which is a sequence of threads. - *unlock* is a function called after the threads are started, even if an - exception was raised; an example would be :meth:`threading.Event.set`. - ``start_threads`` will attempt to join the started threads upon exit. - - -.. function:: threading_cleanup(*original_values) - - Cleanup up threads not specified in *original_values*. Designed to emit - a warning if a test leaves running threads in the background. - - -.. function:: threading_setup() - - Return current thread count and copy of dangling threads. - - -.. function:: wait_threads_exit(timeout=None) - - Context manager to wait until all threads created in the ``with`` statement - exit. - - -.. function:: catch_threading_exception() - - Context manager catching :class:`threading.Thread` exception using - :func:`threading.excepthook`. - - Attributes set when an exception is caught: - - * ``exc_type`` - * ``exc_value`` - * ``exc_traceback`` - * ``thread`` - - See :func:`threading.excepthook` documentation. - - These attributes are deleted at the context manager exit. - - Usage:: - - with threading_helper.catch_threading_exception() as cm: - # code spawning a thread which raises an exception - ... - - # check the thread exception, use cm attributes: - # exc_type, exc_value, exc_traceback, thread - ... - - # exc_type, exc_value, exc_traceback, thread attributes of cm no longer - # exists at this point - # (to avoid reference cycles) - - .. versionadded:: 3.8 - - -:mod:`test.support.os_helper` --- Utilities for os tests -======================================================================== - -.. module:: test.support.os_helper - :synopsis: Support for os tests. - -The :mod:`test.support.os_helper` module provides support for os tests. - -.. versionadded:: 3.10 - - -.. data:: FS_NONASCII - - A non-ASCII character encodable by :func:`os.fsencode`. - - -.. data:: SAVEDCWD - - Set to :func:`os.getcwd`. - - -.. data:: TESTFN - - Set to a name that is safe to use as the name of a temporary file. Any - temporary file that is created should be closed and unlinked (removed). - - -.. data:: TESTFN_NONASCII - - Set to a filename containing the :data:`FS_NONASCII` character, if it exists. - This guarantees that if the filename exists, it can be encoded and decoded - with the default filesystem encoding. This allows tests that require a - non-ASCII filename to be easily skipped on platforms where they can't work. - - -.. data:: TESTFN_UNENCODABLE - - Set to a filename (str type) that should not be able to be encoded by file - system encoding in strict mode. It may be ``None`` if it's not possible to - generate such a filename. - - -.. data:: TESTFN_UNDECODABLE - - Set to a filename (bytes type) that should not be able to be decoded by - file system encoding in strict mode. It may be ``None`` if it's not - possible to generate such a filename. - - -.. data:: TESTFN_UNICODE - - Set to a non-ASCII name for a temporary file. - - -.. class:: EnvironmentVarGuard() - - Class used to temporarily set or unset environment variables. Instances can - be used as a context manager and have a complete dictionary interface for - querying/modifying the underlying ``os.environ``. After exit from the - context manager all changes to environment variables done through this - instance will be rolled back. - - .. versionchanged:: 3.1 - Added dictionary interface. - - -.. class:: FakePath(path) - - Simple :term:`path-like object`. It implements the :meth:`__fspath__` - method which just returns the *path* argument. If *path* is an exception, - it will be raised in :meth:`!__fspath__`. - - -.. method:: EnvironmentVarGuard.set(envvar, value) - - Temporarily set the environment variable ``envvar`` to the value of - ``value``. - - -.. method:: EnvironmentVarGuard.unset(envvar) - - Temporarily unset the environment variable ``envvar``. - - -.. function:: can_symlink() - - Return ``True`` if the OS supports symbolic links, ``False`` - otherwise. - - -.. function:: can_xattr() - - Return ``True`` if the OS supports xattr, ``False`` - otherwise. - - -.. function:: change_cwd(path, quiet=False) - - A context manager that temporarily changes the current working - directory to *path* and yields the directory. - - If *quiet* is ``False``, the context manager raises an exception - on error. Otherwise, it issues only a warning and keeps the current - working directory the same. - - -.. function:: create_empty_file(filename) - - Create an empty file with *filename*. If it already exists, truncate it. - - -.. function:: fd_count() - - Count the number of open file descriptors. - - -.. function:: fs_is_case_insensitive(directory) - - Return ``True`` if the file system for *directory* is case-insensitive. - - -.. function:: make_bad_fd() - - Create an invalid file descriptor by opening and closing a temporary file, - and returning its descriptor. - - -.. function:: rmdir(filename) - - Call :func:`os.rmdir` on *filename*. On Windows platforms, this is - wrapped with a wait loop that checks for the existence of the file, - which is needed due to antivirus programs that can hold files open and prevent - deletion. - - -.. function:: rmtree(path) - - Call :func:`shutil.rmtree` on *path* or call :func:`os.lstat` and - :func:`os.rmdir` to remove a path and its contents. As with :func:`rmdir`, - on Windows platforms - this is wrapped with a wait loop that checks for the existence of the files. - - -.. decorator:: skip_unless_symlink - - A decorator for running tests that require support for symbolic links. - - -.. decorator:: skip_unless_xattr - - A decorator for running tests that require support for xattr. - - -.. function:: temp_cwd(name='tempcwd', quiet=False) - - A context manager that temporarily creates a new directory and - changes the current working directory (CWD). - - The context manager creates a temporary directory in the current - directory with name *name* before temporarily changing the current - working directory. If *name* is ``None``, the temporary directory is - created using :func:`tempfile.mkdtemp`. - - If *quiet* is ``False`` and it is not possible to create or change - the CWD, an error is raised. Otherwise, only a warning is raised - and the original CWD is used. - - -.. function:: temp_dir(path=None, quiet=False) - - A context manager that creates a temporary directory at *path* and - yields the directory. - - If *path* is ``None``, the temporary directory is created using - :func:`tempfile.mkdtemp`. If *quiet* is ``False``, the context manager - raises an exception on error. Otherwise, if *path* is specified and - cannot be created, only a warning is issued. - - -.. function:: temp_umask(umask) - - A context manager that temporarily sets the process umask. - - -.. function:: unlink(filename) - - Call :func:`os.unlink` on *filename*. As with :func:`rmdir`, - on Windows platforms, this is - wrapped with a wait loop that checks for the existence of the file. - - -:mod:`test.support.import_helper` --- Utilities for import tests -================================================================ - -.. module:: test.support.import_helper - :synopsis: Support for import tests. - -The :mod:`test.support.import_helper` module provides support for import tests. - -.. versionadded:: 3.10 - - -.. function:: forget(module_name) - - Remove the module named *module_name* from ``sys.modules`` and delete any - byte-compiled files of the module. - - -.. function:: import_fresh_module(name, fresh=(), blocked=(), deprecated=False) - - This function imports and returns a fresh copy of the named Python module - by removing the named module from ``sys.modules`` before doing the import. - Note that unlike :func:`reload`, the original module is not affected by - this operation. - - *fresh* is an iterable of additional module names that are also removed - from the ``sys.modules`` cache before doing the import. - - *blocked* is an iterable of module names that are replaced with ``None`` - in the module cache during the import to ensure that attempts to import - them raise :exc:`ImportError`. - - The named module and any modules named in the *fresh* and *blocked* - parameters are saved before starting the import and then reinserted into - ``sys.modules`` when the fresh import is complete. - - Module and package deprecation messages are suppressed during this import - if *deprecated* is ``True``. - - This function will raise :exc:`ImportError` if the named module cannot be - imported. - - Example use:: - - # Get copies of the warnings module for testing without affecting the - # version being used by the rest of the test suite. One copy uses the - # C implementation, the other is forced to use the pure Python fallback - # implementation - py_warnings = import_fresh_module('warnings', blocked=['_warnings']) - c_warnings = import_fresh_module('warnings', fresh=['_warnings']) - - .. versionadded:: 3.1 - - -.. function:: import_module(name, deprecated=False, *, required_on=()) - - This function imports and returns the named module. Unlike a normal - import, this function raises :exc:`unittest.SkipTest` if the module - cannot be imported. - - Module and package deprecation messages are suppressed during this import - if *deprecated* is ``True``. If a module is required on a platform but - optional for others, set *required_on* to an iterable of platform prefixes - which will be compared against :data:`sys.platform`. - - .. versionadded:: 3.1 - - -.. function:: modules_setup() - - Return a copy of :data:`sys.modules`. - - -.. function:: modules_cleanup(oldmodules) - - Remove modules except for *oldmodules* and ``encodings`` in order to - preserve internal cache. - - -.. function:: unload(name) - - Delete *name* from ``sys.modules``. - - -.. function:: make_legacy_pyc(source) - - Move a :pep:`3147`/:pep:`488` pyc file to its legacy pyc location and return the file - system path to the legacy pyc file. The *source* value is the file system - path to the source file. It does not need to exist, however the PEP - 3147/488 pyc file must exist. - - -.. class:: CleanImport(*module_names) - - A context manager to force import to return a new module reference. This - is useful for testing module-level behaviors, such as the emission of a - :exc:`DeprecationWarning` on import. Example usage:: - - with CleanImport('foo'): - importlib.import_module('foo') # New reference. - - -.. class:: DirsOnSysPath(*paths) - - A context manager to temporarily add directories to :data:`sys.path`. - - This makes a copy of :data:`sys.path`, appends any directories given - as positional arguments, then reverts :data:`sys.path` to the copied - settings when the context ends. - - Note that *all* :data:`sys.path` modifications in the body of the - context manager, including replacement of the object, - will be reverted at the end of the block. - - -:mod:`test.support.warnings_helper` --- Utilities for warnings tests -==================================================================== - -.. module:: test.support.warnings_helper - :synopsis: Support for warnings tests. - -The :mod:`test.support.warnings_helper` module provides support for warnings tests. - -.. versionadded:: 3.10 - - -.. function:: ignore_warnings(*, category) - - Suppress warnings that are instances of *category*, - which must be :exc:`Warning` or a subclass. - Roughly equivalent to :func:`warnings.catch_warnings` - with :meth:`warnings.simplefilter('ignore', category=category) `. - For example:: - - @warning_helper.ignore_warnings(category=DeprecationWarning) - def test_suppress_warning(): - # do something - - .. versionadded:: 3.8 - - -.. function:: check_no_resource_warning(testcase) - - Context manager to check that no :exc:`ResourceWarning` was raised. You - must remove the object which may emit :exc:`ResourceWarning` before the - end of the context manager. - - -.. function:: check_syntax_warning(testcase, statement, errtext='', *, lineno=1, offset=None) - - Test for syntax warning in *statement* by attempting to compile *statement*. - Test also that the :exc:`SyntaxWarning` is emitted only once, and that it - will be converted to a :exc:`SyntaxError` when turned into error. - *testcase* is the :mod:`unittest` instance for the test. *errtext* is the - regular expression which should match the string representation of the - emitted :exc:`SyntaxWarning` and raised :exc:`SyntaxError`. If *lineno* - is not ``None``, compares to the line of the warning and exception. - If *offset* is not ``None``, compares to the offset of the exception. - - .. versionadded:: 3.8 - - -.. function:: check_warnings(*filters, quiet=True) - - A convenience wrapper for :func:`warnings.catch_warnings()` that makes it - easier to test that a warning was correctly raised. It is approximately - equivalent to calling ``warnings.catch_warnings(record=True)`` with - :meth:`warnings.simplefilter` set to ``always`` and with the option to - automatically validate the results that are recorded. - - ``check_warnings`` accepts 2-tuples of the form ``("message regexp", - WarningCategory)`` as positional arguments. If one or more *filters* are - provided, or if the optional keyword argument *quiet* is ``False``, - it checks to make sure the warnings are as expected: each specified filter - must match at least one of the warnings raised by the enclosed code or the - test fails, and if any warnings are raised that do not match any of the - specified filters the test fails. To disable the first of these checks, - set *quiet* to ``True``. - - If no arguments are specified, it defaults to:: - - check_warnings(("", Warning), quiet=True) - - In this case all warnings are caught and no errors are raised. - - On entry to the context manager, a :class:`WarningRecorder` instance is - returned. The underlying warnings list from - :func:`~warnings.catch_warnings` is available via the recorder object's - :attr:`warnings` attribute. As a convenience, the attributes of the object - representing the most recent warning can also be accessed directly through - the recorder object (see example below). If no warning has been raised, - then any of the attributes that would otherwise be expected on an object - representing a warning will return ``None``. - - The recorder object also has a :meth:`reset` method, which clears the - warnings list. - - The context manager is designed to be used like this:: - - with check_warnings(("assertion is always true", SyntaxWarning), - ("", UserWarning)): - exec('assert(False, "Hey!")') - warnings.warn(UserWarning("Hide me!")) - - In this case if either warning was not raised, or some other warning was - raised, :func:`check_warnings` would raise an error. - - When a test needs to look more deeply into the warnings, rather than - just checking whether or not they occurred, code like this can be used:: - - with check_warnings(quiet=True) as w: - warnings.warn("foo") - assert str(w.args[0]) == "foo" - warnings.warn("bar") - assert str(w.args[0]) == "bar" - assert str(w.warnings[0].args[0]) == "foo" - assert str(w.warnings[1].args[0]) == "bar" - w.reset() - assert len(w.warnings) == 0 - - - Here all warnings will be caught, and the test code tests the captured - warnings directly. - - .. versionchanged:: 3.2 - New optional arguments *filters* and *quiet*. - - -.. class:: WarningsRecorder() - - Class used to record warnings for unit tests. See documentation of - :func:`check_warnings` above for more details. diff --git a/Doc/library/text.rst b/Doc/library/text.rst deleted file mode 100644 index 47b678434fc899..00000000000000 --- a/Doc/library/text.rst +++ /dev/null @@ -1,26 +0,0 @@ -.. _stringservices: -.. _textservices: - -************************ -Text Processing Services -************************ - -The modules described in this chapter provide a wide range of string -manipulation operations and other text processing services. - -The :mod:`codecs` module described under :ref:`binaryservices` is also -highly relevant to text processing. In addition, see the documentation for -Python's built-in string type in :ref:`textseq`. - - -.. toctree:: - - string.rst - re.rst - difflib.rst - textwrap.rst - unicodedata.rst - stringprep.rst - readline.rst - rlcompleter.rst - diff --git a/Doc/library/textwrap.rst b/Doc/library/textwrap.rst deleted file mode 100644 index 7445410f91808c..00000000000000 --- a/Doc/library/textwrap.rst +++ /dev/null @@ -1,310 +0,0 @@ -:mod:`textwrap` --- Text wrapping and filling -============================================= - -.. module:: textwrap - :synopsis: Text wrapping and filling - -.. moduleauthor:: Greg Ward -.. sectionauthor:: Greg Ward - -**Source code:** :source:`Lib/textwrap.py` - --------------- - -The :mod:`textwrap` module provides some convenience functions, -as well as :class:`TextWrapper`, the class that does all the work. -If you're just wrapping or filling one or two text strings, the convenience -functions should be good enough; otherwise, you should use an instance of -:class:`TextWrapper` for efficiency. - -.. function:: wrap(text, width=70, *, initial_indent="", \ - subsequent_indent="", expand_tabs=True, \ - replace_whitespace=True, fix_sentence_endings=False, \ - break_long_words=True, drop_whitespace=True, \ - break_on_hyphens=True, tabsize=8, max_lines=None, \ - placeholder=' [...]') - - Wraps the single paragraph in *text* (a string) so every line is at most - *width* characters long. Returns a list of output lines, without final - newlines. - - Optional keyword arguments correspond to the instance attributes of - :class:`TextWrapper`, documented below. - - See the :meth:`TextWrapper.wrap` method for additional details on how - :func:`wrap` behaves. - - -.. function:: fill(text, width=70, *, initial_indent="", \ - subsequent_indent="", expand_tabs=True, \ - replace_whitespace=True, fix_sentence_endings=False, \ - break_long_words=True, drop_whitespace=True, \ - break_on_hyphens=True, tabsize=8, \ - max_lines=None, placeholder=' [...]') - - Wraps the single paragraph in *text*, and returns a single string containing the - wrapped paragraph. :func:`fill` is shorthand for :: - - "\n".join(wrap(text, ...)) - - In particular, :func:`fill` accepts exactly the same keyword arguments as - :func:`wrap`. - - -.. function:: shorten(text, width, *, fix_sentence_endings=False, \ - break_long_words=True, break_on_hyphens=True, \ - placeholder=' [...]') - - Collapse and truncate the given *text* to fit in the given *width*. - - First the whitespace in *text* is collapsed (all whitespace is replaced by - single spaces). If the result fits in the *width*, it is returned. - Otherwise, enough words are dropped from the end so that the remaining words - plus the *placeholder* fit within *width*:: - - >>> textwrap.shorten("Hello world!", width=12) - 'Hello world!' - >>> textwrap.shorten("Hello world!", width=11) - 'Hello [...]' - >>> textwrap.shorten("Hello world", width=10, placeholder="...") - 'Hello...' - - Optional keyword arguments correspond to the instance attributes of - :class:`TextWrapper`, documented below. Note that the whitespace is - collapsed before the text is passed to the :class:`TextWrapper` :meth:`fill` - function, so changing the value of :attr:`.tabsize`, :attr:`.expand_tabs`, - :attr:`.drop_whitespace`, and :attr:`.replace_whitespace` will have no effect. - - .. versionadded:: 3.4 - -.. function:: dedent(text) - - Remove any common leading whitespace from every line in *text*. - - This can be used to make triple-quoted strings line up with the left edge of the - display, while still presenting them in the source code in indented form. - - Note that tabs and spaces are both treated as whitespace, but they are not - equal: the lines ``" hello"`` and ``"\thello"`` are considered to have no - common leading whitespace. - - Lines containing only whitespace are ignored in the input and normalized to a - single newline character in the output. - - For example:: - - def test(): - # end first line with \ to avoid the empty line! - s = '''\ - hello - world - ''' - print(repr(s)) # prints ' hello\n world\n ' - print(repr(dedent(s))) # prints 'hello\n world\n' - - -.. function:: indent(text, prefix, predicate=None) - - Add *prefix* to the beginning of selected lines in *text*. - - Lines are separated by calling ``text.splitlines(True)``. - - By default, *prefix* is added to all lines that do not consist - solely of whitespace (including any line endings). - - For example:: - - >>> s = 'hello\n\n \nworld' - >>> indent(s, ' ') - ' hello\n\n \n world' - - The optional *predicate* argument can be used to control which lines - are indented. For example, it is easy to add *prefix* to even empty - and whitespace-only lines:: - - >>> print(indent(s, '+ ', lambda line: True)) - + hello - + - + - + world - - .. versionadded:: 3.3 - - -:func:`wrap`, :func:`fill` and :func:`shorten` work by creating a -:class:`TextWrapper` instance and calling a single method on it. That -instance is not reused, so for applications that process many text -strings using :func:`wrap` and/or :func:`fill`, it may be more efficient to -create your own :class:`TextWrapper` object. - -Text is preferably wrapped on whitespaces and right after the hyphens in -hyphenated words; only then will long words be broken if necessary, unless -:attr:`TextWrapper.break_long_words` is set to false. - -.. class:: TextWrapper(**kwargs) - - The :class:`TextWrapper` constructor accepts a number of optional keyword - arguments. Each keyword argument corresponds to an instance attribute, so - for example :: - - wrapper = TextWrapper(initial_indent="* ") - - is the same as :: - - wrapper = TextWrapper() - wrapper.initial_indent = "* " - - You can re-use the same :class:`TextWrapper` object many times, and you can - change any of its options through direct assignment to instance attributes - between uses. - - The :class:`TextWrapper` instance attributes (and keyword arguments to the - constructor) are as follows: - - - .. attribute:: width - - (default: ``70``) The maximum length of wrapped lines. As long as there - are no individual words in the input text longer than :attr:`width`, - :class:`TextWrapper` guarantees that no output line will be longer than - :attr:`width` characters. - - - .. attribute:: expand_tabs - - (default: ``True``) If true, then all tab characters in *text* will be - expanded to spaces using the :meth:`~str.expandtabs` method of *text*. - - - .. attribute:: tabsize - - (default: ``8``) If :attr:`expand_tabs` is true, then all tab characters - in *text* will be expanded to zero or more spaces, depending on the - current column and the given tab size. - - .. versionadded:: 3.3 - - - .. attribute:: replace_whitespace - - (default: ``True``) If true, after tab expansion but before wrapping, - the :meth:`wrap` method will replace each whitespace character - with a single space. The whitespace characters replaced are - as follows: tab, newline, vertical tab, formfeed, and carriage - return (``'\t\n\v\f\r'``). - - .. note:: - - If :attr:`expand_tabs` is false and :attr:`replace_whitespace` is true, - each tab character will be replaced by a single space, which is *not* - the same as tab expansion. - - .. note:: - - If :attr:`replace_whitespace` is false, newlines may appear in the - middle of a line and cause strange output. For this reason, text should - be split into paragraphs (using :meth:`str.splitlines` or similar) - which are wrapped separately. - - - .. attribute:: drop_whitespace - - (default: ``True``) If true, whitespace at the beginning and ending of - every line (after wrapping but before indenting) is dropped. - Whitespace at the beginning of the paragraph, however, is not dropped - if non-whitespace follows it. If whitespace being dropped takes up an - entire line, the whole line is dropped. - - - .. attribute:: initial_indent - - (default: ``''``) String that will be prepended to the first line of - wrapped output. Counts towards the length of the first line. The empty - string is not indented. - - - .. attribute:: subsequent_indent - - (default: ``''``) String that will be prepended to all lines of wrapped - output except the first. Counts towards the length of each line except - the first. - - - .. attribute:: fix_sentence_endings - - (default: ``False``) If true, :class:`TextWrapper` attempts to detect - sentence endings and ensure that sentences are always separated by exactly - two spaces. This is generally desired for text in a monospaced font. - However, the sentence detection algorithm is imperfect: it assumes that a - sentence ending consists of a lowercase letter followed by one of ``'.'``, - ``'!'``, or ``'?'``, possibly followed by one of ``'"'`` or ``"'"``, - followed by a space. One problem with this algorithm is that it is - unable to detect the difference between "Dr." in :: - - [...] Dr. Frankenstein's monster [...] - - and "Spot." in :: - - [...] See Spot. See Spot run [...] - - :attr:`fix_sentence_endings` is false by default. - - Since the sentence detection algorithm relies on ``string.lowercase`` for - the definition of "lowercase letter", and a convention of using two spaces - after a period to separate sentences on the same line, it is specific to - English-language texts. - - - .. attribute:: break_long_words - - (default: ``True``) If true, then words longer than :attr:`width` will be - broken in order to ensure that no lines are longer than :attr:`width`. If - it is false, long words will not be broken, and some lines may be longer - than :attr:`width`. (Long words will be put on a line by themselves, in - order to minimize the amount by which :attr:`width` is exceeded.) - - - .. attribute:: break_on_hyphens - - (default: ``True``) If true, wrapping will occur preferably on whitespaces - and right after hyphens in compound words, as it is customary in English. - If false, only whitespaces will be considered as potentially good places - for line breaks, but you need to set :attr:`break_long_words` to false if - you want truly insecable words. Default behaviour in previous versions - was to always allow breaking hyphenated words. - - - .. attribute:: max_lines - - (default: ``None``) If not ``None``, then the output will contain at most - *max_lines* lines, with *placeholder* appearing at the end of the output. - - .. versionadded:: 3.4 - - - .. index:: single: ...; placeholder - - .. attribute:: placeholder - - (default: ``' [...]'``) String that will appear at the end of the output - text if it has been truncated. - - .. versionadded:: 3.4 - - - :class:`TextWrapper` also provides some public methods, analogous to the - module-level convenience functions: - - .. method:: wrap(text) - - Wraps the single paragraph in *text* (a string) so every line is at most - :attr:`width` characters long. All wrapping options are taken from - instance attributes of the :class:`TextWrapper` instance. Returns a list - of output lines, without final newlines. If the wrapped output has no - content, the returned list is empty. - - - .. method:: fill(text) - - Wraps the single paragraph in *text*, and returns a single string - containing the wrapped paragraph. diff --git a/Doc/library/threading.rst b/Doc/library/threading.rst deleted file mode 100644 index d2c896eacd2e8c..00000000000000 --- a/Doc/library/threading.rst +++ /dev/null @@ -1,1151 +0,0 @@ -:mod:`threading` --- Thread-based parallelism -============================================= - -.. module:: threading - :synopsis: Thread-based parallelism. - -**Source code:** :source:`Lib/threading.py` - --------------- - -This module constructs higher-level threading interfaces on top of the lower -level :mod:`_thread` module. - -.. versionchanged:: 3.7 - This module used to be optional, it is now always available. - -.. seealso:: - - :class:`concurrent.futures.ThreadPoolExecutor` offers a higher level interface - to push tasks to a background thread without blocking execution of the - calling thread, while still being able to retrieve their results when needed. - - :mod:`queue` provides a thread-safe interface for exchanging data between - running threads. - - :mod:`asyncio` offers an alternative approach to achieving task level - concurrency without requiring the use of multiple operating system threads. - -.. note:: - - In the Python 2.x series, this module contained ``camelCase`` names - for some methods and functions. These are deprecated as of Python 3.10, - but they are still supported for compatibility with Python 2.5 and lower. - - -.. impl-detail:: - - In CPython, due to the :term:`Global Interpreter Lock - `, only one thread - can execute Python code at once (even though certain performance-oriented - libraries might overcome this limitation). - If you want your application to make better use of the computational - resources of multi-core machines, you are advised to use - :mod:`multiprocessing` or :class:`concurrent.futures.ProcessPoolExecutor`. - However, threading is still an appropriate model if you want to run - multiple I/O-bound tasks simultaneously. - -.. include:: ../includes/wasm-notavail.rst - -This module defines the following functions: - - -.. function:: active_count() - - Return the number of :class:`Thread` objects currently alive. The returned - count is equal to the length of the list returned by :func:`.enumerate`. - - The function ``activeCount`` is a deprecated alias for this function. - - -.. function:: current_thread() - - Return the current :class:`Thread` object, corresponding to the caller's thread - of control. If the caller's thread of control was not created through the - :mod:`threading` module, a dummy thread object with limited functionality is - returned. - - The function ``currentThread`` is a deprecated alias for this function. - - -.. function:: excepthook(args, /) - - Handle uncaught exception raised by :func:`Thread.run`. - - The *args* argument has the following attributes: - - * *exc_type*: Exception type. - * *exc_value*: Exception value, can be ``None``. - * *exc_traceback*: Exception traceback, can be ``None``. - * *thread*: Thread which raised the exception, can be ``None``. - - If *exc_type* is :exc:`SystemExit`, the exception is silently ignored. - Otherwise, the exception is printed out on :data:`sys.stderr`. - - If this function raises an exception, :func:`sys.excepthook` is called to - handle it. - - :func:`threading.excepthook` can be overridden to control how uncaught - exceptions raised by :func:`Thread.run` are handled. - - Storing *exc_value* using a custom hook can create a reference cycle. It - should be cleared explicitly to break the reference cycle when the - exception is no longer needed. - - Storing *thread* using a custom hook can resurrect it if it is set to an - object which is being finalized. Avoid storing *thread* after the custom - hook completes to avoid resurrecting objects. - - .. seealso:: - :func:`sys.excepthook` handles uncaught exceptions. - - .. versionadded:: 3.8 - -.. data:: __excepthook__ - - Holds the original value of :func:`threading.excepthook`. It is saved so that the - original value can be restored in case they happen to get replaced with - broken or alternative objects. - - .. versionadded:: 3.10 - -.. function:: get_ident() - - Return the 'thread identifier' of the current thread. This is a nonzero - integer. Its value has no direct meaning; it is intended as a magic cookie - to be used e.g. to index a dictionary of thread-specific data. Thread - identifiers may be recycled when a thread exits and another thread is - created. - - .. versionadded:: 3.3 - - -.. function:: get_native_id() - - Return the native integral Thread ID of the current thread assigned by the kernel. - This is a non-negative integer. - Its value may be used to uniquely identify this particular thread system-wide - (until the thread terminates, after which the value may be recycled by the OS). - - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX. - - .. versionadded:: 3.8 - - -.. function:: enumerate() - - Return a list of all :class:`Thread` objects currently active. The list - includes daemonic threads and dummy thread objects created by - :func:`current_thread`. It excludes terminated threads and threads - that have not yet been started. However, the main thread is always part - of the result, even when terminated. - - -.. function:: main_thread() - - Return the main :class:`Thread` object. In normal conditions, the - main thread is the thread from which the Python interpreter was - started. - - .. versionadded:: 3.4 - - -.. function:: settrace(func) - - .. index:: single: trace function - - Set a trace function for all threads started from the :mod:`threading` module. - The *func* will be passed to :func:`sys.settrace` for each thread, before its - :meth:`~Thread.run` method is called. - - -.. function:: gettrace() - - .. index:: - single: trace function - single: debugger - - Get the trace function as set by :func:`settrace`. - - .. versionadded:: 3.10 - - -.. function:: setprofile(func) - - .. index:: single: profile function - - Set a profile function for all threads started from the :mod:`threading` module. - The *func* will be passed to :func:`sys.setprofile` for each thread, before its - :meth:`~Thread.run` method is called. - - -.. function:: getprofile() - - .. index:: single: profile function - - Get the profiler function as set by :func:`setprofile`. - - .. versionadded:: 3.10 - - -.. function:: stack_size([size]) - - Return the thread stack size used when creating new threads. The optional - *size* argument specifies the stack size to be used for subsequently created - threads, and must be 0 (use platform or configured default) or a positive - integer value of at least 32,768 (32 KiB). If *size* is not specified, - 0 is used. If changing the thread stack size is - unsupported, a :exc:`RuntimeError` is raised. If the specified stack size is - invalid, a :exc:`ValueError` is raised and the stack size is unmodified. 32 KiB - is currently the minimum supported stack size value to guarantee sufficient - stack space for the interpreter itself. Note that some platforms may have - particular restrictions on values for the stack size, such as requiring a - minimum stack size > 32 KiB or requiring allocation in multiples of the system - memory page size - platform documentation should be referred to for more - information (4 KiB pages are common; using multiples of 4096 for the stack size is - the suggested approach in the absence of more specific information). - - .. availability:: Windows, pthreads. - - Unix platforms with POSIX threads support. - - -This module also defines the following constant: - -.. data:: TIMEOUT_MAX - - The maximum value allowed for the *timeout* parameter of blocking functions - (:meth:`Lock.acquire`, :meth:`RLock.acquire`, :meth:`Condition.wait`, etc.). - Specifying a timeout greater than this value will raise an - :exc:`OverflowError`. - - .. versionadded:: 3.2 - - -This module defines a number of classes, which are detailed in the sections -below. - -The design of this module is loosely based on Java's threading model. However, -where Java makes locks and condition variables basic behavior of every object, -they are separate objects in Python. Python's :class:`Thread` class supports a -subset of the behavior of Java's Thread class; currently, there are no -priorities, no thread groups, and threads cannot be destroyed, stopped, -suspended, resumed, or interrupted. The static methods of Java's Thread class, -when implemented, are mapped to module-level functions. - -All of the methods described below are executed atomically. - - -Thread-Local Data ------------------ - -Thread-local data is data whose values are thread specific. To manage -thread-local data, just create an instance of :class:`local` (or a -subclass) and store attributes on it:: - - mydata = threading.local() - mydata.x = 1 - -The instance's values will be different for separate threads. - - -.. class:: local() - - A class that represents thread-local data. - - For more details and extensive examples, see the documentation string of the - :mod:`!_threading_local` module: :source:`Lib/_threading_local.py`. - - -.. _thread-objects: - -Thread Objects --------------- - -The :class:`Thread` class represents an activity that is run in a separate -thread of control. There are two ways to specify the activity: by passing a -callable object to the constructor, or by overriding the :meth:`~Thread.run` -method in a subclass. No other methods (except for the constructor) should be -overridden in a subclass. In other words, *only* override the -``__init__()`` and :meth:`~Thread.run` methods of this class. - -Once a thread object is created, its activity must be started by calling the -thread's :meth:`~Thread.start` method. This invokes the :meth:`~Thread.run` -method in a separate thread of control. - -Once the thread's activity is started, the thread is considered 'alive'. It -stops being alive when its :meth:`~Thread.run` method terminates -- either -normally, or by raising an unhandled exception. The :meth:`~Thread.is_alive` -method tests whether the thread is alive. - -Other threads can call a thread's :meth:`~Thread.join` method. This blocks -the calling thread until the thread whose :meth:`~Thread.join` method is -called is terminated. - -A thread has a name. The name can be passed to the constructor, and read or -changed through the :attr:`~Thread.name` attribute. - -If the :meth:`~Thread.run` method raises an exception, -:func:`threading.excepthook` is called to handle it. By default, -:func:`threading.excepthook` ignores silently :exc:`SystemExit`. - -A thread can be flagged as a "daemon thread". The significance of this flag is -that the entire Python program exits when only daemon threads are left. The -initial value is inherited from the creating thread. The flag can be set -through the :attr:`~Thread.daemon` property or the *daemon* constructor -argument. - -.. note:: - Daemon threads are abruptly stopped at shutdown. Their resources (such - as open files, database transactions, etc.) may not be released properly. - If you want your threads to stop gracefully, make them non-daemonic and - use a suitable signalling mechanism such as an :class:`Event`. - -There is a "main thread" object; this corresponds to the initial thread of -control in the Python program. It is not a daemon thread. - -There is the possibility that "dummy thread objects" are created. These are -thread objects corresponding to "alien threads", which are threads of control -started outside the threading module, such as directly from C code. Dummy -thread objects have limited functionality; they are always considered alive and -daemonic, and cannot be :ref:`joined `. They are never deleted, -since it is impossible to detect the termination of alien threads. - - -.. class:: Thread(group=None, target=None, name=None, args=(), kwargs={}, *, \ - daemon=None) - - This constructor should always be called with keyword arguments. Arguments - are: - - *group* should be ``None``; reserved for future extension when a - :class:`!ThreadGroup` class is implemented. - - *target* is the callable object to be invoked by the :meth:`run` method. - Defaults to ``None``, meaning nothing is called. - - *name* is the thread name. By default, a unique name is constructed - of the form "Thread-*N*" where *N* is a small decimal number, - or "Thread-*N* (target)" where "target" is ``target.__name__`` if the - *target* argument is specified. - - *args* is a list or tuple of arguments for the target invocation. Defaults to ``()``. - - *kwargs* is a dictionary of keyword arguments for the target invocation. - Defaults to ``{}``. - - If not ``None``, *daemon* explicitly sets whether the thread is daemonic. - If ``None`` (the default), the daemonic property is inherited from the - current thread. - - If the subclass overrides the constructor, it must make sure to invoke the - base class constructor (``Thread.__init__()``) before doing anything else to - the thread. - - .. versionchanged:: 3.10 - Use the *target* name if *name* argument is omitted. - - .. versionchanged:: 3.3 - Added the *daemon* argument. - - .. method:: start() - - Start the thread's activity. - - It must be called at most once per thread object. It arranges for the - object's :meth:`~Thread.run` method to be invoked in a separate thread - of control. - - This method will raise a :exc:`RuntimeError` if called more than once - on the same thread object. - - .. method:: run() - - Method representing the thread's activity. - - You may override this method in a subclass. The standard :meth:`run` - method invokes the callable object passed to the object's constructor as - the *target* argument, if any, with positional and keyword arguments taken - from the *args* and *kwargs* arguments, respectively. - - Using list or tuple as the *args* argument which passed to the :class:`Thread` - could achieve the same effect. - - Example:: - - >>> from threading import Thread - >>> t = Thread(target=print, args=[1]) - >>> t.run() - 1 - >>> t = Thread(target=print, args=(1,)) - >>> t.run() - 1 - - .. _meth-thread-join: - - .. method:: join(timeout=None) - - Wait until the thread terminates. This blocks the calling thread until - the thread whose :meth:`~Thread.join` method is called terminates -- either - normally or through an unhandled exception -- or until the optional - timeout occurs. - - When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). As :meth:`~Thread.join` always returns ``None``, - you must call :meth:`~Thread.is_alive` after :meth:`~Thread.join` to - decide whether a timeout happened -- if the thread is still alive, the - :meth:`~Thread.join` call timed out. - - When the *timeout* argument is not present or ``None``, the operation will - block until the thread terminates. - - A thread can be joined many times. - - :meth:`~Thread.join` raises a :exc:`RuntimeError` if an attempt is made - to join the current thread as that would cause a deadlock. It is also - an error to :meth:`~Thread.join` a thread before it has been started - and attempts to do so raise the same exception. - - .. attribute:: name - - A string used for identification purposes only. It has no semantics. - Multiple threads may be given the same name. The initial name is set by - the constructor. - - .. method:: getName() - setName() - - Deprecated getter/setter API for :attr:`~Thread.name`; use it directly as a - property instead. - - .. deprecated:: 3.10 - - .. attribute:: ident - - The 'thread identifier' of this thread or ``None`` if the thread has not - been started. This is a nonzero integer. See the :func:`get_ident` - function. Thread identifiers may be recycled when a thread exits and - another thread is created. The identifier is available even after the - thread has exited. - - .. attribute:: native_id - - The Thread ID (``TID``) of this thread, as assigned by the OS (kernel). - This is a non-negative integer, or ``None`` if the thread has not - been started. See the :func:`get_native_id` function. - This value may be used to uniquely identify this particular thread - system-wide (until the thread terminates, after which the value - may be recycled by the OS). - - .. note:: - - Similar to Process IDs, Thread IDs are only valid (guaranteed unique - system-wide) from the time the thread is created until the thread - has been terminated. - - .. availability:: Windows, FreeBSD, Linux, macOS, OpenBSD, NetBSD, AIX, DragonFlyBSD. - - .. versionadded:: 3.8 - - .. method:: is_alive() - - Return whether the thread is alive. - - This method returns ``True`` just before the :meth:`~Thread.run` method - starts until just after the :meth:`~Thread.run` method terminates. The - module function :func:`.enumerate` returns a list of all alive threads. - - .. attribute:: daemon - - A boolean value indicating whether this thread is a daemon thread (``True``) - or not (``False``). This must be set before :meth:`~Thread.start` is called, - otherwise :exc:`RuntimeError` is raised. Its initial value is inherited - from the creating thread; the main thread is not a daemon thread and - therefore all threads created in the main thread default to - :attr:`~Thread.daemon` = ``False``. - - The entire Python program exits when no alive non-daemon threads are left. - - .. method:: isDaemon() - setDaemon() - - Deprecated getter/setter API for :attr:`~Thread.daemon`; use it directly as a - property instead. - - .. deprecated:: 3.10 - - -.. _lock-objects: - -Lock Objects ------------- - -A primitive lock is a synchronization primitive that is not owned by a -particular thread when locked. In Python, it is currently the lowest level -synchronization primitive available, implemented directly by the :mod:`_thread` -extension module. - -A primitive lock is in one of two states, "locked" or "unlocked". It is created -in the unlocked state. It has two basic methods, :meth:`~Lock.acquire` and -:meth:`~Lock.release`. When the state is unlocked, :meth:`~Lock.acquire` -changes the state to locked and returns immediately. When the state is locked, -:meth:`~Lock.acquire` blocks until a call to :meth:`~Lock.release` in another -thread changes it to unlocked, then the :meth:`~Lock.acquire` call resets it -to locked and returns. The :meth:`~Lock.release` method should only be -called in the locked state; it changes the state to unlocked and returns -immediately. If an attempt is made to release an unlocked lock, a -:exc:`RuntimeError` will be raised. - -Locks also support the :ref:`context management protocol `. - -When more than one thread is blocked in :meth:`~Lock.acquire` waiting for the -state to turn to unlocked, only one thread proceeds when a :meth:`~Lock.release` -call resets the state to unlocked; which one of the waiting threads proceeds -is not defined, and may vary across implementations. - -All methods are executed atomically. - - -.. class:: Lock() - - The class implementing primitive lock objects. Once a thread has acquired a - lock, subsequent attempts to acquire it block, until it is released; any - thread may release it. - - Note that ``Lock`` is actually a factory function which returns an instance - of the most efficient version of the concrete Lock class that is supported - by the platform. - - - .. method:: acquire(blocking=True, timeout=-1) - - Acquire a lock, blocking or non-blocking. - - When invoked with the *blocking* argument set to ``True`` (the default), - block until the lock is unlocked, then set it to locked and return ``True``. - - When invoked with the *blocking* argument set to ``False``, do not block. - If a call with *blocking* set to ``True`` would block, return ``False`` - immediately; otherwise, set the lock to locked and return ``True``. - - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. A *timeout* argument of ``-1`` - specifies an unbounded wait. It is forbidden to specify a *timeout* - when *blocking* is ``False``. - - The return value is ``True`` if the lock is acquired successfully, - ``False`` if not (for example if the *timeout* expired). - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - .. versionchanged:: 3.2 - Lock acquisition can now be interrupted by signals on POSIX if the - underlying threading implementation supports it. - - - .. method:: release() - - Release a lock. This can be called from any thread, not only the thread - which has acquired the lock. - - When the lock is locked, reset it to unlocked, and return. If any other threads - are blocked waiting for the lock to become unlocked, allow exactly one of them - to proceed. - - When invoked on an unlocked lock, a :exc:`RuntimeError` is raised. - - There is no return value. - - .. method:: locked() - - Return ``True`` if the lock is acquired. - - - -.. _rlock-objects: - -RLock Objects -------------- - -A reentrant lock is a synchronization primitive that may be acquired multiple -times by the same thread. Internally, it uses the concepts of "owning thread" -and "recursion level" in addition to the locked/unlocked state used by primitive -locks. In the locked state, some thread owns the lock; in the unlocked state, -no thread owns it. - -To lock the lock, a thread calls its :meth:`~RLock.acquire` method; this -returns once the thread owns the lock. To unlock the lock, a thread calls -its :meth:`~Lock.release` method. :meth:`~Lock.acquire`/:meth:`~Lock.release` -call pairs may be nested; only the final :meth:`~Lock.release` (the -:meth:`~Lock.release` of the outermost pair) resets the lock to unlocked and -allows another thread blocked in :meth:`~Lock.acquire` to proceed. - -Reentrant locks also support the :ref:`context management protocol `. - - -.. class:: RLock() - - This class implements reentrant lock objects. A reentrant lock must be - released by the thread that acquired it. Once a thread has acquired a - reentrant lock, the same thread may acquire it again without blocking; the - thread must release it once for each time it has acquired it. - - Note that ``RLock`` is actually a factory function which returns an instance - of the most efficient version of the concrete RLock class that is supported - by the platform. - - - .. method:: acquire(blocking=True, timeout=-1) - - Acquire a lock, blocking or non-blocking. - - When invoked without arguments: if this thread already owns the lock, increment - the recursion level by one, and return immediately. Otherwise, if another - thread owns the lock, block until the lock is unlocked. Once the lock is - unlocked (not owned by any thread), then grab ownership, set the recursion level - to one, and return. If more than one thread is blocked waiting until the lock - is unlocked, only one at a time will be able to grab ownership of the lock. - There is no return value in this case. - - When invoked with the *blocking* argument set to ``True``, do the same thing as when - called without arguments, and return ``True``. - - When invoked with the *blocking* argument set to ``False``, do not block. If a call - without an argument would block, return ``False`` immediately; otherwise, do the - same thing as when called without arguments, and return ``True``. - - When invoked with the floating-point *timeout* argument set to a positive - value, block for at most the number of seconds specified by *timeout* - and as long as the lock cannot be acquired. Return ``True`` if the lock has - been acquired, ``False`` if the timeout has elapsed. - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - - .. method:: release() - - Release a lock, decrementing the recursion level. If after the decrement it is - zero, reset the lock to unlocked (not owned by any thread), and if any other - threads are blocked waiting for the lock to become unlocked, allow exactly one - of them to proceed. If after the decrement the recursion level is still - nonzero, the lock remains locked and owned by the calling thread. - - Only call this method when the calling thread owns the lock. A - :exc:`RuntimeError` is raised if this method is called when the lock is - unlocked. - - There is no return value. - - -.. _condition-objects: - -Condition Objects ------------------ - -A condition variable is always associated with some kind of lock; this can be -passed in or one will be created by default. Passing one in is useful when -several condition variables must share the same lock. The lock is part of -the condition object: you don't have to track it separately. - -A condition variable obeys the :ref:`context management protocol `: -using the ``with`` statement acquires the associated lock for the duration of -the enclosed block. The :meth:`~Condition.acquire` and -:meth:`~Condition.release` methods also call the corresponding methods of -the associated lock. - -Other methods must be called with the associated lock held. The -:meth:`~Condition.wait` method releases the lock, and then blocks until -another thread awakens it by calling :meth:`~Condition.notify` or -:meth:`~Condition.notify_all`. Once awakened, :meth:`~Condition.wait` -re-acquires the lock and returns. It is also possible to specify a timeout. - -The :meth:`~Condition.notify` method wakes up one of the threads waiting for -the condition variable, if any are waiting. The :meth:`~Condition.notify_all` -method wakes up all threads waiting for the condition variable. - -Note: the :meth:`~Condition.notify` and :meth:`~Condition.notify_all` methods -don't release the lock; this means that the thread or threads awakened will -not return from their :meth:`~Condition.wait` call immediately, but only when -the thread that called :meth:`~Condition.notify` or :meth:`~Condition.notify_all` -finally relinquishes ownership of the lock. - -The typical programming style using condition variables uses the lock to -synchronize access to some shared state; threads that are interested in a -particular change of state call :meth:`~Condition.wait` repeatedly until they -see the desired state, while threads that modify the state call -:meth:`~Condition.notify` or :meth:`~Condition.notify_all` when they change -the state in such a way that it could possibly be a desired state for one -of the waiters. For example, the following code is a generic -producer-consumer situation with unlimited buffer capacity:: - - # Consume one item - with cv: - while not an_item_is_available(): - cv.wait() - get_an_available_item() - - # Produce one item - with cv: - make_an_item_available() - cv.notify() - -The ``while`` loop checking for the application's condition is necessary -because :meth:`~Condition.wait` can return after an arbitrary long time, -and the condition which prompted the :meth:`~Condition.notify` call may -no longer hold true. This is inherent to multi-threaded programming. The -:meth:`~Condition.wait_for` method can be used to automate the condition -checking, and eases the computation of timeouts:: - - # Consume an item - with cv: - cv.wait_for(an_item_is_available) - get_an_available_item() - -To choose between :meth:`~Condition.notify` and :meth:`~Condition.notify_all`, -consider whether one state change can be interesting for only one or several -waiting threads. E.g. in a typical producer-consumer situation, adding one -item to the buffer only needs to wake up one consumer thread. - - -.. class:: Condition(lock=None) - - This class implements condition variable objects. A condition variable - allows one or more threads to wait until they are notified by another thread. - - If the *lock* argument is given and not ``None``, it must be a :class:`Lock` - or :class:`RLock` object, and it is used as the underlying lock. Otherwise, - a new :class:`RLock` object is created and used as the underlying lock. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - .. method:: acquire(*args) - - Acquire the underlying lock. This method calls the corresponding method on - the underlying lock; the return value is whatever that method returns. - - .. method:: release() - - Release the underlying lock. This method calls the corresponding method on - the underlying lock; there is no return value. - - .. method:: wait(timeout=None) - - Wait until notified or until a timeout occurs. If the calling thread has - not acquired the lock when this method is called, a :exc:`RuntimeError` is - raised. - - This method releases the underlying lock, and then blocks until it is - awakened by a :meth:`notify` or :meth:`notify_all` call for the same - condition variable in another thread, or until the optional timeout - occurs. Once awakened or timed out, it re-acquires the lock and returns. - - When the *timeout* argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). - - When the underlying lock is an :class:`RLock`, it is not released using - its :meth:`release` method, since this may not actually unlock the lock - when it was acquired multiple times recursively. Instead, an internal - interface of the :class:`RLock` class is used, which really unlocks it - even when it has been recursively acquired several times. Another internal - interface is then used to restore the recursion level when the lock is - reacquired. - - The return value is ``True`` unless a given *timeout* expired, in which - case it is ``False``. - - .. versionchanged:: 3.2 - Previously, the method always returned ``None``. - - .. method:: wait_for(predicate, timeout=None) - - Wait until a condition evaluates to true. *predicate* should be a - callable which result will be interpreted as a boolean value. - A *timeout* may be provided giving the maximum time to wait. - - This utility method may call :meth:`wait` repeatedly until the predicate - is satisfied, or until a timeout occurs. The return value is - the last return value of the predicate and will evaluate to - ``False`` if the method timed out. - - Ignoring the timeout feature, calling this method is roughly equivalent to - writing:: - - while not predicate(): - cv.wait() - - Therefore, the same rules apply as with :meth:`wait`: The lock must be - held when called and is re-acquired on return. The predicate is evaluated - with the lock held. - - .. versionadded:: 3.2 - - .. method:: notify(n=1) - - By default, wake up one thread waiting on this condition, if any. If the - calling thread has not acquired the lock when this method is called, a - :exc:`RuntimeError` is raised. - - This method wakes up at most *n* of the threads waiting for the condition - variable; it is a no-op if no threads are waiting. - - The current implementation wakes up exactly *n* threads, if at least *n* - threads are waiting. However, it's not safe to rely on this behavior. - A future, optimized implementation may occasionally wake up more than - *n* threads. - - Note: an awakened thread does not actually return from its :meth:`wait` - call until it can reacquire the lock. Since :meth:`notify` does not - release the lock, its caller should. - - .. method:: notify_all() - - Wake up all threads waiting on this condition. This method acts like - :meth:`notify`, but wakes up all waiting threads instead of one. If the - calling thread has not acquired the lock when this method is called, a - :exc:`RuntimeError` is raised. - - The method ``notifyAll`` is a deprecated alias for this method. - - -.. _semaphore-objects: - -Semaphore Objects ------------------ - -This is one of the oldest synchronization primitives in the history of computer -science, invented by the early Dutch computer scientist Edsger W. Dijkstra (he -used the names ``P()`` and ``V()`` instead of :meth:`~Semaphore.acquire` and -:meth:`~Semaphore.release`). - -A semaphore manages an internal counter which is decremented by each -:meth:`~Semaphore.acquire` call and incremented by each :meth:`~Semaphore.release` -call. The counter can never go below zero; when :meth:`~Semaphore.acquire` -finds that it is zero, it blocks, waiting until some other thread calls -:meth:`~Semaphore.release`. - -Semaphores also support the :ref:`context management protocol `. - - -.. class:: Semaphore(value=1) - - This class implements semaphore objects. A semaphore manages an atomic - counter representing the number of :meth:`release` calls minus the number of - :meth:`acquire` calls, plus an initial value. The :meth:`acquire` method - blocks if necessary until it can return without making the counter negative. - If not given, *value* defaults to 1. - - The optional argument gives the initial *value* for the internal counter; it - defaults to ``1``. If the *value* given is less than 0, :exc:`ValueError` is - raised. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - .. method:: acquire(blocking=True, timeout=None) - - Acquire a semaphore. - - When invoked without arguments: - - * If the internal counter is larger than zero on entry, decrement it by - one and return ``True`` immediately. - * If the internal counter is zero on entry, block until awoken by a call to - :meth:`~Semaphore.release`. Once awoken (and the counter is greater - than 0), decrement the counter by 1 and return ``True``. Exactly one - thread will be awoken by each call to :meth:`~Semaphore.release`. The - order in which threads are awoken should not be relied on. - - When invoked with *blocking* set to ``False``, do not block. If a call - without an argument would block, return ``False`` immediately; otherwise, do - the same thing as when called without arguments, and return ``True``. - - When invoked with a *timeout* other than ``None``, it will block for at - most *timeout* seconds. If acquire does not complete successfully in - that interval, return ``False``. Return ``True`` otherwise. - - .. versionchanged:: 3.2 - The *timeout* parameter is new. - - .. method:: release(n=1) - - Release a semaphore, incrementing the internal counter by *n*. When it - was zero on entry and other threads are waiting for it to become larger - than zero again, wake up *n* of those threads. - - .. versionchanged:: 3.9 - Added the *n* parameter to release multiple waiting threads at once. - - -.. class:: BoundedSemaphore(value=1) - - Class implementing bounded semaphore objects. A bounded semaphore checks to - make sure its current value doesn't exceed its initial value. If it does, - :exc:`ValueError` is raised. In most situations semaphores are used to guard - resources with limited capacity. If the semaphore is released too many times - it's a sign of a bug. If not given, *value* defaults to 1. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - -.. _semaphore-examples: - -:class:`Semaphore` Example -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Semaphores are often used to guard resources with limited capacity, for example, -a database server. In any situation where the size of the resource is fixed, -you should use a bounded semaphore. Before spawning any worker threads, your -main thread would initialize the semaphore:: - - maxconnections = 5 - # ... - pool_sema = BoundedSemaphore(value=maxconnections) - -Once spawned, worker threads call the semaphore's acquire and release methods -when they need to connect to the server:: - - with pool_sema: - conn = connectdb() - try: - # ... use connection ... - finally: - conn.close() - -The use of a bounded semaphore reduces the chance that a programming error which -causes the semaphore to be released more than it's acquired will go undetected. - - -.. _event-objects: - -Event Objects -------------- - -This is one of the simplest mechanisms for communication between threads: one -thread signals an event and other threads wait for it. - -An event object manages an internal flag that can be set to true with the -:meth:`~Event.set` method and reset to false with the :meth:`~Event.clear` -method. The :meth:`~Event.wait` method blocks until the flag is true. - - -.. class:: Event() - - Class implementing event objects. An event manages a flag that can be set to - true with the :meth:`~Event.set` method and reset to false with the - :meth:`clear` method. The :meth:`wait` method blocks until the flag is true. - The flag is initially false. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - .. method:: is_set() - - Return ``True`` if and only if the internal flag is true. - - The method ``isSet`` is a deprecated alias for this method. - - .. method:: set() - - Set the internal flag to true. All threads waiting for it to become true - are awakened. Threads that call :meth:`wait` once the flag is true will - not block at all. - - .. method:: clear() - - Reset the internal flag to false. Subsequently, threads calling - :meth:`wait` will block until :meth:`.set` is called to set the internal - flag to true again. - - .. method:: wait(timeout=None) - - Block until the internal flag is true. If the internal flag is true on - entry, return immediately. Otherwise, block until another thread calls - :meth:`.set` to set the flag to true, or until the optional timeout occurs. - - When the timeout argument is present and not ``None``, it should be a - floating point number specifying a timeout for the operation in seconds - (or fractions thereof). - - This method returns ``True`` if and only if the internal flag has been set to - true, either before the wait call or after the wait starts, so it will - always return ``True`` except if a timeout is given and the operation - times out. - - .. versionchanged:: 3.1 - Previously, the method always returned ``None``. - - -.. _timer-objects: - -Timer Objects -------------- - -This class represents an action that should be run only after a certain amount -of time has passed --- a timer. :class:`Timer` is a subclass of :class:`Thread` -and as such also functions as an example of creating custom threads. - -Timers are started, as with threads, by calling their :meth:`Timer.start ` -method. The timer can be stopped (before its action has begun) by calling the -:meth:`~Timer.cancel` method. The interval the timer will wait before -executing its action may not be exactly the same as the interval specified by -the user. - -For example:: - - def hello(): - print("hello, world") - - t = Timer(30.0, hello) - t.start() # after 30 seconds, "hello, world" will be printed - - -.. class:: Timer(interval, function, args=None, kwargs=None) - - Create a timer that will run *function* with arguments *args* and keyword - arguments *kwargs*, after *interval* seconds have passed. - If *args* is ``None`` (the default) then an empty list will be used. - If *kwargs* is ``None`` (the default) then an empty dict will be used. - - .. versionchanged:: 3.3 - changed from a factory function to a class. - - .. method:: cancel() - - Stop the timer, and cancel the execution of the timer's action. This will - only work if the timer is still in its waiting stage. - - -Barrier Objects ---------------- - -.. versionadded:: 3.2 - -This class provides a simple synchronization primitive for use by a fixed number -of threads that need to wait for each other. Each of the threads tries to pass -the barrier by calling the :meth:`~Barrier.wait` method and will block until -all of the threads have made their :meth:`~Barrier.wait` calls. At this point, -the threads are released simultaneously. - -The barrier can be reused any number of times for the same number of threads. - -As an example, here is a simple way to synchronize a client and server thread:: - - b = Barrier(2, timeout=5) - - def server(): - start_server() - b.wait() - while True: - connection = accept_connection() - process_server_connection(connection) - - def client(): - b.wait() - while True: - connection = make_connection() - process_client_connection(connection) - - -.. class:: Barrier(parties, action=None, timeout=None) - - Create a barrier object for *parties* number of threads. An *action*, when - provided, is a callable to be called by one of the threads when they are - released. *timeout* is the default timeout value if none is specified for - the :meth:`wait` method. - - .. method:: wait(timeout=None) - - Pass the barrier. When all the threads party to the barrier have called - this function, they are all released simultaneously. If a *timeout* is - provided, it is used in preference to any that was supplied to the class - constructor. - - The return value is an integer in the range 0 to *parties* -- 1, different - for each thread. This can be used to select a thread to do some special - housekeeping, e.g.:: - - i = barrier.wait() - if i == 0: - # Only one thread needs to print this - print("passed the barrier") - - If an *action* was provided to the constructor, one of the threads will - have called it prior to being released. Should this call raise an error, - the barrier is put into the broken state. - - If the call times out, the barrier is put into the broken state. - - This method may raise a :class:`BrokenBarrierError` exception if the - barrier is broken or reset while a thread is waiting. - - .. method:: reset() - - Return the barrier to the default, empty state. Any threads waiting on it - will receive the :class:`BrokenBarrierError` exception. - - Note that using this function may require some external - synchronization if there are other threads whose state is unknown. If a - barrier is broken it may be better to just leave it and create a new one. - - .. method:: abort() - - Put the barrier into a broken state. This causes any active or future - calls to :meth:`wait` to fail with the :class:`BrokenBarrierError`. Use - this for example if one of the threads needs to abort, to avoid deadlocking the - application. - - It may be preferable to simply create the barrier with a sensible - *timeout* value to automatically guard against one of the threads going - awry. - - .. attribute:: parties - - The number of threads required to pass the barrier. - - .. attribute:: n_waiting - - The number of threads currently waiting in the barrier. - - .. attribute:: broken - - A boolean that is ``True`` if the barrier is in the broken state. - - -.. exception:: BrokenBarrierError - - This exception, a subclass of :exc:`RuntimeError`, is raised when the - :class:`Barrier` object is reset or broken. - - -.. _with-locks: - -Using locks, conditions, and semaphores in the :keyword:`!with` statement -------------------------------------------------------------------------- - -All of the objects provided by this module that have ``acquire`` and -``release`` methods can be used as context managers for a :keyword:`with` -statement. The ``acquire`` method will be called when the block is -entered, and ``release`` will be called when the block is exited. Hence, -the following snippet:: - - with some_lock: - # do something... - -is equivalent to:: - - some_lock.acquire() - try: - # do something... - finally: - some_lock.release() - -Currently, :class:`Lock`, :class:`RLock`, :class:`Condition`, -:class:`Semaphore`, and :class:`BoundedSemaphore` objects may be used as -:keyword:`with` statement context managers. diff --git a/Doc/library/time.rst b/Doc/library/time.rst deleted file mode 100644 index 93eceed29d2b5b..00000000000000 --- a/Doc/library/time.rst +++ /dev/null @@ -1,966 +0,0 @@ -:mod:`time` --- Time access and conversions -=========================================== - -.. module:: time - :synopsis: Time access and conversions. - --------------- - -This module provides various time-related functions. For related -functionality, see also the :mod:`datetime` and :mod:`calendar` modules. - -Although this module is always available, -not all functions are available on all platforms. Most of the functions -defined in this module call platform C library functions with the same name. It -may sometimes be helpful to consult the platform documentation, because the -semantics of these functions varies among platforms. - -An explanation of some terminology and conventions is in order. - -.. _epoch: - -.. index:: single: epoch - -* The :dfn:`epoch` is the point where the time starts, the return value of - ``time.gmtime(0)``. It is January 1, 1970, 00:00:00 (UTC) on all platforms. - -.. _leap seconds: https://en.wikipedia.org/wiki/Leap_second - -.. index:: seconds since the epoch - -* The term :dfn:`seconds since the epoch` refers to the total number - of elapsed seconds since the epoch, typically excluding - `leap seconds`_. Leap seconds are excluded from this total on all - POSIX-compliant platforms. - -.. index:: single: Year 2038 - -* The functions in this module may not handle dates and times before the epoch_ or - far in the future. The cut-off point in the future is determined by the C - library; for 32-bit systems, it is typically in 2038. - -.. index:: - single: 2-digit years - -* Function :func:`strptime` can parse 2-digit years when given ``%y`` format - code. When 2-digit years are parsed, they are converted according to the POSIX - and ISO C standards: values 69--99 are mapped to 1969--1999, and values 0--68 - are mapped to 2000--2068. - -.. index:: - single: UTC - single: Coordinated Universal Time - single: Greenwich Mean Time - -* UTC is Coordinated Universal Time (formerly known as Greenwich Mean Time, or - GMT). The acronym UTC is not a mistake but a compromise between English and - French. - -.. index:: single: Daylight Saving Time - -* DST is Daylight Saving Time, an adjustment of the timezone by (usually) one - hour during part of the year. DST rules are magic (determined by local law) and - can change from year to year. The C library has a table containing the local - rules (often it is read from a system file for flexibility) and is the only - source of True Wisdom in this respect. - -* The precision of the various real-time functions may be less than suggested by - the units in which their value or argument is expressed. E.g. on most Unix - systems, the clock "ticks" only 50 or 100 times a second. - -* On the other hand, the precision of :func:`.time` and :func:`sleep` is better - than their Unix equivalents: times are expressed as floating point numbers, - :func:`.time` returns the most accurate time available (using Unix - :c:func:`!gettimeofday` where available), and :func:`sleep` will accept a time - with a nonzero fraction (Unix :c:func:`!select` is used to implement this, where - available). - -* The time value as returned by :func:`gmtime`, :func:`localtime`, and - :func:`strptime`, and accepted by :func:`asctime`, :func:`mktime` and - :func:`strftime`, is a sequence of 9 integers. The return values of - :func:`gmtime`, :func:`localtime`, and :func:`strptime` also offer attribute - names for individual fields. - - See :class:`struct_time` for a description of these objects. - - .. versionchanged:: 3.3 - The :class:`struct_time` type was extended to provide - the :attr:`~struct_time.tm_gmtoff` and :attr:`~struct_time.tm_zone` - attributes when platform supports corresponding - ``struct tm`` members. - - .. versionchanged:: 3.6 - The :class:`struct_time` attributes - :attr:`~struct_time.tm_gmtoff` and :attr:`~struct_time.tm_zone` - are now available on all platforms. - -* Use the following functions to convert between time representations: - - +-------------------------+-------------------------+-------------------------+ - | From | To | Use | - +=========================+=========================+=========================+ - | seconds since the epoch | :class:`struct_time` in | :func:`gmtime` | - | | UTC | | - +-------------------------+-------------------------+-------------------------+ - | seconds since the epoch | :class:`struct_time` in | :func:`localtime` | - | | local time | | - +-------------------------+-------------------------+-------------------------+ - | :class:`struct_time` in | seconds since the epoch | :func:`calendar.timegm` | - | UTC | | | - +-------------------------+-------------------------+-------------------------+ - | :class:`struct_time` in | seconds since the epoch | :func:`mktime` | - | local time | | | - +-------------------------+-------------------------+-------------------------+ - - -.. _time-functions: - -Functions ---------- - -.. function:: asctime([t]) - - Convert a tuple or :class:`struct_time` representing a time as returned by - :func:`gmtime` or :func:`localtime` to a string of the following - form: ``'Sun Jun 20 23:21:05 1993'``. The day field is two characters long - and is space padded if the day is a single digit, - e.g.: ``'Wed Jun 9 04:26:40 1993'``. - - If *t* is not provided, the current time as returned by :func:`localtime` - is used. Locale information is not used by :func:`asctime`. - - .. note:: - - Unlike the C function of the same name, :func:`asctime` does not add a - trailing newline. - -.. function:: pthread_getcpuclockid(thread_id) - - Return the *clk_id* of the thread-specific CPU-time clock for the specified *thread_id*. - - Use :func:`threading.get_ident` or the :attr:`~threading.Thread.ident` - attribute of :class:`threading.Thread` objects to get a suitable value - for *thread_id*. - - .. warning:: - Passing an invalid or expired *thread_id* may result in - undefined behavior, such as segmentation fault. - - .. availability:: Unix - - See the man page for :manpage:`pthread_getcpuclockid(3)` for - further information. - - .. versionadded:: 3.7 - -.. function:: clock_getres(clk_id) - - Return the resolution (precision) of the specified clock *clk_id*. Refer to - :ref:`time-clock-id-constants` for a list of accepted values for *clk_id*. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: clock_gettime(clk_id) -> float - - Return the time of the specified clock *clk_id*. Refer to - :ref:`time-clock-id-constants` for a list of accepted values for *clk_id*. - - Use :func:`clock_gettime_ns` to avoid the precision loss caused by the - :class:`float` type. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: clock_gettime_ns(clk_id) -> int - - Similar to :func:`clock_gettime` but return time as nanoseconds. - - .. availability:: Unix. - - .. versionadded:: 3.7 - - -.. function:: clock_settime(clk_id, time: float) - - Set the time of the specified clock *clk_id*. Currently, - :data:`CLOCK_REALTIME` is the only accepted value for *clk_id*. - - Use :func:`clock_settime_ns` to avoid the precision loss caused by the - :class:`float` type. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. function:: clock_settime_ns(clk_id, time: int) - - Similar to :func:`clock_settime` but set time with nanoseconds. - - .. availability:: Unix. - - .. versionadded:: 3.7 - - -.. function:: ctime([secs]) - - Convert a time expressed in seconds since the epoch_ to a string of a form: - ``'Sun Jun 20 23:21:05 1993'`` representing local time. The day field - is two characters long and is space padded if the day is a single digit, - e.g.: ``'Wed Jun 9 04:26:40 1993'``. - - If *secs* is not provided or :const:`None`, the current time as - returned by :func:`.time` is used. ``ctime(secs)`` is equivalent to - ``asctime(localtime(secs))``. Locale information is not used by - :func:`ctime`. - - -.. function:: get_clock_info(name) - - Get information on the specified clock as a namespace object. - Supported clock names and the corresponding functions to read their value - are: - - * ``'monotonic'``: :func:`time.monotonic` - * ``'perf_counter'``: :func:`time.perf_counter` - * ``'process_time'``: :func:`time.process_time` - * ``'thread_time'``: :func:`time.thread_time` - * ``'time'``: :func:`time.time` - - The result has the following attributes: - - - *adjustable*: ``True`` if the clock can be changed automatically (e.g. by - a NTP daemon) or manually by the system administrator, ``False`` otherwise - - *implementation*: The name of the underlying C function used to get - the clock value. Refer to :ref:`time-clock-id-constants` for possible values. - - *monotonic*: ``True`` if the clock cannot go backward, - ``False`` otherwise - - *resolution*: The resolution of the clock in seconds (:class:`float`) - - .. versionadded:: 3.3 - - -.. function:: gmtime([secs]) - - Convert a time expressed in seconds since the epoch_ to a :class:`struct_time` in - UTC in which the dst flag is always zero. If *secs* is not provided or - :const:`None`, the current time as returned by :func:`.time` is used. Fractions - of a second are ignored. See above for a description of the - :class:`struct_time` object. See :func:`calendar.timegm` for the inverse of this - function. - - -.. function:: localtime([secs]) - - Like :func:`gmtime` but converts to local time. If *secs* is not provided or - :const:`None`, the current time as returned by :func:`.time` is used. The dst - flag is set to ``1`` when DST applies to the given time. - - :func:`localtime` may raise :exc:`OverflowError`, if the timestamp is - outside the range of values supported by the platform C :c:func:`localtime` - or :c:func:`gmtime` functions, and :exc:`OSError` on :c:func:`localtime` or - :c:func:`gmtime` failure. It's common for this to be restricted to years - between 1970 and 2038. - - -.. function:: mktime(t) - - This is the inverse function of :func:`localtime`. Its argument is the - :class:`struct_time` or full 9-tuple (since the dst flag is needed; use ``-1`` - as the dst flag if it is unknown) which expresses the time in *local* time, not - UTC. It returns a floating point number, for compatibility with :func:`.time`. - If the input value cannot be represented as a valid time, either - :exc:`OverflowError` or :exc:`ValueError` will be raised (which depends on - whether the invalid value is caught by Python or the underlying C libraries). - The earliest date for which it can generate a time is platform-dependent. - - -.. function:: monotonic() -> float - - Return the value (in fractional seconds) of a monotonic clock, i.e. a clock - that cannot go backwards. The clock is not affected by system clock updates. - The reference point of the returned value is undefined, so that only the - difference between the results of two calls is valid. - - Use :func:`monotonic_ns` to avoid the precision loss caused by the - :class:`float` type. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.5 - The function is now always available and always system-wide. - - .. versionchanged:: 3.10 - On macOS, the function is now system-wide. - - -.. function:: monotonic_ns() -> int - - Similar to :func:`monotonic`, but return time as nanoseconds. - - .. versionadded:: 3.7 - -.. function:: perf_counter() -> float - - .. index:: - single: benchmarking - - Return the value (in fractional seconds) of a performance counter, i.e. a - clock with the highest available resolution to measure a short duration. It - does include time elapsed during sleep and is system-wide. The reference - point of the returned value is undefined, so that only the difference between - the results of two calls is valid. - - Use :func:`perf_counter_ns` to avoid the precision loss caused by the - :class:`float` type. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.10 - On Windows, the function is now system-wide. - -.. function:: perf_counter_ns() -> int - - Similar to :func:`perf_counter`, but return time as nanoseconds. - - .. versionadded:: 3.7 - - -.. function:: process_time() -> float - - .. index:: - single: CPU time - single: processor time - single: benchmarking - - Return the value (in fractional seconds) of the sum of the system and user - CPU time of the current process. It does not include time elapsed during - sleep. It is process-wide by definition. The reference point of the - returned value is undefined, so that only the difference between the results - of two calls is valid. - - Use :func:`process_time_ns` to avoid the precision loss caused by the - :class:`float` type. - - .. versionadded:: 3.3 - -.. function:: process_time_ns() -> int - - Similar to :func:`process_time` but return time as nanoseconds. - - .. versionadded:: 3.7 - -.. function:: sleep(secs) - - Suspend execution of the calling thread for the given number of seconds. - The argument may be a floating point number to indicate a more precise sleep - time. - - If the sleep is interrupted by a signal and no exception is raised by the - signal handler, the sleep is restarted with a recomputed timeout. - - The suspension time may be longer than requested by an arbitrary amount, - because of the scheduling of other activity in the system. - - On Windows, if *secs* is zero, the thread relinquishes the remainder of its - time slice to any other thread that is ready to run. If there are no other - threads ready to run, the function returns immediately, and the thread - continues execution. On Windows 8.1 and newer the implementation uses - a `high-resolution timer - `_ - which provides resolution of 100 nanoseconds. If *secs* is zero, ``Sleep(0)`` is used. - - Unix implementation: - - * Use ``clock_nanosleep()`` if available (resolution: 1 nanosecond); - * Or use ``nanosleep()`` if available (resolution: 1 nanosecond); - * Or use ``select()`` (resolution: 1 microsecond). - - .. versionchanged:: 3.11 - On Unix, the ``clock_nanosleep()`` and ``nanosleep()`` functions are now - used if available. On Windows, a waitable timer is now used. - - .. versionchanged:: 3.5 - The function now sleeps at least *secs* even if the sleep is interrupted - by a signal, except if the signal handler raises an exception (see - :pep:`475` for the rationale). - - -.. index:: - single: % (percent); datetime format - -.. function:: strftime(format[, t]) - - Convert a tuple or :class:`struct_time` representing a time as returned by - :func:`gmtime` or :func:`localtime` to a string as specified by the *format* - argument. If *t* is not provided, the current time as returned by - :func:`localtime` is used. *format* must be a string. :exc:`ValueError` is - raised if any field in *t* is outside of the allowed range. - - 0 is a legal argument for any position in the time tuple; if it is normally - illegal the value is forced to a correct one. - - The following directives can be embedded in the *format* string. They are shown - without the optional field width and precision specification, and are replaced - by the indicated characters in the :func:`strftime` result: - - +-----------+------------------------------------------------+-------+ - | Directive | Meaning | Notes | - +===========+================================================+=======+ - | ``%a`` | Locale's abbreviated weekday name. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%A`` | Locale's full weekday name. | | - +-----------+------------------------------------------------+-------+ - | ``%b`` | Locale's abbreviated month name. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%B`` | Locale's full month name. | | - +-----------+------------------------------------------------+-------+ - | ``%c`` | Locale's appropriate date and time | | - | | representation. | | - +-----------+------------------------------------------------+-------+ - | ``%d`` | Day of the month as a decimal number [01,31]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%H`` | Hour (24-hour clock) as a decimal number | | - | | [00,23]. | | - +-----------+------------------------------------------------+-------+ - | ``%I`` | Hour (12-hour clock) as a decimal number | | - | | [01,12]. | | - +-----------+------------------------------------------------+-------+ - | ``%j`` | Day of the year as a decimal number [001,366]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%m`` | Month as a decimal number [01,12]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%M`` | Minute as a decimal number [00,59]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%p`` | Locale's equivalent of either AM or PM. | \(1) | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%S`` | Second as a decimal number [00,61]. | \(2) | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%U`` | Week number of the year (Sunday as the first | \(3) | - | | day of the week) as a decimal number [00,53]. | | - | | All days in a new year preceding the first | | - | | Sunday are considered to be in week 0. | | - | | | | - | | | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%w`` | Weekday as a decimal number [0(Sunday),6]. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%W`` | Week number of the year (Monday as the first | \(3) | - | | day of the week) as a decimal number [00,53]. | | - | | All days in a new year preceding the first | | - | | Monday are considered to be in week 0. | | - | | | | - | | | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%x`` | Locale's appropriate date representation. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%X`` | Locale's appropriate time representation. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%y`` | Year without century as a decimal number | | - | | [00,99]. | | - +-----------+------------------------------------------------+-------+ - | ``%Y`` | Year with century as a decimal number. | | - | | | | - +-----------+------------------------------------------------+-------+ - | ``%z`` | Time zone offset indicating a positive or | | - | | negative time difference from UTC/GMT of the | | - | | form +HHMM or -HHMM, where H represents decimal| | - | | hour digits and M represents decimal minute | | - | | digits [-23:59, +23:59]. [1]_ | | - +-----------+------------------------------------------------+-------+ - | ``%Z`` | Time zone name (no characters if no time zone | | - | | exists). Deprecated. [1]_ | | - +-----------+------------------------------------------------+-------+ - | ``%%`` | A literal ``'%'`` character. | | - +-----------+------------------------------------------------+-------+ - - Notes: - - (1) - When used with the :func:`strptime` function, the ``%p`` directive only affects - the output hour field if the ``%I`` directive is used to parse the hour. - - .. _leap-second: - - (2) - The range really is ``0`` to ``61``; value ``60`` is valid in - timestamps representing `leap seconds`_ and value ``61`` is supported - for historical reasons. - - (3) - When used with the :func:`strptime` function, ``%U`` and ``%W`` are only used in - calculations when the day of the week and the year are specified. - - Here is an example, a format for dates compatible with that specified in the - :rfc:`2822` Internet email standard. [1]_ :: - - >>> from time import gmtime, strftime - >>> strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime()) - 'Thu, 28 Jun 2001 14:17:15 +0000' - - Additional directives may be supported on certain platforms, but only the - ones listed here have a meaning standardized by ANSI C. To see the full set - of format codes supported on your platform, consult the :manpage:`strftime(3)` - documentation. - - On some platforms, an optional field width and precision specification can - immediately follow the initial ``'%'`` of a directive in the following order; - this is also not portable. The field width is normally 2 except for ``%j`` where - it is 3. - - -.. index:: - single: % (percent); datetime format - -.. function:: strptime(string[, format]) - - Parse a string representing a time according to a format. The return value - is a :class:`struct_time` as returned by :func:`gmtime` or - :func:`localtime`. - - The *format* parameter uses the same directives as those used by - :func:`strftime`; it defaults to ``"%a %b %d %H:%M:%S %Y"`` which matches the - formatting returned by :func:`ctime`. If *string* cannot be parsed according - to *format*, or if it has excess data after parsing, :exc:`ValueError` is - raised. The default values used to fill in any missing data when more - accurate values cannot be inferred are ``(1900, 1, 1, 0, 0, 0, 0, 1, -1)``. - Both *string* and *format* must be strings. - - For example: - - >>> import time - >>> time.strptime("30 Nov 00", "%d %b %y") # doctest: +NORMALIZE_WHITESPACE - time.struct_time(tm_year=2000, tm_mon=11, tm_mday=30, tm_hour=0, tm_min=0, - tm_sec=0, tm_wday=3, tm_yday=335, tm_isdst=-1) - - Support for the ``%Z`` directive is based on the values contained in ``tzname`` - and whether ``daylight`` is true. Because of this, it is platform-specific - except for recognizing UTC and GMT which are always known (and are considered to - be non-daylight savings timezones). - - Only the directives specified in the documentation are supported. Because - ``strftime()`` is implemented per platform it can sometimes offer more - directives than those listed. But ``strptime()`` is independent of any platform - and thus does not necessarily support all directives available that are not - documented as supported. - - -.. class:: struct_time - - The type of the time value sequence returned by :func:`gmtime`, - :func:`localtime`, and :func:`strptime`. It is an object with a :term:`named - tuple` interface: values can be accessed by index and by attribute name. The - following values are present: - - .. list-table:: - - * - Index - - Attribute - - Values - - * - 0 - - .. attribute:: tm_year - - (for example, 1993) - - * - 1 - - .. attribute:: tm_mon - - range [1, 12] - - * - 2 - - .. attribute:: tm_day - - range [1, 31] - - * - 3 - - .. attribute:: tm_hour - - range [0, 23] - - * - 4 - - .. attribute:: tm_min - - range [0, 59] - - * - 5 - - .. attribute:: tm_sec - - range [0, 61]; see :ref:`Note (2) ` in :func:`strftime` - - * - 6 - - .. attribute:: tm_wday - - range [0, 6]; Monday is 0 - - * - 7 - - .. attribute:: tm_yday - - range [1, 366] - - * - 8 - - .. attribute:: tm_isdst - - 0, 1 or -1; see below - - * - N/A - - .. attribute:: tm_zone - - abbreviation of timezone name - - * - N/A - - .. attribute:: tm_gmtoff - - offset east of UTC in seconds - - Note that unlike the C structure, the month value is a range of [1, 12], not - [0, 11]. - - In calls to :func:`mktime`, :attr:`tm_isdst` may be set to 1 when daylight - savings time is in effect, and 0 when it is not. A value of -1 indicates that - this is not known, and will usually result in the correct state being filled in. - - When a tuple with an incorrect length is passed to a function expecting a - :class:`struct_time`, or having elements of the wrong type, a - :exc:`TypeError` is raised. - -.. function:: time() -> float - - Return the time in seconds since the epoch_ as a floating point - number. The handling of `leap seconds`_ is platform dependent. - On Windows and most Unix systems, the leap seconds are not counted towards - the time in seconds since the epoch_. This is commonly referred to as `Unix - time `_. - - Note that even though the time is always returned as a floating point - number, not all systems provide time with a better precision than 1 second. - While this function normally returns non-decreasing values, it can return a - lower value than a previous call if the system clock has been set back - between the two calls. - - The number returned by :func:`.time` may be converted into a more common - time format (i.e. year, month, day, hour, etc...) in UTC by passing it to - :func:`gmtime` function or in local time by passing it to the - :func:`localtime` function. In both cases a - :class:`struct_time` object is returned, from which the components - of the calendar date may be accessed as attributes. - - Use :func:`time_ns` to avoid the precision loss caused by the :class:`float` - type. - - -.. function:: time_ns() -> int - - Similar to :func:`~time.time` but returns time as an integer number of - nanoseconds since the epoch_. - - .. versionadded:: 3.7 - - -.. function:: thread_time() -> float - - .. index:: - single: CPU time - single: processor time - single: benchmarking - - Return the value (in fractional seconds) of the sum of the system and user - CPU time of the current thread. It does not include time elapsed during - sleep. It is thread-specific by definition. The reference point of the - returned value is undefined, so that only the difference between the results - of two calls in the same thread is valid. - - Use :func:`thread_time_ns` to avoid the precision loss caused by the - :class:`float` type. - - .. availability:: Linux, Unix, Windows. - - Unix systems supporting ``CLOCK_THREAD_CPUTIME_ID``. - - .. versionadded:: 3.7 - - -.. function:: thread_time_ns() -> int - - Similar to :func:`thread_time` but return time as nanoseconds. - - .. versionadded:: 3.7 - - -.. function:: tzset() - - Reset the time conversion rules used by the library routines. The environment - variable :envvar:`TZ` specifies how this is done. It will also set the variables - ``tzname`` (from the :envvar:`TZ` environment variable), ``timezone`` (non-DST - seconds West of UTC), ``altzone`` (DST seconds west of UTC) and ``daylight`` - (to 0 if this timezone does not have any daylight saving time rules, or to - nonzero if there is a time, past, present or future when daylight saving time - applies). - - .. availability:: Unix. - - .. note:: - - Although in many cases, changing the :envvar:`TZ` environment variable may - affect the output of functions like :func:`localtime` without calling - :func:`tzset`, this behavior should not be relied on. - - The :envvar:`TZ` environment variable should contain no whitespace. - - The standard format of the :envvar:`TZ` environment variable is (whitespace - added for clarity):: - - std offset [dst [offset [,start[/time], end[/time]]]] - - Where the components are: - - ``std`` and ``dst`` - Three or more alphanumerics giving the timezone abbreviations. These will be - propagated into time.tzname - - ``offset`` - The offset has the form: ``± hh[:mm[:ss]]``. This indicates the value - added the local time to arrive at UTC. If preceded by a '-', the timezone - is east of the Prime Meridian; otherwise, it is west. If no offset follows - dst, summer time is assumed to be one hour ahead of standard time. - - ``start[/time], end[/time]`` - Indicates when to change to and back from DST. The format of the - start and end dates are one of the following: - - :samp:`J{n}` - The Julian day *n* (1 <= *n* <= 365). Leap days are not counted, so in - all years February 28 is day 59 and March 1 is day 60. - - :samp:`{n}` - The zero-based Julian day (0 <= *n* <= 365). Leap days are counted, and - it is possible to refer to February 29. - - :samp:`M{m}.{n}.{d}` - The *d*'th day (0 <= *d* <= 6) of week *n* of month *m* of the year (1 - <= *n* <= 5, 1 <= *m* <= 12, where week 5 means "the last *d* day in - month *m*" which may occur in either the fourth or the fifth - week). Week 1 is the first week in which the *d*'th day occurs. Day - zero is a Sunday. - - ``time`` has the same format as ``offset`` except that no leading sign - ('-' or '+') is allowed. The default, if time is not given, is 02:00:00. - - :: - - >>> os.environ['TZ'] = 'EST+05EDT,M4.1.0,M10.5.0' - >>> time.tzset() - >>> time.strftime('%X %x %Z') - '02:07:36 05/08/03 EDT' - >>> os.environ['TZ'] = 'AEST-10AEDT-11,M10.5.0,M3.5.0' - >>> time.tzset() - >>> time.strftime('%X %x %Z') - '16:08:12 05/08/03 AEST' - - On many Unix systems (including \*BSD, Linux, Solaris, and Darwin), it is more - convenient to use the system's zoneinfo (:manpage:`tzfile(5)`) database to - specify the timezone rules. To do this, set the :envvar:`TZ` environment - variable to the path of the required timezone datafile, relative to the root of - the systems 'zoneinfo' timezone database, usually located at - :file:`/usr/share/zoneinfo`. For example, ``'US/Eastern'``, - ``'Australia/Melbourne'``, ``'Egypt'`` or ``'Europe/Amsterdam'``. :: - - >>> os.environ['TZ'] = 'US/Eastern' - >>> time.tzset() - >>> time.tzname - ('EST', 'EDT') - >>> os.environ['TZ'] = 'Egypt' - >>> time.tzset() - >>> time.tzname - ('EET', 'EEST') - - -.. _time-clock-id-constants: - -Clock ID Constants ------------------- - -These constants are used as parameters for :func:`clock_getres` and -:func:`clock_gettime`. - -.. data:: CLOCK_BOOTTIME - - Identical to :data:`CLOCK_MONOTONIC`, except it also includes any time that - the system is suspended. - - This allows applications to get a suspend-aware monotonic clock without - having to deal with the complications of :data:`CLOCK_REALTIME`, which may - have discontinuities if the time is changed using ``settimeofday()`` or - similar. - - .. availability:: Linux >= 2.6.39. - - .. versionadded:: 3.7 - - -.. data:: CLOCK_HIGHRES - - The Solaris OS has a ``CLOCK_HIGHRES`` timer that attempts to use an optimal - hardware source, and may give close to nanosecond resolution. - ``CLOCK_HIGHRES`` is the nonadjustable, high-resolution clock. - - .. availability:: Solaris. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_MONOTONIC - - Clock that cannot be set and represents monotonic time since some unspecified - starting point. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_MONOTONIC_RAW - - Similar to :data:`CLOCK_MONOTONIC`, but provides access to a raw - hardware-based time that is not subject to NTP adjustments. - - .. availability:: Linux >= 2.6.28, macOS >= 10.12. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_PROCESS_CPUTIME_ID - - High-resolution per-process timer from the CPU. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_PROF - - High-resolution per-process timer from the CPU. - - .. availability:: FreeBSD, NetBSD >= 7, OpenBSD. - - .. versionadded:: 3.7 - -.. data:: CLOCK_TAI - - `International Atomic Time `_ - - The system must have a current leap second table in order for this to give - the correct answer. PTP or NTP software can maintain a leap second table. - - .. availability:: Linux. - - .. versionadded:: 3.9 - -.. data:: CLOCK_THREAD_CPUTIME_ID - - Thread-specific CPU-time clock. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. data:: CLOCK_UPTIME - - Time whose absolute value is the time the system has been running and not - suspended, providing accurate uptime measurement, both absolute and - interval. - - .. availability:: FreeBSD, OpenBSD >= 5.5. - - .. versionadded:: 3.7 - - -.. data:: CLOCK_UPTIME_RAW - - Clock that increments monotonically, tracking the time since an arbitrary - point, unaffected by frequency or time adjustments and not incremented while - the system is asleep. - - .. availability:: macOS >= 10.12. - - .. versionadded:: 3.8 - -The following constant is the only parameter that can be sent to -:func:`clock_settime`. - - -.. data:: CLOCK_REALTIME - - System-wide real-time clock. Setting this clock requires appropriate - privileges. - - .. availability:: Unix. - - .. versionadded:: 3.3 - - -.. _time-timezone-constants: - -Timezone Constants -------------------- - -.. data:: altzone - - The offset of the local DST timezone, in seconds west of UTC, if one is defined. - This is negative if the local DST timezone is east of UTC (as in Western Europe, - including the UK). Only use this if ``daylight`` is nonzero. See note below. - -.. data:: daylight - - Nonzero if a DST timezone is defined. See note below. - -.. data:: timezone - - The offset of the local (non-DST) timezone, in seconds west of UTC (negative in - most of Western Europe, positive in the US, zero in the UK). See note below. - -.. data:: tzname - - A tuple of two strings: the first is the name of the local non-DST timezone, the - second is the name of the local DST timezone. If no DST timezone is defined, - the second string should not be used. See note below. - -.. note:: - - For the above Timezone constants (:data:`altzone`, :data:`daylight`, :data:`timezone`, - and :data:`tzname`), the value is determined by the timezone rules in effect - at module load time or the last time :func:`tzset` is called and may be incorrect - for times in the past. It is recommended to use the :attr:`~struct_time.tm_gmtoff` and - :attr:`~struct_time.tm_zone` results from :func:`localtime` to obtain timezone information. - - -.. seealso:: - - Module :mod:`datetime` - More object-oriented interface to dates and times. - - Module :mod:`locale` - Internationalization services. The locale setting affects the interpretation - of many format specifiers in :func:`strftime` and :func:`strptime`. - - Module :mod:`calendar` - General calendar-related functions. :func:`~calendar.timegm` is the - inverse of :func:`gmtime` from this module. - -.. rubric:: Footnotes - -.. [1] The use of ``%Z`` is now deprecated, but the ``%z`` escape that expands to the - preferred hour/minute offset is not supported by all ANSI C libraries. Also, a - strict reading of the original 1982 :rfc:`822` standard calls for a two-digit - year (``%y`` rather than ``%Y``), but practice moved to 4-digit years long before the - year 2000. After that, :rfc:`822` became obsolete and the 4-digit year has - been first recommended by :rfc:`1123` and then mandated by :rfc:`2822`. diff --git a/Doc/library/timeit.rst b/Doc/library/timeit.rst deleted file mode 100644 index c9afb7d802e84f..00000000000000 --- a/Doc/library/timeit.rst +++ /dev/null @@ -1,381 +0,0 @@ -:mod:`timeit` --- Measure execution time of small code snippets -=============================================================== - -.. module:: timeit - :synopsis: Measure the execution time of small code snippets. - -**Source code:** :source:`Lib/timeit.py` - -.. index:: - single: Benchmarking - single: Performance - --------------- - -This module provides a simple way to time small bits of Python code. It has both -a :ref:`timeit-command-line-interface` as well as a :ref:`callable ` -one. It avoids a number of common traps for measuring execution times. -See also Tim Peters' introduction to the "Algorithms" chapter in the second -edition of *Python Cookbook*, published by O'Reilly. - - -Basic Examples --------------- - -The following example shows how the :ref:`timeit-command-line-interface` -can be used to compare three different expressions: - -.. code-block:: shell-session - - $ python3 -m timeit '"-".join(str(n) for n in range(100))' - 10000 loops, best of 5: 30.2 usec per loop - $ python3 -m timeit '"-".join([str(n) for n in range(100)])' - 10000 loops, best of 5: 27.5 usec per loop - $ python3 -m timeit '"-".join(map(str, range(100)))' - 10000 loops, best of 5: 23.2 usec per loop - -This can be achieved from the :ref:`python-interface` with:: - - >>> import timeit - >>> timeit.timeit('"-".join(str(n) for n in range(100))', number=10000) - 0.3018611848820001 - >>> timeit.timeit('"-".join([str(n) for n in range(100)])', number=10000) - 0.2727368790656328 - >>> timeit.timeit('"-".join(map(str, range(100)))', number=10000) - 0.23702679807320237 - -A callable can also be passed from the :ref:`python-interface`:: - - >>> timeit.timeit(lambda: "-".join(map(str, range(100))), number=10000) - 0.19665591977536678 - -Note however that :func:`.timeit` will automatically determine the number of -repetitions only when the command-line interface is used. In the -:ref:`timeit-examples` section you can find more advanced examples. - - -.. _python-interface: - -Python Interface ----------------- - -The module defines three convenience functions and a public class: - - -.. function:: timeit(stmt='pass', setup='pass', timer=, number=1000000, globals=None) - - Create a :class:`Timer` instance with the given statement, *setup* code and - *timer* function and run its :meth:`.timeit` method with *number* executions. - The optional *globals* argument specifies a namespace in which to execute the - code. - - .. versionchanged:: 3.5 - The optional *globals* parameter was added. - - -.. function:: repeat(stmt='pass', setup='pass', timer=, repeat=5, number=1000000, globals=None) - - Create a :class:`Timer` instance with the given statement, *setup* code and - *timer* function and run its :meth:`.repeat` method with the given *repeat* - count and *number* executions. The optional *globals* argument specifies a - namespace in which to execute the code. - - .. versionchanged:: 3.5 - The optional *globals* parameter was added. - - .. versionchanged:: 3.7 - Default value of *repeat* changed from 3 to 5. - - -.. function:: default_timer() - - The default timer, which is always time.perf_counter(), returns float seconds. - An alternative, time.perf_counter_ns, returns integer nanoseconds. - - .. versionchanged:: 3.3 - :func:`time.perf_counter` is now the default timer. - - -.. class:: Timer(stmt='pass', setup='pass', timer=, globals=None) - - Class for timing execution speed of small code snippets. - - The constructor takes a statement to be timed, an additional statement used - for setup, and a timer function. Both statements default to ``'pass'``; - the timer function is platform-dependent (see the module doc string). - *stmt* and *setup* may also contain multiple statements separated by ``;`` - or newlines, as long as they don't contain multi-line string literals. The - statement will by default be executed within timeit's namespace; this behavior - can be controlled by passing a namespace to *globals*. - - To measure the execution time of the first statement, use the :meth:`.timeit` - method. The :meth:`.repeat` and :meth:`.autorange` methods are convenience - methods to call :meth:`.timeit` multiple times. - - The execution time of *setup* is excluded from the overall timed execution run. - - The *stmt* and *setup* parameters can also take objects that are callable - without arguments. This will embed calls to them in a timer function that - will then be executed by :meth:`.timeit`. Note that the timing overhead is a - little larger in this case because of the extra function calls. - - .. versionchanged:: 3.5 - The optional *globals* parameter was added. - - .. method:: Timer.timeit(number=1000000) - - Time *number* executions of the main statement. This executes the setup - statement once, and then returns the time it takes to execute the main - statement a number of times. The default timer returns seconds as a float. - The argument is the number of times through the loop, defaulting to one - million. The main statement, the setup statement and the timer function - to be used are passed to the constructor. - - .. note:: - - By default, :meth:`.timeit` temporarily turns off :term:`garbage - collection` during the timing. The advantage of this approach is that - it makes independent timings more comparable. The disadvantage is - that GC may be an important component of the performance of the - function being measured. If so, GC can be re-enabled as the first - statement in the *setup* string. For example:: - - timeit.Timer('for i in range(10): oct(i)', 'gc.enable()').timeit() - - - .. method:: Timer.autorange(callback=None) - - Automatically determine how many times to call :meth:`.timeit`. - - This is a convenience function that calls :meth:`.timeit` repeatedly - so that the total time >= 0.2 second, returning the eventual - (number of loops, time taken for that number of loops). It calls - :meth:`.timeit` with increasing numbers from the sequence 1, 2, 5, - 10, 20, 50, ... until the time taken is at least 0.2 seconds. - - If *callback* is given and is not ``None``, it will be called after - each trial with two arguments: ``callback(number, time_taken)``. - - .. versionadded:: 3.6 - - - .. method:: Timer.repeat(repeat=5, number=1000000) - - Call :meth:`.timeit` a few times. - - This is a convenience function that calls the :meth:`.timeit` repeatedly, - returning a list of results. The first argument specifies how many times - to call :meth:`.timeit`. The second argument specifies the *number* - argument for :meth:`.timeit`. - - .. note:: - - It's tempting to calculate mean and standard deviation from the result - vector and report these. However, this is not very useful. - In a typical case, the lowest value gives a lower bound for how fast - your machine can run the given code snippet; higher values in the - result vector are typically not caused by variability in Python's - speed, but by other processes interfering with your timing accuracy. - So the :func:`min` of the result is probably the only number you - should be interested in. After that, you should look at the entire - vector and apply common sense rather than statistics. - - .. versionchanged:: 3.7 - Default value of *repeat* changed from 3 to 5. - - - .. method:: Timer.print_exc(file=None) - - Helper to print a traceback from the timed code. - - Typical use:: - - t = Timer(...) # outside the try/except - try: - t.timeit(...) # or t.repeat(...) - except Exception: - t.print_exc() - - The advantage over the standard traceback is that source lines in the - compiled template will be displayed. The optional *file* argument directs - where the traceback is sent; it defaults to :data:`sys.stderr`. - - -.. _timeit-command-line-interface: - -Command-Line Interface ----------------------- - -When called as a program from the command line, the following form is used:: - - python -m timeit [-n N] [-r N] [-u U] [-s S] [-h] [statement ...] - -Where the following options are understood: - -.. program:: timeit - -.. option:: -n N, --number=N - - how many times to execute 'statement' - -.. option:: -r N, --repeat=N - - how many times to repeat the timer (default 5) - -.. option:: -s S, --setup=S - - statement to be executed once initially (default ``pass``) - -.. option:: -p, --process - - measure process time, not wallclock time, using :func:`time.process_time` - instead of :func:`time.perf_counter`, which is the default - - .. versionadded:: 3.3 - -.. option:: -u, --unit=U - - specify a time unit for timer output; can select ``nsec``, ``usec``, ``msec``, or ``sec`` - - .. versionadded:: 3.5 - -.. option:: -v, --verbose - - print raw timing results; repeat for more digits precision - -.. option:: -h, --help - - print a short usage message and exit - -A multi-line statement may be given by specifying each line as a separate -statement argument; indented lines are possible by enclosing an argument in -quotes and using leading spaces. Multiple :option:`-s` options are treated -similarly. - -If :option:`-n` is not given, a suitable number of loops is calculated by trying -increasing numbers from the sequence 1, 2, 5, 10, 20, 50, ... until the total -time is at least 0.2 seconds. - -:func:`default_timer` measurements can be affected by other programs running on -the same machine, so the best thing to do when accurate timing is necessary is -to repeat the timing a few times and use the best time. The :option:`-r` -option is good for this; the default of 5 repetitions is probably enough in -most cases. You can use :func:`time.process_time` to measure CPU time. - -.. note:: - - There is a certain baseline overhead associated with executing a pass statement. - The code here doesn't try to hide it, but you should be aware of it. The - baseline overhead can be measured by invoking the program without arguments, - and it might differ between Python versions. - - -.. _timeit-examples: - -Examples --------- - -It is possible to provide a setup statement that is executed only once at the beginning: - -.. code-block:: shell-session - - $ python -m timeit -s 'text = "sample string"; char = "g"' 'char in text' - 5000000 loops, best of 5: 0.0877 usec per loop - $ python -m timeit -s 'text = "sample string"; char = "g"' 'text.find(char)' - 1000000 loops, best of 5: 0.342 usec per loop - -In the output, there are three fields. The loop count, which tells you how many -times the statement body was run per timing loop repetition. The repetition -count ('best of 5') which tells you how many times the timing loop was -repeated, and finally the time the statement body took on average within the -best repetition of the timing loop. That is, the time the fastest repetition -took divided by the loop count. - -:: - - >>> import timeit - >>> timeit.timeit('char in text', setup='text = "sample string"; char = "g"') - 0.41440500499993504 - >>> timeit.timeit('text.find(char)', setup='text = "sample string"; char = "g"') - 1.7246671520006203 - -The same can be done using the :class:`Timer` class and its methods:: - - >>> import timeit - >>> t = timeit.Timer('char in text', setup='text = "sample string"; char = "g"') - >>> t.timeit() - 0.3955516149999312 - >>> t.repeat() - [0.40183617287970225, 0.37027556854118704, 0.38344867356679524, 0.3712595970846668, 0.37866875250654886] - - -The following examples show how to time expressions that contain multiple lines. -Here we compare the cost of using :func:`hasattr` vs. :keyword:`try`/:keyword:`except` -to test for missing and present object attributes: - -.. code-block:: shell-session - - $ python -m timeit 'try:' ' str.__bool__' 'except AttributeError:' ' pass' - 20000 loops, best of 5: 15.7 usec per loop - $ python -m timeit 'if hasattr(str, "__bool__"): pass' - 50000 loops, best of 5: 4.26 usec per loop - - $ python -m timeit 'try:' ' int.__bool__' 'except AttributeError:' ' pass' - 200000 loops, best of 5: 1.43 usec per loop - $ python -m timeit 'if hasattr(int, "__bool__"): pass' - 100000 loops, best of 5: 2.23 usec per loop - -:: - - >>> import timeit - >>> # attribute is missing - >>> s = """\ - ... try: - ... str.__bool__ - ... except AttributeError: - ... pass - ... """ - >>> timeit.timeit(stmt=s, number=100000) - 0.9138244460009446 - >>> s = "if hasattr(str, '__bool__'): pass" - >>> timeit.timeit(stmt=s, number=100000) - 0.5829014980008651 - >>> - >>> # attribute is present - >>> s = """\ - ... try: - ... int.__bool__ - ... except AttributeError: - ... pass - ... """ - >>> timeit.timeit(stmt=s, number=100000) - 0.04215312199994514 - >>> s = "if hasattr(int, '__bool__'): pass" - >>> timeit.timeit(stmt=s, number=100000) - 0.08588060699912603 - - -To give the :mod:`timeit` module access to functions you define, you can pass a -*setup* parameter which contains an import statement:: - - def test(): - """Stupid test function""" - L = [i for i in range(100)] - - if __name__ == '__main__': - import timeit - print(timeit.timeit("test()", setup="from __main__ import test")) - -Another option is to pass :func:`globals` to the *globals* parameter, which will cause the code -to be executed within your current global namespace. This can be more convenient -than individually specifying imports:: - - def f(x): - return x**2 - def g(x): - return x**4 - def h(x): - return x**8 - - import timeit - print(timeit.timeit('[func(42) for func in (f,g,h)]', globals=globals())) diff --git a/Doc/library/tk.rst b/Doc/library/tk.rst deleted file mode 100644 index 3dc2130539c2cf..00000000000000 --- a/Doc/library/tk.rst +++ /dev/null @@ -1,47 +0,0 @@ -.. _tkinter: - -********************************* -Graphical User Interfaces with Tk -********************************* - -.. index:: - single: GUI - single: Graphical User Interface - single: Tkinter - single: Tk - -Tk/Tcl has long been an integral part of Python. It provides a robust and -platform independent windowing toolkit, that is available to Python programmers -using the :mod:`tkinter` package, and its extension, the :mod:`tkinter.tix` and -the :mod:`tkinter.ttk` modules. - -The :mod:`tkinter` package is a thin object-oriented layer on top of Tcl/Tk. To -use :mod:`tkinter`, you don't need to write Tcl code, but you will need to -consult the Tk documentation, and occasionally the Tcl documentation. -:mod:`tkinter` is a set of wrappers that implement the Tk widgets as Python -classes. - -:mod:`tkinter`'s chief virtues are that it is fast, and that it usually comes -bundled with Python. Although its standard documentation is weak, good -material is available, which includes: references, tutorials, a book and -others. :mod:`tkinter` is also famous for having an outdated look and feel, -which has been vastly improved in Tk 8.5. Nevertheless, there are many other -GUI libraries that you could be interested in. The Python wiki lists several -alternative `GUI frameworks and tools `_. - -.. toctree:: - - tkinter.rst - tkinter.colorchooser.rst - tkinter.font.rst - dialog.rst - tkinter.messagebox.rst - tkinter.scrolledtext.rst - tkinter.dnd.rst - tkinter.ttk.rst - tkinter.tix.rst - idle.rst - -.. Other sections I have in mind are - Tkinter internals - Freezing Tkinter applications diff --git a/Doc/library/tk_msg.png b/Doc/library/tk_msg.png deleted file mode 100644 index 6495e63e84650d..00000000000000 Binary files a/Doc/library/tk_msg.png and /dev/null differ diff --git a/Doc/library/tkinter.colorchooser.rst b/Doc/library/tkinter.colorchooser.rst deleted file mode 100644 index 6e8479c1dea1e2..00000000000000 --- a/Doc/library/tkinter.colorchooser.rst +++ /dev/null @@ -1,29 +0,0 @@ -:mod:`tkinter.colorchooser` --- Color choosing dialog -===================================================== - -.. module:: tkinter.colorchooser - :platform: Tk - :synopsis: Color choosing dialog - -**Source code:** :source:`Lib/tkinter/colorchooser.py` - --------------- - -The :mod:`tkinter.colorchooser` module provides the :class:`Chooser` class -as an interface to the native color picker dialog. ``Chooser`` implements -a modal color choosing dialog window. The ``Chooser`` class inherits from -the :class:`~tkinter.commondialog.Dialog` class. - -.. class:: Chooser(master=None, **options) - -.. function:: askcolor(color=None, **options) - - Create a color choosing dialog. A call to this method will show the window, - wait for the user to make a selection, and return the selected color (or - ``None``) to the caller. - - -.. seealso:: - - Module :mod:`tkinter.commondialog` - Tkinter standard dialog module diff --git a/Doc/library/tkinter.dnd.rst b/Doc/library/tkinter.dnd.rst deleted file mode 100644 index 02de0fd331958d..00000000000000 --- a/Doc/library/tkinter.dnd.rst +++ /dev/null @@ -1,64 +0,0 @@ -:mod:`tkinter.dnd` --- Drag and drop support -============================================ - -.. module:: tkinter.dnd - :platform: Tk - :synopsis: Tkinter drag-and-drop interface - -**Source code:** :source:`Lib/tkinter/dnd.py` - --------------- - -.. note:: This is experimental and due to be deprecated when it is replaced - with the Tk DND. - -The :mod:`tkinter.dnd` module provides drag-and-drop support for objects within -a single application, within the same window or between windows. To enable an -object to be dragged, you must create an event binding for it that starts the -drag-and-drop process. Typically, you bind a ButtonPress event to a callback -function that you write (see :ref:`Bindings-and-Events`). The function should -call :func:`dnd_start`, where 'source' is the object to be dragged, and 'event' -is the event that invoked the call (the argument to your callback function). - -Selection of a target object occurs as follows: - -#. Top-down search of area under mouse for target widget - - * Target widget should have a callable *dnd_accept* attribute - * If *dnd_accept* is not present or returns None, search moves to parent widget - * If no target widget is found, then the target object is None - -2. Call to *.dnd_leave(source, event)* -#. Call to *.dnd_enter(source, event)* -#. Call to *.dnd_commit(source, event)* to notify of drop -#. Call to *.dnd_end(target, event)* to signal end of drag-and-drop - - -.. class:: DndHandler(source, event) - - The *DndHandler* class handles drag-and-drop events tracking Motion and - ButtonRelease events on the root of the event widget. - - .. method:: cancel(event=None) - - Cancel the drag-and-drop process. - - .. method:: finish(event, commit=0) - - Execute end of drag-and-drop functions. - - .. method:: on_motion(event) - - Inspect area below mouse for target objects while drag is performed. - - .. method:: on_release(event) - - Signal end of drag when the release pattern is triggered. - -.. function:: dnd_start(source, event) - - Factory function for drag-and-drop process. - -.. seealso:: - - :ref:`Bindings-and-Events` diff --git a/Doc/library/tkinter.font.rst b/Doc/library/tkinter.font.rst deleted file mode 100644 index c7c2b7b566cf8f..00000000000000 --- a/Doc/library/tkinter.font.rst +++ /dev/null @@ -1,99 +0,0 @@ -:mod:`tkinter.font` --- Tkinter font wrapper -============================================ - -.. module:: tkinter.font - :platform: Tk - :synopsis: Tkinter font-wrapping class - -**Source code:** :source:`Lib/tkinter/font.py` - --------------- - -The :mod:`tkinter.font` module provides the :class:`Font` class for creating -and using named fonts. - -The different font weights and slants are: - -.. data:: NORMAL - BOLD - ITALIC - ROMAN - -.. class:: Font(root=None, font=None, name=None, exists=False, **options) - - The :class:`Font` class represents a named font. *Font* instances are given - unique names and can be specified by their family, size, and style - configuration. Named fonts are Tk's method of creating and identifying - fonts as a single object, rather than specifying a font by its attributes - with each occurrence. - - arguments: - - | *font* - font specifier tuple (family, size, options) - | *name* - unique font name - | *exists* - self points to existing named font if true - - additional keyword options (ignored if *font* is specified): - - | *family* - font family i.e. Courier, Times - | *size* - font size - | If *size* is positive it is interpreted as size in points. - | If *size* is a negative number its absolute value is treated - | as size in pixels. - | *weight* - font emphasis (NORMAL, BOLD) - | *slant* - ROMAN, ITALIC - | *underline* - font underlining (0 - none, 1 - underline) - | *overstrike* - font strikeout (0 - none, 1 - strikeout) - - .. method:: actual(option=None, displayof=None) - - Return the attributes of the font. - - .. method:: cget(option) - - Retrieve an attribute of the font. - - .. method:: config(**options) - - Modify attributes of the font. - - .. method:: copy() - - Return new instance of the current font. - - .. method:: measure(text, displayof=None) - - Return amount of space the text would occupy on the specified display - when formatted in the current font. If no display is specified then the - main application window is assumed. - - .. method:: metrics(*options, **kw) - - Return font-specific data. - Options include: - - *ascent* - distance between baseline and highest point that a - character of the font can occupy - - *descent* - distance between baseline and lowest point that a - character of the font can occupy - - *linespace* - minimum vertical separation necessary between any two - characters of the font that ensures no vertical overlap between lines. - - *fixed* - 1 if font is fixed-width else 0 - -.. function:: families(root=None, displayof=None) - - Return the different font families. - -.. function:: names(root=None) - - Return the names of defined fonts. - -.. function:: nametofont(name, root=None) - - Return a :class:`Font` representation of a tk named font. - - .. versionchanged:: 3.10 - The *root* parameter was added. diff --git a/Doc/library/tkinter.messagebox.rst b/Doc/library/tkinter.messagebox.rst deleted file mode 100644 index 56090a0a0e424b..00000000000000 --- a/Doc/library/tkinter.messagebox.rst +++ /dev/null @@ -1,194 +0,0 @@ -:mod:`tkinter.messagebox` --- Tkinter message prompts -===================================================== - -.. module:: tkinter.messagebox - :platform: Tk - :synopsis: Various types of alert dialogs - -**Source code:** :source:`Lib/tkinter/messagebox.py` - --------------- - -The :mod:`tkinter.messagebox` module provides a template base class as well as -a variety of convenience methods for commonly used configurations. The message -boxes are modal and will return a subset of (``True``, ``False``, ``None``, -:data:`OK`, :data:`CANCEL`, :data:`YES`, :data:`NO`) based on -the user's selection. Common message box styles and layouts include but are not -limited to: - -.. figure:: tk_msg.png - -.. class:: Message(master=None, **options) - - Create a message window with an application-specified message, an icon - and a set of buttons. - Each of the buttons in the message window is identified by a unique symbolic name (see the *type* options). - - The following options are supported: - - *command* - Specifies the function to invoke when the user closes the dialog. - The name of the button clicked by the user to close the dialog is - passed as argument. - This is only available on macOS. - - *default* - Gives the :ref:`symbolic name ` of the default button - for this message window (:data:`OK`, :data:`CANCEL`, and so on). - If this option is not specified, the first button in the dialog will - be made the default. - - *detail* - Specifies an auxiliary message to the main message given by the - *message* option. - The message detail will be presented beneath the main message and, - where supported by the OS, in a less emphasized font than the main - message. - - *icon* - Specifies an :ref:`icon ` to display. - If this option is not specified, then the :data:`INFO` icon will be - displayed. - - *message* - Specifies the message to display in this message box. - The default value is an empty string. - - *parent* - Makes the specified window the logical parent of the message box. - The message box is displayed on top of its parent window. - - *title* - Specifies a string to display as the title of the message box. - This option is ignored on macOS, where platform guidelines forbid - the use of a title on this kind of dialog. - - *type* - Arranges for a :ref:`predefined set of buttons ` - to be displayed. - - .. method:: show(**options) - - Display a message window and wait for the user to select one of the buttons. Then return the symbolic name of the selected button. - Keyword arguments can override options specified in the constructor. - - -**Information message box** - -.. function:: showinfo(title=None, message=None, **options) - - Creates and displays an information message box with the specified title - and message. - -**Warning message boxes** - -.. function:: showwarning(title=None, message=None, **options) - - Creates and displays a warning message box with the specified title - and message. - -.. function:: showerror(title=None, message=None, **options) - - Creates and displays an error message box with the specified title - and message. - -**Question message boxes** - -.. function:: askquestion(title=None, message=None, *, type=YESNO, **options) - - Ask a question. By default shows buttons :data:`YES` and :data:`NO`. - Returns the symbolic name of the selected button. - -.. function:: askokcancel(title=None, message=None, **options) - - Ask if operation should proceed. Shows buttons :data:`OK` and :data:`CANCEL`. - Returns ``True`` if the answer is ok and ``False`` otherwise. - -.. function:: askretrycancel(title=None, message=None, **options) - - Ask if operation should be retried. Shows buttons :data:`RETRY` and :data:`CANCEL`. - Return ``True`` if the answer is yes and ``False`` otherwise. - -.. function:: askyesno(title=None, message=None, **options) - - Ask a question. Shows buttons :data:`YES` and :data:`NO`. - Returns ``True`` if the answer is yes and ``False`` otherwise. - -.. function:: askyesnocancel(title=None, message=None, **options) - - Ask a question. Shows buttons :data:`YES`, :data:`NO` and :data:`CANCEL`. - Return ``True`` if the answer is yes, ``None`` if cancelled, and ``False`` - otherwise. - - -.. _messagebox-buttons: - -Symbolic names of buttons: - -.. data:: ABORT - :value: 'abort' -.. data:: RETRY - :value: 'retry' -.. data:: IGNORE - :value: 'ignore' -.. data:: OK - :value: 'ok' -.. data:: CANCEL - :value: 'cancel' -.. data:: YES - :value: 'yes' -.. data:: NO - :value: 'no' - -.. _messagebox-types: - -Predefined sets of buttons: - -.. data:: ABORTRETRYIGNORE - :value: 'abortretryignore' - - Displays three buttons whose symbolic names are :data:`ABORT`, - :data:`RETRY` and :data:`IGNORE`. - -.. data:: OK - :value: 'ok' - :noindex: - - Displays one button whose symbolic name is :data:`OK`. - -.. data:: OKCANCEL - :value: 'okcancel' - - Displays two buttons whose symbolic names are :data:`OK` and - :data:`CANCEL`. - -.. data:: RETRYCANCEL - :value: 'retrycancel' - - Displays two buttons whose symbolic names are :data:`RETRY` and - :data:`CANCEL`. - -.. data:: YESNO - :value: 'yesno' - - Displays two buttons whose symbolic names are :data:`YES` and - :data:`NO`. - -.. data:: YESNOCANCEL - :value: 'yesnocancel' - - Displays three buttons whose symbolic names are :data:`YES`, - :data:`NO` and :data:`CANCEL`. - -.. _messagebox-icons: - -Icon images: - -.. data:: ERROR - :value: 'error' -.. data:: INFO - :value: 'info' -.. data:: QUESTION - :value: 'question' -.. data:: WARNING - :value: 'warning' diff --git a/Doc/library/tkinter.rst b/Doc/library/tkinter.rst deleted file mode 100644 index ee34f2659cf3d8..00000000000000 --- a/Doc/library/tkinter.rst +++ /dev/null @@ -1,1041 +0,0 @@ -:mod:`tkinter` --- Python interface to Tcl/Tk -============================================= - -.. module:: tkinter - :synopsis: Interface to Tcl/Tk for graphical user interfaces - -.. moduleauthor:: Guido van Rossum - -**Source code:** :source:`Lib/tkinter/__init__.py` - --------------- - -The :mod:`tkinter` package ("Tk interface") is the standard Python interface to -the Tcl/Tk GUI toolkit. Both Tk and :mod:`tkinter` are available on most Unix -platforms, including macOS, as well as on Windows systems. - -Running ``python -m tkinter`` from the command line should open a window -demonstrating a simple Tk interface, letting you know that :mod:`tkinter` is -properly installed on your system, and also showing what version of Tcl/Tk is -installed, so you can read the Tcl/Tk documentation specific to that version. - -Tkinter supports a range of Tcl/Tk versions, built either with or -without thread support. The official Python binary release bundles Tcl/Tk 8.6 -threaded. See the source code for the :mod:`_tkinter` module -for more information about supported versions. - -Tkinter is not a thin wrapper, but adds a fair amount of its own logic to -make the experience more pythonic. This documentation will concentrate on these -additions and changes, and refer to the official Tcl/Tk documentation for -details that are unchanged. - -.. note:: - - Tcl/Tk 8.5 (2007) introduced a modern set of themed user interface components - along with a new API to use them. Both old and new APIs are still available. - Most documentation you will find online still uses the old API and - can be woefully outdated. - -.. seealso:: - - * `TkDocs `_ - Extensive tutorial on creating user interfaces with Tkinter. Explains key concepts, - and illustrates recommended approaches using the modern API. - - * `Tkinter 8.5 reference: a GUI for Python `_ - Reference documentation for Tkinter 8.5 detailing available classes, methods, and options. - - Tcl/Tk Resources: - - * `Tk commands `_ - Comprehensive reference to each of the underlying Tcl/Tk commands used by Tkinter. - - * `Tcl/Tk Home Page `_ - Additional documentation, and links to Tcl/Tk core development. - - Books: - - * `Modern Tkinter for Busy Python Developers `_ - By Mark Roseman. (ISBN 978-1999149567) - - * `Python and Tkinter Programming `_ - By Alan Moore. (ISBN 978-1788835886) - - * `Programming Python `_ - By Mark Lutz; has excellent coverage of Tkinter. (ISBN 978-0596158101) - - * `Tcl and the Tk Toolkit (2nd edition) `_ - By John Ousterhout, inventor of Tcl/Tk, and Ken Jones; does not cover Tkinter. (ISBN 978-0321336330) - - -Architecture ------------- - -Tcl/Tk is not a single library but rather consists of a few distinct -modules, each with separate functionality and its own official -documentation. Python's binary releases also ship an add-on module -together with it. - -Tcl - Tcl is a dynamic interpreted programming language, just like Python. Though - it can be used on its own as a general-purpose programming language, it is - most commonly embedded into C applications as a scripting engine or an - interface to the Tk toolkit. The Tcl library has a C interface to - create and manage one or more instances of a Tcl interpreter, run Tcl - commands and scripts in those instances, and add custom commands - implemented in either Tcl or C. Each interpreter has an event queue, - and there are facilities to send events to it and process them. - Unlike Python, Tcl's execution model is designed around cooperative - multitasking, and Tkinter bridges this difference - (see `Threading model`_ for details). - -Tk - Tk is a `Tcl package `_ implemented in C - that adds custom commands to create and manipulate GUI widgets. Each - :class:`Tk` object embeds its own Tcl interpreter instance with Tk loaded into - it. Tk's widgets are very customizable, though at the cost of a dated appearance. - Tk uses Tcl's event queue to generate and process GUI events. - -Ttk - Themed Tk (Ttk) is a newer family of Tk widgets that provide a much better - appearance on different platforms than many of the classic Tk widgets. - Ttk is distributed as part of Tk, starting with Tk version 8.5. Python - bindings are provided in a separate module, :mod:`tkinter.ttk`. - -Internally, Tk and Ttk use facilities of the underlying operating system, -i.e., Xlib on Unix/X11, Cocoa on macOS, GDI on Windows. - -When your Python application uses a class in Tkinter, e.g., to create a widget, -the :mod:`tkinter` module first assembles a Tcl/Tk command string. It passes that -Tcl command string to an internal :mod:`_tkinter` binary module, which then -calls the Tcl interpreter to evaluate it. The Tcl interpreter will then call into the -Tk and/or Ttk packages, which will in turn make calls to Xlib, Cocoa, or GDI. - - -Tkinter Modules ---------------- - -Support for Tkinter is spread across several modules. Most applications will need the -main :mod:`tkinter` module, as well as the :mod:`tkinter.ttk` module, which provides -the modern themed widget set and API:: - - - from tkinter import * - from tkinter import ttk - - -.. class:: Tk(screenName=None, baseName=None, className='Tk', useTk=True, sync=False, use=None) - - Construct a toplevel Tk widget, which is usually the main window of an - application, and initialize a Tcl interpreter for this widget. Each - instance has its own associated Tcl interpreter. - - The :class:`Tk` class is typically instantiated using all default values. - However, the following keyword arguments are currently recognized: - - *screenName* - When given (as a string), sets the :envvar:`DISPLAY` environment - variable. (X11 only) - *baseName* - Name of the profile file. By default, *baseName* is derived from the - program name (``sys.argv[0]``). - *className* - Name of the widget class. Used as a profile file and also as the name - with which Tcl is invoked (*argv0* in *interp*). - *useTk* - If ``True``, initialize the Tk subsystem. The :func:`tkinter.Tcl() ` - function sets this to ``False``. - *sync* - If ``True``, execute all X server commands synchronously, so that errors - are reported immediately. Can be used for debugging. (X11 only) - *use* - Specifies the *id* of the window in which to embed the application, - instead of it being created as an independent toplevel window. *id* must - be specified in the same way as the value for the -use option for - toplevel widgets (that is, it has a form like that returned by - :meth:`winfo_id`). - - Note that on some platforms this will only work correctly if *id* refers - to a Tk frame or toplevel that has its -container option enabled. - - :class:`Tk` reads and interprets profile files, named - :file:`.{className}.tcl` and :file:`.{baseName}.tcl`, into the Tcl - interpreter and calls :func:`exec` on the contents of - :file:`.{className}.py` and :file:`.{baseName}.py`. The path for the - profile files is the :envvar:`HOME` environment variable or, if that - isn't defined, then :data:`os.curdir`. - - .. attribute:: tk - - The Tk application object created by instantiating :class:`Tk`. This - provides access to the Tcl interpreter. Each widget that is attached - the same instance of :class:`Tk` has the same value for its :attr:`tk` - attribute. - - .. attribute:: master - - The widget object that contains this widget. For :class:`Tk`, the - *master* is :const:`None` because it is the main window. The terms - *master* and *parent* are similar and sometimes used interchangeably - as argument names; however, calling :meth:`winfo_parent` returns a - string of the widget name whereas :attr:`master` returns the object. - *parent*/*child* reflects the tree-like relationship while - *master*/*slave* reflects the container structure. - - .. attribute:: children - - The immediate descendants of this widget as a :class:`dict` with the - child widget names as the keys and the child instance objects as the - values. - - -.. function:: Tcl(screenName=None, baseName=None, className='Tk', useTk=False) - - The :func:`Tcl` function is a factory function which creates an object much like - that created by the :class:`Tk` class, except that it does not initialize the Tk - subsystem. This is most often useful when driving the Tcl interpreter in an - environment where one doesn't want to create extraneous toplevel windows, or - where one cannot (such as Unix/Linux systems without an X server). An object - created by the :func:`Tcl` object can have a Toplevel window created (and the Tk - subsystem initialized) by calling its :meth:`loadtk` method. - - -The modules that provide Tk support include: - -:mod:`tkinter` - Main Tkinter module. - -:mod:`tkinter.colorchooser` - Dialog to let the user choose a color. - -:mod:`tkinter.commondialog` - Base class for the dialogs defined in the other modules listed here. - -:mod:`tkinter.filedialog` - Common dialogs to allow the user to specify a file to open or save. - -:mod:`tkinter.font` - Utilities to help work with fonts. - -:mod:`tkinter.messagebox` - Access to standard Tk dialog boxes. - -:mod:`tkinter.scrolledtext` - Text widget with a vertical scroll bar built in. - -:mod:`tkinter.simpledialog` - Basic dialogs and convenience functions. - -:mod:`tkinter.ttk` - Themed widget set introduced in Tk 8.5, providing modern alternatives - for many of the classic widgets in the main :mod:`tkinter` module. - -Additional modules: - -:mod:`_tkinter` - A binary module that contains the low-level interface to Tcl/Tk. - It is automatically imported by the main :mod:`tkinter` module, - and should never be used directly by application programmers. - It is usually a shared library (or DLL), but might in some cases be - statically linked with the Python interpreter. - -:mod:`idlelib` - Python's Integrated Development and Learning Environment (IDLE). Based - on :mod:`tkinter`. - -:mod:`tkinter.constants` - Symbolic constants that can be used in place of strings when passing - various parameters to Tkinter calls. Automatically imported by the - main :mod:`tkinter` module. - -:mod:`tkinter.dnd` - (experimental) Drag-and-drop support for :mod:`tkinter`. This will - become deprecated when it is replaced with the Tk DND. - -:mod:`tkinter.tix` - (deprecated) An older third-party Tcl/Tk package that adds several new - widgets. Better alternatives for most can be found in :mod:`tkinter.ttk`. - -:mod:`turtle` - Turtle graphics in a Tk window. - - -Tkinter Life Preserver ----------------------- - -This section is not designed to be an exhaustive tutorial on either Tk or -Tkinter. For that, refer to one of the external resources noted earlier. -Instead, this section provides a very quick orientation to what a Tkinter -application looks like, identifies foundational Tk concepts, and -explains how the Tkinter wrapper is structured. - -The remainder of this section will help you to identify the classes, -methods, and options you'll need in your Tkinter application, and where to -find more detailed documentation on them, including in the official Tcl/Tk -reference manual. - - -A Hello World Program -^^^^^^^^^^^^^^^^^^^^^ - -We'll start by walking through a "Hello World" application in Tkinter. This -isn't the smallest one we could write, but has enough to illustrate some -key concepts you'll need to know. - -:: - - from tkinter import * - from tkinter import ttk - root = Tk() - frm = ttk.Frame(root, padding=10) - frm.grid() - ttk.Label(frm, text="Hello World!").grid(column=0, row=0) - ttk.Button(frm, text="Quit", command=root.destroy).grid(column=1, row=0) - root.mainloop() - - -After the imports, the next line creates an instance of the :class:`Tk` class, -which initializes Tk and creates its associated Tcl interpreter. It also -creates a toplevel window, known as the root window, which serves as the main -window of the application. - -The following line creates a frame widget, which in this case will contain -a label and a button we'll create next. The frame is fit inside the root -window. - -The next line creates a label widget holding a static text string. The -:meth:`grid` method is used to specify the relative layout (position) of the -label within its containing frame widget, similar to how tables in HTML work. - -A button widget is then created, and placed to the right of the label. When -pressed, it will call the :meth:`destroy` method of the root window. - -Finally, the :meth:`mainloop` method puts everything on the display, and -responds to user input until the program terminates. - - - -Important Tk Concepts -^^^^^^^^^^^^^^^^^^^^^ - -Even this simple program illustrates the following key Tk concepts: - -widgets - A Tkinter user interface is made up of individual *widgets*. Each widget is - represented as a Python object, instantiated from classes like - :class:`ttk.Frame`, :class:`ttk.Label`, and :class:`ttk.Button`. - -widget hierarchy - Widgets are arranged in a *hierarchy*. The label and button were contained - within a frame, which in turn was contained within the root window. When - creating each *child* widget, its *parent* widget is passed as the first - argument to the widget constructor. - -configuration options - Widgets have *configuration options*, which modify their appearance and - behavior, such as the text to display in a label or button. Different - classes of widgets will have different sets of options. - -geometry management - Widgets aren't automatically added to the user interface when they are - created. A *geometry manager* like ``grid`` controls where in the - user interface they are placed. - -event loop - Tkinter reacts to user input, changes from your program, and even refreshes - the display only when actively running an *event loop*. If your program - isn't running the event loop, your user interface won't update. - - -Understanding How Tkinter Wraps Tcl/Tk -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -When your application uses Tkinter's classes and methods, internally Tkinter -is assembling strings representing Tcl/Tk commands, and executing those -commands in the Tcl interpreter attached to your application's :class:`Tk` -instance. - -Whether it's trying to navigate reference documentation, trying to find -the right method or option, adapting some existing code, or debugging your -Tkinter application, there are times that it will be useful to understand -what those underlying Tcl/Tk commands look like. - -To illustrate, here is the Tcl/Tk equivalent of the main part of the Tkinter -script above. - -:: - - ttk::frame .frm -padding 10 - grid .frm - grid [ttk::label .frm.lbl -text "Hello World!"] -column 0 -row 0 - grid [ttk::button .frm.btn -text "Quit" -command "destroy ."] -column 1 -row 0 - - -Tcl's syntax is similar to many shell languages, where the first word is the -command to be executed, with arguments to that command following it, separated -by spaces. Without getting into too many details, notice the following: - -* The commands used to create widgets (like ``ttk::frame``) correspond to - widget classes in Tkinter. - -* Tcl widget options (like ``-text``) correspond to keyword arguments in - Tkinter. - -* Widgets are referred to by a *pathname* in Tcl (like ``.frm.btn``), - whereas Tkinter doesn't use names but object references. - -* A widget's place in the widget hierarchy is encoded in its (hierarchical) - pathname, which uses a ``.`` (dot) as a path separator. The pathname for - the root window is just ``.`` (dot). In Tkinter, the hierarchy is defined - not by pathname but by specifying the parent widget when creating each - child widget. - -* Operations which are implemented as separate *commands* in Tcl (like - ``grid`` or ``destroy``) are represented as *methods* on Tkinter widget - objects. As you'll see shortly, at other times Tcl uses what appear to be - method calls on widget objects, which more closely mirror what would is - used in Tkinter. - - -How do I...? What option does...? -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If you're not sure how to do something in Tkinter, and you can't immediately -find it in the tutorial or reference documentation you're using, there are a -few strategies that can be helpful. - -First, remember that the details of how individual widgets work may vary -across different versions of both Tkinter and Tcl/Tk. If you're searching -documentation, make sure it corresponds to the Python and Tcl/Tk versions -installed on your system. - -When searching for how to use an API, it helps to know the exact name of the -class, option, or method that you're using. Introspection, either in an -interactive Python shell or with :func:`print`, can help you identify what -you need. - -To find out what configuration options are available on any widget, call its -:meth:`configure` method, which returns a dictionary containing a variety of -information about each object, including its default and current values. Use -:meth:`keys` to get just the names of each option. - -:: - - btn = ttk.Button(frm, ...) - print(btn.configure().keys()) - -As most widgets have many configuration options in common, it can be useful -to find out which are specific to a particular widget class. Comparing the -list of options to that of a simpler widget, like a frame, is one way to -do that. - -:: - - print(set(btn.configure().keys()) - set(frm.configure().keys())) - -Similarly, you can find the available methods for a widget object using the -standard :func:`dir` function. If you try it, you'll see there are over 200 -common widget methods, so again identifying those specific to a widget class -is helpful. - -:: - - print(dir(btn)) - print(set(dir(btn)) - set(dir(frm))) - - -Navigating the Tcl/Tk Reference Manual -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -As noted, the official `Tk commands `_ -reference manual (man pages) is often the most accurate description of what -specific operations on widgets do. Even when you know the name of the option -or method that you need, you may still have a few places to look. - -While all operations in Tkinter are implemented as method calls on widget -objects, you've seen that many Tcl/Tk operations appear as commands that -take a widget pathname as its first parameter, followed by optional -parameters, e.g. - -:: - - destroy . - grid .frm.btn -column 0 -row 0 - -Others, however, look more like methods called on a widget object (in fact, -when you create a widget in Tcl/Tk, it creates a Tcl command with the name -of the widget pathname, with the first parameter to that command being the -name of a method to call). - -:: - - .frm.btn invoke - .frm.lbl configure -text "Goodbye" - - -In the official Tcl/Tk reference documentation, you'll find most operations -that look like method calls on the man page for a specific widget (e.g., -you'll find the :meth:`invoke` method on the -`ttk::button `_ -man page), while functions that take a widget as a parameter often have -their own man page (e.g., -`grid `_). - -You'll find many common options and methods in the -`options `_ or -`ttk::widget `_ man -pages, while others are found in the man page for a specific widget class. - -You'll also find that many Tkinter methods have compound names, e.g., -:func:`winfo_x`, :func:`winfo_height`, :func:`winfo_viewable`. You'd find -documentation for all of these in the -`winfo `_ man page. - -.. note:: - Somewhat confusingly, there are also methods on all Tkinter widgets - that don't actually operate on the widget, but operate at a global - scope, independent of any widget. Examples are methods for accessing - the clipboard or the system bell. (They happen to be implemented as - methods in the base :class:`Widget` class that all Tkinter widgets - inherit from). - - -Threading model ---------------- - -Python and Tcl/Tk have very different threading models, which :mod:`tkinter` -tries to bridge. If you use threads, you may need to be aware of this. - -A Python interpreter may have many threads associated with it. In Tcl, multiple -threads can be created, but each thread has a separate Tcl interpreter instance -associated with it. Threads can also create more than one interpreter instance, -though each interpreter instance can be used only by the one thread that created it. - -Each :class:`Tk` object created by :mod:`tkinter` contains a Tcl interpreter. -It also keeps track of which thread created that interpreter. Calls to -:mod:`tkinter` can be made from any Python thread. Internally, if a call comes -from a thread other than the one that created the :class:`Tk` object, an event -is posted to the interpreter's event queue, and when executed, the result is -returned to the calling Python thread. - -Tcl/Tk applications are normally event-driven, meaning that after initialization, -the interpreter runs an event loop (i.e. :func:`Tk.mainloop`) and responds to events. -Because it is single-threaded, event handlers must respond quickly, otherwise they -will block other events from being processed. To avoid this, any long-running -computations should not run in an event handler, but are either broken into smaller -pieces using timers, or run in another thread. This is different from many GUI -toolkits where the GUI runs in a completely separate thread from all application -code including event handlers. - -If the Tcl interpreter is not running the event loop and processing events, any -:mod:`tkinter` calls made from threads other than the one running the Tcl -interpreter will fail. - -A number of special cases exist: - -* Tcl/Tk libraries can be built so they are not thread-aware. In this case, - :mod:`tkinter` calls the library from the originating Python thread, even - if this is different than the thread that created the Tcl interpreter. A global - lock ensures only one call occurs at a time. - -* While :mod:`tkinter` allows you to create more than one instance of a :class:`Tk` - object (with its own interpreter), all interpreters that are part of the same - thread share a common event queue, which gets ugly fast. In practice, don't create - more than one instance of :class:`Tk` at a time. Otherwise, it's best to create - them in separate threads and ensure you're running a thread-aware Tcl/Tk build. - -* Blocking event handlers are not the only way to prevent the Tcl interpreter from - reentering the event loop. It is even possible to run multiple nested event loops - or abandon the event loop entirely. If you're doing anything tricky when it comes - to events or threads, be aware of these possibilities. - -* There are a few select :mod:`tkinter` functions that presently work only when - called from the thread that created the Tcl interpreter. - - -Handy Reference ---------------- - - -.. _tkinter-setting-options: - -Setting Options -^^^^^^^^^^^^^^^ - -Options control things like the color and border width of a widget. Options can -be set in three ways: - -At object creation time, using keyword arguments - :: - - fred = Button(self, fg="red", bg="blue") - -After object creation, treating the option name like a dictionary index - :: - - fred["fg"] = "red" - fred["bg"] = "blue" - -Use the config() method to update multiple attrs subsequent to object creation - :: - - fred.config(fg="red", bg="blue") - -For a complete explanation of a given option and its behavior, see the Tk man -pages for the widget in question. - -Note that the man pages list "STANDARD OPTIONS" and "WIDGET SPECIFIC OPTIONS" -for each widget. The former is a list of options that are common to many -widgets, the latter are the options that are idiosyncratic to that particular -widget. The Standard Options are documented on the :manpage:`options(3)` man -page. - -No distinction between standard and widget-specific options is made in this -document. Some options don't apply to some kinds of widgets. Whether a given -widget responds to a particular option depends on the class of the widget; -buttons have a ``command`` option, labels do not. - -The options supported by a given widget are listed in that widget's man page, or -can be queried at runtime by calling the :meth:`config` method without -arguments, or by calling the :meth:`keys` method on that widget. The return -value of these calls is a dictionary whose key is the name of the option as a -string (for example, ``'relief'``) and whose values are 5-tuples. - -Some options, like ``bg`` are synonyms for common options with long names -(``bg`` is shorthand for "background"). Passing the ``config()`` method the name -of a shorthand option will return a 2-tuple, not 5-tuple. The 2-tuple passed -back will contain the name of the synonym and the "real" option (such as -``('bg', 'background')``). - -+-------+---------------------------------+--------------+ -| Index | Meaning | Example | -+=======+=================================+==============+ -| 0 | option name | ``'relief'`` | -+-------+---------------------------------+--------------+ -| 1 | option name for database lookup | ``'relief'`` | -+-------+---------------------------------+--------------+ -| 2 | option class for database | ``'Relief'`` | -| | lookup | | -+-------+---------------------------------+--------------+ -| 3 | default value | ``'raised'`` | -+-------+---------------------------------+--------------+ -| 4 | current value | ``'groove'`` | -+-------+---------------------------------+--------------+ - -Example:: - - >>> print(fred.config()) - {'relief': ('relief', 'relief', 'Relief', 'raised', 'groove')} - -Of course, the dictionary printed will include all the options available and -their values. This is meant only as an example. - - -The Packer -^^^^^^^^^^ - -.. index:: single: packing (widgets) - -The packer is one of Tk's geometry-management mechanisms. Geometry managers -are used to specify the relative positioning of widgets within their container - -their mutual *master*. In contrast to the more cumbersome *placer* (which is -used less commonly, and we do not cover here), the packer takes qualitative -relationship specification - *above*, *to the left of*, *filling*, etc - and -works everything out to determine the exact placement coordinates for you. - -The size of any *master* widget is determined by the size of the "slave widgets" -inside. The packer is used to control where slave widgets appear inside the -master into which they are packed. You can pack widgets into frames, and frames -into other frames, in order to achieve the kind of layout you desire. -Additionally, the arrangement is dynamically adjusted to accommodate incremental -changes to the configuration, once it is packed. - -Note that widgets do not appear until they have had their geometry specified -with a geometry manager. It's a common early mistake to leave out the geometry -specification, and then be surprised when the widget is created but nothing -appears. A widget will appear only after it has had, for example, the packer's -:meth:`pack` method applied to it. - -The pack() method can be called with keyword-option/value pairs that control -where the widget is to appear within its container, and how it is to behave when -the main application window is resized. Here are some examples:: - - fred.pack() # defaults to side = "top" - fred.pack(side="left") - fred.pack(expand=1) - - -Packer Options -^^^^^^^^^^^^^^ - -For more extensive information on the packer and the options that it can take, -see the man pages and page 183 of John Ousterhout's book. - -anchor - Anchor type. Denotes where the packer is to place each slave in its parcel. - -expand - Boolean, ``0`` or ``1``. - -fill - Legal values: ``'x'``, ``'y'``, ``'both'``, ``'none'``. - -ipadx and ipady - A distance - designating internal padding on each side of the slave widget. - -padx and pady - A distance - designating external padding on each side of the slave widget. - -side - Legal values are: ``'left'``, ``'right'``, ``'top'``, ``'bottom'``. - - -Coupling Widget Variables -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The current-value setting of some widgets (like text entry widgets) can be -connected directly to application variables by using special options. These -options are ``variable``, ``textvariable``, ``onvalue``, ``offvalue``, and -``value``. This connection works both ways: if the variable changes for any -reason, the widget it's connected to will be updated to reflect the new value. - -Unfortunately, in the current implementation of :mod:`tkinter` it is not -possible to hand over an arbitrary Python variable to a widget through a -``variable`` or ``textvariable`` option. The only kinds of variables for which -this works are variables that are subclassed from a class called Variable, -defined in :mod:`tkinter`. - -There are many useful subclasses of Variable already defined: -:class:`StringVar`, :class:`IntVar`, :class:`DoubleVar`, and -:class:`BooleanVar`. To read the current value of such a variable, call the -:meth:`get` method on it, and to change its value you call the :meth:`!set` -method. If you follow this protocol, the widget will always track the value of -the variable, with no further intervention on your part. - -For example:: - - import tkinter as tk - - class App(tk.Frame): - def __init__(self, master): - super().__init__(master) - self.pack() - - self.entrythingy = tk.Entry() - self.entrythingy.pack() - - # Create the application variable. - self.contents = tk.StringVar() - # Set it to some value. - self.contents.set("this is a variable") - # Tell the entry widget to watch this variable. - self.entrythingy["textvariable"] = self.contents - - # Define a callback for when the user hits return. - # It prints the current value of the variable. - self.entrythingy.bind('', - self.print_contents) - - def print_contents(self, event): - print("Hi. The current entry content is:", - self.contents.get()) - - root = tk.Tk() - myapp = App(root) - myapp.mainloop() - -The Window Manager -^^^^^^^^^^^^^^^^^^ - -.. index:: single: window manager (widgets) - -In Tk, there is a utility command, ``wm``, for interacting with the window -manager. Options to the ``wm`` command allow you to control things like titles, -placement, icon bitmaps, and the like. In :mod:`tkinter`, these commands have -been implemented as methods on the :class:`Wm` class. Toplevel widgets are -subclassed from the :class:`Wm` class, and so can call the :class:`Wm` methods -directly. - -To get at the toplevel window that contains a given widget, you can often just -refer to the widget's master. Of course if the widget has been packed inside of -a frame, the master won't represent a toplevel window. To get at the toplevel -window that contains an arbitrary widget, you can call the :meth:`_root` method. -This method begins with an underscore to denote the fact that this function is -part of the implementation, and not an interface to Tk functionality. - -Here are some examples of typical usage:: - - import tkinter as tk - - class App(tk.Frame): - def __init__(self, master=None): - super().__init__(master) - self.pack() - - # create the application - myapp = App() - - # - # here are method calls to the window manager class - # - myapp.master.title("My Do-Nothing Application") - myapp.master.maxsize(1000, 400) - - # start the program - myapp.mainloop() - - -Tk Option Data Types -^^^^^^^^^^^^^^^^^^^^ - -.. index:: single: Tk Option Data Types - -anchor - Legal values are points of the compass: ``"n"``, ``"ne"``, ``"e"``, ``"se"``, - ``"s"``, ``"sw"``, ``"w"``, ``"nw"``, and also ``"center"``. - -bitmap - There are eight built-in, named bitmaps: ``'error'``, ``'gray25'``, - ``'gray50'``, ``'hourglass'``, ``'info'``, ``'questhead'``, ``'question'``, - ``'warning'``. To specify an X bitmap filename, give the full path to the file, - preceded with an ``@``, as in ``"@/usr/contrib/bitmap/gumby.bit"``. - -boolean - You can pass integers 0 or 1 or the strings ``"yes"`` or ``"no"``. - -callback - This is any Python function that takes no arguments. For example:: - - def print_it(): - print("hi there") - fred["command"] = print_it - -color - Colors can be given as the names of X colors in the rgb.txt file, or as strings - representing RGB values in 4 bit: ``"#RGB"``, 8 bit: ``"#RRGGBB"``, 12 bit: - ``"#RRRGGGBBB"``, or 16 bit: ``"#RRRRGGGGBBBB"`` ranges, where R,G,B here - represent any legal hex digit. See page 160 of Ousterhout's book for details. - -cursor - The standard X cursor names from :file:`cursorfont.h` can be used, without the - ``XC_`` prefix. For example to get a hand cursor (:const:`XC_hand2`), use the - string ``"hand2"``. You can also specify a bitmap and mask file of your own. - See page 179 of Ousterhout's book. - -distance - Screen distances can be specified in either pixels or absolute distances. - Pixels are given as numbers and absolute distances as strings, with the trailing - character denoting units: ``c`` for centimetres, ``i`` for inches, ``m`` for - millimetres, ``p`` for printer's points. For example, 3.5 inches is expressed - as ``"3.5i"``. - -font - Tk uses a list font name format, such as ``{courier 10 bold}``. Font sizes with - positive numbers are measured in points; sizes with negative numbers are - measured in pixels. - -geometry - This is a string of the form ``widthxheight``, where width and height are - measured in pixels for most widgets (in characters for widgets displaying text). - For example: ``fred["geometry"] = "200x100"``. - -justify - Legal values are the strings: ``"left"``, ``"center"``, ``"right"``, and - ``"fill"``. - -region - This is a string with four space-delimited elements, each of which is a legal - distance (see above). For example: ``"2 3 4 5"`` and ``"3i 2i 4.5i 2i"`` and - ``"3c 2c 4c 10.43c"`` are all legal regions. - -relief - Determines what the border style of a widget will be. Legal values are: - ``"raised"``, ``"sunken"``, ``"flat"``, ``"groove"``, and ``"ridge"``. - -scrollcommand - This is almost always the :meth:`!set` method of some scrollbar widget, but can - be any widget method that takes a single argument. - -wrap - Must be one of: ``"none"``, ``"char"``, or ``"word"``. - -.. _Bindings-and-Events: - -Bindings and Events -^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: bind (widgets) - single: events (widgets) - -The bind method from the widget command allows you to watch for certain events -and to have a callback function trigger when that event type occurs. The form -of the bind method is:: - - def bind(self, sequence, func, add=''): - -where: - -sequence - is a string that denotes the target kind of event. (See the - :manpage:`bind(3tk)` man page, and page 201 of John Ousterhout's book, - :title-reference:`Tcl and the Tk Toolkit (2nd edition)`, for details). - -func - is a Python function, taking one argument, to be invoked when the event occurs. - An Event instance will be passed as the argument. (Functions deployed this way - are commonly known as *callbacks*.) - -add - is optional, either ``''`` or ``'+'``. Passing an empty string denotes that - this binding is to replace any other bindings that this event is associated - with. Passing a ``'+'`` means that this function is to be added to the list - of functions bound to this event type. - -For example:: - - def turn_red(self, event): - event.widget["activeforeground"] = "red" - - self.button.bind("", self.turn_red) - -Notice how the widget field of the event is being accessed in the -``turn_red()`` callback. This field contains the widget that caught the X -event. The following table lists the other event fields you can access, and how -they are denoted in Tk, which can be useful when referring to the Tk man pages. - -+----+---------------------+----+---------------------+ -| Tk | Tkinter Event Field | Tk | Tkinter Event Field | -+====+=====================+====+=====================+ -| %f | focus | %A | char | -+----+---------------------+----+---------------------+ -| %h | height | %E | send_event | -+----+---------------------+----+---------------------+ -| %k | keycode | %K | keysym | -+----+---------------------+----+---------------------+ -| %s | state | %N | keysym_num | -+----+---------------------+----+---------------------+ -| %t | time | %T | type | -+----+---------------------+----+---------------------+ -| %w | width | %W | widget | -+----+---------------------+----+---------------------+ -| %x | x | %X | x_root | -+----+---------------------+----+---------------------+ -| %y | y | %Y | y_root | -+----+---------------------+----+---------------------+ - - -The index Parameter -^^^^^^^^^^^^^^^^^^^ - -A number of widgets require "index" parameters to be passed. These are used to -point at a specific place in a Text widget, or to particular characters in an -Entry widget, or to particular menu items in a Menu widget. - -Entry widget indexes (index, view index, etc.) - Entry widgets have options that refer to character positions in the text being - displayed. You can use these :mod:`tkinter` functions to access these special - points in text widgets: - -Text widget indexes - The index notation for Text widgets is very rich and is best described in the Tk - man pages. - -Menu indexes (menu.invoke(), menu.entryconfig(), etc.) - Some options and methods for menus manipulate specific menu entries. Anytime a - menu index is needed for an option or a parameter, you may pass in: - - * an integer which refers to the numeric position of the entry in the widget, - counted from the top, starting with 0; - - * the string ``"active"``, which refers to the menu position that is currently - under the cursor; - - * the string ``"last"`` which refers to the last menu item; - - * An integer preceded by ``@``, as in ``@6``, where the integer is interpreted - as a y pixel coordinate in the menu's coordinate system; - - * the string ``"none"``, which indicates no menu entry at all, most often used - with menu.activate() to deactivate all entries, and finally, - - * a text string that is pattern matched against the label of the menu entry, as - scanned from the top of the menu to the bottom. Note that this index type is - considered after all the others, which means that matches for menu items - labelled ``last``, ``active``, or ``none`` may be interpreted as the above - literals, instead. - - -Images -^^^^^^ - -Images of different formats can be created through the corresponding subclass -of :class:`tkinter.Image`: - -* :class:`BitmapImage` for images in XBM format. - -* :class:`PhotoImage` for images in PGM, PPM, GIF and PNG formats. The latter - is supported starting with Tk 8.6. - -Either type of image is created through either the ``file`` or the ``data`` -option (other options are available as well). - -The image object can then be used wherever an ``image`` option is supported by -some widget (e.g. labels, buttons, menus). In these cases, Tk will not keep a -reference to the image. When the last Python reference to the image object is -deleted, the image data is deleted as well, and Tk will display an empty box -wherever the image was used. - -.. seealso:: - - The `Pillow `_ package adds support for - formats such as BMP, JPEG, TIFF, and WebP, among others. - -.. _tkinter-file-handlers: - -File Handlers -------------- - -Tk allows you to register and unregister a callback function which will be -called from the Tk mainloop when I/O is possible on a file descriptor. -Only one handler may be registered per file descriptor. Example code:: - - import tkinter - widget = tkinter.Tk() - mask = tkinter.READABLE | tkinter.WRITABLE - widget.tk.createfilehandler(file, mask, callback) - ... - widget.tk.deletefilehandler(file) - -This feature is not available on Windows. - -Since you don't know how many bytes are available for reading, you may not -want to use the :class:`~io.BufferedIOBase` or :class:`~io.TextIOBase` -:meth:`~io.BufferedIOBase.read` or :meth:`~io.IOBase.readline` methods, -since these will insist on reading a predefined number of bytes. -For sockets, the :meth:`~socket.socket.recv` or -:meth:`~socket.socket.recvfrom` methods will work fine; for other files, -use raw reads or ``os.read(file.fileno(), maxbytecount)``. - - -.. method:: Widget.tk.createfilehandler(file, mask, func) - - Registers the file handler callback function *func*. The *file* argument - may either be an object with a :meth:`~io.IOBase.fileno` method (such as - a file or socket object), or an integer file descriptor. The *mask* - argument is an ORed combination of any of the three constants below. - The callback is called as follows:: - - callback(file, mask) - - -.. method:: Widget.tk.deletefilehandler(file) - - Unregisters a file handler. - - -.. data:: READABLE - WRITABLE - EXCEPTION - - Constants used in the *mask* arguments. diff --git a/Doc/library/tkinter.scrolledtext.rst b/Doc/library/tkinter.scrolledtext.rst deleted file mode 100644 index d20365baa38690..00000000000000 --- a/Doc/library/tkinter.scrolledtext.rst +++ /dev/null @@ -1,37 +0,0 @@ -:mod:`tkinter.scrolledtext` --- Scrolled Text Widget -==================================================== - -.. module:: tkinter.scrolledtext - :platform: Tk - :synopsis: Text widget with a vertical scroll bar. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/tkinter/scrolledtext.py` - --------------- - -The :mod:`tkinter.scrolledtext` module provides a class of the same name which -implements a basic text widget which has a vertical scroll bar configured to do -the "right thing." Using the :class:`ScrolledText` class is a lot easier than -setting up a text widget and scroll bar directly. - -The text widget and scrollbar are packed together in a :class:`Frame`, and the -methods of the :class:`Grid` and :class:`Pack` geometry managers are acquired -from the :class:`Frame` object. This allows the :class:`ScrolledText` widget to -be used directly to achieve most normal geometry management behavior. - -Should more specific control be necessary, the following attributes are -available: - -.. class:: ScrolledText(master=None, **kw) - - - .. attribute:: frame - - The frame which surrounds the text and scroll bar widgets. - - - .. attribute:: vbar - - The scroll bar widget. diff --git a/Doc/library/tkinter.tix.rst b/Doc/library/tkinter.tix.rst deleted file mode 100644 index c86fcfa6a3f46d..00000000000000 --- a/Doc/library/tkinter.tix.rst +++ /dev/null @@ -1,583 +0,0 @@ -:mod:`tkinter.tix` --- Extension widgets for Tk -=============================================== - -.. module:: tkinter.tix - :synopsis: Tk Extension Widgets for Tkinter - -.. sectionauthor:: Mike Clarkson - -**Source code:** :source:`Lib/tkinter/tix.py` - -.. index:: single: Tix - -.. deprecated:: 3.6 - This Tk extension is unmaintained and should not be used in new code. Use - :mod:`tkinter.ttk` instead. - --------------- - -The :mod:`tkinter.tix` (Tk Interface Extension) module provides an additional -rich set of widgets. Although the standard Tk library has many useful widgets, -they are far from complete. The :mod:`tkinter.tix` library provides most of the -commonly needed widgets that are missing from standard Tk: :class:`HList`, -:class:`ComboBox`, :class:`Control` (a.k.a. SpinBox) and an assortment of -scrollable widgets. -:mod:`tkinter.tix` also includes many more widgets that are generally useful in -a wide range of applications: :class:`NoteBook`, :class:`FileEntry`, -:class:`PanedWindow`, etc; there are more than 40 of them. - -With all these new widgets, you can introduce new interaction techniques into -applications, creating more useful and more intuitive user interfaces. You can -design your application by choosing the most appropriate widgets to match the -special needs of your application and users. - -.. seealso:: - - `Tix Homepage `_ - The home page for :mod:`Tix`. This includes links to additional documentation - and downloads. - - `Tix Man Pages `_ - On-line version of the man pages and reference material. - - `Tix Programming Guide `_ - On-line version of the programmer's reference material. - - `Tix Development Applications `_ - Tix applications for development of Tix and Tkinter programs. Tide applications - work under Tk or Tkinter, and include :program:`TixInspect`, an inspector to - remotely modify and debug Tix/Tk/Tkinter applications. - - -Using Tix ---------- - - -.. class:: Tk(screenName=None, baseName=None, className='Tix') - - Toplevel widget of Tix which represents mostly the main window of an - application. It has an associated Tcl interpreter. - - Classes in the :mod:`tkinter.tix` module subclasses the classes in the - :mod:`tkinter`. The former imports the latter, so to use :mod:`tkinter.tix` - with Tkinter, all you need to do is to import one module. In general, you - can just import :mod:`tkinter.tix`, and replace the toplevel call to - :class:`tkinter.Tk` with :class:`tix.Tk`:: - - from tkinter import tix - from tkinter.constants import * - root = tix.Tk() - -To use :mod:`tkinter.tix`, you must have the Tix widgets installed, usually -alongside your installation of the Tk widgets. To test your installation, try -the following:: - - from tkinter import tix - root = tix.Tk() - root.tk.eval('package require Tix') - - -Tix Widgets ------------ - -`Tix `_ -introduces over 40 widget classes to the :mod:`tkinter` repertoire. - - -Basic Widgets -^^^^^^^^^^^^^ - - -.. class:: Balloon() - - A `Balloon - `_ that - pops up over a widget to provide help. When the user moves the cursor inside a - widget to which a Balloon widget has been bound, a small pop-up window with a - descriptive message will be shown on the screen. - -.. Python Demo of: -.. \ulink{Balloon}{https://tix.sourceforge.net/dist/current/demos/samples/Balloon.tcl} - - -.. class:: ButtonBox() - - The `ButtonBox - `_ - widget creates a box of buttons, such as is commonly used for ``Ok Cancel``. - -.. Python Demo of: -.. \ulink{ButtonBox}{https://tix.sourceforge.net/dist/current/demos/samples/BtnBox.tcl} - - -.. class:: ComboBox() - - The `ComboBox - `_ - widget is similar to the combo box control in MS Windows. The user can select a - choice by either typing in the entry subwidget or selecting from the listbox - subwidget. - -.. Python Demo of: -.. \ulink{ComboBox}{https://tix.sourceforge.net/dist/current/demos/samples/ComboBox.tcl} - - -.. class:: Control() - - The `Control - `_ - widget is also known as the :class:`SpinBox` widget. The user can adjust the - value by pressing the two arrow buttons or by entering the value directly into - the entry. The new value will be checked against the user-defined upper and - lower limits. - -.. Python Demo of: -.. \ulink{Control}{https://tix.sourceforge.net/dist/current/demos/samples/Control.tcl} - - -.. class:: LabelEntry() - - The `LabelEntry - `_ - widget packages an entry widget and a label into one mega widget. It can - be used to simplify the creation of "entry-form" type of interface. - -.. Python Demo of: -.. \ulink{LabelEntry}{https://tix.sourceforge.net/dist/current/demos/samples/LabEntry.tcl} - - -.. class:: LabelFrame() - - The `LabelFrame - `_ - widget packages a frame widget and a label into one mega widget. To create - widgets inside a LabelFrame widget, one creates the new widgets relative to the - :attr:`frame` subwidget and manage them inside the :attr:`frame` subwidget. - -.. Python Demo of: -.. \ulink{LabelFrame}{https://tix.sourceforge.net/dist/current/demos/samples/LabFrame.tcl} - - -.. class:: Meter() - - The `Meter - `_ widget - can be used to show the progress of a background job which may take a long time - to execute. - -.. Python Demo of: -.. \ulink{Meter}{https://tix.sourceforge.net/dist/current/demos/samples/Meter.tcl} - - -.. class:: OptionMenu() - - The `OptionMenu - `_ - creates a menu button of options. - -.. Python Demo of: -.. \ulink{OptionMenu}{https://tix.sourceforge.net/dist/current/demos/samples/OptMenu.tcl} - - -.. class:: PopupMenu() - - The `PopupMenu - `_ - widget can be used as a replacement of the ``tk_popup`` command. The advantage - of the :mod:`Tix` :class:`PopupMenu` widget is it requires less application code - to manipulate. - -.. Python Demo of: -.. \ulink{PopupMenu}{https://tix.sourceforge.net/dist/current/demos/samples/PopMenu.tcl} - - -.. class:: Select() - - The `Select - `_ widget - is a container of button subwidgets. It can be used to provide radio-box or - check-box style of selection options for the user. - -.. Python Demo of: -.. \ulink{Select}{https://tix.sourceforge.net/dist/current/demos/samples/Select.tcl} - - -.. class:: StdButtonBox() - - The `StdButtonBox - `_ - widget is a group of standard buttons for Motif-like dialog boxes. - -.. Python Demo of: -.. \ulink{StdButtonBox}{https://tix.sourceforge.net/dist/current/demos/samples/StdBBox.tcl} - - -File Selectors -^^^^^^^^^^^^^^ - - -.. class:: DirList() - - The `DirList - `_ - widget displays a list view of a directory, its previous directories and its - sub-directories. The user can choose one of the directories displayed in the - list or change to another directory. - -.. Python Demo of: -.. \ulink{DirList}{https://tix.sourceforge.net/dist/current/demos/samples/DirList.tcl} - - -.. class:: DirTree() - - The `DirTree - `_ - widget displays a tree view of a directory, its previous directories and its - sub-directories. The user can choose one of the directories displayed in the - list or change to another directory. - -.. Python Demo of: -.. \ulink{DirTree}{https://tix.sourceforge.net/dist/current/demos/samples/DirTree.tcl} - - -.. class:: DirSelectDialog() - - The `DirSelectDialog - `_ - widget presents the directories in the file system in a dialog window. The user - can use this dialog window to navigate through the file system to select the - desired directory. - -.. Python Demo of: -.. \ulink{DirSelectDialog}{https://tix.sourceforge.net/dist/current/demos/samples/DirDlg.tcl} - - -.. class:: DirSelectBox() - - The :class:`DirSelectBox` is similar to the standard Motif(TM) - directory-selection box. It is generally used for the user to choose a - directory. DirSelectBox stores the directories mostly recently selected into - a ComboBox widget so that they can be quickly selected again. - - -.. class:: ExFileSelectBox() - - The `ExFileSelectBox - `_ - widget is usually embedded in a tixExFileSelectDialog widget. It provides a - convenient method for the user to select files. The style of the - :class:`ExFileSelectBox` widget is very similar to the standard file dialog on - MS Windows 3.1. - -.. Python Demo of: -.. \ulink{ExFileSelectDialog}{https://tix.sourceforge.net/dist/current/demos/samples/EFileDlg.tcl} - - -.. class:: FileSelectBox() - - The `FileSelectBox - `_ - is similar to the standard Motif(TM) file-selection box. It is generally used - for the user to choose a file. FileSelectBox stores the files mostly recently - selected into a :class:`ComboBox` widget so that they can be quickly selected - again. - -.. Python Demo of: -.. \ulink{FileSelectDialog}{https://tix.sourceforge.net/dist/current/demos/samples/FileDlg.tcl} - - -.. class:: FileEntry() - - The `FileEntry - `_ - widget can be used to input a filename. The user can type in the filename - manually. Alternatively, the user can press the button widget that sits next to - the entry, which will bring up a file selection dialog. - -.. Python Demo of: -.. \ulink{FileEntry}{https://tix.sourceforge.net/dist/current/demos/samples/FileEnt.tcl} - - -Hierarchical ListBox -^^^^^^^^^^^^^^^^^^^^ - - -.. class:: HList() - - The `HList - `_ widget - can be used to display any data that have a hierarchical structure, for example, - file system directory trees. The list entries are indented and connected by - branch lines according to their places in the hierarchy. - -.. Python Demo of: -.. \ulink{HList}{https://tix.sourceforge.net/dist/current/demos/samples/HList1.tcl} - - -.. class:: CheckList() - - The `CheckList - `_ - widget displays a list of items to be selected by the user. CheckList acts - similarly to the Tk checkbutton or radiobutton widgets, except it is capable of - handling many more items than checkbuttons or radiobuttons. - -.. Python Demo of: -.. \ulink{ CheckList}{https://tix.sourceforge.net/dist/current/demos/samples/ChkList.tcl} -.. Python Demo of: -.. \ulink{ScrolledHList (1)}{https://tix.sourceforge.net/dist/current/demos/samples/SHList.tcl} -.. Python Demo of: -.. \ulink{ScrolledHList (2)}{https://tix.sourceforge.net/dist/current/demos/samples/SHList2.tcl} - - -.. class:: Tree() - - The `Tree - `_ widget - can be used to display hierarchical data in a tree form. The user can adjust the - view of the tree by opening or closing parts of the tree. - -.. Python Demo of: -.. \ulink{Tree}{https://tix.sourceforge.net/dist/current/demos/samples/Tree.tcl} -.. Python Demo of: -.. \ulink{Tree (Dynamic)}{https://tix.sourceforge.net/dist/current/demos/samples/DynTree.tcl} - - -Tabular ListBox -^^^^^^^^^^^^^^^ - - -.. class:: TList() - - The `TList - `_ widget - can be used to display data in a tabular format. The list entries of a - :class:`TList` widget are similar to the entries in the Tk listbox widget. The - main differences are (1) the :class:`TList` widget can display the list entries - in a two dimensional format and (2) you can use graphical images as well as - multiple colors and fonts for the list entries. - -.. Python Demo of: -.. \ulink{ScrolledTList (1)}{https://tix.sourceforge.net/dist/current/demos/samples/STList1.tcl} -.. Python Demo of: -.. \ulink{ScrolledTList (2)}{https://tix.sourceforge.net/dist/current/demos/samples/STList2.tcl} -.. Grid has yet to be added to Python -.. \subsubsection{Grid Widget} -.. Python Demo of: -.. \ulink{Simple Grid}{https://tix.sourceforge.net/dist/current/demos/samples/SGrid0.tcl} -.. Python Demo of: -.. \ulink{ScrolledGrid}{https://tix.sourceforge.net/dist/current/demos/samples/SGrid1.tcl} -.. Python Demo of: -.. \ulink{Editable Grid}{https://tix.sourceforge.net/dist/current/demos/samples/EditGrid.tcl} - - -Manager Widgets -^^^^^^^^^^^^^^^ - - -.. class:: PanedWindow() - - The `PanedWindow - `_ - widget allows the user to interactively manipulate the sizes of several panes. - The panes can be arranged either vertically or horizontally. The user changes - the sizes of the panes by dragging the resize handle between two panes. - -.. Python Demo of: -.. \ulink{PanedWindow}{https://tix.sourceforge.net/dist/current/demos/samples/PanedWin.tcl} - - -.. class:: ListNoteBook() - - The `ListNoteBook - `_ - widget is very similar to the :class:`TixNoteBook` widget: it can be used to - display many windows in a limited space using a notebook metaphor. The notebook - is divided into a stack of pages (windows). At one time only one of these pages - can be shown. The user can navigate through these pages by choosing the name of - the desired page in the :attr:`hlist` subwidget. - -.. Python Demo of: -.. \ulink{ListNoteBook}{https://tix.sourceforge.net/dist/current/demos/samples/ListNBK.tcl} - - -.. class:: NoteBook() - - The `NoteBook - `_ - widget can be used to display many windows in a limited space using a notebook - metaphor. The notebook is divided into a stack of pages. At one time only one of - these pages can be shown. The user can navigate through these pages by choosing - the visual "tabs" at the top of the NoteBook widget. - -.. Python Demo of: -.. \ulink{NoteBook}{https://tix.sourceforge.net/dist/current/demos/samples/NoteBook.tcl} - -.. \subsubsection{Scrolled Widgets} -.. Python Demo of: -.. \ulink{ScrolledListBox}{https://tix.sourceforge.net/dist/current/demos/samples/SListBox.tcl} -.. Python Demo of: -.. \ulink{ScrolledText}{https://tix.sourceforge.net/dist/current/demos/samples/SText.tcl} -.. Python Demo of: -.. \ulink{ScrolledWindow}{https://tix.sourceforge.net/dist/current/demos/samples/SWindow.tcl} -.. Python Demo of: -.. \ulink{Canvas Object View}{https://tix.sourceforge.net/dist/current/demos/samples/CObjView.tcl} - - -Image Types -^^^^^^^^^^^ - -The :mod:`tkinter.tix` module adds: - -* `pixmap `_ - capabilities to all :mod:`tkinter.tix` and :mod:`tkinter` widgets to create - color images from XPM files. - - .. Python Demo of: - .. \ulink{XPM Image In Button}{https://tix.sourceforge.net/dist/current/demos/samples/Xpm.tcl} - .. Python Demo of: - .. \ulink{XPM Image In Menu}{https://tix.sourceforge.net/dist/current/demos/samples/Xpm1.tcl} - -* `Compound - `_ image - types can be used to create images that consists of multiple horizontal lines; - each line is composed of a series of items (texts, bitmaps, images or spaces) - arranged from left to right. For example, a compound image can be used to - display a bitmap and a text string simultaneously in a Tk :class:`Button` - widget. - - .. Python Demo of: - .. \ulink{Compound Image In Buttons}{https://tix.sourceforge.net/dist/current/demos/samples/CmpImg.tcl} - .. Python Demo of: - .. \ulink{Compound Image In NoteBook}{https://tix.sourceforge.net/dist/current/demos/samples/CmpImg2.tcl} - .. Python Demo of: - .. \ulink{Compound Image Notebook Color Tabs}{https://tix.sourceforge.net/dist/current/demos/samples/CmpImg4.tcl} - .. Python Demo of: - .. \ulink{Compound Image Icons}{https://tix.sourceforge.net/dist/current/demos/samples/CmpImg3.tcl} - - -Miscellaneous Widgets -^^^^^^^^^^^^^^^^^^^^^ - - -.. class:: InputOnly() - - The `InputOnly - `_ - widgets are to accept inputs from the user, which can be done with the ``bind`` - command (Unix only). - - -Form Geometry Manager -^^^^^^^^^^^^^^^^^^^^^ - -In addition, :mod:`tkinter.tix` augments :mod:`tkinter` by providing: - - -.. class:: Form() - - The `Form - `_ geometry - manager based on attachment rules for all Tk widgets. - - -Tix Commands ------------- - - -.. class:: tixCommand() - - The `tix commands - `_ provide - access to miscellaneous elements of :mod:`Tix`'s internal state and the - :mod:`Tix` application context. Most of the information manipulated by these - methods pertains to the application as a whole, or to a screen or display, - rather than to a particular window. - - To view the current settings, the common usage is:: - - from tkinter import tix - root = tix.Tk() - print(root.tix_configure()) - - -.. method:: tixCommand.tix_configure(cnf=None, **kw) - - Query or modify the configuration options of the Tix application context. If no - option is specified, returns a dictionary all of the available options. If - option is specified with no value, then the method returns a list describing the - one named option (this list will be identical to the corresponding sublist of - the value returned if no option is specified). If one or more option-value - pairs are specified, then the method modifies the given option(s) to have the - given value(s); in this case the method returns an empty string. Option may be - any of the configuration options. - - -.. method:: tixCommand.tix_cget(option) - - Returns the current value of the configuration option given by *option*. Option - may be any of the configuration options. - - -.. method:: tixCommand.tix_getbitmap(name) - - Locates a bitmap file of the name ``name.xpm`` or ``name`` in one of the bitmap - directories (see the :meth:`tix_addbitmapdir` method). By using - :meth:`tix_getbitmap`, you can avoid hard coding the pathnames of the bitmap - files in your application. When successful, it returns the complete pathname of - the bitmap file, prefixed with the character ``@``. The returned value can be - used to configure the ``bitmap`` option of the Tk and Tix widgets. - - -.. method:: tixCommand.tix_addbitmapdir(directory) - - Tix maintains a list of directories under which the :meth:`tix_getimage` and - :meth:`tix_getbitmap` methods will search for image files. The standard bitmap - directory is :file:`$TIX_LIBRARY/bitmaps`. The :meth:`tix_addbitmapdir` method - adds *directory* into this list. By using this method, the image files of an - applications can also be located using the :meth:`tix_getimage` or - :meth:`tix_getbitmap` method. - - -.. method:: tixCommand.tix_filedialog([dlgclass]) - - Returns the file selection dialog that may be shared among different calls from - this application. This method will create a file selection dialog widget when - it is called the first time. This dialog will be returned by all subsequent - calls to :meth:`tix_filedialog`. An optional dlgclass parameter can be passed - as a string to specified what type of file selection dialog widget is desired. - Possible options are ``tix``, ``FileSelectDialog`` or ``tixExFileSelectDialog``. - - -.. method:: tixCommand.tix_getimage(self, name) - - Locates an image file of the name :file:`name.xpm`, :file:`name.xbm` or - :file:`name.ppm` in one of the bitmap directories (see the - :meth:`tix_addbitmapdir` method above). If more than one file with the same name - (but different extensions) exist, then the image type is chosen according to the - depth of the X display: xbm images are chosen on monochrome displays and color - images are chosen on color displays. By using :meth:`tix_getimage`, you can - avoid hard coding the pathnames of the image files in your application. When - successful, this method returns the name of the newly created image, which can - be used to configure the ``image`` option of the Tk and Tix widgets. - - -.. method:: tixCommand.tix_option_get(name) - - Gets the options maintained by the Tix scheme mechanism. - - -.. method:: tixCommand.tix_resetoptions(newScheme, newFontSet[, newScmPrio]) - - Resets the scheme and fontset of the Tix application to *newScheme* and - *newFontSet*, respectively. This affects only those widgets created after this - call. Therefore, it is best to call the resetoptions method before the creation - of any widgets in a Tix application. - - The optional parameter *newScmPrio* can be given to reset the priority level of - the Tk options set by the Tix schemes. - - Because of the way Tk handles the X option database, after Tix has been has - imported and inited, it is not possible to reset the color schemes and font sets - using the :meth:`tix_config` method. Instead, the :meth:`tix_resetoptions` - method must be used. diff --git a/Doc/library/tkinter.ttk.rst b/Doc/library/tkinter.ttk.rst deleted file mode 100644 index dc31a1a4c1850a..00000000000000 --- a/Doc/library/tkinter.ttk.rst +++ /dev/null @@ -1,1525 +0,0 @@ -:mod:`tkinter.ttk` --- Tk themed widgets -======================================== - -.. module:: tkinter.ttk - :synopsis: Tk themed widget set - -.. sectionauthor:: Guilherme Polo - -**Source code:** :source:`Lib/tkinter/ttk.py` - -.. index:: single: ttk - --------------- - -The :mod:`tkinter.ttk` module provides access to the Tk themed widget set, -introduced in Tk 8.5. It provides additional benefits including anti-aliased font -rendering under X11 and window transparency (requiring a composition -window manager on X11). - -The basic idea for :mod:`tkinter.ttk` is to separate, to the extent possible, -the code implementing a widget's behavior from the code implementing its -appearance. - - -.. seealso:: - - `Tk Widget Styling Support `_ - A document introducing theming support for Tk - - -Using Ttk ---------- - -To start using Ttk, import its module:: - - from tkinter import ttk - -To override the basic Tk widgets, the import should follow the Tk import:: - - from tkinter import * - from tkinter.ttk import * - -That code causes several :mod:`tkinter.ttk` widgets (:class:`Button`, -:class:`Checkbutton`, :class:`Entry`, :class:`Frame`, :class:`Label`, -:class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, -:class:`Radiobutton`, :class:`Scale` and :class:`Scrollbar`) to -automatically replace the Tk widgets. - -This has the direct benefit of using the new widgets which gives a better -look and feel across platforms; however, the replacement widgets are not -completely compatible. The main difference is that widget options such as -"fg", "bg" and others related to widget styling are no -longer present in Ttk widgets. Instead, use the :class:`ttk.Style` class -for improved styling effects. - - -.. seealso:: - - `Converting existing applications to use Tile widgets `_ - A monograph (using Tcl terminology) about differences typically - encountered when moving applications to use the new widgets. - - -Ttk Widgets ------------ - -Ttk comes with 18 widgets, twelve of which already existed in tkinter: -:class:`Button`, :class:`Checkbutton`, :class:`Entry`, :class:`Frame`, -:class:`Label`, :class:`LabelFrame`, :class:`Menubutton`, :class:`PanedWindow`, -:class:`Radiobutton`, :class:`Scale`, :class:`Scrollbar`, and :class:`Spinbox`. -The other six are new: :class:`Combobox`, :class:`Notebook`, -:class:`Progressbar`, :class:`Separator`, :class:`Sizegrip` and -:class:`Treeview`. And all them are subclasses of :class:`Widget`. - -Using the Ttk widgets gives the application an improved look and feel. -As discussed above, there are differences in how the styling is coded. - -Tk code:: - - l1 = tkinter.Label(text="Test", fg="black", bg="white") - l2 = tkinter.Label(text="Test", fg="black", bg="white") - - -Ttk code:: - - style = ttk.Style() - style.configure("BW.TLabel", foreground="black", background="white") - - l1 = ttk.Label(text="Test", style="BW.TLabel") - l2 = ttk.Label(text="Test", style="BW.TLabel") - -For more information about TtkStyling_, see the :class:`Style` class -documentation. - -Widget ------- - -:class:`ttk.Widget` defines standard options and methods supported by Tk -themed widgets and is not supposed to be directly instantiated. - - -Standard Options -^^^^^^^^^^^^^^^^ - -All the :mod:`ttk` Widgets accept the following options: - -.. tabularcolumns:: |l|L| - -+-----------+--------------------------------------------------------------+ -| Option | Description | -+===========+==============================================================+ -| class | Specifies the window class. The class is used when querying | -| | the option database for the window's other options, to | -| | determine the default bindtags for the window, and to select | -| | the widget's default layout and style. This option is | -| | read-only, and may only be specified when the window is | -| | created. | -+-----------+--------------------------------------------------------------+ -| cursor | Specifies the mouse cursor to be used for the widget. If set | -| | to the empty string (the default), the cursor is inherited | -| | for the parent widget. | -+-----------+--------------------------------------------------------------+ -| takefocus | Determines whether the window accepts the focus during | -| | keyboard traversal. 0, 1 or an empty string is returned. | -| | If 0 is returned, it means that the window should be skipped | -| | entirely during keyboard traversal. If 1, it means that the | -| | window should receive the input focus as long as it is | -| | viewable. And an empty string means that the traversal | -| | scripts make the decision about whether or not to focus | -| | on the window. | -+-----------+--------------------------------------------------------------+ -| style | May be used to specify a custom widget style. | -+-----------+--------------------------------------------------------------+ - - -Scrollable Widget Options -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The following options are supported by widgets that are controlled by a -scrollbar. - -.. tabularcolumns:: |l|L| - -+----------------+---------------------------------------------------------+ -| Option | Description | -+================+=========================================================+ -| xscrollcommand | Used to communicate with horizontal scrollbars. | -| | | -| | When the view in the widget's window change, the widget | -| | will generate a Tcl command based on the scrollcommand. | -| | | -| | Usually this option consists of the method | -| | :meth:`Scrollbar.set` of some scrollbar. This will cause| -| | the scrollbar to be updated whenever the view in the | -| | window changes. | -+----------------+---------------------------------------------------------+ -| yscrollcommand | Used to communicate with vertical scrollbars. | -| | For some more information, see above. | -+----------------+---------------------------------------------------------+ - - -Label Options -^^^^^^^^^^^^^ - -The following options are supported by labels, buttons and other button-like -widgets. - -.. tabularcolumns:: |l|p{0.7\linewidth}| - -+--------------+-----------------------------------------------------------+ -| Option | Description | -+==============+===========================================================+ -| text | Specifies a text string to be displayed inside the widget.| -+--------------+-----------------------------------------------------------+ -| textvariable | Specifies a name whose value will be used in place of the | -| | text option resource. | -+--------------+-----------------------------------------------------------+ -| underline | If set, specifies the index (0-based) of a character to | -| | underline in the text string. The underline character is | -| | used for mnemonic activation. | -+--------------+-----------------------------------------------------------+ -| image | Specifies an image to display. This is a list of 1 or more| -| | elements. The first element is the default image name. The| -| | rest of the list if a sequence of statespec/value pairs as| -| | defined by :meth:`Style.map`, specifying different images | -| | to use when the widget is in a particular state or a | -| | combination of states. All images in the list should have | -| | the same size. | -+--------------+-----------------------------------------------------------+ -| compound | Specifies how to display the image relative to the text, | -| | in the case both text and images options are present. | -| | Valid values are: | -| | | -| | * text: display text only | -| | * image: display image only | -| | * top, bottom, left, right: display image above, below, | -| | left of, or right of the text, respectively. | -| | * none: the default. display the image if present, | -| | otherwise the text. | -+--------------+-----------------------------------------------------------+ -| width | If greater than zero, specifies how much space, in | -| | character widths, to allocate for the text label, if less | -| | than zero, specifies a minimum width. If zero or | -| | unspecified, the natural width of the text label is used. | -+--------------+-----------------------------------------------------------+ - - -Compatibility Options -^^^^^^^^^^^^^^^^^^^^^ - -.. tabularcolumns:: |l|L| - -+--------+----------------------------------------------------------------+ -| Option | Description | -+========+================================================================+ -| state | May be set to "normal" or "disabled" to control the "disabled" | -| | state bit. This is a write-only option: setting it changes the | -| | widget state, but the :meth:`Widget.state` method does not | -| | affect this option. | -+--------+----------------------------------------------------------------+ - -Widget States -^^^^^^^^^^^^^ - -The widget state is a bitmap of independent state flags. - -.. tabularcolumns:: |l|L| - -+------------+-------------------------------------------------------------+ -| Flag | Description | -+============+=============================================================+ -| active | The mouse cursor is over the widget and pressing a mouse | -| | button will cause some action to occur | -+------------+-------------------------------------------------------------+ -| disabled | Widget is disabled under program control | -+------------+-------------------------------------------------------------+ -| focus | Widget has keyboard focus | -+------------+-------------------------------------------------------------+ -| pressed | Widget is being pressed | -+------------+-------------------------------------------------------------+ -| selected | "On", "true", or "current" for things like Checkbuttons and | -| | radiobuttons | -+------------+-------------------------------------------------------------+ -| background | Windows and Mac have a notion of an "active" or foreground | -| | window. The *background* state is set for widgets in a | -| | background window, and cleared for those in the foreground | -| | window | -+------------+-------------------------------------------------------------+ -| readonly | Widget should not allow user modification | -+------------+-------------------------------------------------------------+ -| alternate | A widget-specific alternate display format | -+------------+-------------------------------------------------------------+ -| invalid | The widget's value is invalid | -+------------+-------------------------------------------------------------+ - -A state specification is a sequence of state names, optionally prefixed with -an exclamation point indicating that the bit is off. - - -ttk.Widget -^^^^^^^^^^ - -Besides the methods described below, the :class:`ttk.Widget` supports the -methods :meth:`tkinter.Widget.cget` and :meth:`tkinter.Widget.configure`. - -.. class:: Widget - - .. method:: identify(x, y) - - Returns the name of the element at position *x* *y*, or the empty string - if the point does not lie within any element. - - *x* and *y* are pixel coordinates relative to the widget. - - - .. method:: instate(statespec, callback=None, *args, **kw) - - Test the widget's state. If a callback is not specified, returns ``True`` - if the widget state matches *statespec* and ``False`` otherwise. If callback - is specified then it is called with args if widget state matches - *statespec*. - - - .. method:: state(statespec=None) - - Modify or inquire widget state. If *statespec* is specified, sets the - widget state according to it and return a new *statespec* indicating - which flags were changed. If *statespec* is not specified, returns - the currently enabled state flags. - - *statespec* will usually be a list or a tuple. - - -Combobox --------- - -The :class:`ttk.Combobox` widget combines a text field with a pop-down list of -values. This widget is a subclass of :class:`Entry`. - -Besides the methods inherited from :class:`Widget`: :meth:`Widget.cget`, -:meth:`Widget.configure`, :meth:`Widget.identify`, :meth:`Widget.instate` -and :meth:`Widget.state`, and the following inherited from :class:`Entry`: -:meth:`Entry.bbox`, :meth:`Entry.delete`, :meth:`Entry.icursor`, -:meth:`Entry.index`, :meth:`Entry.insert`, :meth:`Entry.selection`, -:meth:`Entry.xview`, it has some other methods, described at -:class:`ttk.Combobox`. - - -Options -^^^^^^^ - -This widget accepts the following specific options: - -.. tabularcolumns:: |l|L| - -+-----------------+--------------------------------------------------------+ -| Option | Description | -+=================+========================================================+ -| exportselection | Boolean value. If set, the widget selection is linked | -| | to the Window Manager selection (which can be returned | -| | by invoking Misc.selection_get, for example). | -+-----------------+--------------------------------------------------------+ -| justify | Specifies how the text is aligned within the widget. | -| | One of "left", "center", or "right". | -+-----------------+--------------------------------------------------------+ -| height | Specifies the height of the pop-down listbox, in rows. | -+-----------------+--------------------------------------------------------+ -| postcommand | A script (possibly registered with Misc.register) that | -| | is called immediately before displaying the values. It | -| | may specify which values to display. | -+-----------------+--------------------------------------------------------+ -| state | One of "normal", "readonly", or "disabled". In the | -| | "readonly" state, the value may not be edited directly,| -| | and the user can only selection of the values from the | -| | dropdown list. In the "normal" state, the text field is| -| | directly editable. In the "disabled" state, no | -| | interaction is possible. | -+-----------------+--------------------------------------------------------+ -| textvariable | Specifies a name whose value is linked to the widget | -| | value. Whenever the value associated with that name | -| | changes, the widget value is updated, and vice versa. | -| | See :class:`tkinter.StringVar`. | -+-----------------+--------------------------------------------------------+ -| values | Specifies the list of values to display in the | -| | drop-down listbox. | -+-----------------+--------------------------------------------------------+ -| width | Specifies an integer value indicating the desired width| -| | of the entry window, in average-size characters of the | -| | widget's font. | -+-----------------+--------------------------------------------------------+ - - -Virtual events -^^^^^^^^^^^^^^ - -The combobox widgets generates a **<>** virtual event -when the user selects an element from the list of values. - - -ttk.Combobox -^^^^^^^^^^^^ - -.. class:: Combobox - - .. method:: current(newindex=None) - - If *newindex* is specified, sets the combobox value to the element - position *newindex*. Otherwise, returns the index of the current value or - -1 if the current value is not in the values list. - - - .. method:: get() - - Returns the current value of the combobox. - - - .. method:: set(value) - - Sets the value of the combobox to *value*. - - -Spinbox -------- -The :class:`ttk.Spinbox` widget is a :class:`ttk.Entry` enhanced with increment -and decrement arrows. It can be used for numbers or lists of string values. -This widget is a subclass of :class:`Entry`. - -Besides the methods inherited from :class:`Widget`: :meth:`Widget.cget`, -:meth:`Widget.configure`, :meth:`Widget.identify`, :meth:`Widget.instate` -and :meth:`Widget.state`, and the following inherited from :class:`Entry`: -:meth:`Entry.bbox`, :meth:`Entry.delete`, :meth:`Entry.icursor`, -:meth:`Entry.index`, :meth:`Entry.insert`, :meth:`Entry.xview`, -it has some other methods, described at :class:`ttk.Spinbox`. - -Options -^^^^^^^ - -This widget accepts the following specific options: - -.. tabularcolumns:: |l|L| - -+----------------------+------------------------------------------------------+ -| Option | Description | -+======================+======================================================+ -| from | Float value. If set, this is the minimum value to | -| | which the decrement button will decrement. Must be | -| | spelled as ``from_`` when used as an argument, since | -| | ``from`` is a Python keyword. | -+----------------------+------------------------------------------------------+ -| to | Float value. If set, this is the maximum value to | -| | which the increment button will increment. | -+----------------------+------------------------------------------------------+ -| increment | Float value. Specifies the amount which the | -| | increment/decrement buttons change the | -| | value. Defaults to 1.0. | -+----------------------+------------------------------------------------------+ -| values | Sequence of string or float values. If specified, | -| | the increment/decrement buttons will cycle through | -| | the items in this sequence rather than incrementing | -| | or decrementing numbers. | -| | | -+----------------------+------------------------------------------------------+ -| wrap | Boolean value. If ``True``, increment and decrement | -| | buttons will cycle from the ``to`` value to the | -| | ``from`` value or the ``from`` value to the ``to`` | -| | value, respectively. | -+----------------------+------------------------------------------------------+ -| format | String value. This specifies the format of numbers | -| | set by the increment/decrement buttons. It must be | -| | in the form "%W.Pf", where W is the padded width of | -| | the value, P is the precision, and '%' and 'f' are | -| | literal. | -+----------------------+------------------------------------------------------+ -| command | Python callable. Will be called with no arguments | -| | whenever either of the increment or decrement buttons| -| | are pressed. | -| | | -+----------------------+------------------------------------------------------+ - - -Virtual events -^^^^^^^^^^^^^^ - -The spinbox widget generates an **<>** virtual event when the -user presses , and a **<>** virtual event when the user -presses . - -ttk.Spinbox -^^^^^^^^^^^^ - -.. class:: Spinbox - - .. method:: get() - - Returns the current value of the spinbox. - - - .. method:: set(value) - - Sets the value of the spinbox to *value*. - - -Notebook --------- - -Ttk Notebook widget manages a collection of windows and displays a single -one at a time. Each child window is associated with a tab, which the user -may select to change the currently displayed window. - - -Options -^^^^^^^ - -This widget accepts the following specific options: - -.. tabularcolumns:: |l|L| - -+---------+----------------------------------------------------------------+ -| Option | Description | -+=========+================================================================+ -| height | If present and greater than zero, specifies the desired height | -| | of the pane area (not including internal padding or tabs). | -| | Otherwise, the maximum height of all panes is used. | -+---------+----------------------------------------------------------------+ -| padding | Specifies the amount of extra space to add around the outside | -| | of the notebook. The padding is a list up to four length | -| | specifications left top right bottom. If fewer than four | -| | elements are specified, bottom defaults to top, right defaults | -| | to left, and top defaults to left. | -+---------+----------------------------------------------------------------+ -| width | If present and greater than zero, specified the desired width | -| | of the pane area (not including internal padding). Otherwise, | -| | the maximum width of all panes is used. | -+---------+----------------------------------------------------------------+ - - -Tab Options -^^^^^^^^^^^ - -There are also specific options for tabs: - -.. tabularcolumns:: |l|L| - -+-----------+--------------------------------------------------------------+ -| Option | Description | -+===========+==============================================================+ -| state | Either "normal", "disabled" or "hidden". If "disabled", then | -| | the tab is not selectable. If "hidden", then the tab is not | -| | shown. | -+-----------+--------------------------------------------------------------+ -| sticky | Specifies how the child window is positioned within the pane | -| | area. Value is a string containing zero or more of the | -| | characters "n", "s", "e" or "w". Each letter refers to a | -| | side (north, south, east or west) that the child window will | -| | stick to, as per the :meth:`grid` geometry manager. | -+-----------+--------------------------------------------------------------+ -| padding | Specifies the amount of extra space to add between the | -| | notebook and this pane. Syntax is the same as for the option | -| | padding used by this widget. | -+-----------+--------------------------------------------------------------+ -| text | Specifies a text to be displayed in the tab. | -+-----------+--------------------------------------------------------------+ -| image | Specifies an image to display in the tab. See the option | -| | image described in :class:`Widget`. | -+-----------+--------------------------------------------------------------+ -| compound | Specifies how to display the image relative to the text, in | -| | the case both options text and image are present. See | -| | `Label Options`_ for legal values. | -+-----------+--------------------------------------------------------------+ -| underline | Specifies the index (0-based) of a character to underline in | -| | the text string. The underlined character is used for | -| | mnemonic activation if :meth:`Notebook.enable_traversal` is | -| | called. | -+-----------+--------------------------------------------------------------+ - - -Tab Identifiers -^^^^^^^^^^^^^^^ - -The tab_id present in several methods of :class:`ttk.Notebook` may take any -of the following forms: - -* An integer between zero and the number of tabs -* The name of a child window -* A positional specification of the form "@x,y", which identifies the tab -* The literal string "current", which identifies the currently selected tab -* The literal string "end", which returns the number of tabs (only valid for - :meth:`Notebook.index`) - - -Virtual Events -^^^^^^^^^^^^^^ - -This widget generates a **<>** virtual event after a new -tab is selected. - - -ttk.Notebook -^^^^^^^^^^^^ - -.. class:: Notebook - - .. method:: add(child, **kw) - - Adds a new tab to the notebook. - - If window is currently managed by the notebook but hidden, it is - restored to its previous position. - - See `Tab Options`_ for the list of available options. - - - .. method:: forget(tab_id) - - Removes the tab specified by *tab_id*, unmaps and unmanages the - associated window. - - - .. method:: hide(tab_id) - - Hides the tab specified by *tab_id*. - - The tab will not be displayed, but the associated window remains - managed by the notebook and its configuration remembered. Hidden tabs - may be restored with the :meth:`add` command. - - - .. method:: identify(x, y) - - Returns the name of the tab element at position *x*, *y*, or the empty - string if none. - - - .. method:: index(tab_id) - - Returns the numeric index of the tab specified by *tab_id*, or the total - number of tabs if *tab_id* is the string "end". - - - .. method:: insert(pos, child, **kw) - - Inserts a pane at the specified position. - - *pos* is either the string "end", an integer index, or the name of a - managed child. If *child* is already managed by the notebook, moves it to - the specified position. - - See `Tab Options`_ for the list of available options. - - - .. method:: select(tab_id=None) - - Selects the specified *tab_id*. - - The associated child window will be displayed, and the - previously selected window (if different) is unmapped. If *tab_id* is - omitted, returns the widget name of the currently selected pane. - - - .. method:: tab(tab_id, option=None, **kw) - - Query or modify the options of the specific *tab_id*. - - If *kw* is not given, returns a dictionary of the tab option values. If - *option* is specified, returns the value of that *option*. Otherwise, - sets the options to the corresponding values. - - - .. method:: tabs() - - Returns a list of windows managed by the notebook. - - - .. method:: enable_traversal() - - Enable keyboard traversal for a toplevel window containing this notebook. - - This will extend the bindings for the toplevel window containing the - notebook as follows: - - * :kbd:`Control-Tab`: selects the tab following the currently selected one. - * :kbd:`Shift-Control-Tab`: selects the tab preceding the currently selected one. - * :kbd:`Alt-K`: where *K* is the mnemonic (underlined) character of any tab, will - select that tab. - - Multiple notebooks in a single toplevel may be enabled for traversal, - including nested notebooks. However, notebook traversal only works - properly if all panes have the notebook they are in as master. - - -Progressbar ------------ - -The :class:`ttk.Progressbar` widget shows the status of a long-running -operation. It can operate in two modes: 1) the determinate mode which shows the -amount completed relative to the total amount of work to be done and 2) the -indeterminate mode which provides an animated display to let the user know that -work is progressing. - - -Options -^^^^^^^ - -This widget accepts the following specific options: - -.. tabularcolumns:: |l|L| - -+----------+---------------------------------------------------------------+ -| Option | Description | -+==========+===============================================================+ -| orient | One of "horizontal" or "vertical". Specifies the orientation | -| | of the progress bar. | -+----------+---------------------------------------------------------------+ -| length | Specifies the length of the long axis of the progress bar | -| | (width if horizontal, height if vertical). | -+----------+---------------------------------------------------------------+ -| mode | One of "determinate" or "indeterminate". | -+----------+---------------------------------------------------------------+ -| maximum | A number specifying the maximum value. Defaults to 100. | -+----------+---------------------------------------------------------------+ -| value | The current value of the progress bar. In "determinate" mode, | -| | this represents the amount of work completed. In | -| | "indeterminate" mode, it is interpreted as modulo *maximum*; | -| | that is, the progress bar completes one "cycle" when its value| -| | increases by *maximum*. | -+----------+---------------------------------------------------------------+ -| variable | A name which is linked to the option value. If specified, the | -| | value of the progress bar is automatically set to the value of| -| | this name whenever the latter is modified. | -+----------+---------------------------------------------------------------+ -| phase | Read-only option. The widget periodically increments the value| -| | of this option whenever its value is greater than 0 and, in | -| | determinate mode, less than maximum. This option may be used | -| | by the current theme to provide additional animation effects. | -+----------+---------------------------------------------------------------+ - - -ttk.Progressbar -^^^^^^^^^^^^^^^ - -.. class:: Progressbar - - .. method:: start(interval=None) - - Begin autoincrement mode: schedules a recurring timer event that calls - :meth:`Progressbar.step` every *interval* milliseconds. If omitted, - *interval* defaults to 50 milliseconds. - - - .. method:: step(amount=None) - - Increments the progress bar's value by *amount*. - - *amount* defaults to 1.0 if omitted. - - - .. method:: stop() - - Stop autoincrement mode: cancels any recurring timer event initiated by - :meth:`Progressbar.start` for this progress bar. - - -Separator ---------- - -The :class:`ttk.Separator` widget displays a horizontal or vertical separator -bar. - -It has no other methods besides the ones inherited from :class:`ttk.Widget`. - - -Options -^^^^^^^ - -This widget accepts the following specific option: - -.. tabularcolumns:: |l|L| - -+--------+----------------------------------------------------------------+ -| Option | Description | -+========+================================================================+ -| orient | One of "horizontal" or "vertical". Specifies the orientation of| -| | the separator. | -+--------+----------------------------------------------------------------+ - - -Sizegrip --------- - -The :class:`ttk.Sizegrip` widget (also known as a grow box) allows the user to -resize the containing toplevel window by pressing and dragging the grip. - -This widget has neither specific options nor specific methods, besides the -ones inherited from :class:`ttk.Widget`. - - -Platform-specific notes -^^^^^^^^^^^^^^^^^^^^^^^ - -* On macOS, toplevel windows automatically include a built-in size grip - by default. Adding a :class:`Sizegrip` is harmless, since the built-in - grip will just mask the widget. - - -Bugs -^^^^ - -* If the containing toplevel's position was specified relative to the right - or bottom of the screen (e.g. ....), the :class:`Sizegrip` widget will - not resize the window. -* This widget supports only "southeast" resizing. - - -Treeview --------- - -The :class:`ttk.Treeview` widget displays a hierarchical collection of items. -Each item has a textual label, an optional image, and an optional list of data -values. The data values are displayed in successive columns after the tree -label. - -The order in which data values are displayed may be controlled by setting -the widget option ``displaycolumns``. The tree widget can also display column -headings. Columns may be accessed by number or symbolic names listed in the -widget option columns. See `Column Identifiers`_. - -Each item is identified by a unique name. The widget will generate item IDs -if they are not supplied by the caller. There is a distinguished root item, -named ``{}``. The root item itself is not displayed; its children appear at the -top level of the hierarchy. - -Each item also has a list of tags, which can be used to associate event bindings -with individual items and control the appearance of the item. - -The Treeview widget supports horizontal and vertical scrolling, according to -the options described in `Scrollable Widget Options`_ and the methods -:meth:`Treeview.xview` and :meth:`Treeview.yview`. - - -Options -^^^^^^^ - -This widget accepts the following specific options: - -.. tabularcolumns:: |l|p{0.7\linewidth}| - -+----------------+--------------------------------------------------------+ -| Option | Description | -+================+========================================================+ -| columns | A list of column identifiers, specifying the number of | -| | columns and their names. | -+----------------+--------------------------------------------------------+ -| displaycolumns | A list of column identifiers (either symbolic or | -| | integer indices) specifying which data columns are | -| | displayed and the order in which they appear, or the | -| | string "#all". | -+----------------+--------------------------------------------------------+ -| height | Specifies the number of rows which should be visible. | -| | Note: the requested width is determined from the sum | -| | of the column widths. | -+----------------+--------------------------------------------------------+ -| padding | Specifies the internal padding for the widget. The | -| | padding is a list of up to four length specifications. | -+----------------+--------------------------------------------------------+ -| selectmode | Controls how the built-in class bindings manage the | -| | selection. One of "extended", "browse" or "none". | -| | If set to "extended" (the default), multiple items may | -| | be selected. If "browse", only a single item will be | -| | selected at a time. If "none", the selection will not | -| | be changed. | -| | | -| | Note that the application code and tag bindings can set| -| | the selection however they wish, regardless of the | -| | value of this option. | -+----------------+--------------------------------------------------------+ -| show | A list containing zero or more of the following values,| -| | specifying which elements of the tree to display. | -| | | -| | * tree: display tree labels in column #0. | -| | * headings: display the heading row. | -| | | -| | The default is "tree headings", i.e., show all | -| | elements. | -| | | -| | **Note**: Column #0 always refers to the tree column, | -| | even if show="tree" is not specified. | -+----------------+--------------------------------------------------------+ - - -Item Options -^^^^^^^^^^^^ - -The following item options may be specified for items in the insert and item -widget commands. - -.. tabularcolumns:: |l|L| - -+--------+---------------------------------------------------------------+ -| Option | Description | -+========+===============================================================+ -| text | The textual label to display for the item. | -+--------+---------------------------------------------------------------+ -| image | A Tk Image, displayed to the left of the label. | -+--------+---------------------------------------------------------------+ -| values | The list of values associated with the item. | -| | | -| | Each item should have the same number of values as the widget | -| | option columns. If there are fewer values than columns, the | -| | remaining values are assumed empty. If there are more values | -| | than columns, the extra values are ignored. | -+--------+---------------------------------------------------------------+ -| open | ``True``/``False`` value indicating whether the item's | -| | children should be displayed or hidden. | -+--------+---------------------------------------------------------------+ -| tags | A list of tags associated with this item. | -+--------+---------------------------------------------------------------+ - - -Tag Options -^^^^^^^^^^^ - -The following options may be specified on tags: - -.. tabularcolumns:: |l|L| - -+------------+-----------------------------------------------------------+ -| Option | Description | -+============+===========================================================+ -| foreground | Specifies the text foreground color. | -+------------+-----------------------------------------------------------+ -| background | Specifies the cell or item background color. | -+------------+-----------------------------------------------------------+ -| font | Specifies the font to use when drawing text. | -+------------+-----------------------------------------------------------+ -| image | Specifies the item image, in case the item's image option | -| | is empty. | -+------------+-----------------------------------------------------------+ - - -Column Identifiers -^^^^^^^^^^^^^^^^^^ - -Column identifiers take any of the following forms: - -* A symbolic name from the list of columns option. -* An integer n, specifying the nth data column. -* A string of the form #n, where n is an integer, specifying the nth display - column. - -Notes: - -* Item's option values may be displayed in a different order than the order - in which they are stored. -* Column #0 always refers to the tree column, even if show="tree" is not - specified. - -A data column number is an index into an item's option values list; a display -column number is the column number in the tree where the values are displayed. -Tree labels are displayed in column #0. If option displaycolumns is not set, -then data column n is displayed in column #n+1. Again, **column #0 always -refers to the tree column**. - - -Virtual Events -^^^^^^^^^^^^^^ - -The Treeview widget generates the following virtual events. - -.. tabularcolumns:: |l|L| - -+--------------------+--------------------------------------------------+ -| Event | Description | -+====================+==================================================+ -| <> | Generated whenever the selection changes. | -+--------------------+--------------------------------------------------+ -| <> | Generated just before settings the focus item to | -| | open=True. | -+--------------------+--------------------------------------------------+ -| <> | Generated just after setting the focus item to | -| | open=False. | -+--------------------+--------------------------------------------------+ - -The :meth:`Treeview.focus` and :meth:`Treeview.selection` methods can be used -to determine the affected item or items. - - -ttk.Treeview -^^^^^^^^^^^^ - -.. class:: Treeview - - .. method:: bbox(item, column=None) - - Returns the bounding box (relative to the treeview widget's window) of - the specified *item* in the form (x, y, width, height). - - If *column* is specified, returns the bounding box of that cell. If the - *item* is not visible (i.e., if it is a descendant of a closed item or is - scrolled offscreen), returns an empty string. - - - .. method:: get_children(item=None) - - Returns the list of children belonging to *item*. - - If *item* is not specified, returns root children. - - - .. method:: set_children(item, *newchildren) - - Replaces *item*'s child with *newchildren*. - - Children present in *item* that are not present in *newchildren* are - detached from the tree. No items in *newchildren* may be an ancestor of - *item*. Note that not specifying *newchildren* results in detaching - *item*'s children. - - - .. method:: column(column, option=None, **kw) - - Query or modify the options for the specified *column*. - - If *kw* is not given, returns a dict of the column option values. If - *option* is specified then the value for that *option* is returned. - Otherwise, sets the options to the corresponding values. - - The valid options/values are: - - id - Returns the column name. This is a read-only option. - anchor: One of the standard Tk anchor values. - Specifies how the text in this column should be aligned with respect - to the cell. - minwidth: width - The minimum width of the column in pixels. The treeview widget will - not make the column any smaller than specified by this option when - the widget is resized or the user drags a column. - stretch: ``True``/``False`` - Specifies whether the column's width should be adjusted when - the widget is resized. - width: width - The width of the column in pixels. - - To configure the tree column, call this with column = "#0" - - .. method:: delete(*items) - - Delete all specified *items* and all their descendants. - - The root item may not be deleted. - - - .. method:: detach(*items) - - Unlinks all of the specified *items* from the tree. - - The items and all of their descendants are still present, and may be - reinserted at another point in the tree, but will not be displayed. - - The root item may not be detached. - - - .. method:: exists(item) - - Returns ``True`` if the specified *item* is present in the tree. - - - .. method:: focus(item=None) - - If *item* is specified, sets the focus item to *item*. Otherwise, returns - the current focus item, or '' if there is none. - - - .. method:: heading(column, option=None, **kw) - - Query or modify the heading options for the specified *column*. - - If *kw* is not given, returns a dict of the heading option values. If - *option* is specified then the value for that *option* is returned. - Otherwise, sets the options to the corresponding values. - - The valid options/values are: - - text: text - The text to display in the column heading. - image: imageName - Specifies an image to display to the right of the column heading. - anchor: anchor - Specifies how the heading text should be aligned. One of the standard - Tk anchor values. - command: callback - A callback to be invoked when the heading label is pressed. - - To configure the tree column heading, call this with column = "#0". - - - .. method:: identify(component, x, y) - - Returns a description of the specified *component* under the point given - by *x* and *y*, or the empty string if no such *component* is present at - that position. - - - .. method:: identify_row(y) - - Returns the item ID of the item at position *y*. - - - .. method:: identify_column(x) - - Returns the data column identifier of the cell at position *x*. - - The tree column has ID #0. - - - .. method:: identify_region(x, y) - - Returns one of: - - +-----------+--------------------------------------+ - | region | meaning | - +===========+======================================+ - | heading | Tree heading area. | - +-----------+--------------------------------------+ - | separator | Space between two columns headings. | - +-----------+--------------------------------------+ - | tree | The tree area. | - +-----------+--------------------------------------+ - | cell | A data cell. | - +-----------+--------------------------------------+ - - Availability: Tk 8.6. - - - .. method:: identify_element(x, y) - - Returns the element at position *x*, *y*. - - Availability: Tk 8.6. - - - .. method:: index(item) - - Returns the integer index of *item* within its parent's list of children. - - - .. method:: insert(parent, index, iid=None, **kw) - - Creates a new item and returns the item identifier of the newly created - item. - - *parent* is the item ID of the parent item, or the empty string to create - a new top-level item. *index* is an integer, or the value "end", - specifying where in the list of parent's children to insert the new item. - If *index* is less than or equal to zero, the new node is inserted at - the beginning; if *index* is greater than or equal to the current number - of children, it is inserted at the end. If *iid* is specified, it is used - as the item identifier; *iid* must not already exist in the tree. - Otherwise, a new unique identifier is generated. - - See `Item Options`_ for the list of available points. - - - .. method:: item(item, option=None, **kw) - - Query or modify the options for the specified *item*. - - If no options are given, a dict with options/values for the item is - returned. - If *option* is specified then the value for that option is returned. - Otherwise, sets the options to the corresponding values as given by *kw*. - - - .. method:: move(item, parent, index) - - Moves *item* to position *index* in *parent*'s list of children. - - It is illegal to move an item under one of its descendants. If *index* is - less than or equal to zero, *item* is moved to the beginning; if greater - than or equal to the number of children, it is moved to the end. If *item* - was detached it is reattached. - - - .. method:: next(item) - - Returns the identifier of *item*'s next sibling, or '' if *item* is the - last child of its parent. - - - .. method:: parent(item) - - Returns the ID of the parent of *item*, or '' if *item* is at the top - level of the hierarchy. - - - .. method:: prev(item) - - Returns the identifier of *item*'s previous sibling, or '' if *item* is - the first child of its parent. - - - .. method:: reattach(item, parent, index) - - An alias for :meth:`Treeview.move`. - - - .. method:: see(item) - - Ensure that *item* is visible. - - Sets all of *item*'s ancestors open option to ``True``, and scrolls the - widget if necessary so that *item* is within the visible portion of - the tree. - - - .. method:: selection() - - Returns a tuple of selected items. - - .. versionchanged:: 3.8 - ``selection()`` no longer takes arguments. For changing the selection - state use the following selection methods. - - - .. method:: selection_set(*items) - - *items* becomes the new selection. - - .. versionchanged:: 3.6 - *items* can be passed as separate arguments, not just as a single tuple. - - - .. method:: selection_add(*items) - - Add *items* to the selection. - - .. versionchanged:: 3.6 - *items* can be passed as separate arguments, not just as a single tuple. - - - .. method:: selection_remove(*items) - - Remove *items* from the selection. - - .. versionchanged:: 3.6 - *items* can be passed as separate arguments, not just as a single tuple. - - - .. method:: selection_toggle(*items) - - Toggle the selection state of each item in *items*. - - .. versionchanged:: 3.6 - *items* can be passed as separate arguments, not just as a single tuple. - - - .. method:: set(item, column=None, value=None) - - With one argument, returns a dictionary of column/value pairs for the - specified *item*. With two arguments, returns the current value of the - specified *column*. With three arguments, sets the value of given - *column* in given *item* to the specified *value*. - - - .. method:: tag_bind(tagname, sequence=None, callback=None) - - Bind a callback for the given event *sequence* to the tag *tagname*. - When an event is delivered to an item, the callbacks for each of the - item's tags option are called. - - - .. method:: tag_configure(tagname, option=None, **kw) - - Query or modify the options for the specified *tagname*. - - If *kw* is not given, returns a dict of the option settings for - *tagname*. If *option* is specified, returns the value for that *option* - for the specified *tagname*. Otherwise, sets the options to the - corresponding values for the given *tagname*. - - - .. method:: tag_has(tagname, item=None) - - If *item* is specified, returns 1 or 0 depending on whether the specified - *item* has the given *tagname*. Otherwise, returns a list of all items - that have the specified tag. - - Availability: Tk 8.6 - - - .. method:: xview(*args) - - Query or modify horizontal position of the treeview. - - - .. method:: yview(*args) - - Query or modify vertical position of the treeview. - - -.. _TtkStyling: - -Ttk Styling ------------ - -Each widget in :mod:`ttk` is assigned a style, which specifies the set of -elements making up the widget and how they are arranged, along with dynamic -and default settings for element options. By default the style name is the -same as the widget's class name, but it may be overridden by the widget's style -option. If you don't know the class name of a widget, use the method -:meth:`Misc.winfo_class` (somewidget.winfo_class()). - -.. seealso:: - - `Tcl'2004 conference presentation `_ - This document explains how the theme engine works - - -.. class:: Style - - This class is used to manipulate the style database. - - - .. method:: configure(style, query_opt=None, **kw) - - Query or set the default value of the specified option(s) in *style*. - - Each key in *kw* is an option and each value is a string identifying - the value for that option. - - For example, to change every default button to be a flat button with - some padding and a different background color:: - - from tkinter import ttk - import tkinter - - root = tkinter.Tk() - - ttk.Style().configure("TButton", padding=6, relief="flat", - background="#ccc") - - btn = ttk.Button(text="Sample") - btn.pack() - - root.mainloop() - - - .. method:: map(style, query_opt=None, **kw) - - Query or sets dynamic values of the specified option(s) in *style*. - - Each key in *kw* is an option and each value should be a list or a - tuple (usually) containing statespecs grouped in tuples, lists, or - some other preference. A statespec is a compound of one - or more states and then a value. - - An example may make it more understandable:: - - import tkinter - from tkinter import ttk - - root = tkinter.Tk() - - style = ttk.Style() - style.map("C.TButton", - foreground=[('pressed', 'red'), ('active', 'blue')], - background=[('pressed', '!disabled', 'black'), ('active', 'white')] - ) - - colored_btn = ttk.Button(text="Test", style="C.TButton").pack() - - root.mainloop() - - - Note that the order of the (states, value) sequences for an option does - matter, if the order is changed to ``[('active', 'blue'), ('pressed', - 'red')]`` in the foreground option, for example, the result would be a - blue foreground when the widget were in active or pressed states. - - - .. method:: lookup(style, option, state=None, default=None) - - Returns the value specified for *option* in *style*. - - If *state* is specified, it is expected to be a sequence of one or more - states. If the *default* argument is set, it is used as a fallback value - in case no specification for option is found. - - To check what font a Button uses by default:: - - from tkinter import ttk - - print(ttk.Style().lookup("TButton", "font")) - - - .. method:: layout(style, layoutspec=None) - - Define the widget layout for given *style*. If *layoutspec* is omitted, - return the layout specification for given style. - - *layoutspec*, if specified, is expected to be a list or some other - sequence type (excluding strings), where each item should be a tuple and - the first item is the layout name and the second item should have the - format described in `Layouts`_. - - To understand the format, see the following example (it is not - intended to do anything useful):: - - from tkinter import ttk - import tkinter - - root = tkinter.Tk() - - style = ttk.Style() - style.layout("TMenubutton", [ - ("Menubutton.background", None), - ("Menubutton.button", {"children": - [("Menubutton.focus", {"children": - [("Menubutton.padding", {"children": - [("Menubutton.label", {"side": "left", "expand": 1})] - })] - })] - }), - ]) - - mbtn = ttk.Menubutton(text='Text') - mbtn.pack() - root.mainloop() - - - .. method:: element_create(elementname, etype, *args, **kw) - - Create a new element in the current theme, of the given *etype* which is - expected to be either "image", "from" or "vsapi". The latter is only - available in Tk 8.6a for Windows XP and Vista and is not described here. - - If "image" is used, *args* should contain the default image name followed - by statespec/value pairs (this is the imagespec), and *kw* may have the - following options: - - border=padding - padding is a list of up to four integers, specifying the left, top, - right, and bottom borders, respectively. - - height=height - Specifies a minimum height for the element. If less than zero, the - base image's height is used as a default. - - padding=padding - Specifies the element's interior padding. Defaults to border's value - if not specified. - - sticky=spec - Specifies how the image is placed within the final parcel. spec - contains zero or more characters "n", "s", "w", or "e". - - width=width - Specifies a minimum width for the element. If less than zero, the - base image's width is used as a default. - - If "from" is used as the value of *etype*, - :meth:`element_create` will clone an existing - element. *args* is expected to contain a themename, from which - the element will be cloned, and optionally an element to clone from. - If this element to clone from is not specified, an empty element will - be used. *kw* is discarded. - - - .. method:: element_names() - - Returns the list of elements defined in the current theme. - - - .. method:: element_options(elementname) - - Returns the list of *elementname*'s options. - - - .. method:: theme_create(themename, parent=None, settings=None) - - Create a new theme. - - It is an error if *themename* already exists. If *parent* is specified, - the new theme will inherit styles, elements and layouts from the parent - theme. If *settings* are present they are expected to have the same - syntax used for :meth:`theme_settings`. - - - .. method:: theme_settings(themename, settings) - - Temporarily sets the current theme to *themename*, apply specified - *settings* and then restore the previous theme. - - Each key in *settings* is a style and each value may contain the keys - 'configure', 'map', 'layout' and 'element create' and they are expected - to have the same format as specified by the methods - :meth:`Style.configure`, :meth:`Style.map`, :meth:`Style.layout` and - :meth:`Style.element_create` respectively. - - As an example, let's change the Combobox for the default theme a bit:: - - from tkinter import ttk - import tkinter - - root = tkinter.Tk() - - style = ttk.Style() - style.theme_settings("default", { - "TCombobox": { - "configure": {"padding": 5}, - "map": { - "background": [("active", "green2"), - ("!disabled", "green4")], - "fieldbackground": [("!disabled", "green3")], - "foreground": [("focus", "OliveDrab1"), - ("!disabled", "OliveDrab2")] - } - } - }) - - combo = ttk.Combobox().pack() - - root.mainloop() - - - .. method:: theme_names() - - Returns a list of all known themes. - - - .. method:: theme_use(themename=None) - - If *themename* is not given, returns the theme in use. Otherwise, sets - the current theme to *themename*, refreshes all widgets and emits a - <> event. - - -Layouts -^^^^^^^ - -A layout can be just ``None``, if it takes no options, or a dict of -options specifying how to arrange the element. The layout mechanism -uses a simplified version of the pack geometry manager: given an -initial cavity, each element is allocated a parcel. Valid -options/values are: - -side: whichside - Specifies which side of the cavity to place the element; one of - top, right, bottom or left. If omitted, the element occupies the - entire cavity. - -sticky: nswe - Specifies where the element is placed inside its allocated parcel. - -unit: 0 or 1 - If set to 1, causes the element and all of its descendants to be treated as - a single element for the purposes of :meth:`Widget.identify` et al. It's - used for things like scrollbar thumbs with grips. - -children: [sublayout... ] - Specifies a list of elements to place inside the element. Each - element is a tuple (or other sequence type) where the first item is - the layout name, and the other is a `Layout`_. - -.. _Layout: `Layouts`_ diff --git a/Doc/library/token-list.inc b/Doc/library/token-list.inc deleted file mode 100644 index 1a99f0518d1b47..00000000000000 --- a/Doc/library/token-list.inc +++ /dev/null @@ -1,220 +0,0 @@ -.. Auto-generated by Tools/scripts/generate_token.py -.. data:: ENDMARKER - -.. data:: NAME - -.. data:: NUMBER - -.. data:: STRING - -.. data:: NEWLINE - -.. data:: INDENT - -.. data:: DEDENT - -.. data:: LPAR - - Token value for ``"("``. - -.. data:: RPAR - - Token value for ``")"``. - -.. data:: LSQB - - Token value for ``"["``. - -.. data:: RSQB - - Token value for ``"]"``. - -.. data:: COLON - - Token value for ``":"``. - -.. data:: COMMA - - Token value for ``","``. - -.. data:: SEMI - - Token value for ``";"``. - -.. data:: PLUS - - Token value for ``"+"``. - -.. data:: MINUS - - Token value for ``"-"``. - -.. data:: STAR - - Token value for ``"*"``. - -.. data:: SLASH - - Token value for ``"/"``. - -.. data:: VBAR - - Token value for ``"|"``. - -.. data:: AMPER - - Token value for ``"&"``. - -.. data:: LESS - - Token value for ``"<"``. - -.. data:: GREATER - - Token value for ``">"``. - -.. data:: EQUAL - - Token value for ``"="``. - -.. data:: DOT - - Token value for ``"."``. - -.. data:: PERCENT - - Token value for ``"%"``. - -.. data:: LBRACE - - Token value for ``"{"``. - -.. data:: RBRACE - - Token value for ``"}"``. - -.. data:: EQEQUAL - - Token value for ``"=="``. - -.. data:: NOTEQUAL - - Token value for ``"!="``. - -.. data:: LESSEQUAL - - Token value for ``"<="``. - -.. data:: GREATEREQUAL - - Token value for ``">="``. - -.. data:: TILDE - - Token value for ``"~"``. - -.. data:: CIRCUMFLEX - - Token value for ``"^"``. - -.. data:: LEFTSHIFT - - Token value for ``"<<"``. - -.. data:: RIGHTSHIFT - - Token value for ``">>"``. - -.. data:: DOUBLESTAR - - Token value for ``"**"``. - -.. data:: PLUSEQUAL - - Token value for ``"+="``. - -.. data:: MINEQUAL - - Token value for ``"-="``. - -.. data:: STAREQUAL - - Token value for ``"*="``. - -.. data:: SLASHEQUAL - - Token value for ``"/="``. - -.. data:: PERCENTEQUAL - - Token value for ``"%="``. - -.. data:: AMPEREQUAL - - Token value for ``"&="``. - -.. data:: VBAREQUAL - - Token value for ``"|="``. - -.. data:: CIRCUMFLEXEQUAL - - Token value for ``"^="``. - -.. data:: LEFTSHIFTEQUAL - - Token value for ``"<<="``. - -.. data:: RIGHTSHIFTEQUAL - - Token value for ``">>="``. - -.. data:: DOUBLESTAREQUAL - - Token value for ``"**="``. - -.. data:: DOUBLESLASH - - Token value for ``"//"``. - -.. data:: DOUBLESLASHEQUAL - - Token value for ``"//="``. - -.. data:: AT - - Token value for ``"@"``. - -.. data:: ATEQUAL - - Token value for ``"@="``. - -.. data:: RARROW - - Token value for ``"->"``. - -.. data:: ELLIPSIS - - Token value for ``"..."``. - -.. data:: COLONEQUAL - - Token value for ``":="``. - -.. data:: OP - -.. data:: AWAIT - -.. data:: ASYNC - -.. data:: TYPE_IGNORE - -.. data:: TYPE_COMMENT - -.. data:: SOFT_KEYWORD - -.. data:: ERRORTOKEN - -.. data:: N_TOKENS - -.. data:: NT_OFFSET diff --git a/Doc/library/token.rst b/Doc/library/token.rst deleted file mode 100644 index a1aceba96ce030..00000000000000 --- a/Doc/library/token.rst +++ /dev/null @@ -1,94 +0,0 @@ -:mod:`token` --- Constants used with Python parse trees -======================================================= - -.. module:: token - :synopsis: Constants representing terminal nodes of the parse tree. - -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/token.py` - --------------- - -This module provides constants which represent the numeric values of leaf nodes -of the parse tree (terminal tokens). Refer to the file :file:`Grammar/Tokens` -in the Python distribution for the definitions of the names in the context of -the language grammar. The specific numeric values which the names map to may -change between Python versions. - -The module also provides a mapping from numeric codes to names and some -functions. The functions mirror definitions in the Python C header files. - - -.. data:: tok_name - - Dictionary mapping the numeric values of the constants defined in this module - back to name strings, allowing more human-readable representation of parse trees - to be generated. - - -.. function:: ISTERMINAL(x) - - Return ``True`` for terminal token values. - - -.. function:: ISNONTERMINAL(x) - - Return ``True`` for non-terminal token values. - - -.. function:: ISEOF(x) - - Return ``True`` if *x* is the marker indicating the end of input. - - -The token constants are: - -.. include:: token-list.inc - -The following token type values aren't used by the C tokenizer but are needed for -the :mod:`tokenize` module. - -.. data:: COMMENT - - Token value used to indicate a comment. - - -.. data:: NL - - Token value used to indicate a non-terminating newline. The - :data:`NEWLINE` token indicates the end of a logical line of Python code; - ``NL`` tokens are generated when a logical line of code is continued over - multiple physical lines. - - -.. data:: ENCODING - - Token value that indicates the encoding used to decode the source bytes - into text. The first token returned by :func:`tokenize.tokenize` will - always be an ``ENCODING`` token. - - -.. data:: TYPE_COMMENT - :noindex: - - Token value indicating that a type comment was recognized. Such - tokens are only produced when :func:`ast.parse()` is invoked with - ``type_comments=True``. - - -.. versionchanged:: 3.5 - Added :data:`AWAIT` and :data:`ASYNC` tokens. - -.. versionchanged:: 3.7 - Added :data:`COMMENT`, :data:`NL` and :data:`ENCODING` tokens. - -.. versionchanged:: 3.7 - Removed :data:`AWAIT` and :data:`ASYNC` tokens. "async" and "await" are - now tokenized as :data:`NAME` tokens. - -.. versionchanged:: 3.8 - Added :data:`TYPE_COMMENT`, :data:`TYPE_IGNORE`, :data:`COLONEQUAL`. - Added :data:`AWAIT` and :data:`ASYNC` tokens back (they're needed - to support parsing older Python versions for :func:`ast.parse` with - ``feature_version`` set to 6 or lower). diff --git a/Doc/library/tokenize.rst b/Doc/library/tokenize.rst deleted file mode 100644 index 6f6c4d5371bb7f..00000000000000 --- a/Doc/library/tokenize.rst +++ /dev/null @@ -1,308 +0,0 @@ -:mod:`tokenize` --- Tokenizer for Python source -=============================================== - -.. module:: tokenize - :synopsis: Lexical scanner for Python source code. - -.. moduleauthor:: Ka Ping Yee -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/tokenize.py` - --------------- - -The :mod:`tokenize` module provides a lexical scanner for Python source code, -implemented in Python. The scanner in this module returns comments as tokens -as well, making it useful for implementing "pretty-printers", including -colorizers for on-screen displays. - -To simplify token stream handling, all :ref:`operator ` and -:ref:`delimiter ` tokens and :data:`Ellipsis` are returned using -the generic :data:`~token.OP` token type. The exact -type can be determined by checking the ``exact_type`` property on the -:term:`named tuple` returned from :func:`tokenize.tokenize`. - - -.. warning:: - - Note that the functions in this module are only designed to parse - syntactically valid Python code (code that does not raise when parsed - using :func:`ast.parse`). The behavior of the functions in this module is - **undefined** when providing invalid Python code and it can change at any - point. - -Tokenizing Input ----------------- - -The primary entry point is a :term:`generator`: - -.. function:: tokenize(readline) - - The :func:`.tokenize` generator requires one argument, *readline*, which - must be a callable object which provides the same interface as the - :meth:`io.IOBase.readline` method of file objects. Each call to the - function should return one line of input as bytes. - - The generator produces 5-tuples with these members: the token type; the - token string; a 2-tuple ``(srow, scol)`` of ints specifying the row and - column where the token begins in the source; a 2-tuple ``(erow, ecol)`` of - ints specifying the row and column where the token ends in the source; and - the line on which the token was found. The line passed (the last tuple item) - is the *physical* line. The 5 tuple is returned as a :term:`named tuple` - with the field names: - ``type string start end line``. - - The returned :term:`named tuple` has an additional property named - ``exact_type`` that contains the exact operator type for - :data:`~token.OP` tokens. For all other token types ``exact_type`` - equals the named tuple ``type`` field. - - .. versionchanged:: 3.1 - Added support for named tuples. - - .. versionchanged:: 3.3 - Added support for ``exact_type``. - - :func:`.tokenize` determines the source encoding of the file by looking for a - UTF-8 BOM or encoding cookie, according to :pep:`263`. - -.. function:: generate_tokens(readline) - - Tokenize a source reading unicode strings instead of bytes. - - Like :func:`.tokenize`, the *readline* argument is a callable returning - a single line of input. However, :func:`generate_tokens` expects *readline* - to return a str object rather than bytes. - - The result is an iterator yielding named tuples, exactly like - :func:`.tokenize`. It does not yield an :data:`~token.ENCODING` token. - -All constants from the :mod:`token` module are also exported from -:mod:`tokenize`. - -Another function is provided to reverse the tokenization process. This is -useful for creating tools that tokenize a script, modify the token stream, and -write back the modified script. - - -.. function:: untokenize(iterable) - - Converts tokens back into Python source code. The *iterable* must return - sequences with at least two elements, the token type and the token string. - Any additional sequence elements are ignored. - - The reconstructed script is returned as a single string. The result is - guaranteed to tokenize back to match the input so that the conversion is - lossless and round-trips are assured. The guarantee applies only to the - token type and token string as the spacing between tokens (column - positions) may change. - - It returns bytes, encoded using the :data:`~token.ENCODING` token, which - is the first token sequence output by :func:`.tokenize`. If there is no - encoding token in the input, it returns a str instead. - - -:func:`.tokenize` needs to detect the encoding of source files it tokenizes. The -function it uses to do this is available: - -.. function:: detect_encoding(readline) - - The :func:`detect_encoding` function is used to detect the encoding that - should be used to decode a Python source file. It requires one argument, - readline, in the same way as the :func:`.tokenize` generator. - - It will call readline a maximum of twice, and return the encoding used - (as a string) and a list of any lines (not decoded from bytes) it has read - in. - - It detects the encoding from the presence of a UTF-8 BOM or an encoding - cookie as specified in :pep:`263`. If both a BOM and a cookie are present, - but disagree, a :exc:`SyntaxError` will be raised. Note that if the BOM is found, - ``'utf-8-sig'`` will be returned as an encoding. - - If no encoding is specified, then the default of ``'utf-8'`` will be - returned. - - Use :func:`.open` to open Python source files: it uses - :func:`detect_encoding` to detect the file encoding. - - -.. function:: open(filename) - - Open a file in read only mode using the encoding detected by - :func:`detect_encoding`. - - .. versionadded:: 3.2 - -.. exception:: TokenError - - Raised when either a docstring or expression that may be split over several - lines is not completed anywhere in the file, for example:: - - """Beginning of - docstring - - or:: - - [1, - 2, - 3 - -Note that unclosed single-quoted strings do not cause an error to be -raised. They are tokenized as :data:`~token.ERRORTOKEN`, followed by the -tokenization of their contents. - - -.. _tokenize-cli: - -Command-Line Usage ------------------- - -.. versionadded:: 3.3 - -The :mod:`tokenize` module can be executed as a script from the command line. -It is as simple as: - -.. code-block:: sh - - python -m tokenize [-e] [filename.py] - -The following options are accepted: - -.. program:: tokenize - -.. option:: -h, --help - - show this help message and exit - -.. option:: -e, --exact - - display token names using the exact type - -If :file:`filename.py` is specified its contents are tokenized to stdout. -Otherwise, tokenization is performed on stdin. - -Examples ------------------- - -Example of a script rewriter that transforms float literals into Decimal -objects:: - - from tokenize import tokenize, untokenize, NUMBER, STRING, NAME, OP - from io import BytesIO - - def decistmt(s): - """Substitute Decimals for floats in a string of statements. - - >>> from decimal import Decimal - >>> s = 'print(+21.3e-5*-.1234/81.7)' - >>> decistmt(s) - "print (+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7'))" - - The format of the exponent is inherited from the platform C library. - Known cases are "e-007" (Windows) and "e-07" (not Windows). Since - we're only showing 12 digits, and the 13th isn't close to 5, the - rest of the output should be platform-independent. - - >>> exec(s) #doctest: +ELLIPSIS - -3.21716034272e-0...7 - - Output from calculations with Decimal should be identical across all - platforms. - - >>> exec(decistmt(s)) - -3.217160342717258261933904529E-7 - """ - result = [] - g = tokenize(BytesIO(s.encode('utf-8')).readline) # tokenize the string - for toknum, tokval, _, _, _ in g: - if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens - result.extend([ - (NAME, 'Decimal'), - (OP, '('), - (STRING, repr(tokval)), - (OP, ')') - ]) - else: - result.append((toknum, tokval)) - return untokenize(result).decode('utf-8') - -Example of tokenizing from the command line. The script:: - - def say_hello(): - print("Hello, World!") - - say_hello() - -will be tokenized to the following output where the first column is the range -of the line/column coordinates where the token is found, the second column is -the name of the token, and the final column is the value of the token (if any) - -.. code-block:: shell-session - - $ python -m tokenize hello.py - 0,0-0,0: ENCODING 'utf-8' - 1,0-1,3: NAME 'def' - 1,4-1,13: NAME 'say_hello' - 1,13-1,14: OP '(' - 1,14-1,15: OP ')' - 1,15-1,16: OP ':' - 1,16-1,17: NEWLINE '\n' - 2,0-2,4: INDENT ' ' - 2,4-2,9: NAME 'print' - 2,9-2,10: OP '(' - 2,10-2,25: STRING '"Hello, World!"' - 2,25-2,26: OP ')' - 2,26-2,27: NEWLINE '\n' - 3,0-3,1: NL '\n' - 4,0-4,0: DEDENT '' - 4,0-4,9: NAME 'say_hello' - 4,9-4,10: OP '(' - 4,10-4,11: OP ')' - 4,11-4,12: NEWLINE '\n' - 5,0-5,0: ENDMARKER '' - -The exact token type names can be displayed using the :option:`-e` option: - -.. code-block:: shell-session - - $ python -m tokenize -e hello.py - 0,0-0,0: ENCODING 'utf-8' - 1,0-1,3: NAME 'def' - 1,4-1,13: NAME 'say_hello' - 1,13-1,14: LPAR '(' - 1,14-1,15: RPAR ')' - 1,15-1,16: COLON ':' - 1,16-1,17: NEWLINE '\n' - 2,0-2,4: INDENT ' ' - 2,4-2,9: NAME 'print' - 2,9-2,10: LPAR '(' - 2,10-2,25: STRING '"Hello, World!"' - 2,25-2,26: RPAR ')' - 2,26-2,27: NEWLINE '\n' - 3,0-3,1: NL '\n' - 4,0-4,0: DEDENT '' - 4,0-4,9: NAME 'say_hello' - 4,9-4,10: LPAR '(' - 4,10-4,11: RPAR ')' - 4,11-4,12: NEWLINE '\n' - 5,0-5,0: ENDMARKER '' - -Example of tokenizing a file programmatically, reading unicode -strings instead of bytes with :func:`generate_tokens`:: - - import tokenize - - with tokenize.open('hello.py') as f: - tokens = tokenize.generate_tokens(f.readline) - for token in tokens: - print(token) - -Or reading bytes directly with :func:`.tokenize`:: - - import tokenize - - with open('hello.py', 'rb') as f: - tokens = tokenize.tokenize(f.readline) - for token in tokens: - print(token) diff --git a/Doc/library/trace.rst b/Doc/library/trace.rst deleted file mode 100644 index e9b59a6d186ba2..00000000000000 --- a/Doc/library/trace.rst +++ /dev/null @@ -1,216 +0,0 @@ -:mod:`trace` --- Trace or track Python statement execution -========================================================== - -.. module:: trace - :synopsis: Trace or track Python statement execution. - -**Source code:** :source:`Lib/trace.py` - --------------- - -The :mod:`trace` module allows you to trace program execution, generate -annotated statement coverage listings, print caller/callee relationships and -list functions executed during a program run. It can be used in another program -or from the command line. - -.. seealso:: - - `Coverage.py `_ - A popular third-party coverage tool that provides HTML - output along with advanced features such as branch coverage. - -.. _trace-cli: - -Command-Line Usage ------------------- - -The :mod:`trace` module can be invoked from the command line. It can be as -simple as :: - - python -m trace --count -C . somefile.py ... - -The above will execute :file:`somefile.py` and generate annotated listings of -all Python modules imported during the execution into the current directory. - -.. program:: trace - -.. option:: --help - - Display usage and exit. - -.. option:: --version - - Display the version of the module and exit. - -.. versionadded:: 3.8 - Added ``--module`` option that allows to run an executable module. - -Main options -^^^^^^^^^^^^ - -At least one of the following options must be specified when invoking -:mod:`trace`. The :option:`--listfuncs <-l>` option is mutually exclusive with -the :option:`--trace <-t>` and :option:`--count <-c>` options. When -:option:`--listfuncs <-l>` is provided, neither :option:`--count <-c>` nor -:option:`--trace <-t>` are accepted, and vice versa. - -.. program:: trace - -.. option:: -c, --count - - Produce a set of annotated listing files upon program completion that shows - how many times each statement was executed. See also - :option:`--coverdir <-C>`, :option:`--file <-f>` and - :option:`--no-report <-R>` below. - -.. option:: -t, --trace - - Display lines as they are executed. - -.. option:: -l, --listfuncs - - Display the functions executed by running the program. - -.. option:: -r, --report - - Produce an annotated list from an earlier program run that used the - :option:`--count <-c>` and :option:`--file <-f>` option. This does not - execute any code. - -.. option:: -T, --trackcalls - - Display the calling relationships exposed by running the program. - -Modifiers -^^^^^^^^^ - -.. program:: trace - -.. option:: -f, --file= - - Name of a file to accumulate counts over several tracing runs. Should be - used with the :option:`--count <-c>` option. - -.. option:: -C, --coverdir= - - Directory where the report files go. The coverage report for - ``package.module`` is written to file :file:`{dir}/{package}/{module}.cover`. - -.. option:: -m, --missing - - When generating annotated listings, mark lines which were not executed with - ``>>>>>>``. - -.. option:: -s, --summary - - When using :option:`--count <-c>` or :option:`--report <-r>`, write a brief - summary to stdout for each file processed. - -.. option:: -R, --no-report - - Do not generate annotated listings. This is useful if you intend to make - several runs with :option:`--count <-c>`, and then produce a single set of - annotated listings at the end. - -.. option:: -g, --timing - - Prefix each line with the time since the program started. Only used while - tracing. - -Filters -^^^^^^^ - -These options may be repeated multiple times. - -.. program:: trace - -.. option:: --ignore-module= - - Ignore each of the given module names and its submodules (if it is a - package). The argument can be a list of names separated by a comma. - -.. option:: --ignore-dir= - - Ignore all modules and packages in the named directory and subdirectories. - The argument can be a list of directories separated by :data:`os.pathsep`. - -.. _trace-api: - -Programmatic Interface ----------------------- - -.. class:: Trace(count=1, trace=1, countfuncs=0, countcallers=0, ignoremods=(),\ - ignoredirs=(), infile=None, outfile=None, timing=False) - - Create an object to trace execution of a single statement or expression. All - parameters are optional. *count* enables counting of line numbers. *trace* - enables line execution tracing. *countfuncs* enables listing of the - functions called during the run. *countcallers* enables call relationship - tracking. *ignoremods* is a list of modules or packages to ignore. - *ignoredirs* is a list of directories whose modules or packages should be - ignored. *infile* is the name of the file from which to read stored count - information. *outfile* is the name of the file in which to write updated - count information. *timing* enables a timestamp relative to when tracing was - started to be displayed. - - .. method:: run(cmd) - - Execute the command and gather statistics from the execution with - the current tracing parameters. *cmd* must be a string or code object, - suitable for passing into :func:`exec`. - - .. method:: runctx(cmd, globals=None, locals=None) - - Execute the command and gather statistics from the execution with the - current tracing parameters, in the defined global and local - environments. If not defined, *globals* and *locals* default to empty - dictionaries. - - .. method:: runfunc(func, /, *args, **kwds) - - Call *func* with the given arguments under control of the :class:`Trace` - object with the current tracing parameters. - - .. method:: results() - - Return a :class:`CoverageResults` object that contains the cumulative - results of all previous calls to ``run``, ``runctx`` and ``runfunc`` - for the given :class:`Trace` instance. Does not reset the accumulated - trace results. - -.. class:: CoverageResults - - A container for coverage results, created by :meth:`Trace.results`. Should - not be created directly by the user. - - .. method:: update(other) - - Merge in data from another :class:`CoverageResults` object. - - .. method:: write_results(show_missing=True, summary=False, coverdir=None) - - Write coverage results. Set *show_missing* to show lines that had no - hits. Set *summary* to include in the output the coverage summary per - module. *coverdir* specifies the directory into which the coverage - result files will be output. If ``None``, the results for each source - file are placed in its directory. - -A simple example demonstrating the use of the programmatic interface:: - - import sys - import trace - - # create a Trace object, telling it what to ignore, and whether to - # do tracing or line-counting or both. - tracer = trace.Trace( - ignoredirs=[sys.prefix, sys.exec_prefix], - trace=0, - count=1) - - # run the new command using the given tracer - tracer.run('main()') - - # make a report, placing output in the current directory - r = tracer.results() - r.write_results(show_missing=True, coverdir=".") - diff --git a/Doc/library/traceback.rst b/Doc/library/traceback.rst deleted file mode 100644 index e15fe76dbb0186..00000000000000 --- a/Doc/library/traceback.rst +++ /dev/null @@ -1,573 +0,0 @@ -:mod:`traceback` --- Print or retrieve a stack traceback -======================================================== - -.. module:: traceback - :synopsis: Print or retrieve a stack traceback. - -**Source code:** :source:`Lib/traceback.py` - --------------- - -This module provides a standard interface to extract, format and print stack -traces of Python programs. It exactly mimics the behavior of the Python -interpreter when it prints a stack trace. This is useful when you want to print -stack traces under program control, such as in a "wrapper" around the -interpreter. - -.. index:: pair: object; traceback - -The module uses traceback objects --- these are objects of type :class:`types.TracebackType`, -which are assigned to the ``__traceback__`` field of :class:`BaseException` instances. - -.. seealso:: - - Module :mod:`faulthandler` - Used to dump Python tracebacks explicitly, on a fault, after a timeout, or on a user signal. - - Module :mod:`pdb` - Interactive source code debugger for Python programs. - -The module defines the following functions: - -.. function:: print_tb(tb, limit=None, file=None) - - Print up to *limit* stack trace entries from traceback object *tb* (starting - from the caller's frame) if *limit* is positive. Otherwise, print the last - ``abs(limit)`` entries. If *limit* is omitted or ``None``, all entries are - printed. If *file* is omitted or ``None``, the output goes to - ``sys.stderr``; otherwise it should be an open file or file-like object to - receive the output. - - .. versionchanged:: 3.5 - Added negative *limit* support. - - -.. function:: print_exception(exc, /[, value, tb], limit=None, \ - file=None, chain=True) - - Print exception information and stack trace entries from traceback object - *tb* to *file*. This differs from :func:`print_tb` in the following - ways: - - * if *tb* is not ``None``, it prints a header ``Traceback (most recent - call last):`` - - * it prints the exception type and *value* after the stack trace - - .. index:: single: ^ (caret); marker - - * if *type(value)* is :exc:`SyntaxError` and *value* has the appropriate - format, it prints the line where the syntax error occurred with a caret - indicating the approximate position of the error. - - Since Python 3.10, instead of passing *value* and *tb*, an exception object - can be passed as the first argument. If *value* and *tb* are provided, the - first argument is ignored in order to provide backwards compatibility. - - The optional *limit* argument has the same meaning as for :func:`print_tb`. - If *chain* is true (the default), then chained exceptions (the - :attr:`__cause__` or :attr:`__context__` attributes of the exception) will be - printed as well, like the interpreter itself does when printing an unhandled - exception. - - .. versionchanged:: 3.5 - The *etype* argument is ignored and inferred from the type of *value*. - - .. versionchanged:: 3.10 - The *etype* parameter has been renamed to *exc* and is now - positional-only. - - -.. function:: print_exc(limit=None, file=None, chain=True) - - This is a shorthand for ``print_exception(sys.exception(), limit, file, - chain)``. - - -.. function:: print_last(limit=None, file=None, chain=True) - - This is a shorthand for ``print_exception(sys.last_type, sys.last_value, - sys.last_traceback, limit, file, chain)``. In general it will work only - after an exception has reached an interactive prompt (see - :data:`sys.last_type`). - - -.. function:: print_stack(f=None, limit=None, file=None) - - Print up to *limit* stack trace entries (starting from the invocation - point) if *limit* is positive. Otherwise, print the last ``abs(limit)`` - entries. If *limit* is omitted or ``None``, all entries are printed. - The optional *f* argument can be used to specify an alternate stack frame - to start. The optional *file* argument has the same meaning as for - :func:`print_tb`. - - .. versionchanged:: 3.5 - Added negative *limit* support. - - -.. function:: extract_tb(tb, limit=None) - - Return a :class:`StackSummary` object representing a list of "pre-processed" - stack trace entries extracted from the traceback object *tb*. It is useful - for alternate formatting of stack traces. The optional *limit* argument has - the same meaning as for :func:`print_tb`. A "pre-processed" stack trace - entry is a :class:`FrameSummary` object containing attributes - :attr:`~FrameSummary.filename`, :attr:`~FrameSummary.lineno`, - :attr:`~FrameSummary.name`, and :attr:`~FrameSummary.line` representing the - information that is usually printed for a stack trace. The - :attr:`~FrameSummary.line` is a string with leading and trailing - whitespace stripped; if the source is not available it is ``None``. - - -.. function:: extract_stack(f=None, limit=None) - - Extract the raw traceback from the current stack frame. The return value has - the same format as for :func:`extract_tb`. The optional *f* and *limit* - arguments have the same meaning as for :func:`print_stack`. - - -.. function:: format_list(extracted_list) - - Given a list of tuples or :class:`FrameSummary` objects as returned by - :func:`extract_tb` or :func:`extract_stack`, return a list of strings ready - for printing. Each string in the resulting list corresponds to the item with - the same index in the argument list. Each string ends in a newline; the - strings may contain internal newlines as well, for those items whose source - text line is not ``None``. - - -.. function:: format_exception_only(exc, /[, value]) - - Format the exception part of a traceback using an exception value such as - given by ``sys.last_value``. The return value is a list of strings, each - ending in a newline. The list contains the exception's message, which is - normally a single string; however, for :exc:`SyntaxError` exceptions, it - contains several lines that (when printed) display detailed information - about where the syntax error occurred. Following the message, the list - contains the exception's :attr:`notes `. - - Since Python 3.10, instead of passing *value*, an exception object - can be passed as the first argument. If *value* is provided, the first - argument is ignored in order to provide backwards compatibility. - - .. versionchanged:: 3.10 - The *etype* parameter has been renamed to *exc* and is now - positional-only. - - .. versionchanged:: 3.11 - The returned list now includes any notes attached to the exception. - - -.. function:: format_exception(exc, /[, value, tb], limit=None, chain=True) - - Format a stack trace and the exception information. The arguments have the - same meaning as the corresponding arguments to :func:`print_exception`. The - return value is a list of strings, each ending in a newline and some - containing internal newlines. When these lines are concatenated and printed, - exactly the same text is printed as does :func:`print_exception`. - - .. versionchanged:: 3.5 - The *etype* argument is ignored and inferred from the type of *value*. - - .. versionchanged:: 3.10 - This function's behavior and signature were modified to match - :func:`print_exception`. - - -.. function:: format_exc(limit=None, chain=True) - - This is like ``print_exc(limit)`` but returns a string instead of printing to - a file. - - -.. function:: format_tb(tb, limit=None) - - A shorthand for ``format_list(extract_tb(tb, limit))``. - - -.. function:: format_stack(f=None, limit=None) - - A shorthand for ``format_list(extract_stack(f, limit))``. - -.. function:: clear_frames(tb) - - Clears the local variables of all the stack frames in a traceback *tb* - by calling the :meth:`clear` method of each frame object. - - .. versionadded:: 3.4 - -.. function:: walk_stack(f) - - Walk a stack following ``f.f_back`` from the given frame, yielding the frame - and line number for each frame. If *f* is ``None``, the current stack is - used. This helper is used with :meth:`StackSummary.extract`. - - .. versionadded:: 3.5 - -.. function:: walk_tb(tb) - - Walk a traceback following ``tb_next`` yielding the frame and line number - for each frame. This helper is used with :meth:`StackSummary.extract`. - - .. versionadded:: 3.5 - -The module also defines the following classes: - -:class:`TracebackException` Objects ------------------------------------ - -.. versionadded:: 3.5 - -:class:`TracebackException` objects are created from actual exceptions to -capture data for later printing in a lightweight fashion. - -.. class:: TracebackException(exc_type, exc_value, exc_traceback, *, limit=None, lookup_lines=True, capture_locals=False, compact=False, max_group_width=15, max_group_depth=10) - - Capture an exception for later rendering. *limit*, *lookup_lines* and - *capture_locals* are as for the :class:`StackSummary` class. - - If *compact* is true, only data that is required by :class:`TracebackException`'s - ``format`` method is saved in the class attributes. In particular, the - ``__context__`` field is calculated only if ``__cause__`` is ``None`` and - ``__suppress_context__`` is false. - - Note that when locals are captured, they are also shown in the traceback. - - *max_group_width* and *max_group_depth* control the formatting of exception - groups (see :exc:`BaseExceptionGroup`). The depth refers to the nesting - level of the group, and the width refers to the size of a single exception - group's exceptions array. The formatted output is truncated when either - limit is exceeded. - - .. versionchanged:: 3.10 - Added the *compact* parameter. - - .. versionchanged:: 3.11 - Added the *max_group_width* and *max_group_depth* parameters. - - .. attribute:: __cause__ - - A :class:`TracebackException` of the original ``__cause__``. - - .. attribute:: __context__ - - A :class:`TracebackException` of the original ``__context__``. - - .. attribute:: exceptions - - If ``self`` represents an :exc:`ExceptionGroup`, this field holds a list of - :class:`TracebackException` instances representing the nested exceptions. - Otherwise it is ``None``. - - .. versionadded:: 3.11 - - .. attribute:: __suppress_context__ - - The ``__suppress_context__`` value from the original exception. - - .. attribute:: __notes__ - - The ``__notes__`` value from the original exception, or ``None`` - if the exception does not have any notes. If it is not ``None`` - is it formatted in the traceback after the exception string. - - .. versionadded:: 3.11 - - .. attribute:: stack - - A :class:`StackSummary` representing the traceback. - - .. attribute:: exc_type - - The class of the original traceback. - - .. attribute:: filename - - For syntax errors - the file name where the error occurred. - - .. attribute:: lineno - - For syntax errors - the line number where the error occurred. - - .. attribute:: end_lineno - - For syntax errors - the end line number where the error occurred. - Can be ``None`` if not present. - - .. versionadded:: 3.10 - - .. attribute:: text - - For syntax errors - the text where the error occurred. - - .. attribute:: offset - - For syntax errors - the offset into the text where the error occurred. - - .. attribute:: end_offset - - For syntax errors - the end offset into the text where the error occurred. - Can be ``None`` if not present. - - .. versionadded:: 3.10 - - .. attribute:: msg - - For syntax errors - the compiler error message. - - .. classmethod:: from_exception(exc, *, limit=None, lookup_lines=True, capture_locals=False) - - Capture an exception for later rendering. *limit*, *lookup_lines* and - *capture_locals* are as for the :class:`StackSummary` class. - - Note that when locals are captured, they are also shown in the traceback. - - .. method:: print(*, file=None, chain=True) - - Print to *file* (default ``sys.stderr``) the exception information returned by - :meth:`format`. - - .. versionadded:: 3.11 - - .. method:: format(*, chain=True) - - Format the exception. - - If *chain* is not ``True``, ``__cause__`` and ``__context__`` will not - be formatted. - - The return value is a generator of strings, each ending in a newline and - some containing internal newlines. :func:`~traceback.print_exception` - is a wrapper around this method which just prints the lines to a file. - - .. method:: format_exception_only() - - Format the exception part of the traceback. - - The return value is a generator of strings, each ending in a newline. - - The generator emits the exception's message followed by its notes - (if it has any). The exception message is normally a single string; - however, for :exc:`SyntaxError` exceptions, it consists of several - lines that (when printed) display detailed information about where - the syntax error occurred. - - .. versionchanged:: 3.11 - The exception's notes are now included in the output. - - -:class:`StackSummary` Objects ------------------------------ - -.. versionadded:: 3.5 - -:class:`StackSummary` objects represent a call stack ready for formatting. - -.. class:: StackSummary - - .. classmethod:: extract(frame_gen, *, limit=None, lookup_lines=True, capture_locals=False) - - Construct a :class:`StackSummary` object from a frame generator (such as - is returned by :func:`~traceback.walk_stack` or - :func:`~traceback.walk_tb`). - - If *limit* is supplied, only this many frames are taken from *frame_gen*. - If *lookup_lines* is ``False``, the returned :class:`FrameSummary` - objects will not have read their lines in yet, making the cost of - creating the :class:`StackSummary` cheaper (which may be valuable if it - may not actually get formatted). If *capture_locals* is ``True`` the - local variables in each :class:`FrameSummary` are captured as object - representations. - - .. classmethod:: from_list(a_list) - - Construct a :class:`StackSummary` object from a supplied list of - :class:`FrameSummary` objects or old-style list of tuples. Each tuple - should be a 4-tuple with filename, lineno, name, line as the elements. - - .. method:: format() - - Returns a list of strings ready for printing. Each string in the - resulting list corresponds to a single frame from the stack. - Each string ends in a newline; the strings may contain internal - newlines as well, for those items with source text lines. - - For long sequences of the same frame and line, the first few - repetitions are shown, followed by a summary line stating the exact - number of further repetitions. - - .. versionchanged:: 3.6 - Long sequences of repeated frames are now abbreviated. - - .. method:: format_frame_summary(frame_summary) - - Returns a string for printing one of the frames involved in the stack. - This method is called for each :class:`FrameSummary` object to be - printed by :meth:`StackSummary.format`. If it returns ``None``, the - frame is omitted from the output. - - .. versionadded:: 3.11 - - -:class:`FrameSummary` Objects ------------------------------ - -.. versionadded:: 3.5 - -A :class:`FrameSummary` object represents a single frame in a traceback. - -.. class:: FrameSummary(filename, lineno, name, lookup_line=True, locals=None, line=None) - - Represent a single frame in the traceback or stack that is being formatted - or printed. It may optionally have a stringified version of the frames - locals included in it. If *lookup_line* is ``False``, the source code is not - looked up until the :class:`FrameSummary` has the :attr:`~FrameSummary.line` - attribute accessed (which also happens when casting it to a tuple). - :attr:`~FrameSummary.line` may be directly provided, and will prevent line - lookups happening at all. *locals* is an optional local variable - dictionary, and if supplied the variable representations are stored in the - summary for later display. - -.. _traceback-example: - -Traceback Examples ------------------- - -This simple example implements a basic read-eval-print loop, similar to (but -less useful than) the standard Python interactive interpreter loop. For a more -complete implementation of the interpreter loop, refer to the :mod:`code` -module. :: - - import sys, traceback - - def run_user_code(envdir): - source = input(">>> ") - try: - exec(source, envdir) - except Exception: - print("Exception in user code:") - print("-"*60) - traceback.print_exc(file=sys.stdout) - print("-"*60) - - envdir = {} - while True: - run_user_code(envdir) - - -The following example demonstrates the different ways to print and format the -exception and traceback: - -.. testcode:: - - import sys, traceback - - def lumberjack(): - bright_side_of_life() - - def bright_side_of_life(): - return tuple()[0] - - try: - lumberjack() - except IndexError: - exc = sys.exception() - print("*** print_tb:") - traceback.print_tb(exc.__traceback__, limit=1, file=sys.stdout) - print("*** print_exception:") - traceback.print_exception(exc, limit=2, file=sys.stdout) - print("*** print_exc:") - traceback.print_exc(limit=2, file=sys.stdout) - print("*** format_exc, first and last line:") - formatted_lines = traceback.format_exc().splitlines() - print(formatted_lines[0]) - print(formatted_lines[-1]) - print("*** format_exception:") - print(repr(traceback.format_exception(exc))) - print("*** extract_tb:") - print(repr(traceback.extract_tb(exc.__traceback__))) - print("*** format_tb:") - print(repr(traceback.format_tb(exc.__traceback__))) - print("*** tb_lineno:", exc.__traceback__.tb_lineno) - -The output for the example would look similar to this: - -.. testoutput:: - :options: +NORMALIZE_WHITESPACE - - *** print_tb: - File "", line 10, in - lumberjack() - *** print_exception: - Traceback (most recent call last): - File "", line 10, in - lumberjack() - File "", line 4, in lumberjack - bright_side_of_life() - IndexError: tuple index out of range - *** print_exc: - Traceback (most recent call last): - File "", line 10, in - lumberjack() - File "", line 4, in lumberjack - bright_side_of_life() - IndexError: tuple index out of range - *** format_exc, first and last line: - Traceback (most recent call last): - IndexError: tuple index out of range - *** format_exception: - ['Traceback (most recent call last):\n', - ' File "", line 10, in \n lumberjack()\n', - ' File "", line 4, in lumberjack\n bright_side_of_life()\n', - ' File "", line 7, in bright_side_of_life\n return tuple()[0]\n ~~~~~~~^^^\n', - 'IndexError: tuple index out of range\n'] - *** extract_tb: - [, line 10 in >, - , line 4 in lumberjack>, - , line 7 in bright_side_of_life>] - *** format_tb: - [' File "", line 10, in \n lumberjack()\n', - ' File "", line 4, in lumberjack\n bright_side_of_life()\n', - ' File "", line 7, in bright_side_of_life\n return tuple()[0]\n ~~~~~~~^^^\n'] - *** tb_lineno: 10 - - -The following example shows the different ways to print and format the stack:: - - >>> import traceback - >>> def another_function(): - ... lumberstack() - ... - >>> def lumberstack(): - ... traceback.print_stack() - ... print(repr(traceback.extract_stack())) - ... print(repr(traceback.format_stack())) - ... - >>> another_function() - File "", line 10, in - another_function() - File "", line 3, in another_function - lumberstack() - File "", line 6, in lumberstack - traceback.print_stack() - [('', 10, '', 'another_function()'), - ('', 3, 'another_function', 'lumberstack()'), - ('', 7, 'lumberstack', 'print(repr(traceback.extract_stack()))')] - [' File "", line 10, in \n another_function()\n', - ' File "", line 3, in another_function\n lumberstack()\n', - ' File "", line 8, in lumberstack\n print(repr(traceback.format_stack()))\n'] - - -This last example demonstrates the final few formatting functions: - -.. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> import traceback - >>> traceback.format_list([('spam.py', 3, '', 'spam.eggs()'), - ... ('eggs.py', 42, 'eggs', 'return "bacon"')]) - [' File "spam.py", line 3, in \n spam.eggs()\n', - ' File "eggs.py", line 42, in eggs\n return "bacon"\n'] - >>> an_error = IndexError('tuple index out of range') - >>> traceback.format_exception_only(type(an_error), an_error) - ['IndexError: tuple index out of range\n'] diff --git a/Doc/library/tracemalloc.rst b/Doc/library/tracemalloc.rst deleted file mode 100644 index 68432aeaecbcc1..00000000000000 --- a/Doc/library/tracemalloc.rst +++ /dev/null @@ -1,765 +0,0 @@ -:mod:`tracemalloc` --- Trace memory allocations -=============================================== - -.. module:: tracemalloc - :synopsis: Trace memory allocations. - -.. versionadded:: 3.4 - -**Source code:** :source:`Lib/tracemalloc.py` - --------------- - -The tracemalloc module is a debug tool to trace memory blocks allocated by -Python. It provides the following information: - -* Traceback where an object was allocated -* Statistics on allocated memory blocks per filename and per line number: - total size, number and average size of allocated memory blocks -* Compute the differences between two snapshots to detect memory leaks - -To trace most memory blocks allocated by Python, the module should be started -as early as possible by setting the :envvar:`PYTHONTRACEMALLOC` environment -variable to ``1``, or by using :option:`-X` ``tracemalloc`` command line -option. The :func:`tracemalloc.start` function can be called at runtime to -start tracing Python memory allocations. - -By default, a trace of an allocated memory block only stores the most recent -frame (1 frame). To store 25 frames at startup: set the -:envvar:`PYTHONTRACEMALLOC` environment variable to ``25``, or use the -:option:`-X` ``tracemalloc=25`` command line option. - - -Examples --------- - -Display the top 10 -^^^^^^^^^^^^^^^^^^ - -Display the 10 files allocating the most memory:: - - import tracemalloc - - tracemalloc.start() - - # ... run your application ... - - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics('lineno') - - print("[ Top 10 ]") - for stat in top_stats[:10]: - print(stat) - - -Example of output of the Python test suite:: - - [ Top 10 ] - :716: size=4855 KiB, count=39328, average=126 B - :284: size=521 KiB, count=3199, average=167 B - /usr/lib/python3.4/collections/__init__.py:368: size=244 KiB, count=2315, average=108 B - /usr/lib/python3.4/unittest/case.py:381: size=185 KiB, count=779, average=243 B - /usr/lib/python3.4/unittest/case.py:402: size=154 KiB, count=378, average=416 B - /usr/lib/python3.4/abc.py:133: size=88.7 KiB, count=347, average=262 B - :1446: size=70.4 KiB, count=911, average=79 B - :1454: size=52.0 KiB, count=25, average=2131 B - :5: size=49.7 KiB, count=148, average=344 B - /usr/lib/python3.4/sysconfig.py:411: size=48.0 KiB, count=1, average=48.0 KiB - -We can see that Python loaded ``4855 KiB`` data (bytecode and constants) from -modules and that the :mod:`collections` module allocated ``244 KiB`` to build -:class:`~collections.namedtuple` types. - -See :meth:`Snapshot.statistics` for more options. - - -Compute differences -^^^^^^^^^^^^^^^^^^^ - -Take two snapshots and display the differences:: - - import tracemalloc - tracemalloc.start() - # ... start your application ... - - snapshot1 = tracemalloc.take_snapshot() - # ... call the function leaking memory ... - snapshot2 = tracemalloc.take_snapshot() - - top_stats = snapshot2.compare_to(snapshot1, 'lineno') - - print("[ Top 10 differences ]") - for stat in top_stats[:10]: - print(stat) - -Example of output before/after running some tests of the Python test suite:: - - [ Top 10 differences ] - :716: size=8173 KiB (+4428 KiB), count=71332 (+39369), average=117 B - /usr/lib/python3.4/linecache.py:127: size=940 KiB (+940 KiB), count=8106 (+8106), average=119 B - /usr/lib/python3.4/unittest/case.py:571: size=298 KiB (+298 KiB), count=589 (+589), average=519 B - :284: size=1005 KiB (+166 KiB), count=7423 (+1526), average=139 B - /usr/lib/python3.4/mimetypes.py:217: size=112 KiB (+112 KiB), count=1334 (+1334), average=86 B - /usr/lib/python3.4/http/server.py:848: size=96.0 KiB (+96.0 KiB), count=1 (+1), average=96.0 KiB - /usr/lib/python3.4/inspect.py:1465: size=83.5 KiB (+83.5 KiB), count=109 (+109), average=784 B - /usr/lib/python3.4/unittest/mock.py:491: size=77.7 KiB (+77.7 KiB), count=143 (+143), average=557 B - /usr/lib/python3.4/urllib/parse.py:476: size=71.8 KiB (+71.8 KiB), count=969 (+969), average=76 B - /usr/lib/python3.4/contextlib.py:38: size=67.2 KiB (+67.2 KiB), count=126 (+126), average=546 B - -We can see that Python has loaded ``8173 KiB`` of module data (bytecode and -constants), and that this is ``4428 KiB`` more than had been loaded before the -tests, when the previous snapshot was taken. Similarly, the :mod:`linecache` -module has cached ``940 KiB`` of Python source code to format tracebacks, all -of it since the previous snapshot. - -If the system has little free memory, snapshots can be written on disk using -the :meth:`Snapshot.dump` method to analyze the snapshot offline. Then use the -:meth:`Snapshot.load` method reload the snapshot. - - -Get the traceback of a memory block -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Code to display the traceback of the biggest memory block:: - - import tracemalloc - - # Store 25 frames - tracemalloc.start(25) - - # ... run your application ... - - snapshot = tracemalloc.take_snapshot() - top_stats = snapshot.statistics('traceback') - - # pick the biggest memory block - stat = top_stats[0] - print("%s memory blocks: %.1f KiB" % (stat.count, stat.size / 1024)) - for line in stat.traceback.format(): - print(line) - -Example of output of the Python test suite (traceback limited to 25 frames):: - - 903 memory blocks: 870.1 KiB - File "", line 716 - File "", line 1036 - File "", line 934 - File "", line 1068 - File "", line 619 - File "", line 1581 - File "", line 1614 - File "/usr/lib/python3.4/doctest.py", line 101 - import pdb - File "", line 284 - File "", line 938 - File "", line 1068 - File "", line 619 - File "", line 1581 - File "", line 1614 - File "/usr/lib/python3.4/test/support/__init__.py", line 1728 - import doctest - File "/usr/lib/python3.4/test/test_pickletools.py", line 21 - support.run_doctest(pickletools) - File "/usr/lib/python3.4/test/regrtest.py", line 1276 - test_runner() - File "/usr/lib/python3.4/test/regrtest.py", line 976 - display_failure=not verbose) - File "/usr/lib/python3.4/test/regrtest.py", line 761 - match_tests=ns.match_tests) - File "/usr/lib/python3.4/test/regrtest.py", line 1563 - main() - File "/usr/lib/python3.4/test/__main__.py", line 3 - regrtest.main_in_temp_cwd() - File "/usr/lib/python3.4/runpy.py", line 73 - exec(code, run_globals) - File "/usr/lib/python3.4/runpy.py", line 160 - "__main__", fname, loader, pkg_name) - -We can see that the most memory was allocated in the :mod:`importlib` module to -load data (bytecode and constants) from modules: ``870.1 KiB``. The traceback is -where the :mod:`importlib` loaded data most recently: on the ``import pdb`` -line of the :mod:`doctest` module. The traceback may change if a new module is -loaded. - - -Pretty top -^^^^^^^^^^ - -Code to display the 10 lines allocating the most memory with a pretty output, -ignoring ```` and ```` files:: - - import linecache - import os - import tracemalloc - - def display_top(snapshot, key_type='lineno', limit=10): - snapshot = snapshot.filter_traces(( - tracemalloc.Filter(False, ""), - tracemalloc.Filter(False, ""), - )) - top_stats = snapshot.statistics(key_type) - - print("Top %s lines" % limit) - for index, stat in enumerate(top_stats[:limit], 1): - frame = stat.traceback[0] - print("#%s: %s:%s: %.1f KiB" - % (index, frame.filename, frame.lineno, stat.size / 1024)) - line = linecache.getline(frame.filename, frame.lineno).strip() - if line: - print(' %s' % line) - - other = top_stats[limit:] - if other: - size = sum(stat.size for stat in other) - print("%s other: %.1f KiB" % (len(other), size / 1024)) - total = sum(stat.size for stat in top_stats) - print("Total allocated size: %.1f KiB" % (total / 1024)) - - tracemalloc.start() - - # ... run your application ... - - snapshot = tracemalloc.take_snapshot() - display_top(snapshot) - -Example of output of the Python test suite:: - - Top 10 lines - #1: Lib/base64.py:414: 419.8 KiB - _b85chars2 = [(a + b) for a in _b85chars for b in _b85chars] - #2: Lib/base64.py:306: 419.8 KiB - _a85chars2 = [(a + b) for a in _a85chars for b in _a85chars] - #3: collections/__init__.py:368: 293.6 KiB - exec(class_definition, namespace) - #4: Lib/abc.py:133: 115.2 KiB - cls = super().__new__(mcls, name, bases, namespace) - #5: unittest/case.py:574: 103.1 KiB - testMethod() - #6: Lib/linecache.py:127: 95.4 KiB - lines = fp.readlines() - #7: urllib/parse.py:476: 71.8 KiB - for a in _hexdig for b in _hexdig} - #8: :5: 62.0 KiB - #9: Lib/_weakrefset.py:37: 60.0 KiB - self.data = set() - #10: Lib/base64.py:142: 59.8 KiB - _b32tab2 = [a + b for a in _b32tab for b in _b32tab] - 6220 other: 3602.8 KiB - Total allocated size: 5303.1 KiB - -See :meth:`Snapshot.statistics` for more options. - -Record the current and peak size of all traced memory blocks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The following code computes two sums like ``0 + 1 + 2 + ...`` inefficiently, by -creating a list of those numbers. This list consumes a lot of memory -temporarily. We can use :func:`get_traced_memory` and :func:`reset_peak` to -observe the small memory usage after the sum is computed as well as the peak -memory usage during the computations:: - - import tracemalloc - - tracemalloc.start() - - # Example code: compute a sum with a large temporary list - large_sum = sum(list(range(100000))) - - first_size, first_peak = tracemalloc.get_traced_memory() - - tracemalloc.reset_peak() - - # Example code: compute a sum with a small temporary list - small_sum = sum(list(range(1000))) - - second_size, second_peak = tracemalloc.get_traced_memory() - - print(f"{first_size=}, {first_peak=}") - print(f"{second_size=}, {second_peak=}") - -Output:: - - first_size=664, first_peak=3592984 - second_size=804, second_peak=29704 - -Using :func:`reset_peak` ensured we could accurately record the peak during the -computation of ``small_sum``, even though it is much smaller than the overall -peak size of memory blocks since the :func:`start` call. Without the call to -:func:`reset_peak`, ``second_peak`` would still be the peak from the -computation ``large_sum`` (that is, equal to ``first_peak``). In this case, -both peaks are much higher than the final memory usage, and which suggests we -could optimise (by removing the unnecessary call to :class:`list`, and writing -``sum(range(...))``). - -API ---- - -Functions -^^^^^^^^^ - -.. function:: clear_traces() - - Clear traces of memory blocks allocated by Python. - - See also :func:`stop`. - - -.. function:: get_object_traceback(obj) - - Get the traceback where the Python object *obj* was allocated. - Return a :class:`Traceback` instance, or ``None`` if the :mod:`tracemalloc` - module is not tracing memory allocations or did not trace the allocation of - the object. - - See also :func:`gc.get_referrers` and :func:`sys.getsizeof` functions. - - -.. function:: get_traceback_limit() - - Get the maximum number of frames stored in the traceback of a trace. - - The :mod:`tracemalloc` module must be tracing memory allocations to - get the limit, otherwise an exception is raised. - - The limit is set by the :func:`start` function. - - -.. function:: get_traced_memory() - - Get the current size and peak size of memory blocks traced by the - :mod:`tracemalloc` module as a tuple: ``(current: int, peak: int)``. - - -.. function:: reset_peak() - - Set the peak size of memory blocks traced by the :mod:`tracemalloc` module - to the current size. - - Do nothing if the :mod:`tracemalloc` module is not tracing memory - allocations. - - This function only modifies the recorded peak size, and does not modify or - clear any traces, unlike :func:`clear_traces`. Snapshots taken with - :func:`take_snapshot` before a call to :func:`reset_peak` can be - meaningfully compared to snapshots taken after the call. - - See also :func:`get_traced_memory`. - - .. versionadded:: 3.9 - - -.. function:: get_tracemalloc_memory() - - Get the memory usage in bytes of the :mod:`tracemalloc` module used to store - traces of memory blocks. - Return an :class:`int`. - - -.. function:: is_tracing() - - ``True`` if the :mod:`tracemalloc` module is tracing Python memory - allocations, ``False`` otherwise. - - See also :func:`start` and :func:`stop` functions. - - -.. function:: start(nframe: int=1) - - Start tracing Python memory allocations: install hooks on Python memory - allocators. Collected tracebacks of traces will be limited to *nframe* - frames. By default, a trace of a memory block only stores the most recent - frame: the limit is ``1``. *nframe* must be greater or equal to ``1``. - - You can still read the original number of total frames that composed the - traceback by looking at the :attr:`Traceback.total_nframe` attribute. - - Storing more than ``1`` frame is only useful to compute statistics grouped - by ``'traceback'`` or to compute cumulative statistics: see the - :meth:`Snapshot.compare_to` and :meth:`Snapshot.statistics` methods. - - Storing more frames increases the memory and CPU overhead of the - :mod:`tracemalloc` module. Use the :func:`get_tracemalloc_memory` function - to measure how much memory is used by the :mod:`tracemalloc` module. - - The :envvar:`PYTHONTRACEMALLOC` environment variable - (``PYTHONTRACEMALLOC=NFRAME``) and the :option:`-X` ``tracemalloc=NFRAME`` - command line option can be used to start tracing at startup. - - See also :func:`stop`, :func:`is_tracing` and :func:`get_traceback_limit` - functions. - - -.. function:: stop() - - Stop tracing Python memory allocations: uninstall hooks on Python memory - allocators. Also clears all previously collected traces of memory blocks - allocated by Python. - - Call :func:`take_snapshot` function to take a snapshot of traces before - clearing them. - - See also :func:`start`, :func:`is_tracing` and :func:`clear_traces` - functions. - - -.. function:: take_snapshot() - - Take a snapshot of traces of memory blocks allocated by Python. Return a new - :class:`Snapshot` instance. - - The snapshot does not include memory blocks allocated before the - :mod:`tracemalloc` module started to trace memory allocations. - - Tracebacks of traces are limited to :func:`get_traceback_limit` frames. Use - the *nframe* parameter of the :func:`start` function to store more frames. - - The :mod:`tracemalloc` module must be tracing memory allocations to take a - snapshot, see the :func:`start` function. - - See also the :func:`get_object_traceback` function. - - -DomainFilter -^^^^^^^^^^^^ - -.. class:: DomainFilter(inclusive: bool, domain: int) - - Filter traces of memory blocks by their address space (domain). - - .. versionadded:: 3.6 - - .. attribute:: inclusive - - If *inclusive* is ``True`` (include), match memory blocks allocated - in the address space :attr:`domain`. - - If *inclusive* is ``False`` (exclude), match memory blocks not allocated - in the address space :attr:`domain`. - - .. attribute:: domain - - Address space of a memory block (``int``). Read-only property. - - -Filter -^^^^^^ - -.. class:: Filter(inclusive: bool, filename_pattern: str, lineno: int=None, all_frames: bool=False, domain: int=None) - - Filter on traces of memory blocks. - - See the :func:`fnmatch.fnmatch` function for the syntax of - *filename_pattern*. The ``'.pyc'`` file extension is - replaced with ``'.py'``. - - Examples: - - * ``Filter(True, subprocess.__file__)`` only includes traces of the - :mod:`subprocess` module - * ``Filter(False, tracemalloc.__file__)`` excludes traces of the - :mod:`tracemalloc` module - * ``Filter(False, "")`` excludes empty tracebacks - - - .. versionchanged:: 3.5 - The ``'.pyo'`` file extension is no longer replaced with ``'.py'``. - - .. versionchanged:: 3.6 - Added the :attr:`domain` attribute. - - - .. attribute:: domain - - Address space of a memory block (``int`` or ``None``). - - tracemalloc uses the domain ``0`` to trace memory allocations made by - Python. C extensions can use other domains to trace other resources. - - .. attribute:: inclusive - - If *inclusive* is ``True`` (include), only match memory blocks allocated - in a file with a name matching :attr:`filename_pattern` at line number - :attr:`lineno`. - - If *inclusive* is ``False`` (exclude), ignore memory blocks allocated in - a file with a name matching :attr:`filename_pattern` at line number - :attr:`lineno`. - - .. attribute:: lineno - - Line number (``int``) of the filter. If *lineno* is ``None``, the filter - matches any line number. - - .. attribute:: filename_pattern - - Filename pattern of the filter (``str``). Read-only property. - - .. attribute:: all_frames - - If *all_frames* is ``True``, all frames of the traceback are checked. If - *all_frames* is ``False``, only the most recent frame is checked. - - This attribute has no effect if the traceback limit is ``1``. See the - :func:`get_traceback_limit` function and :attr:`Snapshot.traceback_limit` - attribute. - - -Frame -^^^^^ - -.. class:: Frame - - Frame of a traceback. - - The :class:`Traceback` class is a sequence of :class:`Frame` instances. - - .. attribute:: filename - - Filename (``str``). - - .. attribute:: lineno - - Line number (``int``). - - -Snapshot -^^^^^^^^ - -.. class:: Snapshot - - Snapshot of traces of memory blocks allocated by Python. - - The :func:`take_snapshot` function creates a snapshot instance. - - .. method:: compare_to(old_snapshot: Snapshot, key_type: str, cumulative: bool=False) - - Compute the differences with an old snapshot. Get statistics as a sorted - list of :class:`StatisticDiff` instances grouped by *key_type*. - - See the :meth:`Snapshot.statistics` method for *key_type* and *cumulative* - parameters. - - The result is sorted from the biggest to the smallest by: absolute value - of :attr:`StatisticDiff.size_diff`, :attr:`StatisticDiff.size`, absolute - value of :attr:`StatisticDiff.count_diff`, :attr:`Statistic.count` and - then by :attr:`StatisticDiff.traceback`. - - - .. method:: dump(filename) - - Write the snapshot into a file. - - Use :meth:`load` to reload the snapshot. - - - .. method:: filter_traces(filters) - - Create a new :class:`Snapshot` instance with a filtered :attr:`traces` - sequence, *filters* is a list of :class:`DomainFilter` and - :class:`Filter` instances. If *filters* is an empty list, return a new - :class:`Snapshot` instance with a copy of the traces. - - All inclusive filters are applied at once, a trace is ignored if no - inclusive filters match it. A trace is ignored if at least one exclusive - filter matches it. - - .. versionchanged:: 3.6 - :class:`DomainFilter` instances are now also accepted in *filters*. - - - .. classmethod:: load(filename) - - Load a snapshot from a file. - - See also :meth:`dump`. - - - .. method:: statistics(key_type: str, cumulative: bool=False) - - Get statistics as a sorted list of :class:`Statistic` instances grouped - by *key_type*: - - ===================== ======================== - key_type description - ===================== ======================== - ``'filename'`` filename - ``'lineno'`` filename and line number - ``'traceback'`` traceback - ===================== ======================== - - If *cumulative* is ``True``, cumulate size and count of memory blocks of - all frames of the traceback of a trace, not only the most recent frame. - The cumulative mode can only be used with *key_type* equals to - ``'filename'`` and ``'lineno'``. - - The result is sorted from the biggest to the smallest by: - :attr:`Statistic.size`, :attr:`Statistic.count` and then by - :attr:`Statistic.traceback`. - - - .. attribute:: traceback_limit - - Maximum number of frames stored in the traceback of :attr:`traces`: - result of the :func:`get_traceback_limit` when the snapshot was taken. - - .. attribute:: traces - - Traces of all memory blocks allocated by Python: sequence of - :class:`Trace` instances. - - The sequence has an undefined order. Use the :meth:`Snapshot.statistics` - method to get a sorted list of statistics. - - -Statistic -^^^^^^^^^ - -.. class:: Statistic - - Statistic on memory allocations. - - :func:`Snapshot.statistics` returns a list of :class:`Statistic` instances. - - See also the :class:`StatisticDiff` class. - - .. attribute:: count - - Number of memory blocks (``int``). - - .. attribute:: size - - Total size of memory blocks in bytes (``int``). - - .. attribute:: traceback - - Traceback where the memory block was allocated, :class:`Traceback` - instance. - - -StatisticDiff -^^^^^^^^^^^^^ - -.. class:: StatisticDiff - - Statistic difference on memory allocations between an old and a new - :class:`Snapshot` instance. - - :func:`Snapshot.compare_to` returns a list of :class:`StatisticDiff` - instances. See also the :class:`Statistic` class. - - .. attribute:: count - - Number of memory blocks in the new snapshot (``int``): ``0`` if - the memory blocks have been released in the new snapshot. - - .. attribute:: count_diff - - Difference of number of memory blocks between the old and the new - snapshots (``int``): ``0`` if the memory blocks have been allocated in - the new snapshot. - - .. attribute:: size - - Total size of memory blocks in bytes in the new snapshot (``int``): - ``0`` if the memory blocks have been released in the new snapshot. - - .. attribute:: size_diff - - Difference of total size of memory blocks in bytes between the old and - the new snapshots (``int``): ``0`` if the memory blocks have been - allocated in the new snapshot. - - .. attribute:: traceback - - Traceback where the memory blocks were allocated, :class:`Traceback` - instance. - - -Trace -^^^^^ - -.. class:: Trace - - Trace of a memory block. - - The :attr:`Snapshot.traces` attribute is a sequence of :class:`Trace` - instances. - - .. versionchanged:: 3.6 - Added the :attr:`domain` attribute. - - .. attribute:: domain - - Address space of a memory block (``int``). Read-only property. - - tracemalloc uses the domain ``0`` to trace memory allocations made by - Python. C extensions can use other domains to trace other resources. - - .. attribute:: size - - Size of the memory block in bytes (``int``). - - .. attribute:: traceback - - Traceback where the memory block was allocated, :class:`Traceback` - instance. - - -Traceback -^^^^^^^^^ - -.. class:: Traceback - - Sequence of :class:`Frame` instances sorted from the oldest frame to the - most recent frame. - - A traceback contains at least ``1`` frame. If the ``tracemalloc`` module - failed to get a frame, the filename ``""`` at line number ``0`` is - used. - - When a snapshot is taken, tracebacks of traces are limited to - :func:`get_traceback_limit` frames. See the :func:`take_snapshot` function. - The original number of frames of the traceback is stored in the - :attr:`Traceback.total_nframe` attribute. That allows to know if a traceback - has been truncated by the traceback limit. - - The :attr:`Trace.traceback` attribute is an instance of :class:`Traceback` - instance. - - .. versionchanged:: 3.7 - Frames are now sorted from the oldest to the most recent, instead of most recent to oldest. - - .. attribute:: total_nframe - - Total number of frames that composed the traceback before truncation. - This attribute can be set to ``None`` if the information is not - available. - - .. versionchanged:: 3.9 - The :attr:`Traceback.total_nframe` attribute was added. - - .. method:: format(limit=None, most_recent_first=False) - - Format the traceback as a list of lines. Use the :mod:`linecache` module to - retrieve lines from the source code. If *limit* is set, format the *limit* - most recent frames if *limit* is positive. Otherwise, format the - ``abs(limit)`` oldest frames. If *most_recent_first* is ``True``, the order - of the formatted frames is reversed, returning the most recent frame first - instead of last. - - Similar to the :func:`traceback.format_tb` function, except that - :meth:`.format` does not include newlines. - - Example:: - - print("Traceback (most recent call first):") - for line in traceback: - print(line) - - Output:: - - Traceback (most recent call first): - File "test.py", line 9 - obj = Object() - File "test.py", line 12 - tb = tracemalloc.get_object_traceback(f()) diff --git a/Doc/library/tty.rst b/Doc/library/tty.rst deleted file mode 100644 index 75ba6c4523e5e7..00000000000000 --- a/Doc/library/tty.rst +++ /dev/null @@ -1,43 +0,0 @@ -:mod:`tty` --- Terminal control functions -========================================= - -.. module:: tty - :platform: Unix - :synopsis: Utility functions that perform common terminal control operations. - -.. moduleauthor:: Steen Lumholt -.. sectionauthor:: Moshe Zadka - -**Source code:** :source:`Lib/tty.py` - --------------- - -The :mod:`tty` module defines functions for putting the tty into cbreak and raw -modes. - -.. availability:: Unix. - -Because it requires the :mod:`termios` module, it will work only on Unix. - -The :mod:`tty` module defines the following functions: - - -.. function:: setraw(fd, when=termios.TCSAFLUSH) - - Change the mode of the file descriptor *fd* to raw. If *when* is omitted, it - defaults to :const:`termios.TCSAFLUSH`, and is passed to - :func:`termios.tcsetattr`. - - -.. function:: setcbreak(fd, when=termios.TCSAFLUSH) - - Change the mode of file descriptor *fd* to cbreak. If *when* is omitted, it - defaults to :const:`termios.TCSAFLUSH`, and is passed to - :func:`termios.tcsetattr`. - - -.. seealso:: - - Module :mod:`termios` - Low-level terminal control interface. - diff --git a/Doc/library/tulip_coro.dia b/Doc/library/tulip_coro.dia deleted file mode 100644 index 70a33e3c00cf6e..00000000000000 Binary files a/Doc/library/tulip_coro.dia and /dev/null differ diff --git a/Doc/library/tulip_coro.png b/Doc/library/tulip_coro.png deleted file mode 100644 index aad41c93015d09..00000000000000 Binary files a/Doc/library/tulip_coro.png and /dev/null differ diff --git a/Doc/library/turtle-star.pdf b/Doc/library/turtle-star.pdf deleted file mode 100644 index e354073dd42f5e..00000000000000 Binary files a/Doc/library/turtle-star.pdf and /dev/null differ diff --git a/Doc/library/turtle-star.png b/Doc/library/turtle-star.png deleted file mode 100644 index 0961e1e2153f72..00000000000000 Binary files a/Doc/library/turtle-star.png and /dev/null differ diff --git a/Doc/library/turtle-star.ps b/Doc/library/turtle-star.ps deleted file mode 100644 index 46362cb9f7c8f5..00000000000000 Binary files a/Doc/library/turtle-star.ps and /dev/null differ diff --git a/Doc/library/turtle.rst b/Doc/library/turtle.rst deleted file mode 100644 index b56f84bfe2f942..00000000000000 --- a/Doc/library/turtle.rst +++ /dev/null @@ -1,2763 +0,0 @@ -================================= -:mod:`turtle` --- Turtle graphics -================================= - -.. module:: turtle - :synopsis: An educational framework for simple graphics applications - -.. sectionauthor:: Gregor Lingl - -**Source code:** :source:`Lib/turtle.py` - -.. testsetup:: default - - from turtle import * - turtle = Turtle() - --------------- - -Introduction -============ - -Turtle graphics is an implementation of `the popular geometric drawing tools -introduced in Logo `_, developed by Wally Feurzeig, Seymour Papert and Cynthia Solomon -in 1967. - -.. sidebar:: Turtle star - - Turtle can draw intricate shapes using programs that repeat simple - moves. - - .. image:: turtle-star.* - :align: center - -In Python, turtle graphics provides a representation of a physical "turtle" -(a little robot with a pen) that draws on a sheet of paper on the floor. - -It's an effective and well-proven way for learners to encounter -programming concepts and interaction with software, as it provides instant, -visible feedback. It also provides convenient access to graphical output -in general. - -Turtle drawing was originally created as an educational tool, to be used by -teachers in the classroom. For the programmer who needs to produce some -graphical output it can be a way to do that without the overhead of -introducing more complex or external libraries into their work. - - -.. _turtle-tutorial: - -Tutorial -======== - -New users should start here. In this tutorial we'll explore some of the -basics of turtle drawing. - - -Starting a turtle environment ------------------------------ - -In a Python shell, import all the objects of the ``turtle`` module:: - - from turtle import * - -If you run into a ``No module named '_tkinter'`` error, you'll have to -install the :mod:`Tk interface package ` on your system. - - -Basic drawing -------------- - -Send the turtle forward 100 steps:: - - forward(100) - -You should see (most likely, in a new window on your display) a line -drawn by the turtle, heading East. Change the direction of the turtle, -so that it turns 120 degrees left (anti-clockwise):: - - left(120) - -Let's continue by drawing a triangle:: - - forward(100) - left(120) - forward(100) - -Notice how the turtle, represented by an arrow, points in different -directions as you steer it. - -Experiment with those commands, and also with ``backward()`` and -``right()``. - - -Pen control -~~~~~~~~~~~ - -Try changing the color - for example, ``color('blue')`` - and -width of the line - for example, ``width(3)`` - and then drawing again. - -You can also move the turtle around without drawing, by lifting up the pen: -``up()`` before moving. To start drawing again, use ``down()``. - - -The turtle's position -~~~~~~~~~~~~~~~~~~~~~ - -Send your turtle back to its starting-point (useful if it has disappeared -off-screen):: - - home() - -The home position is at the center of the turtle's screen. If you ever need to -know them, get the turtle's x-y co-ordinates with:: - - pos() - -Home is at ``(0, 0)``. - -And after a while, it will probably help to clear the window so we can start -anew:: - - clearscreen() - - -Making algorithmic patterns ---------------------------- - -Using loops, it's possible to build up geometric patterns:: - - for steps in range(100): - for c in ('blue', 'red', 'green'): - color(c) - forward(steps) - right(30) - - -\ - which of course, are limited only by the imagination! - -Let's draw the star shape at the top of this page. We want red lines, -filled in with yellow:: - - color('red') - fillcolor('yellow') - -Just as ``up()`` and ``down()`` determine whether lines will be drawn, -filling can be turned on and off:: - - begin_fill() - -Next we'll create a loop:: - - while True: - forward(200) - left(170) - if abs(pos()) < 1: - break - -``abs(pos()) < 1`` is a good way to know when the turtle is back at its -home position. - -Finally, complete the filling:: - - end_fill() - -(Note that filling only actually takes place when you give the -``end_fill()`` command.) - - -.. _turtle-how-to: - -How to... -========= - -This section covers some typical turtle use-cases and approaches. - - -Get started as quickly as possible ----------------------------------- - -One of the joys of turtle graphics is the immediate, visual feedback that's -available from simple commands - it's an excellent way to introduce children -to programming ideas, with a minimum of overhead (not just children, of -course). - -The turtle module makes this possible by exposing all its basic functionality -as functions, available with ``from turtle import *``. The :ref:`turtle -graphics tutorial ` covers this approach. - -It's worth noting that many of the turtle commands also have even more terse -equivalents, such as ``fd()`` for :func:`forward`. These are especially -useful when working with learners for whom typing is not a skill. - -.. _note: - - You'll need to have the :mod:`Tk interface package ` installed on - your system for turtle graphics to work. Be warned that this is not - always straightforward, so check this in advance if you're planning to - use turtle graphics with a learner. - - -Use the ``turtle`` module namespace ------------------------------------ - -Using ``from turtle import *`` is convenient - but be warned that it imports a -rather large collection of objects, and if you're doing anything but turtle -graphics you run the risk of a name conflict (this becomes even more an issue -if you're using turtle graphics in a script where other modules might be -imported). - -The solution is to use ``import turtle`` - ``fd()`` becomes -``turtle.fd()``, ``width()`` becomes ``turtle.width()`` and so on. (If typing -"turtle" over and over again becomes tedious, use for example ``import turtle -as t`` instead.) - - -Use turtle graphics in a script -------------------------------- - -It's recommended to use the ``turtle`` module namespace as described -immediately above, for example:: - - import turtle as t - from random import random - - for i in range(100): - steps = int(random() * 100) - angle = int(random() * 360) - t.right(angle) - t.fd(steps) - -Another step is also required though - as soon as the script ends, Python -will also close the turtle's window. Add:: - - t.mainloop() - -to the end of the script. The script will now wait to be dismissed and -will not exit until it is terminated, for example by closing the turtle -graphics window. - - -Use object-oriented turtle graphics ------------------------------------ - -.. seealso:: :ref:`Explanation of the object-oriented interface ` - -Other than for very basic introductory purposes, or for trying things out -as quickly as possible, it's more usual and much more powerful to use the -object-oriented approach to turtle graphics. For example, this allows -multiple turtles on screen at once. - -In this approach, the various turtle commands are methods of objects (mostly of -``Turtle`` objects). You *can* use the object-oriented approach in the shell, -but it would be more typical in a Python script. - -The example above then becomes:: - - from turtle import Turtle - from random import random - - t = Turtle() - for i in range(100): - steps = int(random() * 100) - angle = int(random() * 360) - t.right(angle) - t.fd(steps) - - t.screen.mainloop() - -Note the last line. ``t.screen`` is an instance of the :class:`Screen` -that a Turtle instance exists on; it's created automatically along with -the turtle. - -The turtle's screen can be customised, for example:: - - t.screen.title('Object-oriented turtle demo') - t.screen.bgcolor("orange") - - -Turtle graphics reference -========================= - -.. note:: - - In the following documentation the argument list for functions is given. - Methods, of course, have the additional first argument *self* which is - omitted here. - - -Turtle methods --------------- - -Turtle motion - Move and draw - | :func:`forward` | :func:`fd` - | :func:`backward` | :func:`bk` | :func:`back` - | :func:`right` | :func:`rt` - | :func:`left` | :func:`lt` - | :func:`goto` | :func:`setpos` | :func:`setposition` - | :func:`setx` - | :func:`sety` - | :func:`setheading` | :func:`seth` - | :func:`home` - | :func:`circle` - | :func:`dot` - | :func:`stamp` - | :func:`clearstamp` - | :func:`clearstamps` - | :func:`undo` - | :func:`speed` - - Tell Turtle's state - | :func:`position` | :func:`pos` - | :func:`towards` - | :func:`xcor` - | :func:`ycor` - | :func:`heading` - | :func:`distance` - - Setting and measurement - | :func:`degrees` - | :func:`radians` - -Pen control - Drawing state - | :func:`pendown` | :func:`pd` | :func:`down` - | :func:`penup` | :func:`pu` | :func:`up` - | :func:`pensize` | :func:`width` - | :func:`pen` - | :func:`isdown` - - Color control - | :func:`color` - | :func:`pencolor` - | :func:`fillcolor` - - Filling - | :func:`filling` - | :func:`begin_fill` - | :func:`end_fill` - - More drawing control - | :func:`reset` - | :func:`clear` - | :func:`write` - -Turtle state - Visibility - | :func:`showturtle` | :func:`st` - | :func:`hideturtle` | :func:`ht` - | :func:`isvisible` - - Appearance - | :func:`shape` - | :func:`resizemode` - | :func:`shapesize` | :func:`turtlesize` - | :func:`shearfactor` - | :func:`settiltangle` - | :func:`tiltangle` - | :func:`tilt` - | :func:`shapetransform` - | :func:`get_shapepoly` - -Using events - | :func:`onclick` - | :func:`onrelease` - | :func:`ondrag` - -Special Turtle methods - | :func:`begin_poly` - | :func:`end_poly` - | :func:`get_poly` - | :func:`clone` - | :func:`getturtle` | :func:`getpen` - | :func:`getscreen` - | :func:`setundobuffer` - | :func:`undobufferentries` - - -Methods of TurtleScreen/Screen ------------------------------- - -Window control - | :func:`bgcolor` - | :func:`bgpic` - | :func:`clearscreen` - | :func:`resetscreen` - | :func:`screensize` - | :func:`setworldcoordinates` - -Animation control - | :func:`delay` - | :func:`tracer` - | :func:`update` - -Using screen events - | :func:`listen` - | :func:`onkey` | :func:`onkeyrelease` - | :func:`onkeypress` - | :func:`onclick` | :func:`onscreenclick` - | :func:`ontimer` - | :func:`mainloop` | :func:`done` - -Settings and special methods - | :func:`mode` - | :func:`colormode` - | :func:`getcanvas` - | :func:`getshapes` - | :func:`register_shape` | :func:`addshape` - | :func:`turtles` - | :func:`window_height` - | :func:`window_width` - -Input methods - | :func:`textinput` - | :func:`numinput` - -Methods specific to Screen - | :func:`bye` - | :func:`exitonclick` - | :func:`setup` - | :func:`title` - - -Methods of RawTurtle/Turtle and corresponding functions -======================================================= - -Most of the examples in this section refer to a Turtle instance called -``turtle``. - -Turtle motion -------------- - -.. function:: forward(distance) - fd(distance) - - :param distance: a number (integer or float) - - Move the turtle forward by the specified *distance*, in the direction the - turtle is headed. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.position() - (0.00,0.00) - >>> turtle.forward(25) - >>> turtle.position() - (25.00,0.00) - >>> turtle.forward(-75) - >>> turtle.position() - (-50.00,0.00) - - -.. function:: back(distance) - bk(distance) - backward(distance) - - :param distance: a number - - Move the turtle backward by *distance*, opposite to the direction the - turtle is headed. Do not change the turtle's heading. - - .. doctest:: - :hide: - - >>> turtle.goto(0, 0) - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.position() - (0.00,0.00) - >>> turtle.backward(30) - >>> turtle.position() - (-30.00,0.00) - - -.. function:: right(angle) - rt(angle) - - :param angle: a number (integer or float) - - Turn turtle right by *angle* units. (Units are by default degrees, but - can be set via the :func:`degrees` and :func:`radians` functions.) Angle - orientation depends on the turtle mode, see :func:`mode`. - - .. doctest:: - :skipif: _tkinter is None - :hide: - - >>> turtle.setheading(22) - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.heading() - 22.0 - >>> turtle.right(45) - >>> turtle.heading() - 337.0 - - -.. function:: left(angle) - lt(angle) - - :param angle: a number (integer or float) - - Turn turtle left by *angle* units. (Units are by default degrees, but - can be set via the :func:`degrees` and :func:`radians` functions.) Angle - orientation depends on the turtle mode, see :func:`mode`. - - .. doctest:: - :skipif: _tkinter is None - :hide: - - >>> turtle.setheading(22) - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.heading() - 22.0 - >>> turtle.left(45) - >>> turtle.heading() - 67.0 - - -.. function:: goto(x, y=None) - setpos(x, y=None) - setposition(x, y=None) - - :param x: a number or a pair/vector of numbers - :param y: a number or ``None`` - - If *y* is ``None``, *x* must be a pair of coordinates or a :class:`Vec2D` - (e.g. as returned by :func:`pos`). - - Move turtle to an absolute position. If the pen is down, draw line. Do - not change the turtle's orientation. - - .. doctest:: - :skipif: _tkinter is None - :hide: - - >>> turtle.goto(0, 0) - - .. doctest:: - :skipif: _tkinter is None - - >>> tp = turtle.pos() - >>> tp - (0.00,0.00) - >>> turtle.setpos(60,30) - >>> turtle.pos() - (60.00,30.00) - >>> turtle.setpos((20,80)) - >>> turtle.pos() - (20.00,80.00) - >>> turtle.setpos(tp) - >>> turtle.pos() - (0.00,0.00) - - -.. function:: setx(x) - - :param x: a number (integer or float) - - Set the turtle's first coordinate to *x*, leave second coordinate - unchanged. - - .. doctest:: - :skipif: _tkinter is None - :hide: - - >>> turtle.goto(0, 240) - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.position() - (0.00,240.00) - >>> turtle.setx(10) - >>> turtle.position() - (10.00,240.00) - - -.. function:: sety(y) - - :param y: a number (integer or float) - - Set the turtle's second coordinate to *y*, leave first coordinate unchanged. - - .. doctest:: - :skipif: _tkinter is None - :hide: - - >>> turtle.goto(0, 40) - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.position() - (0.00,40.00) - >>> turtle.sety(-10) - >>> turtle.position() - (0.00,-10.00) - - -.. function:: setheading(to_angle) - seth(to_angle) - - :param to_angle: a number (integer or float) - - Set the orientation of the turtle to *to_angle*. Here are some common - directions in degrees: - - =================== ==================== - standard mode logo mode - =================== ==================== - 0 - east 0 - north - 90 - north 90 - east - 180 - west 180 - south - 270 - south 270 - west - =================== ==================== - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.setheading(90) - >>> turtle.heading() - 90.0 - - -.. function:: home() - - Move turtle to the origin -- coordinates (0,0) -- and set its heading to - its start-orientation (which depends on the mode, see :func:`mode`). - - .. doctest:: - :skipif: _tkinter is None - :hide: - - >>> turtle.setheading(90) - >>> turtle.goto(0, -10) - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.heading() - 90.0 - >>> turtle.position() - (0.00,-10.00) - >>> turtle.home() - >>> turtle.position() - (0.00,0.00) - >>> turtle.heading() - 0.0 - - -.. function:: circle(radius, extent=None, steps=None) - - :param radius: a number - :param extent: a number (or ``None``) - :param steps: an integer (or ``None``) - - Draw a circle with given *radius*. The center is *radius* units left of - the turtle; *extent* -- an angle -- determines which part of the circle - is drawn. If *extent* is not given, draw the entire circle. If *extent* - is not a full circle, one endpoint of the arc is the current pen - position. Draw the arc in counterclockwise direction if *radius* is - positive, otherwise in clockwise direction. Finally the direction of the - turtle is changed by the amount of *extent*. - - As the circle is approximated by an inscribed regular polygon, *steps* - determines the number of steps to use. If not given, it will be - calculated automatically. May be used to draw regular polygons. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.home() - >>> turtle.position() - (0.00,0.00) - >>> turtle.heading() - 0.0 - >>> turtle.circle(50) - >>> turtle.position() - (-0.00,0.00) - >>> turtle.heading() - 0.0 - >>> turtle.circle(120, 180) # draw a semicircle - >>> turtle.position() - (0.00,240.00) - >>> turtle.heading() - 180.0 - - -.. function:: dot(size=None, *color) - - :param size: an integer >= 1 (if given) - :param color: a colorstring or a numeric color tuple - - Draw a circular dot with diameter *size*, using *color*. If *size* is - not given, the maximum of pensize+4 and 2*pensize is used. - - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.home() - >>> turtle.dot() - >>> turtle.fd(50); turtle.dot(20, "blue"); turtle.fd(50) - >>> turtle.position() - (100.00,-0.00) - >>> turtle.heading() - 0.0 - - -.. function:: stamp() - - Stamp a copy of the turtle shape onto the canvas at the current turtle - position. Return a stamp_id for that stamp, which can be used to delete - it by calling ``clearstamp(stamp_id)``. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.color("blue") - >>> turtle.stamp() - 11 - >>> turtle.fd(50) - - -.. function:: clearstamp(stampid) - - :param stampid: an integer, must be return value of previous - :func:`stamp` call - - Delete stamp with given *stampid*. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.position() - (150.00,-0.00) - >>> turtle.color("blue") - >>> astamp = turtle.stamp() - >>> turtle.fd(50) - >>> turtle.position() - (200.00,-0.00) - >>> turtle.clearstamp(astamp) - >>> turtle.position() - (200.00,-0.00) - - -.. function:: clearstamps(n=None) - - :param n: an integer (or ``None``) - - Delete all or first/last *n* of turtle's stamps. If *n* is ``None``, delete - all stamps, if *n* > 0 delete first *n* stamps, else if *n* < 0 delete - last *n* stamps. - - .. doctest:: - - >>> for i in range(8): - ... turtle.stamp(); turtle.fd(30) - 13 - 14 - 15 - 16 - 17 - 18 - 19 - 20 - >>> turtle.clearstamps(2) - >>> turtle.clearstamps(-2) - >>> turtle.clearstamps() - - -.. function:: undo() - - Undo (repeatedly) the last turtle action(s). Number of available - undo actions is determined by the size of the undobuffer. - - .. doctest:: - :skipif: _tkinter is None - - >>> for i in range(4): - ... turtle.fd(50); turtle.lt(80) - ... - >>> for i in range(8): - ... turtle.undo() - - -.. function:: speed(speed=None) - - :param speed: an integer in the range 0..10 or a speedstring (see below) - - Set the turtle's speed to an integer value in the range 0..10. If no - argument is given, return current speed. - - If input is a number greater than 10 or smaller than 0.5, speed is set - to 0. Speedstrings are mapped to speedvalues as follows: - - * "fastest": 0 - * "fast": 10 - * "normal": 6 - * "slow": 3 - * "slowest": 1 - - Speeds from 1 to 10 enforce increasingly faster animation of line drawing - and turtle turning. - - Attention: *speed* = 0 means that *no* animation takes - place. forward/back makes turtle jump and likewise left/right make the - turtle turn instantly. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.speed() - 3 - >>> turtle.speed('normal') - >>> turtle.speed() - 6 - >>> turtle.speed(9) - >>> turtle.speed() - 9 - - -Tell Turtle's state -------------------- - -.. function:: position() - pos() - - Return the turtle's current location (x,y) (as a :class:`Vec2D` vector). - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.pos() - (440.00,-0.00) - - -.. function:: towards(x, y=None) - - :param x: a number or a pair/vector of numbers or a turtle instance - :param y: a number if *x* is a number, else ``None`` - - Return the angle between the line from turtle position to position specified - by (x,y), the vector or the other turtle. This depends on the turtle's start - orientation which depends on the mode - "standard"/"world" or "logo". - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.goto(10, 10) - >>> turtle.towards(0,0) - 225.0 - - -.. function:: xcor() - - Return the turtle's x coordinate. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.home() - >>> turtle.left(50) - >>> turtle.forward(100) - >>> turtle.pos() - (64.28,76.60) - >>> print(round(turtle.xcor(), 5)) - 64.27876 - - -.. function:: ycor() - - Return the turtle's y coordinate. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.home() - >>> turtle.left(60) - >>> turtle.forward(100) - >>> print(turtle.pos()) - (50.00,86.60) - >>> print(round(turtle.ycor(), 5)) - 86.60254 - - -.. function:: heading() - - Return the turtle's current heading (value depends on the turtle mode, see - :func:`mode`). - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.home() - >>> turtle.left(67) - >>> turtle.heading() - 67.0 - - -.. function:: distance(x, y=None) - - :param x: a number or a pair/vector of numbers or a turtle instance - :param y: a number if *x* is a number, else ``None`` - - Return the distance from the turtle to (x,y), the given vector, or the given - other turtle, in turtle step units. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.home() - >>> turtle.distance(30,40) - 50.0 - >>> turtle.distance((30,40)) - 50.0 - >>> joe = Turtle() - >>> joe.forward(77) - >>> turtle.distance(joe) - 77.0 - - -Settings for measurement ------------------------- - -.. function:: degrees(fullcircle=360.0) - - :param fullcircle: a number - - Set angle measurement units, i.e. set number of "degrees" for a full circle. - Default value is 360 degrees. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.home() - >>> turtle.left(90) - >>> turtle.heading() - 90.0 - - Change angle measurement unit to grad (also known as gon, - grade, or gradian and equals 1/100-th of the right angle.) - >>> turtle.degrees(400.0) - >>> turtle.heading() - 100.0 - >>> turtle.degrees(360) - >>> turtle.heading() - 90.0 - - -.. function:: radians() - - Set the angle measurement units to radians. Equivalent to - ``degrees(2*math.pi)``. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.home() - >>> turtle.left(90) - >>> turtle.heading() - 90.0 - >>> turtle.radians() - >>> turtle.heading() - 1.5707963267948966 - - .. doctest:: - :skipif: _tkinter is None - :hide: - - >>> turtle.degrees(360) - - -Pen control ------------ - -Drawing state -~~~~~~~~~~~~~ - -.. function:: pendown() - pd() - down() - - Pull the pen down -- drawing when moving. - - -.. function:: penup() - pu() - up() - - Pull the pen up -- no drawing when moving. - - -.. function:: pensize(width=None) - width(width=None) - - :param width: a positive number - - Set the line thickness to *width* or return it. If resizemode is set to - "auto" and turtleshape is a polygon, that polygon is drawn with the same line - thickness. If no argument is given, the current pensize is returned. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.pensize() - 1 - >>> turtle.pensize(10) # from here on lines of width 10 are drawn - - -.. function:: pen(pen=None, **pendict) - - :param pen: a dictionary with some or all of the below listed keys - :param pendict: one or more keyword-arguments with the below listed keys as keywords - - Return or set the pen's attributes in a "pen-dictionary" with the following - key/value pairs: - - * "shown": True/False - * "pendown": True/False - * "pencolor": color-string or color-tuple - * "fillcolor": color-string or color-tuple - * "pensize": positive number - * "speed": number in range 0..10 - * "resizemode": "auto" or "user" or "noresize" - * "stretchfactor": (positive number, positive number) - * "outline": positive number - * "tilt": number - - This dictionary can be used as argument for a subsequent call to :func:`pen` - to restore the former pen-state. Moreover one or more of these attributes - can be provided as keyword-arguments. This can be used to set several pen - attributes in one statement. - - .. doctest:: - :skipif: _tkinter is None - :options: +NORMALIZE_WHITESPACE - - >>> turtle.pen(fillcolor="black", pencolor="red", pensize=10) - >>> sorted(turtle.pen().items()) - [('fillcolor', 'black'), ('outline', 1), ('pencolor', 'red'), - ('pendown', True), ('pensize', 10), ('resizemode', 'noresize'), - ('shearfactor', 0.0), ('shown', True), ('speed', 9), - ('stretchfactor', (1.0, 1.0)), ('tilt', 0.0)] - >>> penstate=turtle.pen() - >>> turtle.color("yellow", "") - >>> turtle.penup() - >>> sorted(turtle.pen().items())[:3] - [('fillcolor', ''), ('outline', 1), ('pencolor', 'yellow')] - >>> turtle.pen(penstate, fillcolor="green") - >>> sorted(turtle.pen().items())[:3] - [('fillcolor', 'green'), ('outline', 1), ('pencolor', 'red')] - -.. function:: isdown() - - Return ``True`` if pen is down, ``False`` if it's up. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.penup() - >>> turtle.isdown() - False - >>> turtle.pendown() - >>> turtle.isdown() - True - - -Color control -~~~~~~~~~~~~~ - -.. function:: pencolor(*args) - - Return or set the pencolor. - - Four input formats are allowed: - - ``pencolor()`` - Return the current pencolor as color specification string or - as a tuple (see example). May be used as input to another - color/pencolor/fillcolor call. - - ``pencolor(colorstring)`` - Set pencolor to *colorstring*, which is a Tk color specification string, - such as ``"red"``, ``"yellow"``, or ``"#33cc8c"``. - - ``pencolor((r, g, b))`` - Set pencolor to the RGB color represented by the tuple of *r*, *g*, and - *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode, where - colormode is either 1.0 or 255 (see :func:`colormode`). - - ``pencolor(r, g, b)`` - Set pencolor to the RGB color represented by *r*, *g*, and *b*. Each of - *r*, *g*, and *b* must be in the range 0..colormode. - - If turtleshape is a polygon, the outline of that polygon is drawn with the - newly set pencolor. - - .. doctest:: - :skipif: _tkinter is None - - >>> colormode() - 1.0 - >>> turtle.pencolor() - 'red' - >>> turtle.pencolor("brown") - >>> turtle.pencolor() - 'brown' - >>> tup = (0.2, 0.8, 0.55) - >>> turtle.pencolor(tup) - >>> turtle.pencolor() - (0.2, 0.8, 0.5490196078431373) - >>> colormode(255) - >>> turtle.pencolor() - (51.0, 204.0, 140.0) - >>> turtle.pencolor('#32c18f') - >>> turtle.pencolor() - (50.0, 193.0, 143.0) - - -.. function:: fillcolor(*args) - - Return or set the fillcolor. - - Four input formats are allowed: - - ``fillcolor()`` - Return the current fillcolor as color specification string, possibly - in tuple format (see example). May be used as input to another - color/pencolor/fillcolor call. - - ``fillcolor(colorstring)`` - Set fillcolor to *colorstring*, which is a Tk color specification string, - such as ``"red"``, ``"yellow"``, or ``"#33cc8c"``. - - ``fillcolor((r, g, b))`` - Set fillcolor to the RGB color represented by the tuple of *r*, *g*, and - *b*. Each of *r*, *g*, and *b* must be in the range 0..colormode, where - colormode is either 1.0 or 255 (see :func:`colormode`). - - ``fillcolor(r, g, b)`` - Set fillcolor to the RGB color represented by *r*, *g*, and *b*. Each of - *r*, *g*, and *b* must be in the range 0..colormode. - - If turtleshape is a polygon, the interior of that polygon is drawn - with the newly set fillcolor. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.fillcolor("violet") - >>> turtle.fillcolor() - 'violet' - >>> turtle.pencolor() - (50.0, 193.0, 143.0) - >>> turtle.fillcolor((50, 193, 143)) # Integers, not floats - >>> turtle.fillcolor() - (50.0, 193.0, 143.0) - >>> turtle.fillcolor('#ffffff') - >>> turtle.fillcolor() - (255.0, 255.0, 255.0) - - -.. function:: color(*args) - - Return or set pencolor and fillcolor. - - Several input formats are allowed. They use 0 to 3 arguments as - follows: - - ``color()`` - Return the current pencolor and the current fillcolor as a pair of color - specification strings or tuples as returned by :func:`pencolor` and - :func:`fillcolor`. - - ``color(colorstring)``, ``color((r,g,b))``, ``color(r,g,b)`` - Inputs as in :func:`pencolor`, set both, fillcolor and pencolor, to the - given value. - - ``color(colorstring1, colorstring2)``, ``color((r1,g1,b1), (r2,g2,b2))`` - Equivalent to ``pencolor(colorstring1)`` and ``fillcolor(colorstring2)`` - and analogously if the other input format is used. - - If turtleshape is a polygon, outline and interior of that polygon is drawn - with the newly set colors. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.color("red", "green") - >>> turtle.color() - ('red', 'green') - >>> color("#285078", "#a0c8f0") - >>> color() - ((40.0, 80.0, 120.0), (160.0, 200.0, 240.0)) - - -See also: Screen method :func:`colormode`. - - -Filling -~~~~~~~ - -.. doctest:: - :skipif: _tkinter is None - :hide: - - >>> turtle.home() - -.. function:: filling() - - Return fillstate (``True`` if filling, ``False`` else). - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.begin_fill() - >>> if turtle.filling(): - ... turtle.pensize(5) - ... else: - ... turtle.pensize(3) - - - -.. function:: begin_fill() - - To be called just before drawing a shape to be filled. - - -.. function:: end_fill() - - Fill the shape drawn after the last call to :func:`begin_fill`. - - Whether or not overlap regions for self-intersecting polygons - or multiple shapes are filled depends on the operating system graphics, - type of overlap, and number of overlaps. For example, the Turtle star - above may be either all yellow or have some white regions. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.color("black", "red") - >>> turtle.begin_fill() - >>> turtle.circle(80) - >>> turtle.end_fill() - - -More drawing control -~~~~~~~~~~~~~~~~~~~~ - -.. function:: reset() - - Delete the turtle's drawings from the screen, re-center the turtle and set - variables to the default values. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.goto(0,-22) - >>> turtle.left(100) - >>> turtle.position() - (0.00,-22.00) - >>> turtle.heading() - 100.0 - >>> turtle.reset() - >>> turtle.position() - (0.00,0.00) - >>> turtle.heading() - 0.0 - - -.. function:: clear() - - Delete the turtle's drawings from the screen. Do not move turtle. State and - position of the turtle as well as drawings of other turtles are not affected. - - -.. function:: write(arg, move=False, align="left", font=("Arial", 8, "normal")) - - :param arg: object to be written to the TurtleScreen - :param move: True/False - :param align: one of the strings "left", "center" or right" - :param font: a triple (fontname, fontsize, fonttype) - - Write text - the string representation of *arg* - at the current turtle - position according to *align* ("left", "center" or "right") and with the given - font. If *move* is true, the pen is moved to the bottom-right corner of the - text. By default, *move* is ``False``. - - >>> turtle.write("Home = ", True, align="center") - >>> turtle.write((0,0), True) - - -Turtle state ------------- - -Visibility -~~~~~~~~~~ - -.. function:: hideturtle() - ht() - - Make the turtle invisible. It's a good idea to do this while you're in the - middle of doing some complex drawing, because hiding the turtle speeds up the - drawing observably. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.hideturtle() - - -.. function:: showturtle() - st() - - Make the turtle visible. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.showturtle() - - -.. function:: isvisible() - - Return ``True`` if the Turtle is shown, ``False`` if it's hidden. - - >>> turtle.hideturtle() - >>> turtle.isvisible() - False - >>> turtle.showturtle() - >>> turtle.isvisible() - True - - -Appearance -~~~~~~~~~~ - -.. function:: shape(name=None) - - :param name: a string which is a valid shapename - - Set turtle shape to shape with given *name* or, if name is not given, return - name of current shape. Shape with *name* must exist in the TurtleScreen's - shape dictionary. Initially there are the following polygon shapes: "arrow", - "turtle", "circle", "square", "triangle", "classic". To learn about how to - deal with shapes see Screen method :func:`register_shape`. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.shape() - 'classic' - >>> turtle.shape("turtle") - >>> turtle.shape() - 'turtle' - - -.. function:: resizemode(rmode=None) - - :param rmode: one of the strings "auto", "user", "noresize" - - Set resizemode to one of the values: "auto", "user", "noresize". If *rmode* - is not given, return current resizemode. Different resizemodes have the - following effects: - - - "auto": adapts the appearance of the turtle corresponding to the value of pensize. - - "user": adapts the appearance of the turtle according to the values of - stretchfactor and outlinewidth (outline), which are set by - :func:`shapesize`. - - "noresize": no adaption of the turtle's appearance takes place. - - ``resizemode("user")`` is called by :func:`shapesize` when used with arguments. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.resizemode() - 'noresize' - >>> turtle.resizemode("auto") - >>> turtle.resizemode() - 'auto' - - -.. function:: shapesize(stretch_wid=None, stretch_len=None, outline=None) - turtlesize(stretch_wid=None, stretch_len=None, outline=None) - - :param stretch_wid: positive number - :param stretch_len: positive number - :param outline: positive number - - Return or set the pen's attributes x/y-stretchfactors and/or outline. Set - resizemode to "user". If and only if resizemode is set to "user", the turtle - will be displayed stretched according to its stretchfactors: *stretch_wid* is - stretchfactor perpendicular to its orientation, *stretch_len* is - stretchfactor in direction of its orientation, *outline* determines the width - of the shape's outline. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.shapesize() - (1.0, 1.0, 1) - >>> turtle.resizemode("user") - >>> turtle.shapesize(5, 5, 12) - >>> turtle.shapesize() - (5, 5, 12) - >>> turtle.shapesize(outline=8) - >>> turtle.shapesize() - (5, 5, 8) - - -.. function:: shearfactor(shear=None) - - :param shear: number (optional) - - Set or return the current shearfactor. Shear the turtleshape according to - the given shearfactor shear, which is the tangent of the shear angle. - Do *not* change the turtle's heading (direction of movement). - If shear is not given: return the current shearfactor, i. e. the - tangent of the shear angle, by which lines parallel to the - heading of the turtle are sheared. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.shearfactor(0.5) - >>> turtle.shearfactor() - 0.5 - - -.. function:: tilt(angle) - - :param angle: a number - - Rotate the turtleshape by *angle* from its current tilt-angle, but do *not* - change the turtle's heading (direction of movement). - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.reset() - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.tilt(30) - >>> turtle.fd(50) - >>> turtle.tilt(30) - >>> turtle.fd(50) - - -.. function:: settiltangle(angle) - - :param angle: a number - - Rotate the turtleshape to point in the direction specified by *angle*, - regardless of its current tilt-angle. *Do not* change the turtle's heading - (direction of movement). - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.reset() - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.settiltangle(45) - >>> turtle.fd(50) - >>> turtle.settiltangle(-45) - >>> turtle.fd(50) - - .. deprecated:: 3.1 - - -.. function:: tiltangle(angle=None) - - :param angle: a number (optional) - - Set or return the current tilt-angle. If angle is given, rotate the - turtleshape to point in the direction specified by angle, - regardless of its current tilt-angle. Do *not* change the turtle's - heading (direction of movement). - If angle is not given: return the current tilt-angle, i. e. the angle - between the orientation of the turtleshape and the heading of the - turtle (its direction of movement). - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.reset() - >>> turtle.shape("circle") - >>> turtle.shapesize(5,2) - >>> turtle.tilt(45) - >>> turtle.tiltangle() - 45.0 - - -.. function:: shapetransform(t11=None, t12=None, t21=None, t22=None) - - :param t11: a number (optional) - :param t12: a number (optional) - :param t21: a number (optional) - :param t12: a number (optional) - - Set or return the current transformation matrix of the turtle shape. - - If none of the matrix elements are given, return the transformation - matrix as a tuple of 4 elements. - Otherwise set the given elements and transform the turtleshape - according to the matrix consisting of first row t11, t12 and - second row t21, t22. The determinant t11 * t22 - t12 * t21 must not be - zero, otherwise an error is raised. - Modify stretchfactor, shearfactor and tiltangle according to the - given matrix. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle = Turtle() - >>> turtle.shape("square") - >>> turtle.shapesize(4,2) - >>> turtle.shearfactor(-0.5) - >>> turtle.shapetransform() - (4.0, -1.0, -0.0, 2.0) - - -.. function:: get_shapepoly() - - Return the current shape polygon as tuple of coordinate pairs. This - can be used to define a new shape or components of a compound shape. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.shape("square") - >>> turtle.shapetransform(4, -1, 0, 2) - >>> turtle.get_shapepoly() - ((50, -20), (30, 20), (-50, 20), (-30, -20)) - - -Using events ------------- - -.. function:: onclick(fun, btn=1, add=None) - :noindex: - - :param fun: a function with two arguments which will be called with the - coordinates of the clicked point on the canvas - :param btn: number of the mouse-button, defaults to 1 (left mouse button) - :param add: ``True`` or ``False`` -- if ``True``, a new binding will be - added, otherwise it will replace a former binding - - Bind *fun* to mouse-click events on this turtle. If *fun* is ``None``, - existing bindings are removed. Example for the anonymous turtle, i.e. the - procedural way: - - .. doctest:: - :skipif: _tkinter is None - - >>> def turn(x, y): - ... left(180) - ... - >>> onclick(turn) # Now clicking into the turtle will turn it. - >>> onclick(None) # event-binding will be removed - - -.. function:: onrelease(fun, btn=1, add=None) - - :param fun: a function with two arguments which will be called with the - coordinates of the clicked point on the canvas - :param btn: number of the mouse-button, defaults to 1 (left mouse button) - :param add: ``True`` or ``False`` -- if ``True``, a new binding will be - added, otherwise it will replace a former binding - - Bind *fun* to mouse-button-release events on this turtle. If *fun* is - ``None``, existing bindings are removed. - - .. doctest:: - :skipif: _tkinter is None - - >>> class MyTurtle(Turtle): - ... def glow(self,x,y): - ... self.fillcolor("red") - ... def unglow(self,x,y): - ... self.fillcolor("") - ... - >>> turtle = MyTurtle() - >>> turtle.onclick(turtle.glow) # clicking on turtle turns fillcolor red, - >>> turtle.onrelease(turtle.unglow) # releasing turns it to transparent. - - -.. function:: ondrag(fun, btn=1, add=None) - - :param fun: a function with two arguments which will be called with the - coordinates of the clicked point on the canvas - :param btn: number of the mouse-button, defaults to 1 (left mouse button) - :param add: ``True`` or ``False`` -- if ``True``, a new binding will be - added, otherwise it will replace a former binding - - Bind *fun* to mouse-move events on this turtle. If *fun* is ``None``, - existing bindings are removed. - - Remark: Every sequence of mouse-move-events on a turtle is preceded by a - mouse-click event on that turtle. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.ondrag(turtle.goto) - - Subsequently, clicking and dragging the Turtle will move it across - the screen thereby producing handdrawings (if pen is down). - - -Special Turtle methods ----------------------- - -.. function:: begin_poly() - - Start recording the vertices of a polygon. Current turtle position is first - vertex of polygon. - - -.. function:: end_poly() - - Stop recording the vertices of a polygon. Current turtle position is last - vertex of polygon. This will be connected with the first vertex. - - -.. function:: get_poly() - - Return the last recorded polygon. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.home() - >>> turtle.begin_poly() - >>> turtle.fd(100) - >>> turtle.left(20) - >>> turtle.fd(30) - >>> turtle.left(60) - >>> turtle.fd(50) - >>> turtle.end_poly() - >>> p = turtle.get_poly() - >>> register_shape("myFavouriteShape", p) - - -.. function:: clone() - - Create and return a clone of the turtle with same position, heading and - turtle properties. - - .. doctest:: - :skipif: _tkinter is None - - >>> mick = Turtle() - >>> joe = mick.clone() - - -.. function:: getturtle() - getpen() - - Return the Turtle object itself. Only reasonable use: as a function to - return the "anonymous turtle": - - .. doctest:: - :skipif: _tkinter is None - - >>> pet = getturtle() - >>> pet.fd(50) - >>> pet - - - -.. function:: getscreen() - - Return the :class:`TurtleScreen` object the turtle is drawing on. - TurtleScreen methods can then be called for that object. - - .. doctest:: - :skipif: _tkinter is None - - >>> ts = turtle.getscreen() - >>> ts - - >>> ts.bgcolor("pink") - - -.. function:: setundobuffer(size) - - :param size: an integer or ``None`` - - Set or disable undobuffer. If *size* is an integer, an empty undobuffer of - given size is installed. *size* gives the maximum number of turtle actions - that can be undone by the :func:`undo` method/function. If *size* is - ``None``, the undobuffer is disabled. - - .. doctest:: - :skipif: _tkinter is None - - >>> turtle.setundobuffer(42) - - -.. function:: undobufferentries() - - Return number of entries in the undobuffer. - - .. doctest:: - :skipif: _tkinter is None - - >>> while undobufferentries(): - ... undo() - - - -.. _compoundshapes: - -Compound shapes ---------------- - -To use compound turtle shapes, which consist of several polygons of different -color, you must use the helper class :class:`Shape` explicitly as described -below: - -1. Create an empty Shape object of type "compound". -2. Add as many components to this object as desired, using the - :meth:`~Shape.addcomponent` method. - - For example: - - .. doctest:: - :skipif: _tkinter is None - - >>> s = Shape("compound") - >>> poly1 = ((0,0),(10,-5),(0,10),(-10,-5)) - >>> s.addcomponent(poly1, "red", "blue") - >>> poly2 = ((0,0),(10,-5),(-10,-5)) - >>> s.addcomponent(poly2, "blue", "red") - -3. Now add the Shape to the Screen's shapelist and use it: - - .. doctest:: - :skipif: _tkinter is None - - >>> register_shape("myshape", s) - >>> shape("myshape") - - -.. note:: - - The :class:`Shape` class is used internally by the :func:`register_shape` - method in different ways. The application programmer has to deal with the - Shape class *only* when using compound shapes like shown above! - - -Methods of TurtleScreen/Screen and corresponding functions -========================================================== - -Most of the examples in this section refer to a TurtleScreen instance called -``screen``. - -.. doctest:: - :skipif: _tkinter is None - :hide: - - >>> screen = Screen() - -Window control --------------- - -.. function:: bgcolor(*args) - - :param args: a color string or three numbers in the range 0..colormode or a - 3-tuple of such numbers - - - Set or return background color of the TurtleScreen. - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.bgcolor("orange") - >>> screen.bgcolor() - 'orange' - >>> screen.bgcolor("#800080") - >>> screen.bgcolor() - (128.0, 0.0, 128.0) - - -.. function:: bgpic(picname=None) - - :param picname: a string, name of a gif-file or ``"nopic"``, or ``None`` - - Set background image or return name of current backgroundimage. If *picname* - is a filename, set the corresponding image as background. If *picname* is - ``"nopic"``, delete background image, if present. If *picname* is ``None``, - return the filename of the current backgroundimage. :: - - >>> screen.bgpic() - 'nopic' - >>> screen.bgpic("landscape.gif") - >>> screen.bgpic() - "landscape.gif" - - -.. function:: clear() - :noindex: - - .. note:: - This TurtleScreen method is available as a global function only under the - name ``clearscreen``. The global function ``clear`` is a different one - derived from the Turtle method ``clear``. - - -.. function:: clearscreen() - - Delete all drawings and all turtles from the TurtleScreen. Reset the now - empty TurtleScreen to its initial state: white background, no background - image, no event bindings and tracing on. - - -.. function:: reset() - :noindex: - - .. note:: - This TurtleScreen method is available as a global function only under the - name ``resetscreen``. The global function ``reset`` is another one - derived from the Turtle method ``reset``. - - -.. function:: resetscreen() - - Reset all Turtles on the Screen to their initial state. - - -.. function:: screensize(canvwidth=None, canvheight=None, bg=None) - - :param canvwidth: positive integer, new width of canvas in pixels - :param canvheight: positive integer, new height of canvas in pixels - :param bg: colorstring or color-tuple, new background color - - If no arguments are given, return current (canvaswidth, canvasheight). Else - resize the canvas the turtles are drawing on. Do not alter the drawing - window. To observe hidden parts of the canvas, use the scrollbars. With this - method, one can make visible those parts of a drawing which were outside the - canvas before. - - >>> screen.screensize() - (400, 300) - >>> screen.screensize(2000,1500) - >>> screen.screensize() - (2000, 1500) - - e.g. to search for an erroneously escaped turtle ;-) - - -.. function:: setworldcoordinates(llx, lly, urx, ury) - - :param llx: a number, x-coordinate of lower left corner of canvas - :param lly: a number, y-coordinate of lower left corner of canvas - :param urx: a number, x-coordinate of upper right corner of canvas - :param ury: a number, y-coordinate of upper right corner of canvas - - Set up user-defined coordinate system and switch to mode "world" if - necessary. This performs a ``screen.reset()``. If mode "world" is already - active, all drawings are redrawn according to the new coordinates. - - **ATTENTION**: in user-defined coordinate systems angles may appear - distorted. - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.reset() - >>> screen.setworldcoordinates(-50,-7.5,50,7.5) - >>> for _ in range(72): - ... left(10) - ... - >>> for _ in range(8): - ... left(45); fd(2) # a regular octagon - - .. doctest:: - :skipif: _tkinter is None - :hide: - - >>> screen.reset() - >>> for t in turtles(): - ... t.reset() - - -Animation control ------------------ - -.. function:: delay(delay=None) - - :param delay: positive integer - - Set or return the drawing *delay* in milliseconds. (This is approximately - the time interval between two consecutive canvas updates.) The longer the - drawing delay, the slower the animation. - - Optional argument: - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.delay() - 10 - >>> screen.delay(5) - >>> screen.delay() - 5 - - -.. function:: tracer(n=None, delay=None) - - :param n: nonnegative integer - :param delay: nonnegative integer - - Turn turtle animation on/off and set delay for update drawings. If - *n* is given, only each n-th regular screen update is really - performed. (Can be used to accelerate the drawing of complex - graphics.) When called without arguments, returns the currently - stored value of n. Second argument sets delay value (see - :func:`delay`). - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.tracer(8, 25) - >>> dist = 2 - >>> for i in range(200): - ... fd(dist) - ... rt(90) - ... dist += 2 - - -.. function:: update() - - Perform a TurtleScreen update. To be used when tracer is turned off. - -See also the RawTurtle/Turtle method :func:`speed`. - - -Using screen events -------------------- - -.. function:: listen(xdummy=None, ydummy=None) - - Set focus on TurtleScreen (in order to collect key-events). Dummy arguments - are provided in order to be able to pass :func:`listen` to the onclick method. - - -.. function:: onkey(fun, key) - onkeyrelease(fun, key) - - :param fun: a function with no arguments or ``None`` - :param key: a string: key (e.g. "a") or key-symbol (e.g. "space") - - Bind *fun* to key-release event of key. If *fun* is ``None``, event bindings - are removed. Remark: in order to be able to register key-events, TurtleScreen - must have the focus. (See method :func:`listen`.) - - .. doctest:: - :skipif: _tkinter is None - - >>> def f(): - ... fd(50) - ... lt(60) - ... - >>> screen.onkey(f, "Up") - >>> screen.listen() - - -.. function:: onkeypress(fun, key=None) - - :param fun: a function with no arguments or ``None`` - :param key: a string: key (e.g. "a") or key-symbol (e.g. "space") - - Bind *fun* to key-press event of key if key is given, - or to any key-press-event if no key is given. - Remark: in order to be able to register key-events, TurtleScreen - must have focus. (See method :func:`listen`.) - - .. doctest:: - :skipif: _tkinter is None - - >>> def f(): - ... fd(50) - ... - >>> screen.onkey(f, "Up") - >>> screen.listen() - - -.. function:: onclick(fun, btn=1, add=None) - onscreenclick(fun, btn=1, add=None) - - :param fun: a function with two arguments which will be called with the - coordinates of the clicked point on the canvas - :param btn: number of the mouse-button, defaults to 1 (left mouse button) - :param add: ``True`` or ``False`` -- if ``True``, a new binding will be - added, otherwise it will replace a former binding - - Bind *fun* to mouse-click events on this screen. If *fun* is ``None``, - existing bindings are removed. - - Example for a TurtleScreen instance named ``screen`` and a Turtle instance - named ``turtle``: - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.onclick(turtle.goto) # Subsequently clicking into the TurtleScreen will - >>> # make the turtle move to the clicked point. - >>> screen.onclick(None) # remove event binding again - - .. note:: - This TurtleScreen method is available as a global function only under the - name ``onscreenclick``. The global function ``onclick`` is another one - derived from the Turtle method ``onclick``. - - -.. function:: ontimer(fun, t=0) - - :param fun: a function with no arguments - :param t: a number >= 0 - - Install a timer that calls *fun* after *t* milliseconds. - - .. doctest:: - :skipif: _tkinter is None - - >>> running = True - >>> def f(): - ... if running: - ... fd(50) - ... lt(60) - ... screen.ontimer(f, 250) - >>> f() ### makes the turtle march around - >>> running = False - - -.. function:: mainloop() - done() - - Starts event loop - calling Tkinter's mainloop function. - Must be the last statement in a turtle graphics program. - Must *not* be used if a script is run from within IDLE in -n mode - (No subprocess) - for interactive use of turtle graphics. :: - - >>> screen.mainloop() - - -Input methods -------------- - -.. function:: textinput(title, prompt) - - :param title: string - :param prompt: string - - Pop up a dialog window for input of a string. Parameter title is - the title of the dialog window, prompt is a text mostly describing - what information to input. - Return the string input. If the dialog is canceled, return ``None``. :: - - >>> screen.textinput("NIM", "Name of first player:") - - -.. function:: numinput(title, prompt, default=None, minval=None, maxval=None) - - :param title: string - :param prompt: string - :param default: number (optional) - :param minval: number (optional) - :param maxval: number (optional) - - Pop up a dialog window for input of a number. title is the title of the - dialog window, prompt is a text mostly describing what numerical information - to input. default: default value, minval: minimum value for input, - maxval: maximum value for input. - The number input must be in the range minval .. maxval if these are - given. If not, a hint is issued and the dialog remains open for - correction. - Return the number input. If the dialog is canceled, return ``None``. :: - - >>> screen.numinput("Poker", "Your stakes:", 1000, minval=10, maxval=10000) - - -Settings and special methods ----------------------------- - -.. function:: mode(mode=None) - - :param mode: one of the strings "standard", "logo" or "world" - - Set turtle mode ("standard", "logo" or "world") and perform reset. If mode - is not given, current mode is returned. - - Mode "standard" is compatible with old :mod:`turtle`. Mode "logo" is - compatible with most Logo turtle graphics. Mode "world" uses user-defined - "world coordinates". **Attention**: in this mode angles appear distorted if - ``x/y`` unit-ratio doesn't equal 1. - - ============ ========================= =================== - Mode Initial turtle heading positive angles - ============ ========================= =================== - "standard" to the right (east) counterclockwise - "logo" upward (north) clockwise - ============ ========================= =================== - - .. doctest:: - :skipif: _tkinter is None - - >>> mode("logo") # resets turtle heading to north - >>> mode() - 'logo' - - -.. function:: colormode(cmode=None) - - :param cmode: one of the values 1.0 or 255 - - Return the colormode or set it to 1.0 or 255. Subsequently *r*, *g*, *b* - values of color triples have to be in the range 0..*cmode*. - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.colormode(1) - >>> turtle.pencolor(240, 160, 80) - Traceback (most recent call last): - ... - TurtleGraphicsError: bad color sequence: (240, 160, 80) - >>> screen.colormode() - 1.0 - >>> screen.colormode(255) - >>> screen.colormode() - 255 - >>> turtle.pencolor(240,160,80) - - -.. function:: getcanvas() - - Return the Canvas of this TurtleScreen. Useful for insiders who know what to - do with a Tkinter Canvas. - - .. doctest:: - :skipif: _tkinter is None - - >>> cv = screen.getcanvas() - >>> cv - - - -.. function:: getshapes() - - Return a list of names of all currently available turtle shapes. - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.getshapes() - ['arrow', 'blank', 'circle', ..., 'turtle'] - - -.. function:: register_shape(name, shape=None) - addshape(name, shape=None) - - There are three different ways to call this function: - - (1) *name* is the name of a gif-file and *shape* is ``None``: Install the - corresponding image shape. :: - - >>> screen.register_shape("turtle.gif") - - .. note:: - Image shapes *do not* rotate when turning the turtle, so they do not - display the heading of the turtle! - - (2) *name* is an arbitrary string and *shape* is a tuple of pairs of - coordinates: Install the corresponding polygon shape. - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.register_shape("triangle", ((5,-3), (0,5), (-5,-3))) - - (3) *name* is an arbitrary string and *shape* is a (compound) :class:`Shape` - object: Install the corresponding compound shape. - - Add a turtle shape to TurtleScreen's shapelist. Only thusly registered - shapes can be used by issuing the command ``shape(shapename)``. - - -.. function:: turtles() - - Return the list of turtles on the screen. - - .. doctest:: - :skipif: _tkinter is None - - >>> for turtle in screen.turtles(): - ... turtle.color("red") - - -.. function:: window_height() - - Return the height of the turtle window. :: - - >>> screen.window_height() - 480 - - -.. function:: window_width() - - Return the width of the turtle window. :: - - >>> screen.window_width() - 640 - - -.. _screenspecific: - -Methods specific to Screen, not inherited from TurtleScreen ------------------------------------------------------------ - -.. function:: bye() - - Shut the turtlegraphics window. - - -.. function:: exitonclick() - - Bind ``bye()`` method to mouse clicks on the Screen. - - - If the value "using_IDLE" in the configuration dictionary is ``False`` - (default value), also enter mainloop. Remark: If IDLE with the ``-n`` switch - (no subprocess) is used, this value should be set to ``True`` in - :file:`turtle.cfg`. In this case IDLE's own mainloop is active also for the - client script. - - -.. function:: setup(width=_CFG["width"], height=_CFG["height"], startx=_CFG["leftright"], starty=_CFG["topbottom"]) - - Set the size and position of the main window. Default values of arguments - are stored in the configuration dictionary and can be changed via a - :file:`turtle.cfg` file. - - :param width: if an integer, a size in pixels, if a float, a fraction of the - screen; default is 50% of screen - :param height: if an integer, the height in pixels, if a float, a fraction of - the screen; default is 75% of screen - :param startx: if positive, starting position in pixels from the left - edge of the screen, if negative from the right edge, if ``None``, - center window horizontally - :param starty: if positive, starting position in pixels from the top - edge of the screen, if negative from the bottom edge, if ``None``, - center window vertically - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.setup (width=200, height=200, startx=0, starty=0) - >>> # sets window to 200x200 pixels, in upper left of screen - >>> screen.setup(width=.75, height=0.5, startx=None, starty=None) - >>> # sets window to 75% of screen by 50% of screen and centers - - -.. function:: title(titlestring) - - :param titlestring: a string that is shown in the titlebar of the turtle - graphics window - - Set title of turtle window to *titlestring*. - - .. doctest:: - :skipif: _tkinter is None - - >>> screen.title("Welcome to the turtle zoo!") - - -Public classes -============== - - -.. class:: RawTurtle(canvas) - RawPen(canvas) - - :param canvas: a :class:`!tkinter.Canvas`, a :class:`ScrolledCanvas` or a - :class:`TurtleScreen` - - Create a turtle. The turtle has all methods described above as "methods of - Turtle/RawTurtle". - - -.. class:: Turtle() - - Subclass of RawTurtle, has the same interface but draws on a default - :class:`Screen` object created automatically when needed for the first time. - - -.. class:: TurtleScreen(cv) - - :param cv: a :class:`!tkinter.Canvas` - - Provides screen oriented methods like :func:`bgcolor` etc. that are described - above. - -.. class:: Screen() - - Subclass of TurtleScreen, with :ref:`four methods added `. - - -.. class:: ScrolledCanvas(master) - - :param master: some Tkinter widget to contain the ScrolledCanvas, i.e. - a Tkinter-canvas with scrollbars added - - Used by class Screen, which thus automatically provides a ScrolledCanvas as - playground for the turtles. - -.. class:: Shape(type_, data) - - :param type\_: one of the strings "polygon", "image", "compound" - - Data structure modeling shapes. The pair ``(type_, data)`` must follow this - specification: - - - =========== =========== - *type_* *data* - =========== =========== - "polygon" a polygon-tuple, i.e. a tuple of pairs of coordinates - "image" an image (in this form only used internally!) - "compound" ``None`` (a compound shape has to be constructed using the - :meth:`addcomponent` method) - =========== =========== - - .. method:: addcomponent(poly, fill, outline=None) - - :param poly: a polygon, i.e. a tuple of pairs of numbers - :param fill: a color the *poly* will be filled with - :param outline: a color for the poly's outline (if given) - - Example: - - .. doctest:: - :skipif: _tkinter is None - - >>> poly = ((0,0),(10,-5),(0,10),(-10,-5)) - >>> s = Shape("compound") - >>> s.addcomponent(poly, "red", "blue") - >>> # ... add more components and then use register_shape() - - See :ref:`compoundshapes`. - - -.. class:: Vec2D(x, y) - - A two-dimensional vector class, used as a helper class for implementing - turtle graphics. May be useful for turtle graphics programs too. Derived - from tuple, so a vector is a tuple! - - Provides (for *a*, *b* vectors, *k* number): - - * ``a + b`` vector addition - * ``a - b`` vector subtraction - * ``a * b`` inner product - * ``k * a`` and ``a * k`` multiplication with scalar - * ``abs(a)`` absolute value of a - * ``a.rotate(angle)`` rotation - - -.. _turtle-explanation: - -Explanation -=========== - -A turtle object draws on a screen object, and there a number of key classes in -the turtle object-oriented interface that can be used to create them and relate -them to each other. - -A :class:`Turtle` instance will automatically create a :class:`Screen` -instance if one is not already present. - -``Turtle`` is a subclass of :class:`RawTurtle`, which *doesn't* automatically -create a drawing surface - a *canvas* will need to be provided or created for -it. The *canvas* can be a :class:`!tkinter.Canvas`, :class:`ScrolledCanvas` -or :class:`TurtleScreen`. - - -:class:`TurtleScreen` is the basic drawing surface for a -turtle. :class:`Screen` is a subclass of ``TurtleScreen``, and -includes :ref:`some additional methods ` for managing its -appearance (including size and title) and behaviour. ``TurtleScreen``'s -constructor needs a :class:`!tkinter.Canvas` or a -:class:`ScrolledCanvas` as an argument. - -The functional interface for turtle graphics uses the various methods of -``Turtle`` and ``TurtleScreen``/``Screen``. Behind the scenes, a screen -object is automatically created whenever a function derived from a ``Screen`` -method is called. Similarly, a turtle object is automatically created -whenever any of the functions derived from a Turtle method is called. - -To use multiple turtles on a screen, the object-oriented interface must be -used. - - -Help and configuration -====================== - -How to use help ---------------- - -The public methods of the Screen and Turtle classes are documented extensively -via docstrings. So these can be used as online-help via the Python help -facilities: - -- When using IDLE, tooltips show the signatures and first lines of the - docstrings of typed in function-/method calls. - -- Calling :func:`help` on methods or functions displays the docstrings:: - - >>> help(Screen.bgcolor) - Help on method bgcolor in module turtle: - - bgcolor(self, *args) unbound turtle.Screen method - Set or return backgroundcolor of the TurtleScreen. - - Arguments (if given): a color string or three numbers - in the range 0..colormode or a 3-tuple of such numbers. - - - >>> screen.bgcolor("orange") - >>> screen.bgcolor() - "orange" - >>> screen.bgcolor(0.5,0,0.5) - >>> screen.bgcolor() - "#800080" - - >>> help(Turtle.penup) - Help on method penup in module turtle: - - penup(self) unbound turtle.Turtle method - Pull the pen up -- no drawing when moving. - - Aliases: penup | pu | up - - No argument - - >>> turtle.penup() - -- The docstrings of the functions which are derived from methods have a modified - form:: - - >>> help(bgcolor) - Help on function bgcolor in module turtle: - - bgcolor(*args) - Set or return backgroundcolor of the TurtleScreen. - - Arguments (if given): a color string or three numbers - in the range 0..colormode or a 3-tuple of such numbers. - - Example:: - - >>> bgcolor("orange") - >>> bgcolor() - "orange" - >>> bgcolor(0.5,0,0.5) - >>> bgcolor() - "#800080" - - >>> help(penup) - Help on function penup in module turtle: - - penup() - Pull the pen up -- no drawing when moving. - - Aliases: penup | pu | up - - No argument - - Example: - >>> penup() - -These modified docstrings are created automatically together with the function -definitions that are derived from the methods at import time. - - -Translation of docstrings into different languages --------------------------------------------------- - -There is a utility to create a dictionary the keys of which are the method names -and the values of which are the docstrings of the public methods of the classes -Screen and Turtle. - -.. function:: write_docstringdict(filename="turtle_docstringdict") - - :param filename: a string, used as filename - - Create and write docstring-dictionary to a Python script with the given - filename. This function has to be called explicitly (it is not used by the - turtle graphics classes). The docstring dictionary will be written to the - Python script :file:`{filename}.py`. It is intended to serve as a template - for translation of the docstrings into different languages. - -If you (or your students) want to use :mod:`turtle` with online help in your -native language, you have to translate the docstrings and save the resulting -file as e.g. :file:`turtle_docstringdict_german.py`. - -If you have an appropriate entry in your :file:`turtle.cfg` file this dictionary -will be read in at import time and will replace the original English docstrings. - -At the time of this writing there are docstring dictionaries in German and in -Italian. (Requests please to glingl@aon.at.) - - - -How to configure Screen and Turtles ------------------------------------ - -The built-in default configuration mimics the appearance and behaviour of the -old turtle module in order to retain best possible compatibility with it. - -If you want to use a different configuration which better reflects the features -of this module or which better fits to your needs, e.g. for use in a classroom, -you can prepare a configuration file ``turtle.cfg`` which will be read at import -time and modify the configuration according to its settings. - -The built in configuration would correspond to the following ``turtle.cfg``: - -.. code-block:: ini - - width = 0.5 - height = 0.75 - leftright = None - topbottom = None - canvwidth = 400 - canvheight = 300 - mode = standard - colormode = 1.0 - delay = 10 - undobuffersize = 1000 - shape = classic - pencolor = black - fillcolor = black - resizemode = noresize - visible = True - language = english - exampleturtle = turtle - examplescreen = screen - title = Python Turtle Graphics - using_IDLE = False - -Short explanation of selected entries: - -- The first four lines correspond to the arguments of the :func:`Screen.setup ` - method. -- Line 5 and 6 correspond to the arguments of the method - :func:`Screen.screensize `. -- *shape* can be any of the built-in shapes, e.g: arrow, turtle, etc. For more - info try ``help(shape)``. -- If you want to use no fill color (i.e. make the turtle transparent), you have - to write ``fillcolor = ""`` (but all nonempty strings must not have quotes in - the cfg file). -- If you want to reflect the turtle its state, you have to use ``resizemode = - auto``. -- If you set e.g. ``language = italian`` the docstringdict - :file:`turtle_docstringdict_italian.py` will be loaded at import time (if - present on the import path, e.g. in the same directory as :mod:`turtle`). -- The entries *exampleturtle* and *examplescreen* define the names of these - objects as they occur in the docstrings. The transformation of - method-docstrings to function-docstrings will delete these names from the - docstrings. -- *using_IDLE*: Set this to ``True`` if you regularly work with IDLE and its ``-n`` - switch ("no subprocess"). This will prevent :func:`exitonclick` to enter the - mainloop. - -There can be a :file:`turtle.cfg` file in the directory where :mod:`turtle` is -stored and an additional one in the current working directory. The latter will -override the settings of the first one. - -The :file:`Lib/turtledemo` directory contains a :file:`turtle.cfg` file. You can -study it as an example and see its effects when running the demos (preferably -not from within the demo-viewer). - - -:mod:`turtledemo` --- Demo scripts -================================== - -.. module:: turtledemo - :synopsis: A viewer for example turtle scripts - -The :mod:`turtledemo` package includes a set of demo scripts. These -scripts can be run and viewed using the supplied demo viewer as follows:: - - python -m turtledemo - -Alternatively, you can run the demo scripts individually. For example, :: - - python -m turtledemo.bytedesign - -The :mod:`turtledemo` package directory contains: - -- A demo viewer :file:`__main__.py` which can be used to view the sourcecode - of the scripts and run them at the same time. -- Multiple scripts demonstrating different features of the :mod:`turtle` - module. Examples can be accessed via the Examples menu. They can also - be run standalone. -- A :file:`turtle.cfg` file which serves as an example of how to write - and use such files. - -The demo scripts are: - -.. currentmodule:: turtle - -.. tabularcolumns:: |l|L|L| - -+----------------+------------------------------+-----------------------+ -| Name | Description | Features | -+================+==============================+=======================+ -| bytedesign | complex classical | :func:`tracer`, delay,| -| | turtle graphics pattern | :func:`update` | -+----------------+------------------------------+-----------------------+ -| chaos | graphs Verhulst dynamics, | world coordinates | -| | shows that computer's | | -| | computations can generate | | -| | results sometimes against the| | -| | common sense expectations | | -+----------------+------------------------------+-----------------------+ -| clock | analog clock showing time | turtles as clock's | -| | of your computer | hands, ontimer | -+----------------+------------------------------+-----------------------+ -| colormixer | experiment with r, g, b | :func:`ondrag` | -+----------------+------------------------------+-----------------------+ -| forest | 3 breadth-first trees | randomization | -+----------------+------------------------------+-----------------------+ -| fractalcurves | Hilbert & Koch curves | recursion | -+----------------+------------------------------+-----------------------+ -| lindenmayer | ethnomathematics | L-System | -| | (indian kolams) | | -+----------------+------------------------------+-----------------------+ -| minimal_hanoi | Towers of Hanoi | Rectangular Turtles | -| | | as Hanoi discs | -| | | (shape, shapesize) | -+----------------+------------------------------+-----------------------+ -| nim | play the classical nim game | turtles as nimsticks, | -| | with three heaps of sticks | event driven (mouse, | -| | against the computer. | keyboard) | -+----------------+------------------------------+-----------------------+ -| paint | super minimalistic | :func:`onclick` | -| | drawing program | | -+----------------+------------------------------+-----------------------+ -| peace | elementary | turtle: appearance | -| | | and animation | -+----------------+------------------------------+-----------------------+ -| penrose | aperiodic tiling with | :func:`stamp` | -| | kites and darts | | -+----------------+------------------------------+-----------------------+ -| planet_and_moon| simulation of | compound shapes, | -| | gravitational system | :class:`Vec2D` | -+----------------+------------------------------+-----------------------+ -| rosette | a pattern from the wikipedia | :func:`clone`, | -| | article on turtle graphics | :func:`undo` | -+----------------+------------------------------+-----------------------+ -| round_dance | dancing turtles rotating | compound shapes, clone| -| | pairwise in opposite | shapesize, tilt, | -| | direction | get_shapepoly, update | -+----------------+------------------------------+-----------------------+ -| sorting_animate| visual demonstration of | simple alignment, | -| | different sorting methods | randomization | -+----------------+------------------------------+-----------------------+ -| tree | a (graphical) breadth | :func:`clone` | -| | first tree (using generators)| | -+----------------+------------------------------+-----------------------+ -| two_canvases | simple design | turtles on two | -| | | canvases | -+----------------+------------------------------+-----------------------+ -| yinyang | another elementary example | :func:`circle` | -+----------------+------------------------------+-----------------------+ - -Have fun! - - -Changes since Python 2.6 -======================== - -- The methods :func:`Turtle.tracer `, :func:`Turtle.window_width ` and - :func:`Turtle.window_height ` have been eliminated. - Methods with these names and functionality are now available only - as methods of :class:`Screen`. The functions derived from these remain - available. (In fact already in Python 2.6 these methods were merely - duplications of the corresponding - :class:`TurtleScreen`/:class:`Screen` methods.) - -- The method :func:`!Turtle.fill` has been eliminated. - The behaviour of :func:`begin_fill` and :func:`end_fill` - have changed slightly: now every filling process must be completed with an - ``end_fill()`` call. - -- A method :func:`Turtle.filling ` has been added. It returns a boolean - value: ``True`` if a filling process is under way, ``False`` otherwise. - This behaviour corresponds to a ``fill()`` call without arguments in - Python 2.6. - -Changes since Python 3.0 -======================== - -- The :class:`Turtle` methods :func:`shearfactor`, :func:`shapetransform` and - :func:`get_shapepoly` have been added. Thus the full range of - regular linear transforms is now available for transforming turtle shapes. - :func:`tiltangle` has been enhanced in functionality: it now can - be used to get or set the tilt angle. :func:`settiltangle` has been - deprecated. - -- The :class:`Screen` method :func:`onkeypress` has been added as a complement to - :func:`onkey`. As the latter binds actions to the key release event, - an alias: :func:`onkeyrelease` was also added for it. - -- The method :func:`Screen.mainloop ` has been added, - so there is no longer a need to use the standalone :func:`mainloop` function - when working with :class:`Screen` and :class:`Turtle` objects. - -- Two input methods have been added: :func:`Screen.textinput ` and - :func:`Screen.numinput `. These pop up input dialogs and return - strings and numbers respectively. - -- Two example scripts :file:`tdemo_nim.py` and :file:`tdemo_round_dance.py` - have been added to the :file:`Lib/turtledemo` directory. - - -.. doctest:: - :skipif: _tkinter is None - :hide: - - >>> for turtle in turtles(): - ... turtle.reset() - >>> turtle.penup() - >>> turtle.goto(-200,25) - >>> turtle.pendown() - >>> turtle.write("No one expects the Spanish Inquisition!", - ... font=("Arial", 20, "normal")) - >>> turtle.penup() - >>> turtle.goto(-100,-50) - >>> turtle.pendown() - >>> turtle.write("Our two chief Turtles are...", - ... font=("Arial", 16, "normal")) - >>> turtle.penup() - >>> turtle.goto(-450,-75) - >>> turtle.write(str(turtles())) diff --git a/Doc/library/types.rst b/Doc/library/types.rst deleted file mode 100644 index 5625140b50b9d5..00000000000000 --- a/Doc/library/types.rst +++ /dev/null @@ -1,499 +0,0 @@ -:mod:`types` --- Dynamic type creation and names for built-in types -=================================================================== - -.. module:: types - :synopsis: Names for built-in types. - -**Source code:** :source:`Lib/types.py` - --------------- - -This module defines utility functions to assist in dynamic creation of -new types. - -It also defines names for some object types that are used by the standard -Python interpreter, but not exposed as builtins like :class:`int` or -:class:`str` are. - -Finally, it provides some additional type-related utility classes and functions -that are not fundamental enough to be builtins. - - -Dynamic Type Creation ---------------------- - -.. function:: new_class(name, bases=(), kwds=None, exec_body=None) - - Creates a class object dynamically using the appropriate metaclass. - - The first three arguments are the components that make up a class - definition header: the class name, the base classes (in order), the - keyword arguments (such as ``metaclass``). - - The *exec_body* argument is a callback that is used to populate the - freshly created class namespace. It should accept the class namespace - as its sole argument and update the namespace directly with the class - contents. If no callback is provided, it has the same effect as passing - in ``lambda ns: None``. - - .. versionadded:: 3.3 - -.. function:: prepare_class(name, bases=(), kwds=None) - - Calculates the appropriate metaclass and creates the class namespace. - - The arguments are the components that make up a class definition header: - the class name, the base classes (in order) and the keyword arguments - (such as ``metaclass``). - - The return value is a 3-tuple: ``metaclass, namespace, kwds`` - - *metaclass* is the appropriate metaclass, *namespace* is the - prepared class namespace and *kwds* is an updated copy of the passed - in *kwds* argument with any ``'metaclass'`` entry removed. If no *kwds* - argument is passed in, this will be an empty dict. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.6 - - The default value for the ``namespace`` element of the returned - tuple has changed. Now an insertion-order-preserving mapping is - used when the metaclass does not have a ``__prepare__`` method. - -.. seealso:: - - :ref:`metaclasses` - Full details of the class creation process supported by these functions - - :pep:`3115` - Metaclasses in Python 3000 - Introduced the ``__prepare__`` namespace hook - -.. function:: resolve_bases(bases) - - Resolve MRO entries dynamically as specified by :pep:`560`. - - This function looks for items in *bases* that are not instances of - :class:`type`, and returns a tuple where each such object that has - an :meth:`~object.__mro_entries__` method is replaced with an unpacked result of - calling this method. If a *bases* item is an instance of :class:`type`, - or it doesn't have an :meth:`!__mro_entries__` method, then it is included in - the return tuple unchanged. - - .. versionadded:: 3.7 - -.. seealso:: - - :pep:`560` - Core support for typing module and generic types - - -Standard Interpreter Types --------------------------- - -This module provides names for many of the types that are required to -implement a Python interpreter. It deliberately avoids including some of -the types that arise only incidentally during processing such as the -``listiterator`` type. - -Typical use of these names is for :func:`isinstance` or -:func:`issubclass` checks. - - -If you instantiate any of these types, note that signatures may vary between Python versions. - -Standard names are defined for the following types: - -.. data:: NoneType - - The type of :data:`None`. - - .. versionadded:: 3.10 - - -.. data:: FunctionType - LambdaType - - The type of user-defined functions and functions created by - :keyword:`lambda` expressions. - - .. audit-event:: function.__new__ code types.FunctionType - - The audit event only occurs for direct instantiation of function objects, - and is not raised for normal compilation. - - -.. data:: GeneratorType - - The type of :term:`generator`-iterator objects, created by - generator functions. - - -.. data:: CoroutineType - - The type of :term:`coroutine` objects, created by - :keyword:`async def` functions. - - .. versionadded:: 3.5 - - -.. data:: AsyncGeneratorType - - The type of :term:`asynchronous generator`-iterator objects, created by - asynchronous generator functions. - - .. versionadded:: 3.6 - - -.. class:: CodeType(**kwargs) - - .. index:: pair: built-in function; compile - - The type for code objects such as returned by :func:`compile`. - - .. audit-event:: code.__new__ code,filename,name,argcount,posonlyargcount,kwonlyargcount,nlocals,stacksize,flags types.CodeType - - Note that the audited arguments may not match the names or positions - required by the initializer. The audit event only occurs for direct - instantiation of code objects, and is not raised for normal compilation. - - .. method:: CodeType.replace(**kwargs) - - Return a copy of the code object with new values for the specified fields. - - .. versionadded:: 3.8 - -.. data:: CellType - - The type for cell objects: such objects are used as containers for - a function's free variables. - - .. versionadded:: 3.8 - - -.. data:: MethodType - - The type of methods of user-defined class instances. - - -.. data:: BuiltinFunctionType - BuiltinMethodType - - The type of built-in functions like :func:`len` or :func:`sys.exit`, and - methods of built-in classes. (Here, the term "built-in" means "written in - C".) - - -.. data:: WrapperDescriptorType - - The type of methods of some built-in data types and base classes such as - :meth:`object.__init__` or :meth:`object.__lt__`. - - .. versionadded:: 3.7 - - -.. data:: MethodWrapperType - - The type of *bound* methods of some built-in data types and base classes. - For example it is the type of :code:`object().__str__`. - - .. versionadded:: 3.7 - - -.. data:: NotImplementedType - - The type of :data:`NotImplemented`. - - .. versionadded:: 3.10 - - -.. data:: MethodDescriptorType - - The type of methods of some built-in data types such as :meth:`str.join`. - - .. versionadded:: 3.7 - - -.. data:: ClassMethodDescriptorType - - The type of *unbound* class methods of some built-in data types such as - ``dict.__dict__['fromkeys']``. - - .. versionadded:: 3.7 - - -.. class:: ModuleType(name, doc=None) - - The type of :term:`modules `. The constructor takes the name of the - module to be created and optionally its :term:`docstring`. - - .. note:: - Use :func:`importlib.util.module_from_spec` to create a new module if you - wish to set the various import-controlled attributes. - - .. attribute:: __doc__ - - The :term:`docstring` of the module. Defaults to ``None``. - - .. attribute:: __loader__ - - The :term:`loader` which loaded the module. Defaults to ``None``. - - This attribute is to match :attr:`importlib.machinery.ModuleSpec.loader` - as stored in the :attr:`__spec__` object. - - .. note:: - A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferably read from the - :attr:`__spec__` attribute instead or use - ``getattr(module, "__loader__", None)`` if you explicitly need to use - this attribute. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __name__ - - The name of the module. Expected to match - :attr:`importlib.machinery.ModuleSpec.name`. - - .. attribute:: __package__ - - Which :term:`package` a module belongs to. If the module is top-level - (i.e. not a part of any specific package) then the attribute should be set - to ``''``, else it should be set to the name of the package (which can be - :attr:`__name__` if the module is a package itself). Defaults to ``None``. - - This attribute is to match :attr:`importlib.machinery.ModuleSpec.parent` - as stored in the :attr:`__spec__` object. - - .. note:: - A future version of Python may stop setting this attribute by default. - To guard against this potential change, preferably read from the - :attr:`__spec__` attribute instead or use - ``getattr(module, "__package__", None)`` if you explicitly need to use - this attribute. - - .. versionchanged:: 3.4 - Defaults to ``None``. Previously the attribute was optional. - - .. attribute:: __spec__ - - A record of the module's import-system-related state. Expected to be an - instance of :class:`importlib.machinery.ModuleSpec`. - - .. versionadded:: 3.4 - - -.. data:: EllipsisType - - The type of :data:`Ellipsis`. - - .. versionadded:: 3.10 - -.. class:: GenericAlias(t_origin, t_args) - - The type of :ref:`parameterized generics ` such as - ``list[int]``. - - ``t_origin`` should be a non-parameterized generic class, such as ``list``, - ``tuple`` or ``dict``. ``t_args`` should be a :class:`tuple` (possibly of - length 1) of types which parameterize ``t_origin``:: - - >>> from types import GenericAlias - - >>> list[int] == GenericAlias(list, (int,)) - True - >>> dict[str, int] == GenericAlias(dict, (str, int)) - True - - .. versionadded:: 3.9 - - .. versionchanged:: 3.9.2 - This type can now be subclassed. - - .. seealso:: - - :ref:`Generic Alias Types` - In-depth documentation on instances of :class:`!types.GenericAlias` - - :pep:`585` - Type Hinting Generics In Standard Collections - Introducing the :class:`!types.GenericAlias` class - -.. class:: UnionType - - The type of :ref:`union type expressions`. - - .. versionadded:: 3.10 - -.. class:: TracebackType(tb_next, tb_frame, tb_lasti, tb_lineno) - - The type of traceback objects such as found in ``sys.exception().__traceback__``. - - See :ref:`the language reference ` for details of the - available attributes and operations, and guidance on creating tracebacks - dynamically. - - -.. data:: FrameType - - The type of frame objects such as found in ``tb.tb_frame`` if ``tb`` is a - traceback object. - - See :ref:`the language reference ` for details of the - available attributes and operations. - - -.. data:: GetSetDescriptorType - - The type of objects defined in extension modules with ``PyGetSetDef``, such - as ``FrameType.f_locals`` or ``array.array.typecode``. This type is used as - descriptor for object attributes; it has the same purpose as the - :class:`property` type, but for classes defined in extension modules. - - -.. data:: MemberDescriptorType - - The type of objects defined in extension modules with ``PyMemberDef``, such - as ``datetime.timedelta.days``. This type is used as descriptor for simple C - data members which use standard conversion functions; it has the same purpose - as the :class:`property` type, but for classes defined in extension modules. - - .. impl-detail:: - - In other implementations of Python, this type may be identical to - ``GetSetDescriptorType``. - -.. class:: MappingProxyType(mapping) - - Read-only proxy of a mapping. It provides a dynamic view on the mapping's - entries, which means that when the mapping changes, the view reflects these - changes. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.9 - - Updated to support the new union (``|``) operator from :pep:`584`, which - simply delegates to the underlying mapping. - - .. describe:: key in proxy - - Return ``True`` if the underlying mapping has a key *key*, else - ``False``. - - .. describe:: proxy[key] - - Return the item of the underlying mapping with key *key*. Raises a - :exc:`KeyError` if *key* is not in the underlying mapping. - - .. describe:: iter(proxy) - - Return an iterator over the keys of the underlying mapping. This is a - shortcut for ``iter(proxy.keys())``. - - .. describe:: len(proxy) - - Return the number of items in the underlying mapping. - - .. method:: copy() - - Return a shallow copy of the underlying mapping. - - .. method:: get(key[, default]) - - Return the value for *key* if *key* is in the underlying mapping, else - *default*. If *default* is not given, it defaults to ``None``, so that - this method never raises a :exc:`KeyError`. - - .. method:: items() - - Return a new view of the underlying mapping's items (``(key, value)`` - pairs). - - .. method:: keys() - - Return a new view of the underlying mapping's keys. - - .. method:: values() - - Return a new view of the underlying mapping's values. - - .. describe:: reversed(proxy) - - Return a reverse iterator over the keys of the underlying mapping. - - .. versionadded:: 3.9 - - -Additional Utility Classes and Functions ----------------------------------------- - -.. class:: SimpleNamespace - - A simple :class:`object` subclass that provides attribute access to its - namespace, as well as a meaningful repr. - - Unlike :class:`object`, with ``SimpleNamespace`` you can add and remove - attributes. If a ``SimpleNamespace`` object is initialized with keyword - arguments, those are directly added to the underlying namespace. - - The type is roughly equivalent to the following code:: - - class SimpleNamespace: - def __init__(self, /, **kwargs): - self.__dict__.update(kwargs) - - def __repr__(self): - items = (f"{k}={v!r}" for k, v in self.__dict__.items()) - return "{}({})".format(type(self).__name__, ", ".join(items)) - - def __eq__(self, other): - if isinstance(self, SimpleNamespace) and isinstance(other, SimpleNamespace): - return self.__dict__ == other.__dict__ - return NotImplemented - - ``SimpleNamespace`` may be useful as a replacement for ``class NS: pass``. - However, for a structured record type use :func:`~collections.namedtuple` - instead. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.9 - Attribute order in the repr changed from alphabetical to insertion (like - ``dict``). - -.. function:: DynamicClassAttribute(fget=None, fset=None, fdel=None, doc=None) - - Route attribute access on a class to __getattr__. - - This is a descriptor, used to define attributes that act differently when - accessed through an instance and through a class. Instance access remains - normal, but access to an attribute through a class will be routed to the - class's __getattr__ method; this is done by raising AttributeError. - - This allows one to have properties active on an instance, and have virtual - attributes on the class with the same name (see :class:`enum.Enum` for an example). - - .. versionadded:: 3.4 - - -Coroutine Utility Functions ---------------------------- - -.. function:: coroutine(gen_func) - - This function transforms a :term:`generator` function into a - :term:`coroutine function` which returns a generator-based coroutine. - The generator-based coroutine is still a :term:`generator iterator`, - but is also considered to be a :term:`coroutine` object and is - :term:`awaitable`. However, it may not necessarily implement - the :meth:`~object.__await__` method. - - If *gen_func* is a generator function, it will be modified in-place. - - If *gen_func* is not a generator function, it will be wrapped. If it - returns an instance of :class:`collections.abc.Generator`, the instance - will be wrapped in an *awaitable* proxy object. All other types - of objects will be returned as is. - - .. versionadded:: 3.5 diff --git a/Doc/library/typing.rst b/Doc/library/typing.rst deleted file mode 100644 index 0decbaf39749c0..00000000000000 --- a/Doc/library/typing.rst +++ /dev/null @@ -1,3370 +0,0 @@ -======================================== -:mod:`typing` --- Support for type hints -======================================== - -.. testsetup:: * - - import typing - from dataclasses import dataclass - from typing import * - -.. module:: typing - :synopsis: Support for type hints (see :pep:`484`). - -.. versionadded:: 3.5 - -**Source code:** :source:`Lib/typing.py` - -.. note:: - - The Python runtime does not enforce function and variable type annotations. - They can be used by third party tools such as type checkers, IDEs, linters, - etc. - --------------- - -This module provides runtime support for type hints. For the original -specification of the typing system, see :pep:`484`. For a simplified -introduction to type hints, see :pep:`483`. - - -The function below takes and returns a string and is annotated as follows:: - - def greeting(name: str) -> str: - return 'Hello ' + name - -In the function ``greeting``, the argument ``name`` is expected to be of type -:class:`str` and the return type :class:`str`. Subtypes are accepted as -arguments. - -New features are frequently added to the ``typing`` module. -The `typing_extensions `_ package -provides backports of these new features to older versions of Python. - -For a summary of deprecated features and a deprecation timeline, please see -`Deprecation Timeline of Major Features`_. - -.. seealso:: - - `"Typing cheat sheet" `_ - A quick overview of type hints (hosted at the mypy docs) - - "Type System Reference" section of `the mypy docs `_ - The Python typing system is standardised via PEPs, so this reference - should broadly apply to most Python type checkers. (Some parts may still - be specific to mypy.) - - `"Static Typing with Python" `_ - Type-checker-agnostic documentation written by the community detailing - type system features, useful typing related tools and typing best - practices. - -.. _relevant-peps: - -Relevant PEPs -============= - -Since the initial introduction of type hints in :pep:`484` and :pep:`483`, a -number of PEPs have modified and enhanced Python's framework for type -annotations: - -.. raw:: html - -
    - The full list of PEPs - -* :pep:`526`: Syntax for Variable Annotations - *Introducing* syntax for annotating variables outside of function - definitions, and :data:`ClassVar` -* :pep:`544`: Protocols: Structural subtyping (static duck typing) - *Introducing* :class:`Protocol` and the - :func:`@runtime_checkable` decorator -* :pep:`585`: Type Hinting Generics In Standard Collections - *Introducing* :class:`types.GenericAlias` and the ability to use standard - library classes as :ref:`generic types` -* :pep:`586`: Literal Types - *Introducing* :data:`Literal` -* :pep:`589`: TypedDict: Type Hints for Dictionaries with a Fixed Set of Keys - *Introducing* :class:`TypedDict` -* :pep:`591`: Adding a final qualifier to typing - *Introducing* :data:`Final` and the :func:`@final` decorator -* :pep:`593`: Flexible function and variable annotations - *Introducing* :data:`Annotated` -* :pep:`604`: Allow writing union types as ``X | Y`` - *Introducing* :data:`types.UnionType` and the ability to use - the binary-or operator ``|`` to signify a - :ref:`union of types` -* :pep:`612`: Parameter Specification Variables - *Introducing* :class:`ParamSpec` and :data:`Concatenate` -* :pep:`613`: Explicit Type Aliases - *Introducing* :data:`TypeAlias` -* :pep:`646`: Variadic Generics - *Introducing* :data:`TypeVarTuple` -* :pep:`647`: User-Defined Type Guards - *Introducing* :data:`TypeGuard` -* :pep:`655`: Marking individual TypedDict items as required or potentially missing - *Introducing* :data:`Required` and :data:`NotRequired` -* :pep:`673`: Self type - *Introducing* :data:`Self` -* :pep:`675`: Arbitrary Literal String Type - *Introducing* :data:`LiteralString` -* :pep:`681`: Data Class Transforms - *Introducing* the :func:`@dataclass_transform` decorator - -.. raw:: html - -
    -
    - -.. _type-aliases: - -Type aliases -============ - -A type alias is defined by assigning the type to the alias. In this example, -``Vector`` and ``list[float]`` will be treated as interchangeable synonyms:: - - Vector = list[float] - - def scale(scalar: float, vector: Vector) -> Vector: - return [scalar * num for num in vector] - - # passes type checking; a list of floats qualifies as a Vector. - new_vector = scale(2.0, [1.0, -4.2, 5.4]) - -Type aliases are useful for simplifying complex type signatures. For example:: - - from collections.abc import Sequence - - ConnectionOptions = dict[str, str] - Address = tuple[str, int] - Server = tuple[Address, ConnectionOptions] - - def broadcast_message(message: str, servers: Sequence[Server]) -> None: - ... - - # The static type checker will treat the previous type signature as - # being exactly equivalent to this one. - def broadcast_message( - message: str, - servers: Sequence[tuple[tuple[str, int], dict[str, str]]]) -> None: - ... - -Type aliases may be marked with :data:`TypeAlias` to make it explicit that -the statement is a type alias declaration, not a normal variable assignment:: - - from typing import TypeAlias - - Vector: TypeAlias = list[float] - -.. _distinct: - -NewType -======= - -Use the :class:`NewType` helper to create distinct types:: - - from typing import NewType - - UserId = NewType('UserId', int) - some_id = UserId(524313) - -The static type checker will treat the new type as if it were a subclass -of the original type. This is useful in helping catch logical errors:: - - def get_user_name(user_id: UserId) -> str: - ... - - # passes type checking - user_a = get_user_name(UserId(42351)) - - # fails type checking; an int is not a UserId - user_b = get_user_name(-1) - -You may still perform all ``int`` operations on a variable of type ``UserId``, -but the result will always be of type ``int``. This lets you pass in a -``UserId`` wherever an ``int`` might be expected, but will prevent you from -accidentally creating a ``UserId`` in an invalid way:: - - # 'output' is of type 'int', not 'UserId' - output = UserId(23413) + UserId(54341) - -Note that these checks are enforced only by the static type checker. At runtime, -the statement ``Derived = NewType('Derived', Base)`` will make ``Derived`` a -callable that immediately returns whatever parameter you pass it. That means -the expression ``Derived(some_value)`` does not create a new class or introduce -much overhead beyond that of a regular function call. - -More precisely, the expression ``some_value is Derived(some_value)`` is always -true at runtime. - -It is invalid to create a subtype of ``Derived``:: - - from typing import NewType - - UserId = NewType('UserId', int) - - # Fails at runtime and does not pass type checking - class AdminUserId(UserId): pass - -However, it is possible to create a :class:`NewType` based on a 'derived' ``NewType``:: - - from typing import NewType - - UserId = NewType('UserId', int) - - ProUserId = NewType('ProUserId', UserId) - -and typechecking for ``ProUserId`` will work as expected. - -See :pep:`484` for more details. - -.. note:: - - Recall that the use of a type alias declares two types to be *equivalent* to - one another. Doing ``Alias = Original`` will make the static type checker - treat ``Alias`` as being *exactly equivalent* to ``Original`` in all cases. - This is useful when you want to simplify complex type signatures. - - In contrast, ``NewType`` declares one type to be a *subtype* of another. - Doing ``Derived = NewType('Derived', Original)`` will make the static type - checker treat ``Derived`` as a *subclass* of ``Original``, which means a - value of type ``Original`` cannot be used in places where a value of type - ``Derived`` is expected. This is useful when you want to prevent logic - errors with minimal runtime cost. - -.. versionadded:: 3.5.2 - -.. versionchanged:: 3.10 - ``NewType`` is now a class rather than a function. As a result, there is - some additional runtime cost when calling ``NewType`` over a regular - function. - -.. versionchanged:: 3.11 - The performance of calling ``NewType`` has been restored to its level in - Python 3.9. - -.. _annotating-callables: - -Annotating callable objects -=========================== - -Functions -- or other :term:`callable` objects -- can be annotated using -:class:`collections.abc.Callable` or :data:`typing.Callable`. -``Callable[[int], str]`` signifies a function that takes a single parameter -of type :class:`int` and returns a :class:`str`. - -For example: - -.. testcode:: - - from collections.abc import Callable, Awaitable - - def feeder(get_next_item: Callable[[], str]) -> None: - ... # Body - - def async_query(on_success: Callable[[int], None], - on_error: Callable[[int, Exception], None]) -> None: - ... # Body - - async def on_update(value: str) -> None: - ... # Body - - callback: Callable[[str], Awaitable[None]] = on_update - -The subscription syntax must always be used with exactly two values: the -argument list and the return type. The argument list must be a list of types, -a :class:`ParamSpec`, :data:`Concatenate`, or an ellipsis. The return type must -be a single type. - -If a literal ellipsis ``...`` is given as the argument list, it indicates that -a callable with any arbitrary parameter list would be acceptable: - -.. testcode:: - - def concat(x: str, y: str) -> str: - return x + y - - x: Callable[..., str] - x = str # OK - x = concat # Also OK - -``Callable`` cannot express complex signatures such as functions that take a -variadic number of arguments, :ref:`overloaded functions `, or -functions that have keyword-only parameters. However, these signatures can be -expressed by defining a :class:`Protocol` class with a -:meth:`~object.__call__` method: - -.. testcode:: - - from collections.abc import Iterable - from typing import Protocol - - class Combiner(Protocol): - def __call__(self, *vals: bytes, maxlen: int | None = None) -> list[bytes]: ... - - def batch_proc(data: Iterable[bytes], cb_results: Combiner) -> bytes: - for item in data: - ... - - def good_cb(*vals: bytes, maxlen: int | None = None) -> list[bytes]: - ... - def bad_cb(*vals: bytes, maxitems: int | None) -> list[bytes]: - ... - - batch_proc([], good_cb) # OK - batch_proc([], bad_cb) # Error! Argument 2 has incompatible type because of - # different name and kind in the callback - -Callables which take other callables as arguments may indicate that their -parameter types are dependent on each other using :class:`ParamSpec`. -Additionally, if that callable adds or removes arguments from other -callables, the :data:`Concatenate` operator may be used. They -take the form ``Callable[ParamSpecVariable, ReturnType]`` and -``Callable[Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable], ReturnType]`` -respectively. - -.. versionchanged:: 3.10 - ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. - See :pep:`612` for more details. - -.. seealso:: - The documentation for :class:`ParamSpec` and :class:`Concatenate` provides - examples of usage in ``Callable``. - -.. _generics: - -Generics -======== - -Since type information about objects kept in containers cannot be statically -inferred in a generic way, many container classes in the standard library support -subscription to denote the expected types of container elements. - -.. testcode:: - - from collections.abc import Mapping, Sequence - - class Employee: ... - - # Sequence[Employee] indicates that all elements in the sequence - # must be instances of "Employee". - # Mapping[str, str] indicates that all keys and all values in the mapping - # must be strings. - def notify_by_email(employees: Sequence[Employee], - overrides: Mapping[str, str]) -> None: ... - -Generics can be parameterized by using a factory available in typing -called :class:`TypeVar`. - -:: - - from collections.abc import Sequence - from typing import TypeVar - - T = TypeVar('T') # Declare type variable "T" - - def first(l: Sequence[T]) -> T: # Function is generic over the TypeVar "T" - return l[0] - -.. _annotating-tuples: - -Annotating tuples -================= - -For most containers in Python, the typing system assumes that all elements in -the container will be of the same type. For example:: - - from collections.abc import Mapping - - # Type checker will infer that all elements in ``x`` are meant to be ints - x: list[int] = [] - - # Type checker error: ``list`` only accepts a single type argument: - y: list[int, str] = [1, 'foo'] - - # Type checker will infer that all keys in ``z`` are meant to be strings, - # and that all values in ``z`` are meant to be either strings or ints - z: Mapping[str, str | int] = {} - -:class:`list` only accepts one type argument, so a type checker would emit an -error on the ``y`` assignment above. Similarly, -:class:`~collections.abc.Mapping` only accepts two type arguments: the first -indicates the type of the keys, and the second indicates the type of the -values. - -Unlike most other Python containers, however, it is common in idiomatic Python -code for tuples to have elements which are not all of the same type. For this -reason, tuples are special-cased in Python's typing system. :class:`tuple` -accepts *any number* of type arguments:: - - # OK: ``x`` is assigned to a tuple of length 1 where the sole element is an int - x: tuple[int] = (5,) - - # OK: ``y`` is assigned to a tuple of length 2; - # element 1 is an int, element 2 is a str - y: tuple[int, str] = (5, "foo") - - # Error: the type annotation indicates a tuple of length 1, - # but ``z`` has been assigned to a tuple of length 3 - z: tuple[int] = (1, 2, 3) - -To denote a tuple which could be of *any* length, and in which all elements are -of the same type ``T``, use ``tuple[T, ...]``. To denote an empty tuple, use -``tuple[()]``. Using plain ``tuple`` as an annotation is equivalent to using -``tuple[Any, ...]``:: - - x: tuple[int, ...] = (1, 2) - # These reassignments are OK: ``tuple[int, ...]`` indicates x can be of any length - x = (1, 2, 3) - x = () - # This reassignment is an error: all elements in ``x`` must be ints - x = ("foo", "bar") - - # ``y`` can only ever be assigned to an empty tuple - y: tuple[()] = () - - z: tuple = ("foo", "bar") - # These reassignments are OK: plain ``tuple`` is equivalent to ``tuple[Any, ...]`` - z = (1, 2, 3) - z = () - -.. _type-of-class-objects: - -The type of class objects -========================= - -A variable annotated with ``C`` may accept a value of type ``C``. In -contrast, a variable annotated with ``type[C]`` (or -:class:`typing.Type[C] `) may accept values that are classes -themselves -- specifically, it will accept the *class object* of ``C``. For -example:: - - a = 3 # Has type ``int`` - b = int # Has type ``type[int]`` - c = type(a) # Also has type ``type[int]`` - -Note that ``type[C]`` is covariant:: - - class User: ... - class ProUser(User): ... - class TeamUser(User): ... - - def make_new_user(user_class: type[User]) -> User: - # ... - return user_class() - - make_new_user(User) # OK - make_new_user(ProUser) # Also OK: ``type[ProUser]`` is a subtype of ``type[User]`` - make_new_user(TeamUser) # Still fine - make_new_user(User()) # Error: expected ``type[User]`` but got ``User`` - make_new_user(int) # Error: ``type[int]`` is not a subtype of ``type[User]`` - -The only legal parameters for :class:`type` are classes, :data:`Any`, -:ref:`type variables `, and unions of any of these types. -For example:: - - def new_non_team_user(user_class: type[BasicUser | ProUser]): ... - - new_non_team_user(BasicUser) # OK - new_non_team_user(ProUser) # OK - new_non_team_user(TeamUser) # Error: ``type[TeamUser]`` is not a subtype - # of ``type[BasicUser | ProUser]`` - new_non_team_user(User) # Also an error - -``type[Any]`` is equivalent to :class:`type`, which is the root of Python's -:ref:`metaclass hierarchy `. - -.. _user-defined-generics: - -User-defined generic types -========================== - -A user-defined class can be defined as a generic class. - -:: - - from typing import TypeVar, Generic - from logging import Logger - - T = TypeVar('T') - - class LoggedVar(Generic[T]): - def __init__(self, value: T, name: str, logger: Logger) -> None: - self.name = name - self.logger = logger - self.value = value - - def set(self, new: T) -> None: - self.log('Set ' + repr(self.value)) - self.value = new - - def get(self) -> T: - self.log('Get ' + repr(self.value)) - return self.value - - def log(self, message: str) -> None: - self.logger.info('%s: %s', self.name, message) - -``Generic[T]`` as a base class defines that the class ``LoggedVar`` takes a -single type parameter ``T`` . This also makes ``T`` valid as a type within the -class body. - -The :class:`Generic` base class defines :meth:`~object.__class_getitem__` so -that ``LoggedVar[T]`` is valid as a type:: - - from collections.abc import Iterable - - def zero_all_vars(vars: Iterable[LoggedVar[int]]) -> None: - for var in vars: - var.set(0) - -A generic type can have any number of type variables. All varieties of -:class:`TypeVar` are permissible as parameters for a generic type:: - - from typing import TypeVar, Generic, Sequence - - T = TypeVar('T', contravariant=True) - B = TypeVar('B', bound=Sequence[bytes], covariant=True) - S = TypeVar('S', int, str) - - class WeirdTrio(Generic[T, B, S]): - ... - -Each type variable argument to :class:`Generic` must be distinct. -This is thus invalid:: - - from typing import TypeVar, Generic - ... - - T = TypeVar('T') - - class Pair(Generic[T, T]): # INVALID - ... - -You can use multiple inheritance with :class:`Generic`:: - - from collections.abc import Sized - from typing import TypeVar, Generic - - T = TypeVar('T') - - class LinkedList(Sized, Generic[T]): - ... - -When inheriting from generic classes, some type parameters could be fixed:: - - from collections.abc import Mapping - from typing import TypeVar - - T = TypeVar('T') - - class MyDict(Mapping[str, T]): - ... - -In this case ``MyDict`` has a single parameter, ``T``. - -Using a generic class without specifying type parameters assumes -:data:`Any` for each position. In the following example, ``MyIterable`` is -not generic but implicitly inherits from ``Iterable[Any]``: - -.. testcode:: - - from collections.abc import Iterable - - class MyIterable(Iterable): # Same as Iterable[Any] - ... - -User-defined generic type aliases are also supported. Examples:: - - from collections.abc import Iterable - from typing import TypeVar - S = TypeVar('S') - Response = Iterable[S] | int - - # Return type here is same as Iterable[str] | int - def response(query: str) -> Response[str]: - ... - - T = TypeVar('T', int, float, complex) - Vec = Iterable[tuple[T, T]] - - def inproduct(v: Vec[T]) -> T: # Same as Iterable[tuple[T, T]] - return sum(x*y for x, y in v) - -.. versionchanged:: 3.7 - :class:`Generic` no longer has a custom metaclass. - -User-defined generics for parameter expressions are also supported via parameter -specification variables in the form ``Generic[P]``. The behavior is consistent -with type variables' described above as parameter specification variables are -treated by the typing module as a specialized type variable. The one exception -to this is that a list of types can be used to substitute a :class:`ParamSpec`:: - - >>> from typing import Generic, ParamSpec, TypeVar - - >>> T = TypeVar('T') - >>> P = ParamSpec('P') - - >>> class Z(Generic[T, P]): ... - ... - >>> Z[int, [dict, float]] - __main__.Z[int, (, )] - -Furthermore, a generic with only one parameter specification variable will accept -parameter lists in the forms ``X[[Type1, Type2, ...]]`` and also -``X[Type1, Type2, ...]`` for aesthetic reasons. Internally, the latter is converted -to the former, so the following are equivalent:: - - >>> class X(Generic[P]): ... - ... - >>> X[int, str] - __main__.X[(, )] - >>> X[[int, str]] - __main__.X[(, )] - -Note that generics with :class:`ParamSpec` may not have correct -``__parameters__`` after substitution in some cases because they -are intended primarily for static type checking. - -.. versionchanged:: 3.10 - :class:`Generic` can now be parameterized over parameter expressions. - See :class:`ParamSpec` and :pep:`612` for more details. - -A user-defined generic class can have ABCs as base classes without a metaclass -conflict. Generic metaclasses are not supported. The outcome of parameterizing -generics is cached, and most types in the typing module are :term:`hashable` and -comparable for equality. - - -The :data:`Any` type -==================== - -A special kind of type is :data:`Any`. A static type checker will treat -every type as being compatible with :data:`Any` and :data:`Any` as being -compatible with every type. - -This means that it is possible to perform any operation or method call on a -value of type :data:`Any` and assign it to any variable:: - - from typing import Any - - a: Any = None - a = [] # OK - a = 2 # OK - - s: str = '' - s = a # OK - - def foo(item: Any) -> int: - # Passes type checking; 'item' could be any type, - # and that type might have a 'bar' method - item.bar() - ... - -Notice that no type checking is performed when assigning a value of type -:data:`Any` to a more precise type. For example, the static type checker did -not report an error when assigning ``a`` to ``s`` even though ``s`` was -declared to be of type :class:`str` and receives an :class:`int` value at -runtime! - -Furthermore, all functions without a return type or parameter types will -implicitly default to using :data:`Any`:: - - def legacy_parser(text): - ... - return data - - # A static type checker will treat the above - # as having the same signature as: - def legacy_parser(text: Any) -> Any: - ... - return data - -This behavior allows :data:`Any` to be used as an *escape hatch* when you -need to mix dynamically and statically typed code. - -Contrast the behavior of :data:`Any` with the behavior of :class:`object`. -Similar to :data:`Any`, every type is a subtype of :class:`object`. However, -unlike :data:`Any`, the reverse is not true: :class:`object` is *not* a -subtype of every other type. - -That means when the type of a value is :class:`object`, a type checker will -reject almost all operations on it, and assigning it to a variable (or using -it as a return value) of a more specialized type is a type error. For example:: - - def hash_a(item: object) -> int: - # Fails type checking; an object does not have a 'magic' method. - item.magic() - ... - - def hash_b(item: Any) -> int: - # Passes type checking - item.magic() - ... - - # Passes type checking, since ints and strs are subclasses of object - hash_a(42) - hash_a("foo") - - # Passes type checking, since Any is compatible with all types - hash_b(42) - hash_b("foo") - -Use :class:`object` to indicate that a value could be any type in a typesafe -manner. Use :data:`Any` to indicate that a value is dynamically typed. - - -Nominal vs structural subtyping -=============================== - -Initially :pep:`484` defined the Python static type system as using -*nominal subtyping*. This means that a class ``A`` is allowed where -a class ``B`` is expected if and only if ``A`` is a subclass of ``B``. - -This requirement previously also applied to abstract base classes, such as -:class:`~collections.abc.Iterable`. The problem with this approach is that a class had -to be explicitly marked to support them, which is unpythonic and unlike -what one would normally do in idiomatic dynamically typed Python code. -For example, this conforms to :pep:`484`:: - - from collections.abc import Sized, Iterable, Iterator - - class Bucket(Sized, Iterable[int]): - ... - def __len__(self) -> int: ... - def __iter__(self) -> Iterator[int]: ... - -:pep:`544` allows to solve this problem by allowing users to write -the above code without explicit base classes in the class definition, -allowing ``Bucket`` to be implicitly considered a subtype of both ``Sized`` -and ``Iterable[int]`` by static type checkers. This is known as -*structural subtyping* (or static duck-typing):: - - from collections.abc import Iterator, Iterable - - class Bucket: # Note: no base classes - ... - def __len__(self) -> int: ... - def __iter__(self) -> Iterator[int]: ... - - def collect(items: Iterable[int]) -> int: ... - result = collect(Bucket()) # Passes type check - -Moreover, by subclassing a special class :class:`Protocol`, a user -can define new custom protocols to fully enjoy structural subtyping -(see examples below). - -Module contents -=============== - -The ``typing`` module defines the following classes, functions and decorators. - -Special typing primitives -------------------------- - -Special types -""""""""""""" - -These can be used as types in annotations. They do not support subscription -using ``[]``. - -.. data:: Any - - Special type indicating an unconstrained type. - - * Every type is compatible with :data:`Any`. - * :data:`Any` is compatible with every type. - - .. versionchanged:: 3.11 - :data:`Any` can now be used as a base class. This can be useful for - avoiding type checker errors with classes that can duck type anywhere or - are highly dynamic. - -.. data:: AnyStr - - A :ref:`constrained type variable `. - - Definition:: - - AnyStr = TypeVar('AnyStr', str, bytes) - - ``AnyStr`` is meant to be used for functions that may accept :class:`str` or - :class:`bytes` arguments but cannot allow the two to mix. - - For example:: - - def concat(a: AnyStr, b: AnyStr) -> AnyStr: - return a + b - - concat("foo", "bar") # OK, output has type 'str' - concat(b"foo", b"bar") # OK, output has type 'bytes' - concat("foo", b"bar") # Error, cannot mix str and bytes - - Note that, despite its name, ``AnyStr`` has nothing to do with the - :class:`Any` type, nor does it mean "any string". In particular, ``AnyStr`` - and ``str | bytes`` are different from each other and have different use - cases:: - - # Invalid use of AnyStr: - # The type variable is used only once in the function signature, - # so cannot be "solved" by the type checker - def greet_bad(cond: bool) -> AnyStr: - return "hi there!" if cond else b"greetings!" - - # The better way of annotating this function: - def greet_proper(cond: bool) -> str | bytes: - return "hi there!" if cond else b"greetings!" - -.. data:: LiteralString - - Special type that includes only literal strings. - - Any string - literal is compatible with ``LiteralString``, as is another - ``LiteralString``. However, an object typed as just ``str`` is not. - A string created by composing ``LiteralString``-typed objects - is also acceptable as a ``LiteralString``. - - Example: - - .. testcode:: - - def run_query(sql: LiteralString) -> None: - ... - - def caller(arbitrary_string: str, literal_string: LiteralString) -> None: - run_query("SELECT * FROM students") # OK - run_query(literal_string) # OK - run_query("SELECT * FROM " + literal_string) # OK - run_query(arbitrary_string) # type checker error - run_query( # type checker error - f"SELECT * FROM students WHERE name = {arbitrary_string}" - ) - - ``LiteralString`` is useful for sensitive APIs where arbitrary user-generated - strings could generate problems. For example, the two cases above - that generate type checker errors could be vulnerable to an SQL - injection attack. - - See :pep:`675` for more details. - - .. versionadded:: 3.11 - -.. data:: Never - - The `bottom type `_, - a type that has no members. - - This can be used to define a function that should never be - called, or a function that never returns:: - - from typing import Never - - def never_call_me(arg: Never) -> None: - pass - - def int_or_str(arg: int | str) -> None: - never_call_me(arg) # type checker error - match arg: - case int(): - print("It's an int") - case str(): - print("It's a str") - case _: - never_call_me(arg) # OK, arg is of type Never - - .. versionadded:: 3.11 - - On older Python versions, :data:`NoReturn` may be used to express the - same concept. ``Never`` was added to make the intended meaning more explicit. - -.. data:: NoReturn - - Special type indicating that a function never returns. - - For example:: - - from typing import NoReturn - - def stop() -> NoReturn: - raise RuntimeError('no way') - - ``NoReturn`` can also be used as a - `bottom type `_, a type that - has no values. Starting in Python 3.11, the :data:`Never` type should - be used for this concept instead. Type checkers should treat the two - equivalently. - - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.2 - -.. data:: Self - - Special type to represent the current enclosed class. - - For example:: - - from typing import Self, reveal_type - - class Foo: - def return_self(self) -> Self: - ... - return self - - class SubclassOfFoo(Foo): pass - - reveal_type(Foo().return_self()) # Revealed type is "Foo" - reveal_type(SubclassOfFoo().return_self()) # Revealed type is "SubclassOfFoo" - - This annotation is semantically equivalent to the following, - albeit in a more succinct fashion:: - - from typing import TypeVar - - Self = TypeVar("Self", bound="Foo") - - class Foo: - def return_self(self: Self) -> Self: - ... - return self - - In general, if something returns ``self``, as in the above examples, you - should use ``Self`` as the return annotation. If ``Foo.return_self`` was - annotated as returning ``"Foo"``, then the type checker would infer the - object returned from ``SubclassOfFoo.return_self`` as being of type ``Foo`` - rather than ``SubclassOfFoo``. - - Other common use cases include: - - - :class:`classmethod`\s that are used as alternative constructors and return instances - of the ``cls`` parameter. - - Annotating an :meth:`~object.__enter__` method which returns self. - - You should not use ``Self`` as the return annotation if the method is not - guaranteed to return an instance of a subclass when the class is - subclassed:: - - class Eggs: - # Self would be an incorrect return annotation here, - # as the object returned is always an instance of Eggs, - # even in subclasses - def returns_eggs(self) -> "Eggs": - return Eggs() - - See :pep:`673` for more details. - - .. versionadded:: 3.11 - -.. data:: TypeAlias - - Special annotation for explicitly declaring a :ref:`type alias `. - - For example:: - - from typing import TypeAlias - - Factors: TypeAlias = list[int] - - ``TypeAlias`` is particularly useful for annotating - aliases that make use of forward references, as it can be hard for type - checkers to distinguish these from normal variable assignments: - - .. testcode:: - - from typing import Generic, TypeAlias, TypeVar - - T = TypeVar("T") - - # "Box" does not exist yet, - # so we have to use quotes for the forward reference. - # Using ``TypeAlias`` tells the type checker that this is a type alias declaration, - # not a variable assignment to a string. - BoxOfStrings: TypeAlias = "Box[str]" - - class Box(Generic[T]): - @classmethod - def make_box_of_strings(cls) -> BoxOfStrings: ... - - See :pep:`613` for more details. - - .. versionadded:: 3.10 - -Special forms -""""""""""""" - -These can be used as types in annotations. They all support subscription using -``[]``, but each has a unique syntax. - -.. data:: Union - - Union type; ``Union[X, Y]`` is equivalent to ``X | Y`` and means either X or Y. - - To define a union, use e.g. ``Union[int, str]`` or the shorthand ``int | str``. Using that shorthand is recommended. Details: - - * The arguments must be types and there must be at least one. - - * Unions of unions are flattened, e.g.:: - - Union[Union[int, str], float] == Union[int, str, float] - - * Unions of a single argument vanish, e.g.:: - - Union[int] == int # The constructor actually returns int - - * Redundant arguments are skipped, e.g.:: - - Union[int, str, int] == Union[int, str] == int | str - - * When comparing unions, the argument order is ignored, e.g.:: - - Union[int, str] == Union[str, int] - - * You cannot subclass or instantiate a ``Union``. - - * You cannot write ``Union[X][Y]``. - - .. versionchanged:: 3.7 - Don't remove explicit subclasses from unions at runtime. - - .. versionchanged:: 3.10 - Unions can now be written as ``X | Y``. See - :ref:`union type expressions`. - -.. data:: Optional - - ``Optional[X]`` is equivalent to ``X | None`` (or ``Union[X, None]``). - - Note that this is not the same concept as an optional argument, - which is one that has a default. An optional argument with a - default does not require the ``Optional`` qualifier on its type - annotation just because it is optional. For example:: - - def foo(arg: int = 0) -> None: - ... - - On the other hand, if an explicit value of ``None`` is allowed, the - use of ``Optional`` is appropriate, whether the argument is optional - or not. For example:: - - def foo(arg: Optional[int] = None) -> None: - ... - - .. versionchanged:: 3.10 - Optional can now be written as ``X | None``. See - :ref:`union type expressions`. - -.. data:: Concatenate - - Special form for annotating higher-order functions. - - ``Concatenate`` can be used in conjunction with :ref:`Callable ` and - :class:`ParamSpec` to annotate a higher-order callable which adds, removes, - or transforms parameters of another - callable. Usage is in the form - ``Concatenate[Arg1Type, Arg2Type, ..., ParamSpecVariable]``. ``Concatenate`` - is currently only valid when used as the first argument to a :ref:`Callable `. - The last parameter to ``Concatenate`` must be a :class:`ParamSpec` or - ellipsis (``...``). - - For example, to annotate a decorator ``with_lock`` which provides a - :class:`threading.Lock` to the decorated function, ``Concatenate`` can be - used to indicate that ``with_lock`` expects a callable which takes in a - ``Lock`` as the first argument, and returns a callable with a different type - signature. In this case, the :class:`ParamSpec` indicates that the returned - callable's parameter types are dependent on the parameter types of the - callable being passed in:: - - from collections.abc import Callable - from threading import Lock - from typing import Concatenate, ParamSpec, TypeVar - - P = ParamSpec('P') - R = TypeVar('R') - - # Use this lock to ensure that only one thread is executing a function - # at any time. - my_lock = Lock() - - def with_lock(f: Callable[Concatenate[Lock, P], R]) -> Callable[P, R]: - '''A type-safe decorator which provides a lock.''' - def inner(*args: P.args, **kwargs: P.kwargs) -> R: - # Provide the lock as the first argument. - return f(my_lock, *args, **kwargs) - return inner - - @with_lock - def sum_threadsafe(lock: Lock, numbers: list[float]) -> float: - '''Add a list of numbers together in a thread-safe manner.''' - with lock: - return sum(numbers) - - # We don't need to pass in the lock ourselves thanks to the decorator. - sum_threadsafe([1.1, 2.2, 3.3]) - - .. versionadded:: 3.10 - - .. seealso:: - - * :pep:`612` -- Parameter Specification Variables (the PEP which introduced - ``ParamSpec`` and ``Concatenate``) - * :class:`ParamSpec` - * :ref:`annotating-callables` - -.. data:: Literal - - Special typing form to define "literal types". - - ``Literal`` can be used to indicate to type checkers that the - annotated object has a value equivalent to one of the - provided literals. - - For example:: - - def validate_simple(data: Any) -> Literal[True]: # always returns True - ... - - Mode: TypeAlias = Literal['r', 'rb', 'w', 'wb'] - def open_helper(file: str, mode: Mode) -> str: - ... - - open_helper('/some/path', 'r') # Passes type check - open_helper('/other/path', 'typo') # Error in type checker - - ``Literal[...]`` cannot be subclassed. At runtime, an arbitrary value - is allowed as type argument to ``Literal[...]``, but type checkers may - impose restrictions. See :pep:`586` for more details about literal types. - - .. versionadded:: 3.8 - - .. versionchanged:: 3.9.1 - ``Literal`` now de-duplicates parameters. Equality comparisons of - ``Literal`` objects are no longer order dependent. ``Literal`` objects - will now raise a :exc:`TypeError` exception during equality comparisons - if one of their parameters are not :term:`hashable`. - -.. data:: ClassVar - - Special type construct to mark class variables. - - As introduced in :pep:`526`, a variable annotation wrapped in ClassVar - indicates that a given attribute is intended to be used as a class variable - and should not be set on instances of that class. Usage:: - - class Starship: - stats: ClassVar[dict[str, int]] = {} # class variable - damage: int = 10 # instance variable - - :data:`ClassVar` accepts only types and cannot be further subscribed. - - :data:`ClassVar` is not a class itself, and should not - be used with :func:`isinstance` or :func:`issubclass`. - :data:`ClassVar` does not change Python runtime behavior, but - it can be used by third-party type checkers. For example, a type checker - might flag the following code as an error:: - - enterprise_d = Starship(3000) - enterprise_d.stats = {} # Error, setting class variable on instance - Starship.stats = {} # This is OK - - .. versionadded:: 3.5.3 - -.. data:: Final - - Special typing construct to indicate final names to type checkers. - - Final names cannot be reassigned in any scope. Final names declared in class - scopes cannot be overridden in subclasses. - - For example:: - - MAX_SIZE: Final = 9000 - MAX_SIZE += 1 # Error reported by type checker - - class Connection: - TIMEOUT: Final[int] = 10 - - class FastConnector(Connection): - TIMEOUT = 1 # Error reported by type checker - - There is no runtime checking of these properties. See :pep:`591` for - more details. - - .. versionadded:: 3.8 - -.. data:: Required - - Special typing construct to mark a :class:`TypedDict` key as required. - - This is mainly useful for ``total=False`` TypedDicts. See :class:`TypedDict` - and :pep:`655` for more details. - - .. versionadded:: 3.11 - -.. data:: NotRequired - - Special typing construct to mark a :class:`TypedDict` key as potentially - missing. - - See :class:`TypedDict` and :pep:`655` for more details. - - .. versionadded:: 3.11 - -.. data:: Annotated - - Special typing form to add context-specific metadata to an annotation. - - Add metadata ``x`` to a given type ``T`` by using the annotation - ``Annotated[T, x]``. Metadata added using ``Annotated`` can be used by - static analysis tools or at runtime. At runtime, the metadata is stored - in a :attr:`!__metadata__` attribute. - - If a library or tool encounters an annotation ``Annotated[T, x]`` and has - no special logic for the metadata, it should ignore the metadata and simply - treat the annotation as ``T``. As such, ``Annotated`` can be useful for code - that wants to use annotations for purposes outside Python's static typing - system. - - Using ``Annotated[T, x]`` as an annotation still allows for static - typechecking of ``T``, as type checkers will simply ignore the metadata ``x``. - In this way, ``Annotated`` differs from the - :func:`@no_type_check ` decorator, which can also be used for - adding annotations outside the scope of the typing system, but - completely disables typechecking for a function or class. - - The responsibility of how to interpret the metadata - lies with the tool or library encountering an - ``Annotated`` annotation. A tool or library encountering an ``Annotated`` type - can scan through the metadata elements to determine if they are of interest - (e.g., using :func:`isinstance`). - - .. describe:: Annotated[, ] - - Here is an example of how you might use ``Annotated`` to add metadata to - type annotations if you were doing range analysis: - - .. testcode:: - - @dataclass - class ValueRange: - lo: int - hi: int - - T1 = Annotated[int, ValueRange(-10, 5)] - T2 = Annotated[T1, ValueRange(-20, 3)] - - Details of the syntax: - - * The first argument to ``Annotated`` must be a valid type - - * Multiple metadata elements can be supplied (``Annotated`` supports variadic - arguments):: - - @dataclass - class ctype: - kind: str - - Annotated[int, ValueRange(3, 10), ctype("char")] - - It is up to the tool consuming the annotations to decide whether the - client is allowed to add multiple metadata elements to one annotation and how to - merge those annotations. - - * ``Annotated`` must be subscripted with at least two arguments ( - ``Annotated[int]`` is not valid) - - * The order of the metadata elements is preserved and matters for equality - checks:: - - assert Annotated[int, ValueRange(3, 10), ctype("char")] != Annotated[ - int, ctype("char"), ValueRange(3, 10) - ] - - * Nested ``Annotated`` types are flattened. The order of the metadata elements - starts with the innermost annotation:: - - assert Annotated[Annotated[int, ValueRange(3, 10)], ctype("char")] == Annotated[ - int, ValueRange(3, 10), ctype("char") - ] - - * Duplicated metadata elements are not removed:: - - assert Annotated[int, ValueRange(3, 10)] != Annotated[ - int, ValueRange(3, 10), ValueRange(3, 10) - ] - - * ``Annotated`` can be used with nested and generic aliases: - - .. testcode:: - - @dataclass - class MaxLen: - value: int - - T = TypeVar("T") - Vec: TypeAlias = Annotated[list[tuple[T, T]], MaxLen(10)] - - assert Vec[int] == Annotated[list[tuple[int, int]], MaxLen(10)] - - * ``Annotated`` cannot be used with an unpacked :class:`TypeVarTuple`:: - - Variadic: TypeAlias = Annotated[*Ts, Ann1] # NOT valid - - This would be equivalent to:: - - Annotated[T1, T2, T3, ..., Ann1] - - where ``T1``, ``T2``, etc. are :class:`TypeVars `. This would be - invalid: only one type should be passed to Annotated. - - * By default, :func:`get_type_hints` strips the metadata from annotations. - Pass ``include_extras=True`` to have the metadata preserved: - - .. doctest:: - - >>> from typing import Annotated, get_type_hints - >>> def func(x: Annotated[int, "metadata"]) -> None: pass - ... - >>> get_type_hints(func) - {'x': , 'return': } - >>> get_type_hints(func, include_extras=True) - {'x': typing.Annotated[int, 'metadata'], 'return': } - - * At runtime, the metadata associated with an ``Annotated`` type can be - retrieved via the :attr:`!__metadata__` attribute: - - .. doctest:: - - >>> from typing import Annotated - >>> X = Annotated[int, "very", "important", "metadata"] - >>> X - typing.Annotated[int, 'very', 'important', 'metadata'] - >>> X.__metadata__ - ('very', 'important', 'metadata') - - .. seealso:: - - :pep:`593` - Flexible function and variable annotations - The PEP introducing ``Annotated`` to the standard library. - - .. versionadded:: 3.9 - - -.. data:: TypeGuard - - Special typing construct for marking user-defined type guard functions. - - ``TypeGuard`` can be used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. - At runtime, functions marked this way should return a boolean. - - ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static - type checkers to determine a more precise type of an expression within a - program's code flow. Usually type narrowing is done by analyzing - conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard":: - - def is_str(val: str | float): - # "isinstance" type guard - if isinstance(val, str): - # Type of ``val`` is narrowed to ``str`` - ... - else: - # Else, type of ``val`` is narrowed to ``float``. - ... - - Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. - - Using ``-> TypeGuard`` tells the static type checker that for a given - function: - - 1. The return value is a boolean. - 2. If the return value is ``True``, the type of its argument - is the type inside ``TypeGuard``. - - For example:: - - def is_str_list(val: list[object]) -> TypeGuard[list[str]]: - '''Determines whether all objects in the list are strings''' - return all(isinstance(x, str) for x in val) - - def func1(val: list[object]): - if is_str_list(val): - # Type of ``val`` is narrowed to ``list[str]``. - print(" ".join(val)) - else: - # Type of ``val`` remains as ``list[object]``. - print("Not a list of strings!") - - If ``is_str_list`` is a class or instance method, then the type in - ``TypeGuard`` maps to the type of the second parameter after ``cls`` or - ``self``. - - In short, the form ``def foo(arg: TypeA) -> TypeGuard[TypeB]: ...``, - means that if ``foo(arg)`` returns ``True``, then ``arg`` narrows from - ``TypeA`` to ``TypeB``. - - .. note:: - - ``TypeB`` need not be a narrower form of ``TypeA`` -- it can even be a - wider form. The main reason is to allow for things like - narrowing ``list[object]`` to ``list[str]`` even though the latter - is not a subtype of the former, since ``list`` is invariant. - The responsibility of writing type-safe type guards is left to the user. - - ``TypeGuard`` also works with type variables. See :pep:`647` for more details. - - .. versionadded:: 3.10 - - -.. data:: Unpack - - Typing operator to conceptually mark an object as having been unpacked. - - For example, using the unpack operator ``*`` on a - :ref:`type variable tuple ` is equivalent to using ``Unpack`` - to mark the type variable tuple as having been unpacked:: - - Ts = TypeVarTuple('Ts') - tup: tuple[*Ts] - # Effectively does: - tup: tuple[Unpack[Ts]] - - In fact, ``Unpack`` can be used interchangeably with ``*`` in the context - of :class:`typing.TypeVarTuple ` and - :class:`builtins.tuple ` types. You might see ``Unpack`` being used - explicitly in older versions of Python, where ``*`` couldn't be used in - certain places:: - - # In older versions of Python, TypeVarTuple and Unpack - # are located in the `typing_extensions` backports package. - from typing_extensions import TypeVarTuple, Unpack - - Ts = TypeVarTuple('Ts') - tup: tuple[*Ts] # Syntax error on Python <= 3.10! - tup: tuple[Unpack[Ts]] # Semantically equivalent, and backwards-compatible - - .. versionadded:: 3.11 - -Building generic types -"""""""""""""""""""""" - -The following classes should not be used directly as annotations. -Their intended purpose is to be building blocks -for creating generic types. - -.. class:: Generic - - Abstract base class for generic types. - - A generic type is typically declared by inheriting from an - instantiation of this class with one or more type variables. - For example, a generic mapping type might be defined as:: - - class Mapping(Generic[KT, VT]): - def __getitem__(self, key: KT) -> VT: - ... - # Etc. - - This class can then be used as follows:: - - X = TypeVar('X') - Y = TypeVar('Y') - - def lookup_name(mapping: Mapping[X, Y], key: X, default: Y) -> Y: - try: - return mapping[key] - except KeyError: - return default - -.. _typevar: - -.. class:: TypeVar(name, *constraints, bound=None, covariant=False, contravariant=False) - - Type variable. - - Usage:: - - T = TypeVar('T') # Can be anything - S = TypeVar('S', bound=str) # Can be any subtype of str - A = TypeVar('A', str, bytes) # Must be exactly str or bytes - - Type variables exist primarily for the benefit of static type - checkers. They serve as the parameters for generic types as well - as for generic function and type alias definitions. - See :class:`Generic` for more - information on generic types. Generic functions work as follows:: - - def repeat(x: T, n: int) -> Sequence[T]: - """Return a list containing n references to x.""" - return [x]*n - - - def print_capitalized(x: S) -> S: - """Print x capitalized, and return x.""" - print(x.capitalize()) - return x - - - def concatenate(x: A, y: A) -> A: - """Add two strings or bytes objects together.""" - return x + y - - Note that type variables can be *bound*, *constrained*, or neither, but - cannot be both bound *and* constrained. - - Type variables may be marked covariant or contravariant by passing - ``covariant=True`` or ``contravariant=True``. See :pep:`484` for more - details. By default, type variables are invariant. - - Bound type variables and constrained type variables have different - semantics in several important ways. Using a *bound* type variable means - that the ``TypeVar`` will be solved using the most specific type possible:: - - x = print_capitalized('a string') - reveal_type(x) # revealed type is str - - class StringSubclass(str): - pass - - y = print_capitalized(StringSubclass('another string')) - reveal_type(y) # revealed type is StringSubclass - - z = print_capitalized(45) # error: int is not a subtype of str - - Type variables can be bound to concrete types, abstract types (ABCs or - protocols), and even unions of types:: - - U = TypeVar('U', bound=str|bytes) # Can be any subtype of the union str|bytes - V = TypeVar('V', bound=SupportsAbs) # Can be anything with an __abs__ method - - .. _typing-constrained-typevar: - - Using a *constrained* type variable, however, means that the ``TypeVar`` - can only ever be solved as being exactly one of the constraints given:: - - a = concatenate('one', 'two') - reveal_type(a) # revealed type is str - - b = concatenate(StringSubclass('one'), StringSubclass('two')) - reveal_type(b) # revealed type is str, despite StringSubclass being passed in - - c = concatenate('one', b'two') # error: type variable 'A' can be either str or bytes in a function call, but not both - - At runtime, ``isinstance(x, T)`` will raise :exc:`TypeError`. - - .. attribute:: __name__ - - The name of the type variable. - - .. attribute:: __covariant__ - - Whether the type var has been marked as covariant. - - .. attribute:: __contravariant__ - - Whether the type var has been marked as contravariant. - - .. attribute:: __bound__ - - The bound of the type variable, if any. - - .. attribute:: __constraints__ - - A tuple containing the constraints of the type variable, if any. - -.. _typevartuple: - -.. class:: TypeVarTuple(name) - - Type variable tuple. A specialized form of :ref:`type variable ` - that enables *variadic* generics. - - Usage:: - - T = TypeVar("T") - Ts = TypeVarTuple("Ts") - - def move_first_element_to_last(tup: tuple[T, *Ts]) -> tuple[*Ts, T]: - return (*tup[1:], tup[0]) - - A normal type variable enables parameterization with a single type. A type - variable tuple, in contrast, allows parameterization with an - *arbitrary* number of types by acting like an *arbitrary* number of type - variables wrapped in a tuple. For example:: - - # T is bound to int, Ts is bound to () - # Return value is (1,), which has type tuple[int] - move_first_element_to_last(tup=(1,)) - - # T is bound to int, Ts is bound to (str,) - # Return value is ('spam', 1), which has type tuple[str, int] - move_first_element_to_last(tup=(1, 'spam')) - - # T is bound to int, Ts is bound to (str, float) - # Return value is ('spam', 3.0, 1), which has type tuple[str, float, int] - move_first_element_to_last(tup=(1, 'spam', 3.0)) - - # This fails to type check (and fails at runtime) - # because tuple[()] is not compatible with tuple[T, *Ts] - # (at least one element is required) - move_first_element_to_last(tup=()) - - Note the use of the unpacking operator ``*`` in ``tuple[T, *Ts]``. - Conceptually, you can think of ``Ts`` as a tuple of type variables - ``(T1, T2, ...)``. ``tuple[T, *Ts]`` would then become - ``tuple[T, *(T1, T2, ...)]``, which is equivalent to - ``tuple[T, T1, T2, ...]``. (Note that in older versions of Python, you might - see this written using :data:`Unpack ` instead, as - ``Unpack[Ts]``.) - - Type variable tuples must *always* be unpacked. This helps distinguish type - variable tuples from normal type variables:: - - x: Ts # Not valid - x: tuple[Ts] # Not valid - x: tuple[*Ts] # The correct way to do it - - Type variable tuples can be used in the same contexts as normal type - variables. For example, in class definitions, arguments, and return types:: - - Shape = TypeVarTuple("Shape") - class Array(Generic[*Shape]): - def __getitem__(self, key: tuple[*Shape]) -> float: ... - def __abs__(self) -> "Array[*Shape]": ... - def get_shape(self) -> tuple[*Shape]: ... - - Type variable tuples can be happily combined with normal type variables: - - .. testcode:: - - DType = TypeVar('DType') - Shape = TypeVarTuple('Shape') - - class Array(Generic[DType, *Shape]): # This is fine - pass - - class Array2(Generic[*Shape, DType]): # This would also be fine - pass - - class Height: ... - class Width: ... - - float_array_1d: Array[float, Height] = Array() # Totally fine - int_array_2d: Array[int, Height, Width] = Array() # Yup, fine too - - However, note that at most one type variable tuple may appear in a single - list of type arguments or type parameters:: - - x: tuple[*Ts, *Ts] # Not valid - class Array(Generic[*Shape, *Shape]): # Not valid - pass - - Finally, an unpacked type variable tuple can be used as the type annotation - of ``*args``:: - - def call_soon( - callback: Callable[[*Ts], None], - *args: *Ts - ) -> None: - ... - callback(*args) - - In contrast to non-unpacked annotations of ``*args`` - e.g. ``*args: int``, - which would specify that *all* arguments are ``int`` - ``*args: *Ts`` - enables reference to the types of the *individual* arguments in ``*args``. - Here, this allows us to ensure the types of the ``*args`` passed - to ``call_soon`` match the types of the (positional) arguments of - ``callback``. - - See :pep:`646` for more details on type variable tuples. - - .. attribute:: __name__ - - The name of the type variable tuple. - - .. versionadded:: 3.11 - -.. class:: ParamSpec(name, *, bound=None, covariant=False, contravariant=False) - - Parameter specification variable. A specialized version of - :ref:`type variables `. - - Usage:: - - P = ParamSpec('P') - - Parameter specification variables exist primarily for the benefit of static - type checkers. They are used to forward the parameter types of one - callable to another callable -- a pattern commonly found in higher order - functions and decorators. They are only valid when used in ``Concatenate``, - or as the first argument to ``Callable``, or as parameters for user-defined - Generics. See :class:`Generic` for more information on generic types. - - For example, to add basic logging to a function, one can create a decorator - ``add_logging`` to log function calls. The parameter specification variable - tells the type checker that the callable passed into the decorator and the - new callable returned by it have inter-dependent type parameters:: - - from collections.abc import Callable - from typing import TypeVar, ParamSpec - import logging - - T = TypeVar('T') - P = ParamSpec('P') - - def add_logging(f: Callable[P, T]) -> Callable[P, T]: - '''A type-safe decorator to add logging to a function.''' - def inner(*args: P.args, **kwargs: P.kwargs) -> T: - logging.info(f'{f.__name__} was called') - return f(*args, **kwargs) - return inner - - @add_logging - def add_two(x: float, y: float) -> float: - '''Add two numbers together.''' - return x + y - - Without ``ParamSpec``, the simplest way to annotate this previously was to - use a :class:`TypeVar` with bound ``Callable[..., Any]``. However this - causes two problems: - - 1. The type checker can't type check the ``inner`` function because - ``*args`` and ``**kwargs`` have to be typed :data:`Any`. - 2. :func:`~cast` may be required in the body of the ``add_logging`` - decorator when returning the ``inner`` function, or the static type - checker must be told to ignore the ``return inner``. - - .. attribute:: args - .. attribute:: kwargs - - Since ``ParamSpec`` captures both positional and keyword parameters, - ``P.args`` and ``P.kwargs`` can be used to split a ``ParamSpec`` into its - components. ``P.args`` represents the tuple of positional parameters in a - given call and should only be used to annotate ``*args``. ``P.kwargs`` - represents the mapping of keyword parameters to their values in a given call, - and should be only be used to annotate ``**kwargs``. Both - attributes require the annotated parameter to be in scope. At runtime, - ``P.args`` and ``P.kwargs`` are instances respectively of - :class:`ParamSpecArgs` and :class:`ParamSpecKwargs`. - - .. attribute:: __name__ - - The name of the parameter specification. - - Parameter specification variables created with ``covariant=True`` or - ``contravariant=True`` can be used to declare covariant or contravariant - generic types. The ``bound`` argument is also accepted, similar to - :class:`TypeVar`. However the actual semantics of these keywords are yet to - be decided. - - .. versionadded:: 3.10 - - .. note:: - Only parameter specification variables defined in global scope can - be pickled. - - .. seealso:: - * :pep:`612` -- Parameter Specification Variables (the PEP which introduced - ``ParamSpec`` and ``Concatenate``) - * :data:`Concatenate` - * :ref:`annotating-callables` - -.. data:: ParamSpecArgs -.. data:: ParamSpecKwargs - - Arguments and keyword arguments attributes of a :class:`ParamSpec`. The - ``P.args`` attribute of a ``ParamSpec`` is an instance of ``ParamSpecArgs``, - and ``P.kwargs`` is an instance of ``ParamSpecKwargs``. They are intended - for runtime introspection and have no special meaning to static type checkers. - - Calling :func:`get_origin` on either of these objects will return the - original ``ParamSpec``: - - .. doctest:: - - >>> from typing import ParamSpec - >>> P = ParamSpec("P") - >>> get_origin(P.args) is P - True - >>> get_origin(P.kwargs) is P - True - - .. versionadded:: 3.10 - - -Other special directives -"""""""""""""""""""""""" - -These functions and classes should not be used directly as annotations. -Their intended purpose is to be building blocks for creating and declaring -types. - -.. class:: NamedTuple - - Typed version of :func:`collections.namedtuple`. - - Usage:: - - class Employee(NamedTuple): - name: str - id: int - - This is equivalent to:: - - Employee = collections.namedtuple('Employee', ['name', 'id']) - - To give a field a default value, you can assign to it in the class body:: - - class Employee(NamedTuple): - name: str - id: int = 3 - - employee = Employee('Guido') - assert employee.id == 3 - - Fields with a default value must come after any fields without a default. - - The resulting class has an extra attribute ``__annotations__`` giving a - dict that maps the field names to the field types. (The field names are in - the ``_fields`` attribute and the default values are in the - ``_field_defaults`` attribute, both of which are part of the :func:`~collections.namedtuple` - API.) - - ``NamedTuple`` subclasses can also have docstrings and methods:: - - class Employee(NamedTuple): - """Represents an employee.""" - name: str - id: int = 3 - - def __repr__(self) -> str: - return f'' - - ``NamedTuple`` subclasses can be generic:: - - class Group(NamedTuple, Generic[T]): - key: T - group: list[T] - - Backward-compatible usage:: - - Employee = NamedTuple('Employee', [('name', str), ('id', int)]) - - .. versionchanged:: 3.6 - Added support for :pep:`526` variable annotation syntax. - - .. versionchanged:: 3.6.1 - Added support for default values, methods, and docstrings. - - .. versionchanged:: 3.8 - The ``_field_types`` and ``__annotations__`` attributes are - now regular dictionaries instead of instances of ``OrderedDict``. - - .. versionchanged:: 3.9 - Removed the ``_field_types`` attribute in favor of the more - standard ``__annotations__`` attribute which has the same information. - - .. versionchanged:: 3.11 - Added support for generic namedtuples. - -.. class:: NewType(name, tp) - - Helper class to create low-overhead :ref:`distinct types `. - - A ``NewType`` is considered a distinct type by a typechecker. At runtime, - however, calling a ``NewType`` returns its argument unchanged. - - Usage:: - - UserId = NewType('UserId', int) # Declare the NewType "UserId" - first_user = UserId(1) # "UserId" returns the argument unchanged at runtime - - .. attribute:: __module__ - - The module in which the new type is defined. - - .. attribute:: __name__ - - The name of the new type. - - .. attribute:: __supertype__ - - The type that the new type is based on. - - .. versionadded:: 3.5.2 - - .. versionchanged:: 3.10 - ``NewType`` is now a class rather than a function. - -.. class:: Protocol(Generic) - - Base class for protocol classes. - - Protocol classes are defined like this:: - - class Proto(Protocol): - def meth(self) -> int: - ... - - Such classes are primarily used with static type checkers that recognize - structural subtyping (static duck-typing), for example:: - - class C: - def meth(self) -> int: - return 0 - - def func(x: Proto) -> int: - return x.meth() - - func(C()) # Passes static type check - - See :pep:`544` for more details. Protocol classes decorated with - :func:`runtime_checkable` (described later) act as simple-minded runtime - protocols that check only the presence of given attributes, ignoring their - type signatures. - - Protocol classes can be generic, for example:: - - T = TypeVar("T") - - class GenProto(Protocol[T]): - def meth(self) -> T: - ... - - .. versionadded:: 3.8 - -.. decorator:: runtime_checkable - - Mark a protocol class as a runtime protocol. - - Such a protocol can be used with :func:`isinstance` and :func:`issubclass`. - This raises :exc:`TypeError` when applied to a non-protocol class. This - allows a simple-minded structural check, very similar to "one trick ponies" - in :mod:`collections.abc` such as :class:`~collections.abc.Iterable`. For example:: - - @runtime_checkable - class Closable(Protocol): - def close(self): ... - - assert isinstance(open('/some/file'), Closable) - - @runtime_checkable - class Named(Protocol): - name: str - - import threading - assert isinstance(threading.Thread(name='Bob'), Named) - - .. note:: - - :func:`!runtime_checkable` will check only the presence of the required - methods or attributes, not their type signatures or types. - For example, :class:`ssl.SSLObject` - is a class, therefore it passes an :func:`issubclass` - check against :ref:`Callable `. However, the - ``ssl.SSLObject.__init__`` method exists only to raise a - :exc:`TypeError` with a more informative message, therefore making - it impossible to call (instantiate) :class:`ssl.SSLObject`. - - .. note:: - - An :func:`isinstance` check against a runtime-checkable protocol can be - surprisingly slow compared to an ``isinstance()`` check against - a non-protocol class. Consider using alternative idioms such as - :func:`hasattr` calls for structural checks in performance-sensitive - code. - - .. versionadded:: 3.8 - - -.. class:: TypedDict(dict) - - Special construct to add type hints to a dictionary. - At runtime it is a plain :class:`dict`. - - ``TypedDict`` declares a dictionary type that expects all of its - instances to have a certain set of keys, where each key is - associated with a value of a consistent type. This expectation - is not checked at runtime but is only enforced by type checkers. - Usage:: - - class Point2D(TypedDict): - x: int - y: int - label: str - - a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK - b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check - - assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') - - To allow using this feature with older versions of Python that do not - support :pep:`526`, ``TypedDict`` supports two additional equivalent - syntactic forms: - - * Using a literal :class:`dict` as the second argument:: - - Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) - - * Using keyword arguments:: - - Point2D = TypedDict('Point2D', x=int, y=int, label=str) - - .. deprecated-removed:: 3.11 3.13 - The keyword-argument syntax is deprecated in 3.11 and will be removed - in 3.13. It may also be unsupported by static type checkers. - - The functional syntax should also be used when any of the keys are not valid - :ref:`identifiers `, for example because they are keywords or contain hyphens. - Example:: - - # raises SyntaxError - class Point2D(TypedDict): - in: int # 'in' is a keyword - x-y: int # name with hyphens - - # OK, functional syntax - Point2D = TypedDict('Point2D', {'in': int, 'x-y': int}) - - By default, all keys must be present in a ``TypedDict``. It is possible to - mark individual keys as non-required using :data:`NotRequired`:: - - class Point2D(TypedDict): - x: int - y: int - label: NotRequired[str] - - # Alternative syntax - Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': NotRequired[str]}) - - This means that a ``Point2D`` ``TypedDict`` can have the ``label`` - key omitted. - - It is also possible to mark all keys as non-required by default - by specifying a totality of ``False``:: - - class Point2D(TypedDict, total=False): - x: int - y: int - - # Alternative syntax - Point2D = TypedDict('Point2D', {'x': int, 'y': int}, total=False) - - This means that a ``Point2D`` ``TypedDict`` can have any of the keys - omitted. A type checker is only expected to support a literal ``False`` or - ``True`` as the value of the ``total`` argument. ``True`` is the default, - and makes all items defined in the class body required. - - Individual keys of a ``total=False`` ``TypedDict`` can be marked as - required using :data:`Required`:: - - class Point2D(TypedDict, total=False): - x: Required[int] - y: Required[int] - label: str - - # Alternative syntax - Point2D = TypedDict('Point2D', { - 'x': Required[int], - 'y': Required[int], - 'label': str - }, total=False) - - It is possible for a ``TypedDict`` type to inherit from one or more other ``TypedDict`` types - using the class-based syntax. - Usage:: - - class Point3D(Point2D): - z: int - - ``Point3D`` has three items: ``x``, ``y`` and ``z``. It is equivalent to this - definition:: - - class Point3D(TypedDict): - x: int - y: int - z: int - - A ``TypedDict`` cannot inherit from a non-\ ``TypedDict`` class, - except for :class:`Generic`. For example:: - - class X(TypedDict): - x: int - - class Y(TypedDict): - y: int - - class Z(object): pass # A non-TypedDict class - - class XY(X, Y): pass # OK - - class XZ(X, Z): pass # raises TypeError - - A ``TypedDict`` can be generic: - - .. testcode:: - - T = TypeVar("T") - - class Group(TypedDict, Generic[T]): - key: T - group: list[T] - - A ``TypedDict`` can be introspected via annotations dicts - (see :ref:`annotations-howto` for more information on annotations best practices), - :attr:`__total__`, :attr:`__required_keys__`, and :attr:`__optional_keys__`. - - .. attribute:: __total__ - - ``Point2D.__total__`` gives the value of the ``total`` argument. - Example: - - .. doctest:: - - >>> from typing import TypedDict - >>> class Point2D(TypedDict): pass - >>> Point2D.__total__ - True - >>> class Point2D(TypedDict, total=False): pass - >>> Point2D.__total__ - False - >>> class Point3D(Point2D): pass - >>> Point3D.__total__ - True - - This attribute reflects *only* the value of the ``total`` argument - to the current ``TypedDict`` class, not whether the class is semantically - total. For example, a ``TypedDict`` with ``__total__`` set to True may - have keys marked with :data:`NotRequired`, or it may inherit from another - ``TypedDict`` with ``total=False``. Therefore, it is generally better to use - :attr:`__required_keys__` and :attr:`__optional_keys__` for introspection. - - .. attribute:: __required_keys__ - - .. versionadded:: 3.9 - - .. attribute:: __optional_keys__ - - ``Point2D.__required_keys__`` and ``Point2D.__optional_keys__`` return - :class:`frozenset` objects containing required and non-required keys, respectively. - - Keys marked with :data:`Required` will always appear in ``__required_keys__`` - and keys marked with :data:`NotRequired` will always appear in ``__optional_keys__``. - - For backwards compatibility with Python 3.10 and below, - it is also possible to use inheritance to declare both required and - non-required keys in the same ``TypedDict`` . This is done by declaring a - ``TypedDict`` with one value for the ``total`` argument and then - inheriting from it in another ``TypedDict`` with a different value for - ``total``: - - .. doctest:: - - >>> class Point2D(TypedDict, total=False): - ... x: int - ... y: int - ... - >>> class Point3D(Point2D): - ... z: int - ... - >>> Point3D.__required_keys__ == frozenset({'z'}) - True - >>> Point3D.__optional_keys__ == frozenset({'x', 'y'}) - True - - .. versionadded:: 3.9 - - .. note:: - - If ``from __future__ import annotations`` is used or if annotations - are given as strings, annotations are not evaluated when the - ``TypedDict`` is defined. Therefore, the runtime introspection that - ``__required_keys__`` and ``__optional_keys__`` rely on may not work - properly, and the values of the attributes may be incorrect. - - See :pep:`589` for more examples and detailed rules of using ``TypedDict``. - - .. versionadded:: 3.8 - - .. versionchanged:: 3.11 - Added support for marking individual keys as :data:`Required` or :data:`NotRequired`. - See :pep:`655`. - - .. versionchanged:: 3.11 - Added support for generic ``TypedDict``\ s. - -Protocols ---------- - -The following protocols are provided by the typing module. All are decorated -with :func:`@runtime_checkable `. - -.. class:: SupportsAbs - - An ABC with one abstract method ``__abs__`` that is covariant - in its return type. - -.. class:: SupportsBytes - - An ABC with one abstract method ``__bytes__``. - -.. class:: SupportsComplex - - An ABC with one abstract method ``__complex__``. - -.. class:: SupportsFloat - - An ABC with one abstract method ``__float__``. - -.. class:: SupportsIndex - - An ABC with one abstract method ``__index__``. - - .. versionadded:: 3.8 - -.. class:: SupportsInt - - An ABC with one abstract method ``__int__``. - -.. class:: SupportsRound - - An ABC with one abstract method ``__round__`` - that is covariant in its return type. - -ABCs for working with IO ------------------------- - -.. class:: IO - TextIO - BinaryIO - - Generic type ``IO[AnyStr]`` and its subclasses ``TextIO(IO[str])`` - and ``BinaryIO(IO[bytes])`` - represent the types of I/O streams such as returned by - :func:`open`. - -Functions and decorators ------------------------- - -.. function:: cast(typ, val) - - Cast a value to a type. - - This returns the value unchanged. To the type checker this - signals that the return value has the designated type, but at - runtime we intentionally don't check anything (we want this - to be as fast as possible). - -.. function:: assert_type(val, typ, /) - - Ask a static type checker to confirm that *val* has an inferred type of *typ*. - - At runtime this does nothing: it returns the first argument unchanged with no - checks or side effects, no matter the actual type of the argument. - - When a static type checker encounters a call to ``assert_type()``, it - emits an error if the value is not of the specified type:: - - def greet(name: str) -> None: - assert_type(name, str) # OK, inferred type of `name` is `str` - assert_type(name, int) # type checker error - - This function is useful for ensuring the type checker's understanding of a - script is in line with the developer's intentions:: - - def complex_function(arg: object): - # Do some complex type-narrowing logic, - # after which we hope the inferred type will be `int` - ... - # Test whether the type checker correctly understands our function - assert_type(arg, int) - - .. versionadded:: 3.11 - -.. function:: assert_never(arg, /) - - Ask a static type checker to confirm that a line of code is unreachable. - - Example:: - - def int_or_str(arg: int | str) -> None: - match arg: - case int(): - print("It's an int") - case str(): - print("It's a str") - case _ as unreachable: - assert_never(unreachable) - - Here, the annotations allow the type checker to infer that the - last case can never execute, because ``arg`` is either - an :class:`int` or a :class:`str`, and both options are covered by - earlier cases. - - If a type checker finds that a call to ``assert_never()`` is - reachable, it will emit an error. For example, if the type annotation - for ``arg`` was instead ``int | str | float``, the type checker would - emit an error pointing out that ``unreachable`` is of type :class:`float`. - For a call to ``assert_never`` to pass type checking, the inferred type of - the argument passed in must be the bottom type, :data:`Never`, and nothing - else. - - At runtime, this throws an exception when called. - - .. seealso:: - `Unreachable Code and Exhaustiveness Checking - `__ has more - information about exhaustiveness checking with static typing. - - .. versionadded:: 3.11 - -.. function:: reveal_type(obj, /) - - Reveal the inferred static type of an expression. - - When a static type checker encounters a call to this function, - it emits a diagnostic with the type of the argument. For example:: - - x: int = 1 - reveal_type(x) # Revealed type is "builtins.int" - - This can be useful when you want to debug how your type checker - handles a particular piece of code. - - The function returns its argument unchanged, which allows using - it within an expression:: - - x = reveal_type(1) # Revealed type is "builtins.int" - - Most type checkers support ``reveal_type()`` anywhere, even if the - name is not imported from ``typing``. Importing the name from - ``typing`` allows your code to run without runtime errors and - communicates intent more clearly. - - At runtime, this function prints the runtime type of its argument to stderr - and returns it unchanged:: - - x = reveal_type(1) # prints "Runtime type is int" - print(x) # prints "1" - - .. versionadded:: 3.11 - -.. decorator:: dataclass_transform(*, eq_default=True, order_default=False, \ - kw_only_default=False, \ - field_specifiers=(), **kwargs) - - Decorator to mark an object as providing - :func:`dataclass `-like behavior. - - ``dataclass_transform`` may be used to - decorate a class, metaclass, or a function that is itself a decorator. - The presence of ``@dataclass_transform()`` tells a static type checker that the - decorated object performs runtime "magic" that - transforms a class in a similar way to - :func:`@dataclasses.dataclass `. - - Example usage with a decorator function: - - .. testcode:: - - T = TypeVar("T") - - @dataclass_transform() - def create_model(cls: type[T]) -> type[T]: - ... - return cls - - @create_model - class CustomerModel: - id: int - name: str - - On a base class:: - - @dataclass_transform() - class ModelBase: ... - - class CustomerModel(ModelBase): - id: int - name: str - - On a metaclass:: - - @dataclass_transform() - class ModelMeta(type): ... - - class ModelBase(metaclass=ModelMeta): ... - - class CustomerModel(ModelBase): - id: int - name: str - - The ``CustomerModel`` classes defined above will - be treated by type checkers similarly to classes created with - :func:`@dataclasses.dataclass `. - For example, type checkers will assume these classes have - ``__init__`` methods that accept ``id`` and ``name``. - - The decorated class, metaclass, or function may accept the following bool - arguments which type checkers will assume have the same effect as they - would have on the - :func:`@dataclasses.dataclass` decorator: ``init``, - ``eq``, ``order``, ``unsafe_hash``, ``frozen``, ``match_args``, - ``kw_only``, and ``slots``. It must be possible for the value of these - arguments (``True`` or ``False``) to be statically evaluated. - - The arguments to the ``dataclass_transform`` decorator can be used to - customize the default behaviors of the decorated class, metaclass, or - function: - - :param bool eq_default: - Indicates whether the ``eq`` parameter is assumed to be - ``True`` or ``False`` if it is omitted by the caller. - Defaults to ``True``. - - :param bool order_default: - Indicates whether the ``order`` parameter is - assumed to be ``True`` or ``False`` if it is omitted by the caller. - Defaults to ``False``. - - :param bool kw_only_default: - Indicates whether the ``kw_only`` parameter is - assumed to be ``True`` or ``False`` if it is omitted by the caller. - Defaults to ``False``. - - :param field_specifiers: - Specifies a static list of supported classes - or functions that describe fields, similar to :func:`dataclasses.field`. - Defaults to ``()``. - :type field_specifiers: tuple[Callable[..., Any], ...] - - :param Any \**kwargs: - Arbitrary other keyword arguments are accepted in order to allow for - possible future extensions. - - Type checkers recognize the following optional parameters on field - specifiers: - - .. list-table:: **Recognised parameters for field specifiers** - :header-rows: 1 - :widths: 20 80 - - * - Parameter name - - Description - * - ``init`` - - Indicates whether the field should be included in the - synthesized ``__init__`` method. If unspecified, ``init`` defaults to - ``True``. - * - ``default`` - - Provides the default value for the field. - * - ``default_factory`` - - Provides a runtime callback that returns the - default value for the field. If neither ``default`` nor - ``default_factory`` are specified, the field is assumed to have no - default value and must be provided a value when the class is - instantiated. - * - ``factory`` - - An alias for the ``default_factory`` parameter on field specifiers. - * - ``kw_only`` - - Indicates whether the field should be marked as - keyword-only. If ``True``, the field will be keyword-only. If - ``False``, it will not be keyword-only. If unspecified, the value of - the ``kw_only`` parameter on the object decorated with - ``dataclass_transform`` will be used, or if that is unspecified, the - value of ``kw_only_default`` on ``dataclass_transform`` will be used. - * - ``alias`` - - Provides an alternative name for the field. This alternative - name is used in the synthesized ``__init__`` method. - - At runtime, this decorator records its arguments in the - ``__dataclass_transform__`` attribute on the decorated object. - It has no other runtime effect. - - See :pep:`681` for more details. - - .. versionadded:: 3.11 - -.. _overload: - -.. decorator:: overload - - Decorator for creating overloaded functions and methods. - - The ``@overload`` decorator allows describing functions and methods - that support multiple different combinations of argument types. A series - of ``@overload``-decorated definitions must be followed by exactly one - non-``@overload``-decorated definition (for the same function/method). - - ``@overload``-decorated definitions are for the benefit of the - type checker only, since they will be overwritten by the - non-``@overload``-decorated definition. The non-``@overload``-decorated - definition, meanwhile, will be used at - runtime but should be ignored by a type checker. At runtime, calling - an ``@overload``-decorated function directly will raise - :exc:`NotImplementedError`. - - An example of overload that gives a more - precise type than can be expressed using a union or a type variable: - - .. testcode:: - - @overload - def process(response: None) -> None: - ... - @overload - def process(response: int) -> tuple[int, str]: - ... - @overload - def process(response: bytes) -> str: - ... - def process(response): - ... # actual implementation goes here - - See :pep:`484` for more details and comparison with other typing semantics. - - .. versionchanged:: 3.11 - Overloaded functions can now be introspected at runtime using - :func:`get_overloads`. - - -.. function:: get_overloads(func) - - Return a sequence of :func:`@overload `-decorated definitions for - *func*. - - *func* is the function object for the implementation of the - overloaded function. For example, given the definition of ``process`` in - the documentation for :func:`@overload `, - ``get_overloads(process)`` will return a sequence of three function objects - for the three defined overloads. If called on a function with no overloads, - ``get_overloads()`` returns an empty sequence. - - ``get_overloads()`` can be used for introspecting an overloaded function at - runtime. - - .. versionadded:: 3.11 - - -.. function:: clear_overloads() - - Clear all registered overloads in the internal registry. - - This can be used to reclaim the memory used by the registry. - - .. versionadded:: 3.11 - - -.. decorator:: final - - Decorator to indicate final methods and final classes. - - Decorating a method with ``@final`` indicates to a type checker that the - method cannot be overridden in a subclass. Decorating a class with ``@final`` - indicates that it cannot be subclassed. - - For example:: - - class Base: - @final - def done(self) -> None: - ... - class Sub(Base): - def done(self) -> None: # Error reported by type checker - ... - - @final - class Leaf: - ... - class Other(Leaf): # Error reported by type checker - ... - - There is no runtime checking of these properties. See :pep:`591` for - more details. - - .. versionadded:: 3.8 - - .. versionchanged:: 3.11 - The decorator will now attempt to set a ``__final__`` attribute to ``True`` - on the decorated object. Thus, a check like - ``if getattr(obj, "__final__", False)`` can be used at runtime - to determine whether an object ``obj`` has been marked as final. - If the decorated object does not support setting attributes, - the decorator returns the object unchanged without raising an exception. - - -.. decorator:: no_type_check - - Decorator to indicate that annotations are not type hints. - - This works as a class or function :term:`decorator`. With a class, it - applies recursively to all methods and classes defined in that class - (but not to methods defined in its superclasses or subclasses). Type - checkers will ignore all annotations in a function or class with this - decorator. - - ``@no_type_check`` mutates the decorated object in place. - -.. decorator:: no_type_check_decorator - - Decorator to give another decorator the :func:`no_type_check` effect. - - This wraps the decorator with something that wraps the decorated - function in :func:`no_type_check`. - -.. decorator:: type_check_only - - Decorator to mark a class or function as unavailable at runtime. - - This decorator is itself not available at runtime. It is mainly - intended to mark classes that are defined in type stub files if - an implementation returns an instance of a private class:: - - @type_check_only - class Response: # private or not available at runtime - code: int - def get_header(self, name: str) -> str: ... - - def fetch_response() -> Response: ... - - Note that returning instances of private classes is not recommended. - It is usually preferable to make such classes public. - -Introspection helpers ---------------------- - -.. function:: get_type_hints(obj, globalns=None, localns=None, include_extras=False) - - Return a dictionary containing type hints for a function, method, module - or class object. - - This is often the same as ``obj.__annotations__``. In addition, - forward references encoded as string literals are handled by evaluating - them in ``globals`` and ``locals`` namespaces. For a class ``C``, return - a dictionary constructed by merging all the ``__annotations__`` along - ``C.__mro__`` in reverse order. - - The function recursively replaces all ``Annotated[T, ...]`` with ``T``, - unless ``include_extras`` is set to ``True`` (see :class:`Annotated` for - more information). For example: - - .. testcode:: - - class Student(NamedTuple): - name: Annotated[str, 'some marker'] - - assert get_type_hints(Student) == {'name': str} - assert get_type_hints(Student, include_extras=False) == {'name': str} - assert get_type_hints(Student, include_extras=True) == { - 'name': Annotated[str, 'some marker'] - } - - .. note:: - - :func:`get_type_hints` does not work with imported - :ref:`type aliases ` that include forward references. - Enabling postponed evaluation of annotations (:pep:`563`) may remove - the need for most forward references. - - .. versionchanged:: 3.9 - Added ``include_extras`` parameter as part of :pep:`593`. - See the documentation on :data:`Annotated` for more information. - - .. versionchanged:: 3.11 - Previously, ``Optional[t]`` was added for function and method annotations - if a default value equal to ``None`` was set. - Now the annotation is returned unchanged. - -.. function:: get_origin(tp) - - Get the unsubscripted version of a type: for a typing object of the form - ``X[Y, Z, ...]`` return ``X``. - - If ``X`` is a typing-module alias for a builtin or - :mod:`collections` class, it will be normalized to the original class. - If ``X`` is an instance of :class:`ParamSpecArgs` or :class:`ParamSpecKwargs`, - return the underlying :class:`ParamSpec`. - Return ``None`` for unsupported objects. - - Examples: - - .. testcode:: - - assert get_origin(str) is None - assert get_origin(Dict[str, int]) is dict - assert get_origin(Union[int, str]) is Union - P = ParamSpec('P') - assert get_origin(P.args) is P - assert get_origin(P.kwargs) is P - - .. versionadded:: 3.8 - -.. function:: get_args(tp) - - Get type arguments with all substitutions performed: for a typing object - of the form ``X[Y, Z, ...]`` return ``(Y, Z, ...)``. - - If ``X`` is a union or :class:`Literal` contained in another - generic type, the order of ``(Y, Z, ...)`` may be different from the order - of the original arguments ``[Y, Z, ...]`` due to type caching. - Return ``()`` for unsupported objects. - - Examples: - - .. testcode:: - - assert get_args(int) == () - assert get_args(Dict[int, str]) == (int, str) - assert get_args(Union[int, str]) == (int, str) - - .. versionadded:: 3.8 - -.. function:: is_typeddict(tp) - - Check if a type is a :class:`TypedDict`. - - For example: - - .. testcode:: - - class Film(TypedDict): - title: str - year: int - - assert is_typeddict(Film) - assert not is_typeddict(list | str) - - # TypedDict is a factory for creating typed dicts, - # not a typed dict itself - assert not is_typeddict(TypedDict) - - .. versionadded:: 3.10 - -.. class:: ForwardRef - - Class used for internal typing representation of string forward references. - - For example, ``List["SomeClass"]`` is implicitly transformed into - ``List[ForwardRef("SomeClass")]``. ``ForwardRef`` should not be instantiated by - a user, but may be used by introspection tools. - - .. note:: - :pep:`585` generic types such as ``list["SomeClass"]`` will not be - implicitly transformed into ``list[ForwardRef("SomeClass")]`` and thus - will not automatically resolve to ``list[SomeClass]``. - - .. versionadded:: 3.7.4 - -Constant --------- - -.. data:: TYPE_CHECKING - - A special constant that is assumed to be ``True`` by 3rd party static - type checkers. It is ``False`` at runtime. - - Usage:: - - if TYPE_CHECKING: - import expensive_mod - - def fun(arg: 'expensive_mod.SomeType') -> None: - local_var: expensive_mod.AnotherType = other_fun() - - The first type annotation must be enclosed in quotes, making it a - "forward reference", to hide the ``expensive_mod`` reference from the - interpreter runtime. Type annotations for local variables are not - evaluated, so the second annotation does not need to be enclosed in quotes. - - .. note:: - - If ``from __future__ import annotations`` is used, - annotations are not evaluated at function definition time. - Instead, they are stored as strings in ``__annotations__``. - This makes it unnecessary to use quotes around the annotation - (see :pep:`563`). - - .. versionadded:: 3.5.2 - -.. _generic-concrete-collections: -.. _deprecated-typing-aliases: - -Deprecated aliases ------------------- - -This module defines several deprecated aliases to pre-existing -standard library classes. These were originally included in the typing -module in order to support parameterizing these generic classes using ``[]``. -However, the aliases became redundant in Python 3.9 when the -corresponding pre-existing classes were enhanced to support ``[]`` (see -:pep:`585`). - -The redundant types are deprecated as of Python 3.9. However, while the aliases -may be removed at some point, removal of these aliases is not currently -planned. As such, no deprecation warnings are currently issued by the -interpreter for these aliases. - -If at some point it is decided to remove these deprecated aliases, a -deprecation warning will be issued by the interpreter for at least two releases -prior to removal. The aliases are guaranteed to remain in the typing module -without deprecation warnings until at least Python 3.14. - -Type checkers are encouraged to flag uses of the deprecated types if the -program they are checking targets a minimum Python version of 3.9 or newer. - -.. _corresponding-to-built-in-types: - -Aliases to built-in types -""""""""""""""""""""""""" - -.. class:: Dict(dict, MutableMapping[KT, VT]) - - Deprecated alias to :class:`dict`. - - Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`Mapping` - rather than to use :class:`dict` or :class:`!typing.Dict`. - - This type can be used as follows:: - - def count_words(text: str) -> Dict[str, int]: - ... - - .. deprecated:: 3.9 - :class:`builtins.dict ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: List(list, MutableSequence[T]) - - Deprecated alias to :class:`list`. - - Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`Sequence` or - :class:`Iterable` rather than to use :class:`list` or :class:`!typing.List`. - - This type may be used as follows:: - - T = TypeVar('T', int, float) - - def vec2(x: T, y: T) -> List[T]: - return [x, y] - - def keep_positives(vector: Sequence[T]) -> List[T]: - return [item for item in vector if item > 0] - - .. deprecated:: 3.9 - :class:`builtins.list ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Set(set, MutableSet[T]) - - Deprecated alias to :class:`builtins.set `. - - Note that to annotate arguments, it is preferred - to use an abstract collection type such as :class:`AbstractSet` - rather than to use :class:`set` or :class:`!typing.Set`. - - .. deprecated:: 3.9 - :class:`builtins.set ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: FrozenSet(frozenset, AbstractSet[T_co]) - - Deprecated alias to :class:`builtins.frozenset `. - - .. deprecated:: 3.9 - :class:`builtins.frozenset ` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. data:: Tuple - - Deprecated alias for :class:`tuple`. - - :class:`tuple` and ``Tuple`` are special-cased in the type system; see - :ref:`annotating-tuples` for more details. - - .. deprecated:: 3.9 - :class:`builtins.tuple ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Type(Generic[CT_co]) - - Deprecated alias to :class:`type`. - - See :ref:`type-of-class-objects` for details on using :class:`type` or - ``typing.Type`` in type annotations. - - .. versionadded:: 3.5.2 - - .. deprecated:: 3.9 - :class:`builtins.type ` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. _corresponding-to-types-in-collections: - -Aliases to types in :mod:`collections` -"""""""""""""""""""""""""""""""""""""" - -.. class:: DefaultDict(collections.defaultdict, MutableMapping[KT, VT]) - - Deprecated alias to :class:`collections.defaultdict`. - - .. versionadded:: 3.5.2 - - .. deprecated:: 3.9 - :class:`collections.defaultdict` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: OrderedDict(collections.OrderedDict, MutableMapping[KT, VT]) - - Deprecated alias to :class:`collections.OrderedDict`. - - .. versionadded:: 3.7.2 - - .. deprecated:: 3.9 - :class:`collections.OrderedDict` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: ChainMap(collections.ChainMap, MutableMapping[KT, VT]) - - Deprecated alias to :class:`collections.ChainMap`. - - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 - - .. deprecated:: 3.9 - :class:`collections.ChainMap` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Counter(collections.Counter, Dict[T, int]) - - Deprecated alias to :class:`collections.Counter`. - - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 - - .. deprecated:: 3.9 - :class:`collections.Counter` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Deque(deque, MutableSequence[T]) - - Deprecated alias to :class:`collections.deque`. - - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.1 - - .. deprecated:: 3.9 - :class:`collections.deque` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. _other-concrete-types: - -Aliases to other concrete types -""""""""""""""""""""""""""""""" - -.. class:: Pattern - Match - - Deprecated aliases corresponding to the return types from - :func:`re.compile` and :func:`re.match`. - - These types (and the corresponding functions) are generic over - :data:`AnyStr`. ``Pattern`` can be specialised as ``Pattern[str]`` or - ``Pattern[bytes]``; ``Match`` can be specialised as ``Match[str]`` or - ``Match[bytes]``. - - .. deprecated-removed:: 3.8 3.13 - The ``typing.re`` namespace is deprecated and will be removed. - These types should be directly imported from ``typing`` instead. - - .. deprecated:: 3.9 - Classes ``Pattern`` and ``Match`` from :mod:`re` now support ``[]``. - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Text - - Deprecated alias for :class:`str`. - - ``Text`` is provided to supply a forward - compatible path for Python 2 code: in Python 2, ``Text`` is an alias for - ``unicode``. - - Use ``Text`` to indicate that a value must contain a unicode string in - a manner that is compatible with both Python 2 and Python 3:: - - def add_unicode_checkmark(text: Text) -> Text: - return text + u' \u2713' - - .. versionadded:: 3.5.2 - - .. deprecated:: 3.11 - Python 2 is no longer supported, and most type checkers also no longer - support type checking Python 2 code. Removal of the alias is not - currently planned, but users are encouraged to use - :class:`str` instead of ``Text``. - -.. _abstract-base-classes: -.. _corresponding-to-collections-in-collections-abc: - -Aliases to container ABCs in :mod:`collections.abc` -""""""""""""""""""""""""""""""""""""""""""""""""""" - -.. class:: AbstractSet(Collection[T_co]) - - Deprecated alias to :class:`collections.abc.Set`. - - .. deprecated:: 3.9 - :class:`collections.abc.Set` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: ByteString(Sequence[int]) - - This type represents the types :class:`bytes`, :class:`bytearray`, - and :class:`memoryview` of byte sequences. - - .. deprecated-removed:: 3.9 3.14 - Prefer ``typing_extensions.Buffer``, or a union like ``bytes | bytearray | memoryview``. - -.. class:: Collection(Sized, Iterable[T_co], Container[T_co]) - - Deprecated alias to :class:`collections.abc.Collection`. - - .. versionadded:: 3.6.0 - - .. deprecated:: 3.9 - :class:`collections.abc.Collection` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Container(Generic[T_co]) - - Deprecated alias to :class:`collections.abc.Container`. - - .. deprecated:: 3.9 - :class:`collections.abc.Container` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: ItemsView(MappingView, AbstractSet[tuple[KT_co, VT_co]]) - - Deprecated alias to :class:`collections.abc.ItemsView`. - - .. deprecated:: 3.9 - :class:`collections.abc.ItemsView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: KeysView(MappingView, AbstractSet[KT_co]) - - Deprecated alias to :class:`collections.abc.KeysView`. - - .. deprecated:: 3.9 - :class:`collections.abc.KeysView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Mapping(Collection[KT], Generic[KT, VT_co]) - - Deprecated alias to :class:`collections.abc.Mapping`. - - This type can be used as follows:: - - def get_position_in_index(word_list: Mapping[str, int], word: str) -> int: - return word_list[word] - - .. deprecated:: 3.9 - :class:`collections.abc.Mapping` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: MappingView(Sized) - - Deprecated alias to :class:`collections.abc.MappingView`. - - .. deprecated:: 3.9 - :class:`collections.abc.MappingView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: MutableMapping(Mapping[KT, VT]) - - Deprecated alias to :class:`collections.abc.MutableMapping`. - - .. deprecated:: 3.9 - :class:`collections.abc.MutableMapping` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: MutableSequence(Sequence[T]) - - Deprecated alias to :class:`collections.abc.MutableSequence`. - - .. deprecated:: 3.9 - :class:`collections.abc.MutableSequence` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: MutableSet(AbstractSet[T]) - - Deprecated alias to :class:`collections.abc.MutableSet`. - - .. deprecated:: 3.9 - :class:`collections.abc.MutableSet` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Sequence(Reversible[T_co], Collection[T_co]) - - Deprecated alias to :class:`collections.abc.Sequence`. - - .. deprecated:: 3.9 - :class:`collections.abc.Sequence` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: ValuesView(MappingView, Collection[_VT_co]) - - Deprecated alias to :class:`collections.abc.ValuesView`. - - .. deprecated:: 3.9 - :class:`collections.abc.ValuesView` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. _asynchronous-programming: - -Aliases to asynchronous ABCs in :mod:`collections.abc` -"""""""""""""""""""""""""""""""""""""""""""""""""""""" - -.. class:: Coroutine(Awaitable[ReturnType], Generic[YieldType, SendType, ReturnType]) - - Deprecated alias to :class:`collections.abc.Coroutine`. - - The variance and order of type variables - correspond to those of :class:`Generator`, for example:: - - from collections.abc import Coroutine - c: Coroutine[list[str], str, int] # Some coroutine defined elsewhere - x = c.send('hi') # Inferred type of 'x' is list[str] - async def bar() -> None: - y = await c # Inferred type of 'y' is int - - .. versionadded:: 3.5.3 - - .. deprecated:: 3.9 - :class:`collections.abc.Coroutine` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: AsyncGenerator(AsyncIterator[YieldType], Generic[YieldType, SendType]) - - Deprecated alias to :class:`collections.abc.AsyncGenerator`. - - An async generator can be annotated by the generic type - ``AsyncGenerator[YieldType, SendType]``. For example:: - - async def echo_round() -> AsyncGenerator[int, float]: - sent = yield 0 - while sent >= 0.0: - rounded = await round(sent) - sent = yield rounded - - Unlike normal generators, async generators cannot return a value, so there - is no ``ReturnType`` type parameter. As with :class:`Generator`, the - ``SendType`` behaves contravariantly. - - If your generator will only yield values, set the ``SendType`` to - ``None``:: - - async def infinite_stream(start: int) -> AsyncGenerator[int, None]: - while True: - yield start - start = await increment(start) - - Alternatively, annotate your generator as having a return type of - either ``AsyncIterable[YieldType]`` or ``AsyncIterator[YieldType]``:: - - async def infinite_stream(start: int) -> AsyncIterator[int]: - while True: - yield start - start = await increment(start) - - .. versionadded:: 3.6.1 - - .. deprecated:: 3.9 - :class:`collections.abc.AsyncGenerator` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: AsyncIterable(Generic[T_co]) - - Deprecated alias to :class:`collections.abc.AsyncIterable`. - - .. versionadded:: 3.5.2 - - .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: AsyncIterator(AsyncIterable[T_co]) - - Deprecated alias to :class:`collections.abc.AsyncIterator`. - - .. versionadded:: 3.5.2 - - .. deprecated:: 3.9 - :class:`collections.abc.AsyncIterator` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Awaitable(Generic[T_co]) - - Deprecated alias to :class:`collections.abc.Awaitable`. - - .. versionadded:: 3.5.2 - - .. deprecated:: 3.9 - :class:`collections.abc.Awaitable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. _corresponding-to-other-types-in-collections-abc: - -Aliases to other ABCs in :mod:`collections.abc` -""""""""""""""""""""""""""""""""""""""""""""""" - -.. class:: Iterable(Generic[T_co]) - - Deprecated alias to :class:`collections.abc.Iterable`. - - .. deprecated:: 3.9 - :class:`collections.abc.Iterable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Iterator(Iterable[T_co]) - - Deprecated alias to :class:`collections.abc.Iterator`. - - .. deprecated:: 3.9 - :class:`collections.abc.Iterator` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. data:: Callable - - Deprecated alias to :class:`collections.abc.Callable`. - - See :ref:`annotating-callables` for details on how to use - :class:`collections.abc.Callable` and ``typing.Callable`` in type annotations. - - .. deprecated:: 3.9 - :class:`collections.abc.Callable` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - - .. versionchanged:: 3.10 - ``Callable`` now supports :class:`ParamSpec` and :data:`Concatenate`. - See :pep:`612` for more details. - -.. class:: Generator(Iterator[YieldType], Generic[YieldType, SendType, ReturnType]) - - Deprecated alias to :class:`collections.abc.Generator`. - - A generator can be annotated by the generic type - ``Generator[YieldType, SendType, ReturnType]``. For example:: - - def echo_round() -> Generator[int, float, str]: - sent = yield 0 - while sent >= 0: - sent = yield round(sent) - return 'Done' - - Note that unlike many other generics in the typing module, the ``SendType`` - of :class:`Generator` behaves contravariantly, not covariantly or - invariantly. - - If your generator will only yield values, set the ``SendType`` and - ``ReturnType`` to ``None``:: - - def infinite_stream(start: int) -> Generator[int, None, None]: - while True: - yield start - start += 1 - - Alternatively, annotate your generator as having a return type of - either ``Iterable[YieldType]`` or ``Iterator[YieldType]``:: - - def infinite_stream(start: int) -> Iterator[int]: - while True: - yield start - start += 1 - - .. deprecated:: 3.9 - :class:`collections.abc.Generator` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Hashable - - Alias to :class:`collections.abc.Hashable`. - -.. class:: Reversible(Iterable[T_co]) - - Deprecated alias to :class:`collections.abc.Reversible`. - - .. deprecated:: 3.9 - :class:`collections.abc.Reversible` now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: Sized - - Alias to :class:`collections.abc.Sized`. - -.. _context-manager-types: - -Aliases to :mod:`contextlib` ABCs -""""""""""""""""""""""""""""""""" - -.. class:: ContextManager(Generic[T_co]) - - Deprecated alias to :class:`contextlib.AbstractContextManager`. - - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.0 - - .. deprecated:: 3.9 - :class:`contextlib.AbstractContextManager` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -.. class:: AsyncContextManager(Generic[T_co]) - - Deprecated alias to :class:`contextlib.AbstractAsyncContextManager`. - - .. versionadded:: 3.5.4 - .. versionadded:: 3.6.2 - - .. deprecated:: 3.9 - :class:`contextlib.AbstractAsyncContextManager` - now supports subscripting (``[]``). - See :pep:`585` and :ref:`types-genericalias`. - -Deprecation Timeline of Major Features -====================================== - -Certain features in ``typing`` are deprecated and may be removed in a future -version of Python. The following table summarizes major deprecations for your -convenience. This is subject to change, and not all deprecations are listed. - -.. list-table:: - :header-rows: 1 - - * - Feature - - Deprecated in - - Projected removal - - PEP/issue - * - ``typing.io`` and ``typing.re`` submodules - - 3.8 - - 3.13 - - :issue:`38291` - * - ``typing`` versions of standard collections - - 3.9 - - Undecided (see :ref:`deprecated-typing-aliases` for more information) - - :pep:`585` - * - :class:`typing.ByteString` - - 3.9 - - 3.14 - - :gh:`91896` - * - :data:`typing.Text` - - 3.11 - - Undecided - - :gh:`92332` diff --git a/Doc/library/unicodedata.rst b/Doc/library/unicodedata.rst deleted file mode 100644 index 6276f6382a06ea..00000000000000 --- a/Doc/library/unicodedata.rst +++ /dev/null @@ -1,180 +0,0 @@ -:mod:`unicodedata` --- Unicode Database -======================================= - -.. module:: unicodedata - :synopsis: Access the Unicode Database. - -.. moduleauthor:: Marc-André Lemburg -.. sectionauthor:: Marc-André Lemburg -.. sectionauthor:: Martin v. Löwis - -.. index:: - single: Unicode - single: character - pair: Unicode; database - --------------- - -This module provides access to the Unicode Character Database (UCD) which -defines character properties for all Unicode characters. The data contained in -this database is compiled from the `UCD version 14.0.0 -`_. - -The module uses the same names and symbols as defined by Unicode -Standard Annex #44, `"Unicode Character Database" -`_. It defines the -following functions: - - -.. function:: lookup(name) - - Look up character by name. If a character with the given name is found, return - the corresponding character. If not found, :exc:`KeyError` is raised. - - .. versionchanged:: 3.3 - Support for name aliases [#]_ and named sequences [#]_ has been added. - - -.. function:: name(chr[, default]) - - Returns the name assigned to the character *chr* as a string. If no - name is defined, *default* is returned, or, if not given, :exc:`ValueError` is - raised. - - -.. function:: decimal(chr[, default]) - - Returns the decimal value assigned to the character *chr* as integer. - If no such value is defined, *default* is returned, or, if not given, - :exc:`ValueError` is raised. - - -.. function:: digit(chr[, default]) - - Returns the digit value assigned to the character *chr* as integer. - If no such value is defined, *default* is returned, or, if not given, - :exc:`ValueError` is raised. - - -.. function:: numeric(chr[, default]) - - Returns the numeric value assigned to the character *chr* as float. - If no such value is defined, *default* is returned, or, if not given, - :exc:`ValueError` is raised. - - -.. function:: category(chr) - - Returns the general category assigned to the character *chr* as - string. - - -.. function:: bidirectional(chr) - - Returns the bidirectional class assigned to the character *chr* as - string. If no such value is defined, an empty string is returned. - - -.. function:: combining(chr) - - Returns the canonical combining class assigned to the character *chr* - as integer. Returns ``0`` if no combining class is defined. - - -.. function:: east_asian_width(chr) - - Returns the east asian width assigned to the character *chr* as - string. - - -.. function:: mirrored(chr) - - Returns the mirrored property assigned to the character *chr* as - integer. Returns ``1`` if the character has been identified as a "mirrored" - character in bidirectional text, ``0`` otherwise. - - -.. function:: decomposition(chr) - - Returns the character decomposition mapping assigned to the character - *chr* as string. An empty string is returned in case no such mapping is - defined. - - -.. function:: normalize(form, unistr) - - Return the normal form *form* for the Unicode string *unistr*. Valid values for - *form* are 'NFC', 'NFKC', 'NFD', and 'NFKD'. - - The Unicode standard defines various normalization forms of a Unicode string, - based on the definition of canonical equivalence and compatibility equivalence. - In Unicode, several characters can be expressed in various way. For example, the - character U+00C7 (LATIN CAPITAL LETTER C WITH CEDILLA) can also be expressed as - the sequence U+0043 (LATIN CAPITAL LETTER C) U+0327 (COMBINING CEDILLA). - - For each character, there are two normal forms: normal form C and normal form D. - Normal form D (NFD) is also known as canonical decomposition, and translates - each character into its decomposed form. Normal form C (NFC) first applies a - canonical decomposition, then composes pre-combined characters again. - - In addition to these two forms, there are two additional normal forms based on - compatibility equivalence. In Unicode, certain characters are supported which - normally would be unified with other characters. For example, U+2160 (ROMAN - NUMERAL ONE) is really the same thing as U+0049 (LATIN CAPITAL LETTER I). - However, it is supported in Unicode for compatibility with existing character - sets (e.g. gb2312). - - The normal form KD (NFKD) will apply the compatibility decomposition, i.e. - replace all compatibility characters with their equivalents. The normal form KC - (NFKC) first applies the compatibility decomposition, followed by the canonical - composition. - - Even if two unicode strings are normalized and look the same to - a human reader, if one has combining characters and the other - doesn't, they may not compare equal. - -.. function:: is_normalized(form, unistr) - - Return whether the Unicode string *unistr* is in the normal form *form*. Valid - values for *form* are 'NFC', 'NFKC', 'NFD', and 'NFKD'. - - .. versionadded:: 3.8 - - -In addition, the module exposes the following constant: - -.. data:: unidata_version - - The version of the Unicode database used in this module. - - -.. data:: ucd_3_2_0 - - This is an object that has the same methods as the entire module, but uses the - Unicode database version 3.2 instead, for applications that require this - specific version of the Unicode database (such as IDNA). - -Examples: - - >>> import unicodedata - >>> unicodedata.lookup('LEFT CURLY BRACKET') - '{' - >>> unicodedata.name('/') - 'SOLIDUS' - >>> unicodedata.decimal('9') - 9 - >>> unicodedata.decimal('a') - Traceback (most recent call last): - File "", line 1, in - ValueError: not a decimal - >>> unicodedata.category('A') # 'L'etter, 'u'ppercase - 'Lu' - >>> unicodedata.bidirectional('\u0660') # 'A'rabic, 'N'umber - 'AN' - - -.. rubric:: Footnotes - -.. [#] https://www.unicode.org/Public/14.0.0/ucd/NameAliases.txt - -.. [#] https://www.unicode.org/Public/14.0.0/ucd/NamedSequences.txt diff --git a/Doc/library/unittest.mock-examples.rst b/Doc/library/unittest.mock-examples.rst deleted file mode 100644 index 34f343ebacdbb7..00000000000000 --- a/Doc/library/unittest.mock-examples.rst +++ /dev/null @@ -1,1346 +0,0 @@ -:mod:`unittest.mock` --- getting started -======================================== - -.. moduleauthor:: Michael Foord -.. currentmodule:: unittest.mock - -.. versionadded:: 3.3 - - -.. _getting-started: - - -.. testsetup:: - - import asyncio - import unittest - from unittest.mock import Mock, MagicMock, AsyncMock, patch, call, sentinel - - class SomeClass: - attribute = 'this is a doctest' - - @staticmethod - def static_method(): - pass - -Using Mock ----------- - -Mock Patching Methods -~~~~~~~~~~~~~~~~~~~~~ - -Common uses for :class:`Mock` objects include: - -* Patching methods -* Recording method calls on objects - -You might want to replace a method on an object to check that -it is called with the correct arguments by another part of the system: - - >>> real = SomeClass() - >>> real.method = MagicMock(name='method') - >>> real.method(3, 4, 5, key='value') - - -Once our mock has been used (``real.method`` in this example) it has methods -and attributes that allow you to make assertions about how it has been used. - -.. note:: - - In most of these examples the :class:`Mock` and :class:`MagicMock` classes - are interchangeable. As the ``MagicMock`` is the more capable class it makes - a sensible one to use by default. - -Once the mock has been called its :attr:`~Mock.called` attribute is set to -``True``. More importantly we can use the :meth:`~Mock.assert_called_with` or -:meth:`~Mock.assert_called_once_with` method to check that it was called with -the correct arguments. - -This example tests that calling ``ProductionClass().method`` results in a call to -the ``something`` method: - - >>> class ProductionClass: - ... def method(self): - ... self.something(1, 2, 3) - ... def something(self, a, b, c): - ... pass - ... - >>> real = ProductionClass() - >>> real.something = MagicMock() - >>> real.method() - >>> real.something.assert_called_once_with(1, 2, 3) - - - -Mock for Method Calls on an Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -In the last example we patched a method directly on an object to check that it -was called correctly. Another common use case is to pass an object into a -method (or some part of the system under test) and then check that it is used -in the correct way. - -The simple ``ProductionClass`` below has a ``closer`` method. If it is called with -an object then it calls ``close`` on it. - - >>> class ProductionClass: - ... def closer(self, something): - ... something.close() - ... - -So to test it we need to pass in an object with a ``close`` method and check -that it was called correctly. - - >>> real = ProductionClass() - >>> mock = Mock() - >>> real.closer(mock) - >>> mock.close.assert_called_with() - -We don't have to do any work to provide the 'close' method on our mock. -Accessing close creates it. So, if 'close' hasn't already been called then -accessing it in the test will create it, but :meth:`~Mock.assert_called_with` -will raise a failure exception. - - -Mocking Classes -~~~~~~~~~~~~~~~ - -A common use case is to mock out classes instantiated by your code under test. -When you patch a class, then that class is replaced with a mock. Instances -are created by *calling the class*. This means you access the "mock instance" -by looking at the return value of the mocked class. - -In the example below we have a function ``some_function`` that instantiates ``Foo`` -and calls a method on it. The call to :func:`patch` replaces the class ``Foo`` with a -mock. The ``Foo`` instance is the result of calling the mock, so it is configured -by modifying the mock :attr:`~Mock.return_value`. :: - - >>> def some_function(): - ... instance = module.Foo() - ... return instance.method() - ... - >>> with patch('module.Foo') as mock: - ... instance = mock.return_value - ... instance.method.return_value = 'the result' - ... result = some_function() - ... assert result == 'the result' - - -Naming your mocks -~~~~~~~~~~~~~~~~~ - -It can be useful to give your mocks a name. The name is shown in the repr of -the mock and can be helpful when the mock appears in test failure messages. The -name is also propagated to attributes or methods of the mock: - - >>> mock = MagicMock(name='foo') - >>> mock - - >>> mock.method - - - -Tracking all Calls -~~~~~~~~~~~~~~~~~~ - -Often you want to track more than a single call to a method. The -:attr:`~Mock.mock_calls` attribute records all calls -to child attributes of the mock - and also to their children. - - >>> mock = MagicMock() - >>> mock.method() - - >>> mock.attribute.method(10, x=53) - - >>> mock.mock_calls - [call.method(), call.attribute.method(10, x=53)] - -If you make an assertion about ``mock_calls`` and any unexpected methods -have been called, then the assertion will fail. This is useful because as well -as asserting that the calls you expected have been made, you are also checking -that they were made in the right order and with no additional calls: - -You use the :data:`call` object to construct lists for comparing with -``mock_calls``: - - >>> expected = [call.method(), call.attribute.method(10, x=53)] - >>> mock.mock_calls == expected - True - -However, parameters to calls that return mocks are not recorded, which means it is not -possible to track nested calls where the parameters used to create ancestors are important: - - >>> m = Mock() - >>> m.factory(important=True).deliver() - - >>> m.mock_calls[-1] == call.factory(important=False).deliver() - True - - -Setting Return Values and Attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Setting the return values on a mock object is trivially easy: - - >>> mock = Mock() - >>> mock.return_value = 3 - >>> mock() - 3 - -Of course you can do the same for methods on the mock: - - >>> mock = Mock() - >>> mock.method.return_value = 3 - >>> mock.method() - 3 - -The return value can also be set in the constructor: - - >>> mock = Mock(return_value=3) - >>> mock() - 3 - -If you need an attribute setting on your mock, just do it: - - >>> mock = Mock() - >>> mock.x = 3 - >>> mock.x - 3 - -Sometimes you want to mock up a more complex situation, like for example -``mock.connection.cursor().execute("SELECT 1")``. If we wanted this call to -return a list, then we have to configure the result of the nested call. - -We can use :data:`call` to construct the set of calls in a "chained call" like -this for easy assertion afterwards: - - >>> mock = Mock() - >>> cursor = mock.connection.cursor.return_value - >>> cursor.execute.return_value = ['foo'] - >>> mock.connection.cursor().execute("SELECT 1") - ['foo'] - >>> expected = call.connection.cursor().execute("SELECT 1").call_list() - >>> mock.mock_calls - [call.connection.cursor(), call.connection.cursor().execute('SELECT 1')] - >>> mock.mock_calls == expected - True - -It is the call to ``.call_list()`` that turns our call object into a list of -calls representing the chained calls. - - -Raising exceptions with mocks -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A useful attribute is :attr:`~Mock.side_effect`. If you set this to an -exception class or instance then the exception will be raised when the mock -is called. - - >>> mock = Mock(side_effect=Exception('Boom!')) - >>> mock() - Traceback (most recent call last): - ... - Exception: Boom! - - -Side effect functions and iterables -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -``side_effect`` can also be set to a function or an iterable. The use case for -``side_effect`` as an iterable is where your mock is going to be called several -times, and you want each call to return a different value. When you set -``side_effect`` to an iterable every call to the mock returns the next value -from the iterable: - - >>> mock = MagicMock(side_effect=[4, 5, 6]) - >>> mock() - 4 - >>> mock() - 5 - >>> mock() - 6 - - -For more advanced use cases, like dynamically varying the return values -depending on what the mock is called with, ``side_effect`` can be a function. -The function will be called with the same arguments as the mock. Whatever the -function returns is what the call returns: - - >>> vals = {(1, 2): 1, (2, 3): 2} - >>> def side_effect(*args): - ... return vals[args] - ... - >>> mock = MagicMock(side_effect=side_effect) - >>> mock(1, 2) - 1 - >>> mock(2, 3) - 2 - - -Mocking asynchronous iterators -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock -:ref:`async-iterators` through ``__aiter__``. The :attr:`~Mock.return_value` -attribute of ``__aiter__`` can be used to set the return values to be used for -iteration. - - >>> mock = MagicMock() # AsyncMock also works here - >>> mock.__aiter__.return_value = [1, 2, 3] - >>> async def main(): - ... return [i async for i in mock] - ... - >>> asyncio.run(main()) - [1, 2, 3] - - -Mocking asynchronous context manager -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Since Python 3.8, ``AsyncMock`` and ``MagicMock`` have support to mock -:ref:`async-context-managers` through ``__aenter__`` and ``__aexit__``. -By default, ``__aenter__`` and ``__aexit__`` are ``AsyncMock`` instances that -return an async function. - - >>> class AsyncContextManager: - ... async def __aenter__(self): - ... return self - ... async def __aexit__(self, exc_type, exc, tb): - ... pass - ... - >>> mock_instance = MagicMock(AsyncContextManager()) # AsyncMock also works here - >>> async def main(): - ... async with mock_instance as result: - ... pass - ... - >>> asyncio.run(main()) - >>> mock_instance.__aenter__.assert_awaited_once() - >>> mock_instance.__aexit__.assert_awaited_once() - - -Creating a Mock from an Existing Object -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -One problem with over use of mocking is that it couples your tests to the -implementation of your mocks rather than your real code. Suppose you have a -class that implements ``some_method``. In a test for another class, you -provide a mock of this object that *also* provides ``some_method``. If later -you refactor the first class, so that it no longer has ``some_method`` - then -your tests will continue to pass even though your code is now broken! - -:class:`Mock` allows you to provide an object as a specification for the mock, -using the *spec* keyword argument. Accessing methods / attributes on the -mock that don't exist on your specification object will immediately raise an -attribute error. If you change the implementation of your specification, then -tests that use that class will start failing immediately without you having to -instantiate the class in those tests. - - >>> mock = Mock(spec=SomeClass) - >>> mock.old_method() - Traceback (most recent call last): - ... - AttributeError: object has no attribute 'old_method' - -Using a specification also enables a smarter matching of calls made to the -mock, regardless of whether some parameters were passed as positional or -named arguments:: - - >>> def f(a, b, c): pass - ... - >>> mock = Mock(spec=f) - >>> mock(1, 2, 3) - - >>> mock.assert_called_with(a=1, b=2, c=3) - -If you want this smarter matching to also work with method calls on the mock, -you can use :ref:`auto-speccing `. - -If you want a stronger form of specification that prevents the setting -of arbitrary attributes as well as the getting of them then you can use -*spec_set* instead of *spec*. - - -Using side_effect to return per file content -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -:func:`mock_open` is used to patch :func:`open` method. :attr:`~Mock.side_effect` -can be used to return a new Mock object per call. This can be used to return different -contents per file stored in a dictionary:: - - DEFAULT = "default" - data_dict = {"file1": "data1", - "file2": "data2"} - - def open_side_effect(name): - return mock_open(read_data=data_dict.get(name, DEFAULT))() - - with patch("builtins.open", side_effect=open_side_effect): - with open("file1") as file1: - assert file1.read() == "data1" - - with open("file2") as file2: - assert file2.read() == "data2" - - with open("file3") as file2: - assert file2.read() == "default" - - -Patch Decorators ----------------- - -.. note:: - - With :func:`patch` it matters that you patch objects in the namespace where - they are looked up. This is normally straightforward, but for a quick guide - read :ref:`where to patch `. - - -A common need in tests is to patch a class attribute or a module attribute, -for example patching a builtin or patching a class in a module to test that it -is instantiated. Modules and classes are effectively global, so patching on -them has to be undone after the test or the patch will persist into other -tests and cause hard to diagnose problems. - -mock provides three convenient decorators for this: :func:`patch`, :func:`patch.object` and -:func:`patch.dict`. ``patch`` takes a single string, of the form -``package.module.Class.attribute`` to specify the attribute you are patching. It -also optionally takes a value that you want the attribute (or class or -whatever) to be replaced with. 'patch.object' takes an object and the name of -the attribute you would like patched, plus optionally the value to patch it -with. - -``patch.object``:: - - >>> original = SomeClass.attribute - >>> @patch.object(SomeClass, 'attribute', sentinel.attribute) - ... def test(): - ... assert SomeClass.attribute == sentinel.attribute - ... - >>> test() - >>> assert SomeClass.attribute == original - - >>> @patch('package.module.attribute', sentinel.attribute) - ... def test(): - ... from package.module import attribute - ... assert attribute is sentinel.attribute - ... - >>> test() - -If you are patching a module (including :mod:`builtins`) then use :func:`patch` -instead of :func:`patch.object`: - - >>> mock = MagicMock(return_value=sentinel.file_handle) - >>> with patch('builtins.open', mock): - ... handle = open('filename', 'r') - ... - >>> mock.assert_called_with('filename', 'r') - >>> assert handle == sentinel.file_handle, "incorrect file handle returned" - -The module name can be 'dotted', in the form ``package.module`` if needed:: - - >>> @patch('package.module.ClassName.attribute', sentinel.attribute) - ... def test(): - ... from package.module import ClassName - ... assert ClassName.attribute == sentinel.attribute - ... - >>> test() - -A nice pattern is to actually decorate test methods themselves: - - >>> class MyTest(unittest.TestCase): - ... @patch.object(SomeClass, 'attribute', sentinel.attribute) - ... def test_something(self): - ... self.assertEqual(SomeClass.attribute, sentinel.attribute) - ... - >>> original = SomeClass.attribute - >>> MyTest('test_something').test_something() - >>> assert SomeClass.attribute == original - -If you want to patch with a Mock, you can use :func:`patch` with only one argument -(or :func:`patch.object` with two arguments). The mock will be created for you and -passed into the test function / method: - - >>> class MyTest(unittest.TestCase): - ... @patch.object(SomeClass, 'static_method') - ... def test_something(self, mock_method): - ... SomeClass.static_method() - ... mock_method.assert_called_with() - ... - >>> MyTest('test_something').test_something() - -You can stack up multiple patch decorators using this pattern:: - - >>> class MyTest(unittest.TestCase): - ... @patch('package.module.ClassName1') - ... @patch('package.module.ClassName2') - ... def test_something(self, MockClass2, MockClass1): - ... self.assertIs(package.module.ClassName1, MockClass1) - ... self.assertIs(package.module.ClassName2, MockClass2) - ... - >>> MyTest('test_something').test_something() - -When you nest patch decorators the mocks are passed in to the decorated -function in the same order they applied (the normal *Python* order that -decorators are applied). This means from the bottom up, so in the example -above the mock for ``test_module.ClassName2`` is passed in first. - -There is also :func:`patch.dict` for setting values in a dictionary just -during a scope and restoring the dictionary to its original state when the test -ends: - - >>> foo = {'key': 'value'} - >>> original = foo.copy() - >>> with patch.dict(foo, {'newkey': 'newvalue'}, clear=True): - ... assert foo == {'newkey': 'newvalue'} - ... - >>> assert foo == original - -``patch``, ``patch.object`` and ``patch.dict`` can all be used as context managers. - -Where you use :func:`patch` to create a mock for you, you can get a reference to the -mock using the "as" form of the with statement: - - >>> class ProductionClass: - ... def method(self): - ... pass - ... - >>> with patch.object(ProductionClass, 'method') as mock_method: - ... mock_method.return_value = None - ... real = ProductionClass() - ... real.method(1, 2, 3) - ... - >>> mock_method.assert_called_with(1, 2, 3) - - -As an alternative ``patch``, ``patch.object`` and ``patch.dict`` can be used as -class decorators. When used in this way it is the same as applying the -decorator individually to every method whose name starts with "test". - - -.. _further-examples: - -Further Examples ----------------- - - -Here are some more examples for some slightly more advanced scenarios. - - -Mocking chained calls -~~~~~~~~~~~~~~~~~~~~~ - -Mocking chained calls is actually straightforward with mock once you -understand the :attr:`~Mock.return_value` attribute. When a mock is called for -the first time, or you fetch its ``return_value`` before it has been called, a -new :class:`Mock` is created. - -This means that you can see how the object returned from a call to a mocked -object has been used by interrogating the ``return_value`` mock: - - >>> mock = Mock() - >>> mock().foo(a=2, b=3) - - >>> mock.return_value.foo.assert_called_with(a=2, b=3) - -From here it is a simple step to configure and then make assertions about -chained calls. Of course another alternative is writing your code in a more -testable way in the first place... - -So, suppose we have some code that looks a little bit like this: - - >>> class Something: - ... def __init__(self): - ... self.backend = BackendProvider() - ... def method(self): - ... response = self.backend.get_endpoint('foobar').create_call('spam', 'eggs').start_call() - ... # more code - -Assuming that ``BackendProvider`` is already well tested, how do we test -``method()``? Specifically, we want to test that the code section ``# more -code`` uses the response object in the correct way. - -As this chain of calls is made from an instance attribute we can monkey patch -the ``backend`` attribute on a ``Something`` instance. In this particular case -we are only interested in the return value from the final call to -``start_call`` so we don't have much configuration to do. Let's assume the -object it returns is 'file-like', so we'll ensure that our response object -uses the builtin :func:`open` as its ``spec``. - -To do this we create a mock instance as our mock backend and create a mock -response object for it. To set the response as the return value for that final -``start_call`` we could do this:: - - mock_backend.get_endpoint.return_value.create_call.return_value.start_call.return_value = mock_response - -We can do that in a slightly nicer way using the :meth:`~Mock.configure_mock` -method to directly set the return value for us:: - - >>> something = Something() - >>> mock_response = Mock(spec=open) - >>> mock_backend = Mock() - >>> config = {'get_endpoint.return_value.create_call.return_value.start_call.return_value': mock_response} - >>> mock_backend.configure_mock(**config) - -With these we monkey patch the "mock backend" in place and can make the real -call:: - - >>> something.backend = mock_backend - >>> something.method() - -Using :attr:`~Mock.mock_calls` we can check the chained call with a single -assert. A chained call is several calls in one line of code, so there will be -several entries in ``mock_calls``. We can use :meth:`call.call_list` to create -this list of calls for us:: - - >>> chained = call.get_endpoint('foobar').create_call('spam', 'eggs').start_call() - >>> call_list = chained.call_list() - >>> assert mock_backend.mock_calls == call_list - - -Partial mocking -~~~~~~~~~~~~~~~ - -In some tests I wanted to mock out a call to :meth:`datetime.date.today` -to return a known date, but I didn't want to prevent the code under test from -creating new date objects. Unfortunately :class:`datetime.date` is written in C, and -so I couldn't just monkey-patch out the static :meth:`datetime.date.today` method. - -I found a simple way of doing this that involved effectively wrapping the date -class with a mock, but passing through calls to the constructor to the real -class (and returning real instances). - -The :func:`patch decorator ` is used here to -mock out the ``date`` class in the module under test. The :attr:`~Mock.side_effect` -attribute on the mock date class is then set to a lambda function that returns -a real date. When the mock date class is called a real date will be -constructed and returned by ``side_effect``. :: - - >>> from datetime import date - >>> with patch('mymodule.date') as mock_date: - ... mock_date.today.return_value = date(2010, 10, 8) - ... mock_date.side_effect = lambda *args, **kw: date(*args, **kw) - ... - ... assert mymodule.date.today() == date(2010, 10, 8) - ... assert mymodule.date(2009, 6, 8) == date(2009, 6, 8) - -Note that we don't patch :class:`datetime.date` globally, we patch ``date`` in the -module that *uses* it. See :ref:`where to patch `. - -When ``date.today()`` is called a known date is returned, but calls to the -``date(...)`` constructor still return normal dates. Without this you can find -yourself having to calculate an expected result using exactly the same -algorithm as the code under test, which is a classic testing anti-pattern. - -Calls to the date constructor are recorded in the ``mock_date`` attributes -(``call_count`` and friends) which may also be useful for your tests. - -An alternative way of dealing with mocking dates, or other builtin classes, -is discussed in `this blog entry -`_. - - -Mocking a Generator Method -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -A Python generator is a function or method that uses the :keyword:`yield` statement -to return a series of values when iterated over [#]_. - -A generator method / function is called to return the generator object. It is -the generator object that is then iterated over. The protocol method for -iteration is :meth:`~container.__iter__`, so we can -mock this using a :class:`MagicMock`. - -Here's an example class with an "iter" method implemented as a generator: - - >>> class Foo: - ... def iter(self): - ... for i in [1, 2, 3]: - ... yield i - ... - >>> foo = Foo() - >>> list(foo.iter()) - [1, 2, 3] - - -How would we mock this class, and in particular its "iter" method? - -To configure the values returned from the iteration (implicit in the call to -:class:`list`), we need to configure the object returned by the call to ``foo.iter()``. - - >>> mock_foo = MagicMock() - >>> mock_foo.iter.return_value = iter([1, 2, 3]) - >>> list(mock_foo.iter()) - [1, 2, 3] - -.. [#] There are also generator expressions and more `advanced uses - `_ of generators, but we aren't - concerned about them here. A very good introduction to generators and how - powerful they are is: `Generator Tricks for Systems Programmers - `_. - - -Applying the same patch to every test method -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want several patches in place for multiple test methods the obvious way -is to apply the patch decorators to every method. This can feel like unnecessary -repetition. Instead, you can use :func:`patch` (in all its -various forms) as a class decorator. This applies the patches to all test -methods on the class. A test method is identified by methods whose names start -with ``test``:: - - >>> @patch('mymodule.SomeClass') - ... class MyTest(unittest.TestCase): - ... - ... def test_one(self, MockSomeClass): - ... self.assertIs(mymodule.SomeClass, MockSomeClass) - ... - ... def test_two(self, MockSomeClass): - ... self.assertIs(mymodule.SomeClass, MockSomeClass) - ... - ... def not_a_test(self): - ... return 'something' - ... - >>> MyTest('test_one').test_one() - >>> MyTest('test_two').test_two() - >>> MyTest('test_two').not_a_test() - 'something' - -An alternative way of managing patches is to use the :ref:`start-and-stop`. -These allow you to move the patching into your ``setUp`` and ``tearDown`` methods. -:: - - >>> class MyTest(unittest.TestCase): - ... def setUp(self): - ... self.patcher = patch('mymodule.foo') - ... self.mock_foo = self.patcher.start() - ... - ... def test_foo(self): - ... self.assertIs(mymodule.foo, self.mock_foo) - ... - ... def tearDown(self): - ... self.patcher.stop() - ... - >>> MyTest('test_foo').run() - -If you use this technique you must ensure that the patching is "undone" by -calling ``stop``. This can be fiddlier than you might think, because if an -exception is raised in the setUp then tearDown is not called. -:meth:`unittest.TestCase.addCleanup` makes this easier:: - - >>> class MyTest(unittest.TestCase): - ... def setUp(self): - ... patcher = patch('mymodule.foo') - ... self.addCleanup(patcher.stop) - ... self.mock_foo = patcher.start() - ... - ... def test_foo(self): - ... self.assertIs(mymodule.foo, self.mock_foo) - ... - >>> MyTest('test_foo').run() - - -Mocking Unbound Methods -~~~~~~~~~~~~~~~~~~~~~~~ - -Whilst writing tests today I needed to patch an *unbound method* (patching the -method on the class rather than on the instance). I needed self to be passed -in as the first argument because I want to make asserts about which objects -were calling this particular method. The issue is that you can't patch with a -mock for this, because if you replace an unbound method with a mock it doesn't -become a bound method when fetched from the instance, and so it doesn't get -self passed in. The workaround is to patch the unbound method with a real -function instead. The :func:`patch` decorator makes it so simple to -patch out methods with a mock that having to create a real function becomes a -nuisance. - -If you pass ``autospec=True`` to patch then it does the patching with a -*real* function object. This function object has the same signature as the one -it is replacing, but delegates to a mock under the hood. You still get your -mock auto-created in exactly the same way as before. What it means though, is -that if you use it to patch out an unbound method on a class the mocked -function will be turned into a bound method if it is fetched from an instance. -It will have ``self`` passed in as the first argument, which is exactly what I -wanted: - - >>> class Foo: - ... def foo(self): - ... pass - ... - >>> with patch.object(Foo, 'foo', autospec=True) as mock_foo: - ... mock_foo.return_value = 'foo' - ... foo = Foo() - ... foo.foo() - ... - 'foo' - >>> mock_foo.assert_called_once_with(foo) - -If we don't use ``autospec=True`` then the unbound method is patched out -with a Mock instance instead, and isn't called with ``self``. - - -Checking multiple calls with mock -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -mock has a nice API for making assertions about how your mock objects are used. - - >>> mock = Mock() - >>> mock.foo_bar.return_value = None - >>> mock.foo_bar('baz', spam='eggs') - >>> mock.foo_bar.assert_called_with('baz', spam='eggs') - -If your mock is only being called once you can use the -:meth:`~Mock.assert_called_once_with` method that also asserts that the -:attr:`~Mock.call_count` is one. - - >>> mock.foo_bar.assert_called_once_with('baz', spam='eggs') - >>> mock.foo_bar() - >>> mock.foo_bar.assert_called_once_with('baz', spam='eggs') - Traceback (most recent call last): - ... - AssertionError: Expected to be called once. Called 2 times. - -Both ``assert_called_with`` and ``assert_called_once_with`` make assertions about -the *most recent* call. If your mock is going to be called several times, and -you want to make assertions about *all* those calls you can use -:attr:`~Mock.call_args_list`: - - >>> mock = Mock(return_value=None) - >>> mock(1, 2, 3) - >>> mock(4, 5, 6) - >>> mock() - >>> mock.call_args_list - [call(1, 2, 3), call(4, 5, 6), call()] - -The :data:`call` helper makes it easy to make assertions about these calls. You -can build up a list of expected calls and compare it to ``call_args_list``. This -looks remarkably similar to the repr of the ``call_args_list``: - - >>> expected = [call(1, 2, 3), call(4, 5, 6), call()] - >>> mock.call_args_list == expected - True - - -Coping with mutable arguments -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Another situation is rare, but can bite you, is when your mock is called with -mutable arguments. ``call_args`` and ``call_args_list`` store *references* to the -arguments. If the arguments are mutated by the code under test then you can no -longer make assertions about what the values were when the mock was called. - -Here's some example code that shows the problem. Imagine the following functions -defined in 'mymodule':: - - def frob(val): - pass - - def grob(val): - "First frob and then clear val" - frob(val) - val.clear() - -When we try to test that ``grob`` calls ``frob`` with the correct argument look -what happens:: - - >>> with patch('mymodule.frob') as mock_frob: - ... val = {6} - ... mymodule.grob(val) - ... - >>> val - set() - >>> mock_frob.assert_called_with({6}) - Traceback (most recent call last): - ... - AssertionError: Expected: (({6},), {}) - Called with: ((set(),), {}) - -One possibility would be for mock to copy the arguments you pass in. This -could then cause problems if you do assertions that rely on object identity -for equality. - -Here's one solution that uses the :attr:`~Mock.side_effect` -functionality. If you provide a ``side_effect`` function for a mock then -``side_effect`` will be called with the same args as the mock. This gives us an -opportunity to copy the arguments and store them for later assertions. In this -example I'm using *another* mock to store the arguments so that I can use the -mock methods for doing the assertion. Again a helper function sets this up for -me. :: - - >>> from copy import deepcopy - >>> from unittest.mock import Mock, patch, DEFAULT - >>> def copy_call_args(mock): - ... new_mock = Mock() - ... def side_effect(*args, **kwargs): - ... args = deepcopy(args) - ... kwargs = deepcopy(kwargs) - ... new_mock(*args, **kwargs) - ... return DEFAULT - ... mock.side_effect = side_effect - ... return new_mock - ... - >>> with patch('mymodule.frob') as mock_frob: - ... new_mock = copy_call_args(mock_frob) - ... val = {6} - ... mymodule.grob(val) - ... - >>> new_mock.assert_called_with({6}) - >>> new_mock.call_args - call({6}) - -``copy_call_args`` is called with the mock that will be called. It returns a new -mock that we do the assertion on. The ``side_effect`` function makes a copy of -the args and calls our ``new_mock`` with the copy. - -.. note:: - - If your mock is only going to be used once there is an easier way of - checking arguments at the point they are called. You can simply do the - checking inside a ``side_effect`` function. - - >>> def side_effect(arg): - ... assert arg == {6} - ... - >>> mock = Mock(side_effect=side_effect) - >>> mock({6}) - >>> mock(set()) - Traceback (most recent call last): - ... - AssertionError - -An alternative approach is to create a subclass of :class:`Mock` or -:class:`MagicMock` that copies (using :func:`copy.deepcopy`) the arguments. -Here's an example implementation: - - >>> from copy import deepcopy - >>> class CopyingMock(MagicMock): - ... def __call__(self, /, *args, **kwargs): - ... args = deepcopy(args) - ... kwargs = deepcopy(kwargs) - ... return super().__call__(*args, **kwargs) - ... - >>> c = CopyingMock(return_value=None) - >>> arg = set() - >>> c(arg) - >>> arg.add(1) - >>> c.assert_called_with(set()) - >>> c.assert_called_with(arg) - Traceback (most recent call last): - ... - AssertionError: Expected call: mock({1}) - Actual call: mock(set()) - >>> c.foo - - -When you subclass ``Mock`` or ``MagicMock`` all dynamically created attributes, -and the ``return_value`` will use your subclass automatically. That means all -children of a ``CopyingMock`` will also have the type ``CopyingMock``. - - -Nesting Patches -~~~~~~~~~~~~~~~ - -Using patch as a context manager is nice, but if you do multiple patches you -can end up with nested with statements indenting further and further to the -right:: - - >>> class MyTest(unittest.TestCase): - ... - ... def test_foo(self): - ... with patch('mymodule.Foo') as mock_foo: - ... with patch('mymodule.Bar') as mock_bar: - ... with patch('mymodule.Spam') as mock_spam: - ... assert mymodule.Foo is mock_foo - ... assert mymodule.Bar is mock_bar - ... assert mymodule.Spam is mock_spam - ... - >>> original = mymodule.Foo - >>> MyTest('test_foo').test_foo() - >>> assert mymodule.Foo is original - -With unittest ``cleanup`` functions and the :ref:`start-and-stop` we can -achieve the same effect without the nested indentation. A simple helper -method, ``create_patch``, puts the patch in place and returns the created mock -for us:: - - >>> class MyTest(unittest.TestCase): - ... - ... def create_patch(self, name): - ... patcher = patch(name) - ... thing = patcher.start() - ... self.addCleanup(patcher.stop) - ... return thing - ... - ... def test_foo(self): - ... mock_foo = self.create_patch('mymodule.Foo') - ... mock_bar = self.create_patch('mymodule.Bar') - ... mock_spam = self.create_patch('mymodule.Spam') - ... - ... assert mymodule.Foo is mock_foo - ... assert mymodule.Bar is mock_bar - ... assert mymodule.Spam is mock_spam - ... - >>> original = mymodule.Foo - >>> MyTest('test_foo').run() - >>> assert mymodule.Foo is original - - -Mocking a dictionary with MagicMock -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -You may want to mock a dictionary, or other container object, recording all -access to it whilst having it still behave like a dictionary. - -We can do this with :class:`MagicMock`, which will behave like a dictionary, -and using :data:`~Mock.side_effect` to delegate dictionary access to a real -underlying dictionary that is under our control. - -When the :meth:`~object.__getitem__` and :meth:`~object.__setitem__` methods -of our ``MagicMock`` are called -(normal dictionary access) then ``side_effect`` is called with the key (and in -the case of ``__setitem__`` the value too). We can also control what is returned. - -After the ``MagicMock`` has been used we can use attributes like -:data:`~Mock.call_args_list` to assert about how the dictionary was used: - - >>> my_dict = {'a': 1, 'b': 2, 'c': 3} - >>> def getitem(name): - ... return my_dict[name] - ... - >>> def setitem(name, val): - ... my_dict[name] = val - ... - >>> mock = MagicMock() - >>> mock.__getitem__.side_effect = getitem - >>> mock.__setitem__.side_effect = setitem - -.. note:: - - An alternative to using ``MagicMock`` is to use ``Mock`` and *only* provide - the magic methods you specifically want: - - >>> mock = Mock() - >>> mock.__getitem__ = Mock(side_effect=getitem) - >>> mock.__setitem__ = Mock(side_effect=setitem) - - A *third* option is to use ``MagicMock`` but passing in ``dict`` as the *spec* - (or *spec_set*) argument so that the ``MagicMock`` created only has - dictionary magic methods available: - - >>> mock = MagicMock(spec_set=dict) - >>> mock.__getitem__.side_effect = getitem - >>> mock.__setitem__.side_effect = setitem - -With these side effect functions in place, the ``mock`` will behave like a normal -dictionary but recording the access. It even raises a :exc:`KeyError` if you try -to access a key that doesn't exist. - - >>> mock['a'] - 1 - >>> mock['c'] - 3 - >>> mock['d'] - Traceback (most recent call last): - ... - KeyError: 'd' - >>> mock['b'] = 'fish' - >>> mock['d'] = 'eggs' - >>> mock['b'] - 'fish' - >>> mock['d'] - 'eggs' - -After it has been used you can make assertions about the access using the normal -mock methods and attributes: - - >>> mock.__getitem__.call_args_list - [call('a'), call('c'), call('d'), call('b'), call('d')] - >>> mock.__setitem__.call_args_list - [call('b', 'fish'), call('d', 'eggs')] - >>> my_dict - {'a': 1, 'b': 'fish', 'c': 3, 'd': 'eggs'} - - -Mock subclasses and their attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -There are various reasons why you might want to subclass :class:`Mock`. One -reason might be to add helper methods. Here's a silly example: - - >>> class MyMock(MagicMock): - ... def has_been_called(self): - ... return self.called - ... - >>> mymock = MyMock(return_value=None) - >>> mymock - - >>> mymock.has_been_called() - False - >>> mymock() - >>> mymock.has_been_called() - True - -The standard behaviour for ``Mock`` instances is that attributes and the return -value mocks are of the same type as the mock they are accessed on. This ensures -that ``Mock`` attributes are ``Mocks`` and ``MagicMock`` attributes are ``MagicMocks`` -[#]_. So if you're subclassing to add helper methods then they'll also be -available on the attributes and return value mock of instances of your -subclass. - - >>> mymock.foo - - >>> mymock.foo.has_been_called() - False - >>> mymock.foo() - - >>> mymock.foo.has_been_called() - True - -Sometimes this is inconvenient. For example, `one user -`_ is subclassing mock to -created a `Twisted adaptor -`_. -Having this applied to attributes too actually causes errors. - -``Mock`` (in all its flavours) uses a method called ``_get_child_mock`` to create -these "sub-mocks" for attributes and return values. You can prevent your -subclass being used for attributes by overriding this method. The signature is -that it takes arbitrary keyword arguments (``**kwargs``) which are then passed -onto the mock constructor: - - >>> class Subclass(MagicMock): - ... def _get_child_mock(self, /, **kwargs): - ... return MagicMock(**kwargs) - ... - >>> mymock = Subclass() - >>> mymock.foo - - >>> assert isinstance(mymock, Subclass) - >>> assert not isinstance(mymock.foo, Subclass) - >>> assert not isinstance(mymock(), Subclass) - -.. [#] An exception to this rule are the non-callable mocks. Attributes use the - callable variant because otherwise non-callable mocks couldn't have callable - methods. - - -Mocking imports with patch.dict -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -One situation where mocking can be hard is where you have a local import inside -a function. These are harder to mock because they aren't using an object from -the module namespace that we can patch out. - -Generally local imports are to be avoided. They are sometimes done to prevent -circular dependencies, for which there is *usually* a much better way to solve -the problem (refactor the code) or to prevent "up front costs" by delaying the -import. This can also be solved in better ways than an unconditional local -import (store the module as a class or module attribute and only do the import -on first use). - -That aside there is a way to use ``mock`` to affect the results of an import. -Importing fetches an *object* from the :data:`sys.modules` dictionary. Note that it -fetches an *object*, which need not be a module. Importing a module for the -first time results in a module object being put in ``sys.modules``, so usually -when you import something you get a module back. This need not be the case -however. - -This means you can use :func:`patch.dict` to *temporarily* put a mock in place -in :data:`sys.modules`. Any imports whilst this patch is active will fetch the mock. -When the patch is complete (the decorated function exits, the with statement -body is complete or ``patcher.stop()`` is called) then whatever was there -previously will be restored safely. - -Here's an example that mocks out the 'fooble' module. - - >>> import sys - >>> mock = Mock() - >>> with patch.dict('sys.modules', {'fooble': mock}): - ... import fooble - ... fooble.blob() - ... - - >>> assert 'fooble' not in sys.modules - >>> mock.blob.assert_called_once_with() - -As you can see the ``import fooble`` succeeds, but on exit there is no 'fooble' -left in :data:`sys.modules`. - -This also works for the ``from module import name`` form: - - >>> mock = Mock() - >>> with patch.dict('sys.modules', {'fooble': mock}): - ... from fooble import blob - ... blob.blip() - ... - - >>> mock.blob.blip.assert_called_once_with() - -With slightly more work you can also mock package imports: - - >>> mock = Mock() - >>> modules = {'package': mock, 'package.module': mock.module} - >>> with patch.dict('sys.modules', modules): - ... from package.module import fooble - ... fooble() - ... - - >>> mock.module.fooble.assert_called_once_with() - - -Tracking order of calls and less verbose call assertions -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The :class:`Mock` class allows you to track the *order* of method calls on -your mock objects through the :attr:`~Mock.method_calls` attribute. This -doesn't allow you to track the order of calls between separate mock objects, -however we can use :attr:`~Mock.mock_calls` to achieve the same effect. - -Because mocks track calls to child mocks in ``mock_calls``, and accessing an -arbitrary attribute of a mock creates a child mock, we can create our separate -mocks from a parent one. Calls to those child mock will then all be recorded, -in order, in the ``mock_calls`` of the parent: - - >>> manager = Mock() - >>> mock_foo = manager.foo - >>> mock_bar = manager.bar - - >>> mock_foo.something() - - >>> mock_bar.other.thing() - - - >>> manager.mock_calls - [call.foo.something(), call.bar.other.thing()] - -We can then assert about the calls, including the order, by comparing with -the ``mock_calls`` attribute on the manager mock: - - >>> expected_calls = [call.foo.something(), call.bar.other.thing()] - >>> manager.mock_calls == expected_calls - True - -If ``patch`` is creating, and putting in place, your mocks then you can attach -them to a manager mock using the :meth:`~Mock.attach_mock` method. After -attaching calls will be recorded in ``mock_calls`` of the manager. :: - - >>> manager = MagicMock() - >>> with patch('mymodule.Class1') as MockClass1: - ... with patch('mymodule.Class2') as MockClass2: - ... manager.attach_mock(MockClass1, 'MockClass1') - ... manager.attach_mock(MockClass2, 'MockClass2') - ... MockClass1().foo() - ... MockClass2().bar() - - - >>> manager.mock_calls - [call.MockClass1(), - call.MockClass1().foo(), - call.MockClass2(), - call.MockClass2().bar()] - -If many calls have been made, but you're only interested in a particular -sequence of them then an alternative is to use the -:meth:`~Mock.assert_has_calls` method. This takes a list of calls (constructed -with the :data:`call` object). If that sequence of calls are in -:attr:`~Mock.mock_calls` then the assert succeeds. - - >>> m = MagicMock() - >>> m().foo().bar().baz() - - >>> m.one().two().three() - - >>> calls = call.one().two().three().call_list() - >>> m.assert_has_calls(calls) - -Even though the chained call ``m.one().two().three()`` aren't the only calls that -have been made to the mock, the assert still succeeds. - -Sometimes a mock may have several calls made to it, and you are only interested -in asserting about *some* of those calls. You may not even care about the -order. In this case you can pass ``any_order=True`` to ``assert_has_calls``: - - >>> m = MagicMock() - >>> m(1), m.two(2, 3), m.seven(7), m.fifty('50') - (...) - >>> calls = [call.fifty('50'), call(1), call.seven(7)] - >>> m.assert_has_calls(calls, any_order=True) - - -More complex argument matching -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Using the same basic concept as :data:`ANY` we can implement matchers to do more -complex assertions on objects used as arguments to mocks. - -Suppose we expect some object to be passed to a mock that by default -compares equal based on object identity (which is the Python default for user -defined classes). To use :meth:`~Mock.assert_called_with` we would need to pass -in the exact same object. If we are only interested in some of the attributes -of this object then we can create a matcher that will check these attributes -for us. - -You can see in this example how a 'standard' call to ``assert_called_with`` isn't -sufficient: - - >>> class Foo: - ... def __init__(self, a, b): - ... self.a, self.b = a, b - ... - >>> mock = Mock(return_value=None) - >>> mock(Foo(1, 2)) - >>> mock.assert_called_with(Foo(1, 2)) - Traceback (most recent call last): - ... - AssertionError: Expected: call(<__main__.Foo object at 0x...>) - Actual call: call(<__main__.Foo object at 0x...>) - -A comparison function for our ``Foo`` class might look something like this: - - >>> def compare(self, other): - ... if not type(self) == type(other): - ... return False - ... if self.a != other.a: - ... return False - ... if self.b != other.b: - ... return False - ... return True - ... - -And a matcher object that can use comparison functions like this for its -equality operation would look something like this: - - >>> class Matcher: - ... def __init__(self, compare, some_obj): - ... self.compare = compare - ... self.some_obj = some_obj - ... def __eq__(self, other): - ... return self.compare(self.some_obj, other) - ... - -Putting all this together: - - >>> match_foo = Matcher(compare, Foo(1, 2)) - >>> mock.assert_called_with(match_foo) - -The ``Matcher`` is instantiated with our compare function and the ``Foo`` object -we want to compare against. In ``assert_called_with`` the ``Matcher`` equality -method will be called, which compares the object the mock was called with -against the one we created our matcher with. If they match then -``assert_called_with`` passes, and if they don't an :exc:`AssertionError` is raised: - - >>> match_wrong = Matcher(compare, Foo(3, 4)) - >>> mock.assert_called_with(match_wrong) - Traceback (most recent call last): - ... - AssertionError: Expected: ((,), {}) - Called with: ((,), {}) - -With a bit of tweaking you could have the comparison function raise the -:exc:`AssertionError` directly and provide a more useful failure message. - -As of version 1.5, the Python testing library `PyHamcrest -`_ provides similar functionality, -that may be useful here, in the form of its equality matcher -(`hamcrest.library.integration.match_equality -`_). diff --git a/Doc/library/unittest.mock.rst b/Doc/library/unittest.mock.rst deleted file mode 100644 index b7e03dfd642b39..00000000000000 --- a/Doc/library/unittest.mock.rst +++ /dev/null @@ -1,2771 +0,0 @@ - -:mod:`unittest.mock` --- mock object library -============================================ - -.. module:: unittest.mock - :synopsis: Mock object library. - -.. moduleauthor:: Michael Foord -.. currentmodule:: unittest.mock - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/unittest/mock.py` - --------------- - -:mod:`unittest.mock` is a library for testing in Python. It allows you to -replace parts of your system under test with mock objects and make assertions -about how they have been used. - -:mod:`unittest.mock` provides a core :class:`Mock` class removing the need to -create a host of stubs throughout your test suite. After performing an -action, you can make assertions about which methods / attributes were used -and arguments they were called with. You can also specify return values and -set needed attributes in the normal way. - -Additionally, mock provides a :func:`patch` decorator that handles patching -module and class level attributes within the scope of a test, along with -:const:`sentinel` for creating unique objects. See the `quick guide`_ for -some examples of how to use :class:`Mock`, :class:`MagicMock` and -:func:`patch`. - -Mock is designed for use with :mod:`unittest` and -is based on the 'action -> assertion' pattern instead of 'record -> replay' -used by many mocking frameworks. - -There is a backport of :mod:`unittest.mock` for earlier versions of Python, -available as `mock on PyPI `_. - - -Quick Guide ------------ - -.. testsetup:: - - class ProductionClass: - def method(self, a, b, c): - pass - - class SomeClass: - @staticmethod - def static_method(args): - return args - - @classmethod - def class_method(cls, args): - return args - - -:class:`Mock` and :class:`MagicMock` objects create all attributes and -methods as you access them and store details of how they have been used. You -can configure them, to specify return values or limit what attributes are -available, and then make assertions about how they have been used: - - >>> from unittest.mock import MagicMock - >>> thing = ProductionClass() - >>> thing.method = MagicMock(return_value=3) - >>> thing.method(3, 4, 5, key='value') - 3 - >>> thing.method.assert_called_with(3, 4, 5, key='value') - -:attr:`side_effect` allows you to perform side effects, including raising an -exception when a mock is called: - - >>> mock = Mock(side_effect=KeyError('foo')) - >>> mock() - Traceback (most recent call last): - ... - KeyError: 'foo' - - >>> values = {'a': 1, 'b': 2, 'c': 3} - >>> def side_effect(arg): - ... return values[arg] - ... - >>> mock.side_effect = side_effect - >>> mock('a'), mock('b'), mock('c') - (1, 2, 3) - >>> mock.side_effect = [5, 4, 3, 2, 1] - >>> mock(), mock(), mock() - (5, 4, 3) - -Mock has many other ways you can configure it and control its behaviour. For -example the *spec* argument configures the mock to take its specification -from another object. Attempting to access attributes or methods on the mock -that don't exist on the spec will fail with an :exc:`AttributeError`. - -The :func:`patch` decorator / context manager makes it easy to mock classes or -objects in a module under test. The object you specify will be replaced with a -mock (or other object) during the test and restored when the test ends:: - - >>> from unittest.mock import patch - >>> @patch('module.ClassName2') - ... @patch('module.ClassName1') - ... def test(MockClass1, MockClass2): - ... module.ClassName1() - ... module.ClassName2() - ... assert MockClass1 is module.ClassName1 - ... assert MockClass2 is module.ClassName2 - ... assert MockClass1.called - ... assert MockClass2.called - ... - >>> test() - -.. note:: - - When you nest patch decorators the mocks are passed in to the decorated - function in the same order they applied (the normal *Python* order that - decorators are applied). This means from the bottom up, so in the example - above the mock for ``module.ClassName1`` is passed in first. - - With :func:`patch` it matters that you patch objects in the namespace where they - are looked up. This is normally straightforward, but for a quick guide - read :ref:`where to patch `. - -As well as a decorator :func:`patch` can be used as a context manager in a with -statement: - - >>> with patch.object(ProductionClass, 'method', return_value=None) as mock_method: - ... thing = ProductionClass() - ... thing.method(1, 2, 3) - ... - >>> mock_method.assert_called_once_with(1, 2, 3) - - -There is also :func:`patch.dict` for setting values in a dictionary just -during a scope and restoring the dictionary to its original state when the test -ends: - - >>> foo = {'key': 'value'} - >>> original = foo.copy() - >>> with patch.dict(foo, {'newkey': 'newvalue'}, clear=True): - ... assert foo == {'newkey': 'newvalue'} - ... - >>> assert foo == original - -Mock supports the mocking of Python :ref:`magic methods `. The -easiest way of using magic methods is with the :class:`MagicMock` class. It -allows you to do things like: - - >>> mock = MagicMock() - >>> mock.__str__.return_value = 'foobarbaz' - >>> str(mock) - 'foobarbaz' - >>> mock.__str__.assert_called_with() - -Mock allows you to assign functions (or other Mock instances) to magic methods -and they will be called appropriately. The :class:`MagicMock` class is just a Mock -variant that has all of the magic methods pre-created for you (well, all the -useful ones anyway). - -The following is an example of using magic methods with the ordinary Mock -class: - - >>> mock = Mock() - >>> mock.__str__ = Mock(return_value='wheeeeee') - >>> str(mock) - 'wheeeeee' - -For ensuring that the mock objects in your tests have the same api as the -objects they are replacing, you can use :ref:`auto-speccing `. -Auto-speccing can be done through the *autospec* argument to patch, or the -:func:`create_autospec` function. Auto-speccing creates mock objects that -have the same attributes and methods as the objects they are replacing, and -any functions and methods (including constructors) have the same call -signature as the real object. - -This ensures that your mocks will fail in the same way as your production -code if they are used incorrectly: - - >>> from unittest.mock import create_autospec - >>> def function(a, b, c): - ... pass - ... - >>> mock_function = create_autospec(function, return_value='fishy') - >>> mock_function(1, 2, 3) - 'fishy' - >>> mock_function.assert_called_once_with(1, 2, 3) - >>> mock_function('wrong arguments') - Traceback (most recent call last): - ... - TypeError: () takes exactly 3 arguments (1 given) - -:func:`create_autospec` can also be used on classes, where it copies the signature of -the ``__init__`` method, and on callable objects where it copies the signature of -the ``__call__`` method. - - - -The Mock Class --------------- - -.. testsetup:: - - import asyncio - import inspect - import unittest - from unittest.mock import sentinel, DEFAULT, ANY - from unittest.mock import patch, call, Mock, MagicMock, PropertyMock, AsyncMock - from unittest.mock import mock_open - -:class:`Mock` is a flexible mock object intended to replace the use of stubs and -test doubles throughout your code. Mocks are callable and create attributes as -new mocks when you access them [#]_. Accessing the same attribute will always -return the same mock. Mocks record how you use them, allowing you to make -assertions about what your code has done to them. - -:class:`MagicMock` is a subclass of :class:`Mock` with all the magic methods -pre-created and ready to use. There are also non-callable variants, useful -when you are mocking out objects that aren't callable: -:class:`NonCallableMock` and :class:`NonCallableMagicMock` - -The :func:`patch` decorators makes it easy to temporarily replace classes -in a particular module with a :class:`Mock` object. By default :func:`patch` will create -a :class:`MagicMock` for you. You can specify an alternative class of :class:`Mock` using -the *new_callable* argument to :func:`patch`. - - -.. class:: Mock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) - - Create a new :class:`Mock` object. :class:`Mock` takes several optional arguments - that specify the behaviour of the Mock object: - - * *spec*: This can be either a list of strings or an existing object (a - class or instance) that acts as the specification for the mock object. If - you pass in an object then a list of strings is formed by calling dir on - the object (excluding unsupported magic attributes and methods). - Accessing any attribute not in this list will raise an :exc:`AttributeError`. - - If *spec* is an object (rather than a list of strings) then - :attr:`~instance.__class__` returns the class of the spec object. This - allows mocks to pass :func:`isinstance` tests. - - * *spec_set*: A stricter variant of *spec*. If used, attempting to *set* - or get an attribute on the mock that isn't on the object passed as - *spec_set* will raise an :exc:`AttributeError`. - - * *side_effect*: A function to be called whenever the Mock is called. See - the :attr:`~Mock.side_effect` attribute. Useful for raising exceptions or - dynamically changing return values. The function is called with the same - arguments as the mock, and unless it returns :data:`DEFAULT`, the return - value of this function is used as the return value. - - Alternatively *side_effect* can be an exception class or instance. In - this case the exception will be raised when the mock is called. - - If *side_effect* is an iterable then each call to the mock will return - the next value from the iterable. - - A *side_effect* can be cleared by setting it to ``None``. - - * *return_value*: The value returned when the mock is called. By default - this is a new Mock (created on first access). See the - :attr:`return_value` attribute. - - * *unsafe*: By default, accessing any attribute whose name starts with - *assert*, *assret*, *asert*, *aseert* or *assrt* will raise an - :exc:`AttributeError`. Passing ``unsafe=True`` will allow access to - these attributes. - - .. versionadded:: 3.5 - - * *wraps*: Item for the mock object to wrap. If *wraps* is not ``None`` then - calling the Mock will pass the call through to the wrapped object - (returning the real result). Attribute access on the mock will return a - Mock object that wraps the corresponding attribute of the wrapped - object (so attempting to access an attribute that doesn't exist will - raise an :exc:`AttributeError`). - - If the mock has an explicit *return_value* set then calls are not passed - to the wrapped object and the *return_value* is returned instead. - - * *name*: If the mock has a name then it will be used in the repr of the - mock. This can be useful for debugging. The name is propagated to child - mocks. - - Mocks can also be called with arbitrary keyword arguments. These will be - used to set attributes on the mock after it is created. See the - :meth:`configure_mock` method for details. - - .. method:: assert_called() - - Assert that the mock was called at least once. - - >>> mock = Mock() - >>> mock.method() - - >>> mock.method.assert_called() - - .. versionadded:: 3.6 - - .. method:: assert_called_once() - - Assert that the mock was called exactly once. - - >>> mock = Mock() - >>> mock.method() - - >>> mock.method.assert_called_once() - >>> mock.method() - - >>> mock.method.assert_called_once() - Traceback (most recent call last): - ... - AssertionError: Expected 'method' to have been called once. Called 2 times. - - .. versionadded:: 3.6 - - - .. method:: assert_called_with(*args, **kwargs) - - This method is a convenient way of asserting that the last call has been - made in a particular way: - - >>> mock = Mock() - >>> mock.method(1, 2, 3, test='wow') - - >>> mock.method.assert_called_with(1, 2, 3, test='wow') - - .. method:: assert_called_once_with(*args, **kwargs) - - Assert that the mock was called exactly once and that call was with the - specified arguments. - - >>> mock = Mock(return_value=None) - >>> mock('foo', bar='baz') - >>> mock.assert_called_once_with('foo', bar='baz') - >>> mock('other', bar='values') - >>> mock.assert_called_once_with('other', bar='values') - Traceback (most recent call last): - ... - AssertionError: Expected 'mock' to be called once. Called 2 times. - - - .. method:: assert_any_call(*args, **kwargs) - - assert the mock has been called with the specified arguments. - - The assert passes if the mock has *ever* been called, unlike - :meth:`assert_called_with` and :meth:`assert_called_once_with` that - only pass if the call is the most recent one, and in the case of - :meth:`assert_called_once_with` it must also be the only call. - - >>> mock = Mock(return_value=None) - >>> mock(1, 2, arg='thing') - >>> mock('some', 'thing', 'else') - >>> mock.assert_any_call(1, 2, arg='thing') - - - .. method:: assert_has_calls(calls, any_order=False) - - assert the mock has been called with the specified calls. - The :attr:`mock_calls` list is checked for the calls. - - If *any_order* is false then the calls must be - sequential. There can be extra calls before or after the - specified calls. - - If *any_order* is true then the calls can be in any order, but - they must all appear in :attr:`mock_calls`. - - >>> mock = Mock(return_value=None) - >>> mock(1) - >>> mock(2) - >>> mock(3) - >>> mock(4) - >>> calls = [call(2), call(3)] - >>> mock.assert_has_calls(calls) - >>> calls = [call(4), call(2), call(3)] - >>> mock.assert_has_calls(calls, any_order=True) - - .. method:: assert_not_called() - - Assert the mock was never called. - - >>> m = Mock() - >>> m.hello.assert_not_called() - >>> obj = m.hello() - >>> m.hello.assert_not_called() - Traceback (most recent call last): - ... - AssertionError: Expected 'hello' to not have been called. Called 1 times. - - .. versionadded:: 3.5 - - - .. method:: reset_mock(*, return_value=False, side_effect=False) - - The reset_mock method resets all the call attributes on a mock object: - - >>> mock = Mock(return_value=None) - >>> mock('hello') - >>> mock.called - True - >>> mock.reset_mock() - >>> mock.called - False - - .. versionchanged:: 3.6 - Added two keyword-only arguments to the reset_mock function. - - This can be useful where you want to make a series of assertions that - reuse the same object. Note that :meth:`reset_mock` *doesn't* clear the - return value, :attr:`side_effect` or any child attributes you have - set using normal assignment by default. In case you want to reset - *return_value* or :attr:`side_effect`, then pass the corresponding - parameter as ``True``. Child mocks and the return value mock - (if any) are reset as well. - - .. note:: *return_value*, and :attr:`side_effect` are keyword-only - arguments. - - - .. method:: mock_add_spec(spec, spec_set=False) - - Add a spec to a mock. *spec* can either be an object or a - list of strings. Only attributes on the *spec* can be fetched as - attributes from the mock. - - If *spec_set* is true then only attributes on the spec can be set. - - - .. method:: attach_mock(mock, attribute) - - Attach a mock as an attribute of this one, replacing its name and - parent. Calls to the attached mock will be recorded in the - :attr:`method_calls` and :attr:`mock_calls` attributes of this one. - - - .. method:: configure_mock(**kwargs) - - Set attributes on the mock through keyword arguments. - - Attributes plus return values and side effects can be set on child - mocks using standard dot notation and unpacking a dictionary in the - method call: - - >>> mock = Mock() - >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError} - >>> mock.configure_mock(**attrs) - >>> mock.method() - 3 - >>> mock.other() - Traceback (most recent call last): - ... - KeyError - - The same thing can be achieved in the constructor call to mocks: - - >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError} - >>> mock = Mock(some_attribute='eggs', **attrs) - >>> mock.some_attribute - 'eggs' - >>> mock.method() - 3 - >>> mock.other() - Traceback (most recent call last): - ... - KeyError - - :meth:`configure_mock` exists to make it easier to do configuration - after the mock has been created. - - - .. method:: __dir__() - - :class:`Mock` objects limit the results of ``dir(some_mock)`` to useful results. - For mocks with a *spec* this includes all the permitted attributes - for the mock. - - See :data:`FILTER_DIR` for what this filtering does, and how to - switch it off. - - - .. method:: _get_child_mock(**kw) - - Create the child mocks for attributes and return value. - By default child mocks will be the same type as the parent. - Subclasses of Mock may want to override this to customize the way - child mocks are made. - - For non-callable mocks the callable variant will be used (rather than - any custom subclass). - - - .. attribute:: called - - A boolean representing whether or not the mock object has been called: - - >>> mock = Mock(return_value=None) - >>> mock.called - False - >>> mock() - >>> mock.called - True - - .. attribute:: call_count - - An integer telling you how many times the mock object has been called: - - >>> mock = Mock(return_value=None) - >>> mock.call_count - 0 - >>> mock() - >>> mock() - >>> mock.call_count - 2 - - .. attribute:: return_value - - Set this to configure the value returned by calling the mock: - - >>> mock = Mock() - >>> mock.return_value = 'fish' - >>> mock() - 'fish' - - The default return value is a mock object and you can configure it in - the normal way: - - >>> mock = Mock() - >>> mock.return_value.attribute = sentinel.Attribute - >>> mock.return_value() - - >>> mock.return_value.assert_called_with() - - :attr:`return_value` can also be set in the constructor: - - >>> mock = Mock(return_value=3) - >>> mock.return_value - 3 - >>> mock() - 3 - - - .. attribute:: side_effect - - This can either be a function to be called when the mock is called, - an iterable or an exception (class or instance) to be raised. - - If you pass in a function it will be called with same arguments as the - mock and unless the function returns the :data:`DEFAULT` singleton the - call to the mock will then return whatever the function returns. If the - function returns :data:`DEFAULT` then the mock will return its normal - value (from the :attr:`return_value`). - - If you pass in an iterable, it is used to retrieve an iterator which - must yield a value on every call. This value can either be an exception - instance to be raised, or a value to be returned from the call to the - mock (:data:`DEFAULT` handling is identical to the function case). - - An example of a mock that raises an exception (to test exception - handling of an API): - - >>> mock = Mock() - >>> mock.side_effect = Exception('Boom!') - >>> mock() - Traceback (most recent call last): - ... - Exception: Boom! - - Using :attr:`side_effect` to return a sequence of values: - - >>> mock = Mock() - >>> mock.side_effect = [3, 2, 1] - >>> mock(), mock(), mock() - (3, 2, 1) - - Using a callable: - - >>> mock = Mock(return_value=3) - >>> def side_effect(*args, **kwargs): - ... return DEFAULT - ... - >>> mock.side_effect = side_effect - >>> mock() - 3 - - :attr:`side_effect` can be set in the constructor. Here's an example that - adds one to the value the mock is called with and returns it: - - >>> side_effect = lambda value: value + 1 - >>> mock = Mock(side_effect=side_effect) - >>> mock(3) - 4 - >>> mock(-8) - -7 - - Setting :attr:`side_effect` to ``None`` clears it: - - >>> m = Mock(side_effect=KeyError, return_value=3) - >>> m() - Traceback (most recent call last): - ... - KeyError - >>> m.side_effect = None - >>> m() - 3 - - - .. attribute:: call_args - - This is either ``None`` (if the mock hasn't been called), or the - arguments that the mock was last called with. This will be in the - form of a tuple: the first member, which can also be accessed through - the ``args`` property, is any ordered arguments the mock was - called with (or an empty tuple) and the second member, which can - also be accessed through the ``kwargs`` property, is any keyword - arguments (or an empty dictionary). - - >>> mock = Mock(return_value=None) - >>> print(mock.call_args) - None - >>> mock() - >>> mock.call_args - call() - >>> mock.call_args == () - True - >>> mock(3, 4) - >>> mock.call_args - call(3, 4) - >>> mock.call_args == ((3, 4),) - True - >>> mock.call_args.args - (3, 4) - >>> mock.call_args.kwargs - {} - >>> mock(3, 4, 5, key='fish', next='w00t!') - >>> mock.call_args - call(3, 4, 5, key='fish', next='w00t!') - >>> mock.call_args.args - (3, 4, 5) - >>> mock.call_args.kwargs - {'key': 'fish', 'next': 'w00t!'} - - :attr:`call_args`, along with members of the lists :attr:`call_args_list`, - :attr:`method_calls` and :attr:`mock_calls` are :data:`call` objects. - These are tuples, so they can be unpacked to get at the individual - arguments and make more complex assertions. See - :ref:`calls as tuples `. - - .. versionchanged:: 3.8 - Added ``args`` and ``kwargs`` properties. - - - .. attribute:: call_args_list - - This is a list of all the calls made to the mock object in sequence - (so the length of the list is the number of times it has been - called). Before any calls have been made it is an empty list. The - :data:`call` object can be used for conveniently constructing lists of - calls to compare with :attr:`call_args_list`. - - >>> mock = Mock(return_value=None) - >>> mock() - >>> mock(3, 4) - >>> mock(key='fish', next='w00t!') - >>> mock.call_args_list - [call(), call(3, 4), call(key='fish', next='w00t!')] - >>> expected = [(), ((3, 4),), ({'key': 'fish', 'next': 'w00t!'},)] - >>> mock.call_args_list == expected - True - - Members of :attr:`call_args_list` are :data:`call` objects. These can be - unpacked as tuples to get at the individual arguments. See - :ref:`calls as tuples `. - - - .. attribute:: method_calls - - As well as tracking calls to themselves, mocks also track calls to - methods and attributes, and *their* methods and attributes: - - >>> mock = Mock() - >>> mock.method() - - >>> mock.property.method.attribute() - - >>> mock.method_calls - [call.method(), call.property.method.attribute()] - - Members of :attr:`method_calls` are :data:`call` objects. These can be - unpacked as tuples to get at the individual arguments. See - :ref:`calls as tuples `. - - - .. attribute:: mock_calls - - :attr:`mock_calls` records *all* calls to the mock object, its methods, - magic methods *and* return value mocks. - - >>> mock = MagicMock() - >>> result = mock(1, 2, 3) - >>> mock.first(a=3) - - >>> mock.second() - - >>> int(mock) - 1 - >>> result(1) - - >>> expected = [call(1, 2, 3), call.first(a=3), call.second(), - ... call.__int__(), call()(1)] - >>> mock.mock_calls == expected - True - - Members of :attr:`mock_calls` are :data:`call` objects. These can be - unpacked as tuples to get at the individual arguments. See - :ref:`calls as tuples `. - - .. note:: - - The way :attr:`mock_calls` are recorded means that where nested - calls are made, the parameters of ancestor calls are not recorded - and so will always compare equal: - - >>> mock = MagicMock() - >>> mock.top(a=3).bottom() - - >>> mock.mock_calls - [call.top(a=3), call.top().bottom()] - >>> mock.mock_calls[-1] == call.top(a=-1).bottom() - True - - .. attribute:: __class__ - - Normally the :attr:`__class__` attribute of an object will return its type. - For a mock object with a :attr:`spec`, ``__class__`` returns the spec class - instead. This allows mock objects to pass :func:`isinstance` tests for the - object they are replacing / masquerading as: - - >>> mock = Mock(spec=3) - >>> isinstance(mock, int) - True - - :attr:`__class__` is assignable to, this allows a mock to pass an - :func:`isinstance` check without forcing you to use a spec: - - >>> mock = Mock() - >>> mock.__class__ = dict - >>> isinstance(mock, dict) - True - -.. class:: NonCallableMock(spec=None, wraps=None, name=None, spec_set=None, **kwargs) - - A non-callable version of :class:`Mock`. The constructor parameters have the same - meaning of :class:`Mock`, with the exception of *return_value* and *side_effect* - which have no meaning on a non-callable mock. - -Mock objects that use a class or an instance as a :attr:`spec` or -:attr:`spec_set` are able to pass :func:`isinstance` tests: - - >>> mock = Mock(spec=SomeClass) - >>> isinstance(mock, SomeClass) - True - >>> mock = Mock(spec_set=SomeClass()) - >>> isinstance(mock, SomeClass) - True - -The :class:`Mock` classes have support for mocking magic methods. See :ref:`magic -methods ` for the full details. - -The mock classes and the :func:`patch` decorators all take arbitrary keyword -arguments for configuration. For the :func:`patch` decorators the keywords are -passed to the constructor of the mock being created. The keyword arguments -are for configuring attributes of the mock: - - >>> m = MagicMock(attribute=3, other='fish') - >>> m.attribute - 3 - >>> m.other - 'fish' - -The return value and side effect of child mocks can be set in the same way, -using dotted notation. As you can't use dotted names directly in a call you -have to create a dictionary and unpack it using ``**``: - - >>> attrs = {'method.return_value': 3, 'other.side_effect': KeyError} - >>> mock = Mock(some_attribute='eggs', **attrs) - >>> mock.some_attribute - 'eggs' - >>> mock.method() - 3 - >>> mock.other() - Traceback (most recent call last): - ... - KeyError - -A callable mock which was created with a *spec* (or a *spec_set*) will -introspect the specification object's signature when matching calls to -the mock. Therefore, it can match the actual call's arguments regardless -of whether they were passed positionally or by name:: - - >>> def f(a, b, c): pass - ... - >>> mock = Mock(spec=f) - >>> mock(1, 2, c=3) - - >>> mock.assert_called_with(1, 2, 3) - >>> mock.assert_called_with(a=1, b=2, c=3) - -This applies to :meth:`~Mock.assert_called_with`, -:meth:`~Mock.assert_called_once_with`, :meth:`~Mock.assert_has_calls` and -:meth:`~Mock.assert_any_call`. When :ref:`auto-speccing`, it will also -apply to method calls on the mock object. - -.. versionchanged:: 3.4 - Added signature introspection on specced and autospecced mock objects. - - -.. class:: PropertyMock(*args, **kwargs) - - A mock intended to be used as a property, or other descriptor, on a class. - :class:`PropertyMock` provides :meth:`__get__` and :meth:`__set__` methods - so you can specify a return value when it is fetched. - - Fetching a :class:`PropertyMock` instance from an object calls the mock, with - no args. Setting it calls the mock with the value being set. :: - - >>> class Foo: - ... @property - ... def foo(self): - ... return 'something' - ... @foo.setter - ... def foo(self, value): - ... pass - ... - >>> with patch('__main__.Foo.foo', new_callable=PropertyMock) as mock_foo: - ... mock_foo.return_value = 'mockity-mock' - ... this_foo = Foo() - ... print(this_foo.foo) - ... this_foo.foo = 6 - ... - mockity-mock - >>> mock_foo.mock_calls - [call(), call(6)] - -Because of the way mock attributes are stored you can't directly attach a -:class:`PropertyMock` to a mock object. Instead you can attach it to the mock type -object:: - - >>> m = MagicMock() - >>> p = PropertyMock(return_value=3) - >>> type(m).foo = p - >>> m.foo - 3 - >>> p.assert_called_once_with() - - -.. class:: AsyncMock(spec=None, side_effect=None, return_value=DEFAULT, wraps=None, name=None, spec_set=None, unsafe=False, **kwargs) - - An asynchronous version of :class:`MagicMock`. The :class:`AsyncMock` object will - behave so the object is recognized as an async function, and the result of a - call is an awaitable. - - >>> mock = AsyncMock() - >>> asyncio.iscoroutinefunction(mock) - True - >>> inspect.isawaitable(mock()) # doctest: +SKIP - True - - The result of ``mock()`` is an async function which will have the outcome - of ``side_effect`` or ``return_value`` after it has been awaited: - - - if ``side_effect`` is a function, the async function will return the - result of that function, - - if ``side_effect`` is an exception, the async function will raise the - exception, - - if ``side_effect`` is an iterable, the async function will return the - next value of the iterable, however, if the sequence of result is - exhausted, ``StopAsyncIteration`` is raised immediately, - - if ``side_effect`` is not defined, the async function will return the - value defined by ``return_value``, hence, by default, the async function - returns a new :class:`AsyncMock` object. - - - Setting the *spec* of a :class:`Mock` or :class:`MagicMock` to an async function - will result in a coroutine object being returned after calling. - - >>> async def async_func(): pass - ... - >>> mock = MagicMock(async_func) - >>> mock - - >>> mock() # doctest: +SKIP - - - - Setting the *spec* of a :class:`Mock`, :class:`MagicMock`, or :class:`AsyncMock` - to a class with asynchronous and synchronous functions will automatically - detect the synchronous functions and set them as :class:`MagicMock` (if the - parent mock is :class:`AsyncMock` or :class:`MagicMock`) or :class:`Mock` (if - the parent mock is :class:`Mock`). All asynchronous functions will be - :class:`AsyncMock`. - - >>> class ExampleClass: - ... def sync_foo(): - ... pass - ... async def async_foo(): - ... pass - ... - >>> a_mock = AsyncMock(ExampleClass) - >>> a_mock.sync_foo - - >>> a_mock.async_foo - - >>> mock = Mock(ExampleClass) - >>> mock.sync_foo - - >>> mock.async_foo - - - .. versionadded:: 3.8 - - .. method:: assert_awaited() - - Assert that the mock was awaited at least once. Note that this is separate - from the object having been called, the ``await`` keyword must be used: - - >>> mock = AsyncMock() - >>> async def main(coroutine_mock): - ... await coroutine_mock - ... - >>> coroutine_mock = mock() - >>> mock.called - True - >>> mock.assert_awaited() - Traceback (most recent call last): - ... - AssertionError: Expected mock to have been awaited. - >>> asyncio.run(main(coroutine_mock)) - >>> mock.assert_awaited() - - .. method:: assert_awaited_once() - - Assert that the mock was awaited exactly once. - - >>> mock = AsyncMock() - >>> async def main(): - ... await mock() - ... - >>> asyncio.run(main()) - >>> mock.assert_awaited_once() - >>> asyncio.run(main()) - >>> mock.method.assert_awaited_once() - Traceback (most recent call last): - ... - AssertionError: Expected mock to have been awaited once. Awaited 2 times. - - .. method:: assert_awaited_with(*args, **kwargs) - - Assert that the last await was with the specified arguments. - - >>> mock = AsyncMock() - >>> async def main(*args, **kwargs): - ... await mock(*args, **kwargs) - ... - >>> asyncio.run(main('foo', bar='bar')) - >>> mock.assert_awaited_with('foo', bar='bar') - >>> mock.assert_awaited_with('other') - Traceback (most recent call last): - ... - AssertionError: expected call not found. - Expected: mock('other') - Actual: mock('foo', bar='bar') - - .. method:: assert_awaited_once_with(*args, **kwargs) - - Assert that the mock was awaited exactly once and with the specified - arguments. - - >>> mock = AsyncMock() - >>> async def main(*args, **kwargs): - ... await mock(*args, **kwargs) - ... - >>> asyncio.run(main('foo', bar='bar')) - >>> mock.assert_awaited_once_with('foo', bar='bar') - >>> asyncio.run(main('foo', bar='bar')) - >>> mock.assert_awaited_once_with('foo', bar='bar') - Traceback (most recent call last): - ... - AssertionError: Expected mock to have been awaited once. Awaited 2 times. - - .. method:: assert_any_await(*args, **kwargs) - - Assert the mock has ever been awaited with the specified arguments. - - >>> mock = AsyncMock() - >>> async def main(*args, **kwargs): - ... await mock(*args, **kwargs) - ... - >>> asyncio.run(main('foo', bar='bar')) - >>> asyncio.run(main('hello')) - >>> mock.assert_any_await('foo', bar='bar') - >>> mock.assert_any_await('other') - Traceback (most recent call last): - ... - AssertionError: mock('other') await not found - - .. method:: assert_has_awaits(calls, any_order=False) - - Assert the mock has been awaited with the specified calls. - The :attr:`await_args_list` list is checked for the awaits. - - If *any_order* is false then the awaits must be - sequential. There can be extra calls before or after the - specified awaits. - - If *any_order* is true then the awaits can be in any order, but - they must all appear in :attr:`await_args_list`. - - >>> mock = AsyncMock() - >>> async def main(*args, **kwargs): - ... await mock(*args, **kwargs) - ... - >>> calls = [call("foo"), call("bar")] - >>> mock.assert_has_awaits(calls) - Traceback (most recent call last): - ... - AssertionError: Awaits not found. - Expected: [call('foo'), call('bar')] - Actual: [] - >>> asyncio.run(main('foo')) - >>> asyncio.run(main('bar')) - >>> mock.assert_has_awaits(calls) - - .. method:: assert_not_awaited() - - Assert that the mock was never awaited. - - >>> mock = AsyncMock() - >>> mock.assert_not_awaited() - - .. method:: reset_mock(*args, **kwargs) - - See :func:`Mock.reset_mock`. Also sets :attr:`await_count` to 0, - :attr:`await_args` to None, and clears the :attr:`await_args_list`. - - .. attribute:: await_count - - An integer keeping track of how many times the mock object has been awaited. - - >>> mock = AsyncMock() - >>> async def main(): - ... await mock() - ... - >>> asyncio.run(main()) - >>> mock.await_count - 1 - >>> asyncio.run(main()) - >>> mock.await_count - 2 - - .. attribute:: await_args - - This is either ``None`` (if the mock hasn’t been awaited), or the arguments that - the mock was last awaited with. Functions the same as :attr:`Mock.call_args`. - - >>> mock = AsyncMock() - >>> async def main(*args): - ... await mock(*args) - ... - >>> mock.await_args - >>> asyncio.run(main('foo')) - >>> mock.await_args - call('foo') - >>> asyncio.run(main('bar')) - >>> mock.await_args - call('bar') - - - .. attribute:: await_args_list - - This is a list of all the awaits made to the mock object in sequence (so the - length of the list is the number of times it has been awaited). Before any - awaits have been made it is an empty list. - - >>> mock = AsyncMock() - >>> async def main(*args): - ... await mock(*args) - ... - >>> mock.await_args_list - [] - >>> asyncio.run(main('foo')) - >>> mock.await_args_list - [call('foo')] - >>> asyncio.run(main('bar')) - >>> mock.await_args_list - [call('foo'), call('bar')] - - -Calling -~~~~~~~ - -Mock objects are callable. The call will return the value set as the -:attr:`~Mock.return_value` attribute. The default return value is a new Mock -object; it is created the first time the return value is accessed (either -explicitly or by calling the Mock) - but it is stored and the same one -returned each time. - -Calls made to the object will be recorded in the attributes -like :attr:`~Mock.call_args` and :attr:`~Mock.call_args_list`. - -If :attr:`~Mock.side_effect` is set then it will be called after the call has -been recorded, so if :attr:`side_effect` raises an exception the call is still -recorded. - -The simplest way to make a mock raise an exception when called is to make -:attr:`~Mock.side_effect` an exception class or instance: - - >>> m = MagicMock(side_effect=IndexError) - >>> m(1, 2, 3) - Traceback (most recent call last): - ... - IndexError - >>> m.mock_calls - [call(1, 2, 3)] - >>> m.side_effect = KeyError('Bang!') - >>> m('two', 'three', 'four') - Traceback (most recent call last): - ... - KeyError: 'Bang!' - >>> m.mock_calls - [call(1, 2, 3), call('two', 'three', 'four')] - -If :attr:`side_effect` is a function then whatever that function returns is what -calls to the mock return. The :attr:`side_effect` function is called with the -same arguments as the mock. This allows you to vary the return value of the -call dynamically, based on the input: - - >>> def side_effect(value): - ... return value + 1 - ... - >>> m = MagicMock(side_effect=side_effect) - >>> m(1) - 2 - >>> m(2) - 3 - >>> m.mock_calls - [call(1), call(2)] - -If you want the mock to still return the default return value (a new mock), or -any set return value, then there are two ways of doing this. Either return -:attr:`mock.return_value` from inside :attr:`side_effect`, or return :data:`DEFAULT`: - - >>> m = MagicMock() - >>> def side_effect(*args, **kwargs): - ... return m.return_value - ... - >>> m.side_effect = side_effect - >>> m.return_value = 3 - >>> m() - 3 - >>> def side_effect(*args, **kwargs): - ... return DEFAULT - ... - >>> m.side_effect = side_effect - >>> m() - 3 - -To remove a :attr:`side_effect`, and return to the default behaviour, set the -:attr:`side_effect` to ``None``: - - >>> m = MagicMock(return_value=6) - >>> def side_effect(*args, **kwargs): - ... return 3 - ... - >>> m.side_effect = side_effect - >>> m() - 3 - >>> m.side_effect = None - >>> m() - 6 - -The :attr:`side_effect` can also be any iterable object. Repeated calls to the mock -will return values from the iterable (until the iterable is exhausted and -a :exc:`StopIteration` is raised): - - >>> m = MagicMock(side_effect=[1, 2, 3]) - >>> m() - 1 - >>> m() - 2 - >>> m() - 3 - >>> m() - Traceback (most recent call last): - ... - StopIteration - -If any members of the iterable are exceptions they will be raised instead of -returned:: - - >>> iterable = (33, ValueError, 66) - >>> m = MagicMock(side_effect=iterable) - >>> m() - 33 - >>> m() - Traceback (most recent call last): - ... - ValueError - >>> m() - 66 - - -.. _deleting-attributes: - -Deleting Attributes -~~~~~~~~~~~~~~~~~~~ - -Mock objects create attributes on demand. This allows them to pretend to be -objects of any type. - -You may want a mock object to return ``False`` to a :func:`hasattr` call, or raise an -:exc:`AttributeError` when an attribute is fetched. You can do this by providing -an object as a :attr:`spec` for a mock, but that isn't always convenient. - -You "block" attributes by deleting them. Once deleted, accessing an attribute -will raise an :exc:`AttributeError`. - - >>> mock = MagicMock() - >>> hasattr(mock, 'm') - True - >>> del mock.m - >>> hasattr(mock, 'm') - False - >>> del mock.f - >>> mock.f - Traceback (most recent call last): - ... - AttributeError: f - - -Mock names and the name attribute -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Since "name" is an argument to the :class:`Mock` constructor, if you want your -mock object to have a "name" attribute you can't just pass it in at creation -time. There are two alternatives. One option is to use -:meth:`~Mock.configure_mock`:: - - >>> mock = MagicMock() - >>> mock.configure_mock(name='my_name') - >>> mock.name - 'my_name' - -A simpler option is to simply set the "name" attribute after mock creation:: - - >>> mock = MagicMock() - >>> mock.name = "foo" - - -Attaching Mocks as Attributes -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -When you attach a mock as an attribute of another mock (or as the return -value) it becomes a "child" of that mock. Calls to the child are recorded in -the :attr:`~Mock.method_calls` and :attr:`~Mock.mock_calls` attributes of the -parent. This is useful for configuring child mocks and then attaching them to -the parent, or for attaching mocks to a parent that records all calls to the -children and allows you to make assertions about the order of calls between -mocks: - - >>> parent = MagicMock() - >>> child1 = MagicMock(return_value=None) - >>> child2 = MagicMock(return_value=None) - >>> parent.child1 = child1 - >>> parent.child2 = child2 - >>> child1(1) - >>> child2(2) - >>> parent.mock_calls - [call.child1(1), call.child2(2)] - -The exception to this is if the mock has a name. This allows you to prevent -the "parenting" if for some reason you don't want it to happen. - - >>> mock = MagicMock() - >>> not_a_child = MagicMock(name='not-a-child') - >>> mock.attribute = not_a_child - >>> mock.attribute() - - >>> mock.mock_calls - [] - -Mocks created for you by :func:`patch` are automatically given names. To -attach mocks that have names to a parent you use the :meth:`~Mock.attach_mock` -method:: - - >>> thing1 = object() - >>> thing2 = object() - >>> parent = MagicMock() - >>> with patch('__main__.thing1', return_value=None) as child1: - ... with patch('__main__.thing2', return_value=None) as child2: - ... parent.attach_mock(child1, 'child1') - ... parent.attach_mock(child2, 'child2') - ... child1('one') - ... child2('two') - ... - >>> parent.mock_calls - [call.child1('one'), call.child2('two')] - - -.. [#] The only exceptions are magic methods and attributes (those that have - leading and trailing double underscores). Mock doesn't create these but - instead raises an :exc:`AttributeError`. This is because the interpreter - will often implicitly request these methods, and gets *very* confused to - get a new Mock object when it expects a magic method. If you need magic - method support see :ref:`magic methods `. - - -The patchers ------------- - -The patch decorators are used for patching objects only within the scope of -the function they decorate. They automatically handle the unpatching for you, -even if exceptions are raised. All of these functions can also be used in with -statements or as class decorators. - - -patch -~~~~~ - -.. note:: - - The key is to do the patching in the right namespace. See the section `where to patch`_. - -.. function:: patch(target, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) - - :func:`patch` acts as a function decorator, class decorator or a context - manager. Inside the body of the function or with statement, the *target* - is patched with a *new* object. When the function/with statement exits - the patch is undone. - - If *new* is omitted, then the target is replaced with an - :class:`AsyncMock` if the patched object is an async function or - a :class:`MagicMock` otherwise. - If :func:`patch` is used as a decorator and *new* is - omitted, the created mock is passed in as an extra argument to the - decorated function. If :func:`patch` is used as a context manager the created - mock is returned by the context manager. - - *target* should be a string in the form ``'package.module.ClassName'``. The - *target* is imported and the specified object replaced with the *new* - object, so the *target* must be importable from the environment you are - calling :func:`patch` from. The target is imported when the decorated function - is executed, not at decoration time. - - The *spec* and *spec_set* keyword arguments are passed to the :class:`MagicMock` - if patch is creating one for you. - - In addition you can pass ``spec=True`` or ``spec_set=True``, which causes - patch to pass in the object being mocked as the spec/spec_set object. - - *new_callable* allows you to specify a different class, or callable object, - that will be called to create the *new* object. By default :class:`AsyncMock` - is used for async functions and :class:`MagicMock` for the rest. - - A more powerful form of *spec* is *autospec*. If you set ``autospec=True`` - then the mock will be created with a spec from the object being replaced. - All attributes of the mock will also have the spec of the corresponding - attribute of the object being replaced. Methods and functions being mocked - will have their arguments checked and will raise a :exc:`TypeError` if they are - called with the wrong signature. For mocks - replacing a class, their return value (the 'instance') will have the same - spec as the class. See the :func:`create_autospec` function and - :ref:`auto-speccing`. - - Instead of ``autospec=True`` you can pass ``autospec=some_object`` to use an - arbitrary object as the spec instead of the one being replaced. - - By default :func:`patch` will fail to replace attributes that don't exist. - If you pass in ``create=True``, and the attribute doesn't exist, patch will - create the attribute for you when the patched function is called, and delete - it again after the patched function has exited. This is useful for writing - tests against attributes that your production code creates at runtime. It is - off by default because it can be dangerous. With it switched on you can - write passing tests against APIs that don't actually exist! - - .. note:: - - .. versionchanged:: 3.5 - If you are patching builtins in a module then you don't - need to pass ``create=True``, it will be added by default. - - Patch can be used as a :class:`TestCase` class decorator. It works by - decorating each test method in the class. This reduces the boilerplate - code when your test methods share a common patchings set. :func:`patch` finds - tests by looking for method names that start with ``patch.TEST_PREFIX``. - By default this is ``'test'``, which matches the way :mod:`unittest` finds tests. - You can specify an alternative prefix by setting ``patch.TEST_PREFIX``. - - Patch can be used as a context manager, with the with statement. Here the - patching applies to the indented block after the with statement. If you - use "as" then the patched object will be bound to the name after the - "as"; very useful if :func:`patch` is creating a mock object for you. - - :func:`patch` takes arbitrary keyword arguments. These will be passed to - :class:`AsyncMock` if the patched object is asynchronous, to - :class:`MagicMock` otherwise or to *new_callable* if specified. - - ``patch.dict(...)``, ``patch.multiple(...)`` and ``patch.object(...)`` are - available for alternate use-cases. - -:func:`patch` as function decorator, creating the mock for you and passing it into -the decorated function:: - - >>> @patch('__main__.SomeClass') - ... def function(normal_argument, mock_class): - ... print(mock_class is SomeClass) - ... - >>> function(None) - True - -Patching a class replaces the class with a :class:`MagicMock` *instance*. If the -class is instantiated in the code under test then it will be the -:attr:`~Mock.return_value` of the mock that will be used. - -If the class is instantiated multiple times you could use -:attr:`~Mock.side_effect` to return a new mock each time. Alternatively you -can set the *return_value* to be anything you want. - -To configure return values on methods of *instances* on the patched class -you must do this on the :attr:`return_value`. For example:: - - >>> class Class: - ... def method(self): - ... pass - ... - >>> with patch('__main__.Class') as MockClass: - ... instance = MockClass.return_value - ... instance.method.return_value = 'foo' - ... assert Class() is instance - ... assert Class().method() == 'foo' - ... - -If you use *spec* or *spec_set* and :func:`patch` is replacing a *class*, then the -return value of the created mock will have the same spec. :: - - >>> Original = Class - >>> patcher = patch('__main__.Class', spec=True) - >>> MockClass = patcher.start() - >>> instance = MockClass() - >>> assert isinstance(instance, Original) - >>> patcher.stop() - -The *new_callable* argument is useful where you want to use an alternative -class to the default :class:`MagicMock` for the created mock. For example, if -you wanted a :class:`NonCallableMock` to be used:: - - >>> thing = object() - >>> with patch('__main__.thing', new_callable=NonCallableMock) as mock_thing: - ... assert thing is mock_thing - ... thing() - ... - Traceback (most recent call last): - ... - TypeError: 'NonCallableMock' object is not callable - -Another use case might be to replace an object with an :class:`io.StringIO` instance:: - - >>> from io import StringIO - >>> def foo(): - ... print('Something') - ... - >>> @patch('sys.stdout', new_callable=StringIO) - ... def test(mock_stdout): - ... foo() - ... assert mock_stdout.getvalue() == 'Something\n' - ... - >>> test() - -When :func:`patch` is creating a mock for you, it is common that the first thing -you need to do is to configure the mock. Some of that configuration can be done -in the call to patch. Any arbitrary keywords you pass into the call will be -used to set attributes on the created mock:: - - >>> patcher = patch('__main__.thing', first='one', second='two') - >>> mock_thing = patcher.start() - >>> mock_thing.first - 'one' - >>> mock_thing.second - 'two' - -As well as attributes on the created mock attributes, like the -:attr:`~Mock.return_value` and :attr:`~Mock.side_effect`, of child mocks can -also be configured. These aren't syntactically valid to pass in directly as -keyword arguments, but a dictionary with these as keys can still be expanded -into a :func:`patch` call using ``**``:: - - >>> config = {'method.return_value': 3, 'other.side_effect': KeyError} - >>> patcher = patch('__main__.thing', **config) - >>> mock_thing = patcher.start() - >>> mock_thing.method() - 3 - >>> mock_thing.other() - Traceback (most recent call last): - ... - KeyError - -By default, attempting to patch a function in a module (or a method or an -attribute in a class) that does not exist will fail with :exc:`AttributeError`:: - - >>> @patch('sys.non_existing_attribute', 42) - ... def test(): - ... assert sys.non_existing_attribute == 42 - ... - >>> test() - Traceback (most recent call last): - ... - AttributeError: does not have the attribute 'non_existing_attribute' - -but adding ``create=True`` in the call to :func:`patch` will make the previous example -work as expected:: - - >>> @patch('sys.non_existing_attribute', 42, create=True) - ... def test(mock_stdout): - ... assert sys.non_existing_attribute == 42 - ... - >>> test() - -.. versionchanged:: 3.8 - - :func:`patch` now returns an :class:`AsyncMock` if the target is an async function. - - -patch.object -~~~~~~~~~~~~ - -.. function:: patch.object(target, attribute, new=DEFAULT, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) - - patch the named member (*attribute*) on an object (*target*) with a mock - object. - - :func:`patch.object` can be used as a decorator, class decorator or a context - manager. Arguments *new*, *spec*, *create*, *spec_set*, *autospec* and - *new_callable* have the same meaning as for :func:`patch`. Like :func:`patch`, - :func:`patch.object` takes arbitrary keyword arguments for configuring the mock - object it creates. - - When used as a class decorator :func:`patch.object` honours ``patch.TEST_PREFIX`` - for choosing which methods to wrap. - -You can either call :func:`patch.object` with three arguments or two arguments. The -three argument form takes the object to be patched, the attribute name and the -object to replace the attribute with. - -When calling with the two argument form you omit the replacement object, and a -mock is created for you and passed in as an extra argument to the decorated -function: - - >>> @patch.object(SomeClass, 'class_method') - ... def test(mock_method): - ... SomeClass.class_method(3) - ... mock_method.assert_called_with(3) - ... - >>> test() - -*spec*, *create* and the other arguments to :func:`patch.object` have the same -meaning as they do for :func:`patch`. - - -patch.dict -~~~~~~~~~~ - -.. function:: patch.dict(in_dict, values=(), clear=False, **kwargs) - - Patch a dictionary, or dictionary like object, and restore the dictionary - to its original state after the test. - - *in_dict* can be a dictionary or a mapping like container. If it is a - mapping then it must at least support getting, setting and deleting items - plus iterating over keys. - - *in_dict* can also be a string specifying the name of the dictionary, which - will then be fetched by importing it. - - *values* can be a dictionary of values to set in the dictionary. *values* - can also be an iterable of ``(key, value)`` pairs. - - If *clear* is true then the dictionary will be cleared before the new - values are set. - - :func:`patch.dict` can also be called with arbitrary keyword arguments to set - values in the dictionary. - - .. versionchanged:: 3.8 - - :func:`patch.dict` now returns the patched dictionary when used as a context - manager. - -:func:`patch.dict` can be used as a context manager, decorator or class -decorator: - - >>> foo = {} - >>> @patch.dict(foo, {'newkey': 'newvalue'}) - ... def test(): - ... assert foo == {'newkey': 'newvalue'} - >>> test() - >>> assert foo == {} - -When used as a class decorator :func:`patch.dict` honours -``patch.TEST_PREFIX`` (default to ``'test'``) for choosing which methods to wrap: - - >>> import os - >>> import unittest - >>> from unittest.mock import patch - >>> @patch.dict('os.environ', {'newkey': 'newvalue'}) - ... class TestSample(unittest.TestCase): - ... def test_sample(self): - ... self.assertEqual(os.environ['newkey'], 'newvalue') - -If you want to use a different prefix for your test, you can inform the -patchers of the different prefix by setting ``patch.TEST_PREFIX``. For -more details about how to change the value of see :ref:`test-prefix`. - -:func:`patch.dict` can be used to add members to a dictionary, or simply let a test -change a dictionary, and ensure the dictionary is restored when the test -ends. - - >>> foo = {} - >>> with patch.dict(foo, {'newkey': 'newvalue'}) as patched_foo: - ... assert foo == {'newkey': 'newvalue'} - ... assert patched_foo == {'newkey': 'newvalue'} - ... # You can add, update or delete keys of foo (or patched_foo, it's the same dict) - ... patched_foo['spam'] = 'eggs' - ... - >>> assert foo == {} - >>> assert patched_foo == {} - - >>> import os - >>> with patch.dict('os.environ', {'newkey': 'newvalue'}): - ... print(os.environ['newkey']) - ... - newvalue - >>> assert 'newkey' not in os.environ - -Keywords can be used in the :func:`patch.dict` call to set values in the dictionary: - - >>> mymodule = MagicMock() - >>> mymodule.function.return_value = 'fish' - >>> with patch.dict('sys.modules', mymodule=mymodule): - ... import mymodule - ... mymodule.function('some', 'args') - ... - 'fish' - -:func:`patch.dict` can be used with dictionary like objects that aren't actually -dictionaries. At the very minimum they must support item getting, setting, -deleting and either iteration or membership test. This corresponds to the -magic methods :meth:`~object.__getitem__`, :meth:`__setitem__`, :meth:`__delitem__` and either -:meth:`__iter__` or :meth:`__contains__`. - - >>> class Container: - ... def __init__(self): - ... self.values = {} - ... def __getitem__(self, name): - ... return self.values[name] - ... def __setitem__(self, name, value): - ... self.values[name] = value - ... def __delitem__(self, name): - ... del self.values[name] - ... def __iter__(self): - ... return iter(self.values) - ... - >>> thing = Container() - >>> thing['one'] = 1 - >>> with patch.dict(thing, one=2, two=3): - ... assert thing['one'] == 2 - ... assert thing['two'] == 3 - ... - >>> assert thing['one'] == 1 - >>> assert list(thing) == ['one'] - - -patch.multiple -~~~~~~~~~~~~~~ - -.. function:: patch.multiple(target, spec=None, create=False, spec_set=None, autospec=None, new_callable=None, **kwargs) - - Perform multiple patches in a single call. It takes the object to be - patched (either as an object or a string to fetch the object by importing) - and keyword arguments for the patches:: - - with patch.multiple(settings, FIRST_PATCH='one', SECOND_PATCH='two'): - ... - - Use :data:`DEFAULT` as the value if you want :func:`patch.multiple` to create - mocks for you. In this case the created mocks are passed into a decorated - function by keyword, and a dictionary is returned when :func:`patch.multiple` is - used as a context manager. - - :func:`patch.multiple` can be used as a decorator, class decorator or a context - manager. The arguments *spec*, *spec_set*, *create*, *autospec* and - *new_callable* have the same meaning as for :func:`patch`. These arguments will - be applied to *all* patches done by :func:`patch.multiple`. - - When used as a class decorator :func:`patch.multiple` honours ``patch.TEST_PREFIX`` - for choosing which methods to wrap. - -If you want :func:`patch.multiple` to create mocks for you, then you can use -:data:`DEFAULT` as the value. If you use :func:`patch.multiple` as a decorator -then the created mocks are passed into the decorated function by keyword. :: - - >>> thing = object() - >>> other = object() - - >>> @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) - ... def test_function(thing, other): - ... assert isinstance(thing, MagicMock) - ... assert isinstance(other, MagicMock) - ... - >>> test_function() - -:func:`patch.multiple` can be nested with other ``patch`` decorators, but put arguments -passed by keyword *after* any of the standard arguments created by :func:`patch`:: - - >>> @patch('sys.exit') - ... @patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) - ... def test_function(mock_exit, other, thing): - ... assert 'other' in repr(other) - ... assert 'thing' in repr(thing) - ... assert 'exit' in repr(mock_exit) - ... - >>> test_function() - -If :func:`patch.multiple` is used as a context manager, the value returned by the -context manager is a dictionary where created mocks are keyed by name:: - - >>> with patch.multiple('__main__', thing=DEFAULT, other=DEFAULT) as values: - ... assert 'other' in repr(values['other']) - ... assert 'thing' in repr(values['thing']) - ... assert values['thing'] is thing - ... assert values['other'] is other - ... - - -.. _start-and-stop: - -patch methods: start and stop -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -All the patchers have :meth:`start` and :meth:`stop` methods. These make it simpler to do -patching in ``setUp`` methods or where you want to do multiple patches without -nesting decorators or with statements. - -To use them call :func:`patch`, :func:`patch.object` or :func:`patch.dict` as -normal and keep a reference to the returned ``patcher`` object. You can then -call :meth:`start` to put the patch in place and :meth:`stop` to undo it. - -If you are using :func:`patch` to create a mock for you then it will be returned by -the call to ``patcher.start``. :: - - >>> patcher = patch('package.module.ClassName') - >>> from package import module - >>> original = module.ClassName - >>> new_mock = patcher.start() - >>> assert module.ClassName is not original - >>> assert module.ClassName is new_mock - >>> patcher.stop() - >>> assert module.ClassName is original - >>> assert module.ClassName is not new_mock - - -A typical use case for this might be for doing multiple patches in the ``setUp`` -method of a :class:`TestCase`:: - - >>> class MyTest(unittest.TestCase): - ... def setUp(self): - ... self.patcher1 = patch('package.module.Class1') - ... self.patcher2 = patch('package.module.Class2') - ... self.MockClass1 = self.patcher1.start() - ... self.MockClass2 = self.patcher2.start() - ... - ... def tearDown(self): - ... self.patcher1.stop() - ... self.patcher2.stop() - ... - ... def test_something(self): - ... assert package.module.Class1 is self.MockClass1 - ... assert package.module.Class2 is self.MockClass2 - ... - >>> MyTest('test_something').run() - -.. caution:: - - If you use this technique you must ensure that the patching is "undone" by - calling ``stop``. This can be fiddlier than you might think, because if an - exception is raised in the ``setUp`` then ``tearDown`` is not called. - :meth:`unittest.TestCase.addCleanup` makes this easier:: - - >>> class MyTest(unittest.TestCase): - ... def setUp(self): - ... patcher = patch('package.module.Class') - ... self.MockClass = patcher.start() - ... self.addCleanup(patcher.stop) - ... - ... def test_something(self): - ... assert package.module.Class is self.MockClass - ... - - As an added bonus you no longer need to keep a reference to the ``patcher`` - object. - -It is also possible to stop all patches which have been started by using -:func:`patch.stopall`. - -.. function:: patch.stopall - - Stop all active patches. Only stops patches started with ``start``. - - -.. _patch-builtins: - -patch builtins -~~~~~~~~~~~~~~ -You can patch any builtins within a module. The following example patches -builtin :func:`ord`:: - - >>> @patch('__main__.ord') - ... def test(mock_ord): - ... mock_ord.return_value = 101 - ... print(ord('c')) - ... - >>> test() - 101 - - -.. _test-prefix: - -TEST_PREFIX -~~~~~~~~~~~ - -All of the patchers can be used as class decorators. When used in this way -they wrap every test method on the class. The patchers recognise methods that -start with ``'test'`` as being test methods. This is the same way that the -:class:`unittest.TestLoader` finds test methods by default. - -It is possible that you want to use a different prefix for your tests. You can -inform the patchers of the different prefix by setting ``patch.TEST_PREFIX``:: - - >>> patch.TEST_PREFIX = 'foo' - >>> value = 3 - >>> - >>> @patch('__main__.value', 'not three') - ... class Thing: - ... def foo_one(self): - ... print(value) - ... def foo_two(self): - ... print(value) - ... - >>> - >>> Thing().foo_one() - not three - >>> Thing().foo_two() - not three - >>> value - 3 - - -Nesting Patch Decorators -~~~~~~~~~~~~~~~~~~~~~~~~ - -If you want to perform multiple patches then you can simply stack up the -decorators. - -You can stack up multiple patch decorators using this pattern: - - >>> @patch.object(SomeClass, 'class_method') - ... @patch.object(SomeClass, 'static_method') - ... def test(mock1, mock2): - ... assert SomeClass.static_method is mock1 - ... assert SomeClass.class_method is mock2 - ... SomeClass.static_method('foo') - ... SomeClass.class_method('bar') - ... return mock1, mock2 - ... - >>> mock1, mock2 = test() - >>> mock1.assert_called_once_with('foo') - >>> mock2.assert_called_once_with('bar') - - -Note that the decorators are applied from the bottom upwards. This is the -standard way that Python applies decorators. The order of the created mocks -passed into your test function matches this order. - - -.. _where-to-patch: - -Where to patch -~~~~~~~~~~~~~~ - -:func:`patch` works by (temporarily) changing the object that a *name* points to with -another one. There can be many names pointing to any individual object, so -for patching to work you must ensure that you patch the name used by the system -under test. - -The basic principle is that you patch where an object is *looked up*, which -is not necessarily the same place as where it is defined. A couple of -examples will help to clarify this. - -Imagine we have a project that we want to test with the following structure:: - - a.py - -> Defines SomeClass - - b.py - -> from a import SomeClass - -> some_function instantiates SomeClass - -Now we want to test ``some_function`` but we want to mock out ``SomeClass`` using -:func:`patch`. The problem is that when we import module b, which we will have to -do then it imports ``SomeClass`` from module a. If we use :func:`patch` to mock out -``a.SomeClass`` then it will have no effect on our test; module b already has a -reference to the *real* ``SomeClass`` and it looks like our patching had no -effect. - -The key is to patch out ``SomeClass`` where it is used (or where it is looked up). -In this case ``some_function`` will actually look up ``SomeClass`` in module b, -where we have imported it. The patching should look like:: - - @patch('b.SomeClass') - -However, consider the alternative scenario where instead of ``from a import -SomeClass`` module b does ``import a`` and ``some_function`` uses ``a.SomeClass``. Both -of these import forms are common. In this case the class we want to patch is -being looked up in the module and so we have to patch ``a.SomeClass`` instead:: - - @patch('a.SomeClass') - - -Patching Descriptors and Proxy Objects -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Both patch_ and patch.object_ correctly patch and restore descriptors: class -methods, static methods and properties. You should patch these on the *class* -rather than an instance. They also work with *some* objects -that proxy attribute access, like the `django settings object -`_. - - -MagicMock and magic method support ----------------------------------- - -.. _magic-methods: - -Mocking Magic Methods -~~~~~~~~~~~~~~~~~~~~~ - -:class:`Mock` supports mocking the Python protocol methods, also known as -"magic methods". This allows mock objects to replace containers or other -objects that implement Python protocols. - -Because magic methods are looked up differently from normal methods [#]_, this -support has been specially implemented. This means that only specific magic -methods are supported. The supported list includes *almost* all of them. If -there are any missing that you need please let us know. - -You mock magic methods by setting the method you are interested in to a function -or a mock instance. If you are using a function then it *must* take ``self`` as -the first argument [#]_. - - >>> def __str__(self): - ... return 'fooble' - ... - >>> mock = Mock() - >>> mock.__str__ = __str__ - >>> str(mock) - 'fooble' - - >>> mock = Mock() - >>> mock.__str__ = Mock() - >>> mock.__str__.return_value = 'fooble' - >>> str(mock) - 'fooble' - - >>> mock = Mock() - >>> mock.__iter__ = Mock(return_value=iter([])) - >>> list(mock) - [] - -One use case for this is for mocking objects used as context managers in a -:keyword:`with` statement: - - >>> mock = Mock() - >>> mock.__enter__ = Mock(return_value='foo') - >>> mock.__exit__ = Mock(return_value=False) - >>> with mock as m: - ... assert m == 'foo' - ... - >>> mock.__enter__.assert_called_with() - >>> mock.__exit__.assert_called_with(None, None, None) - -Calls to magic methods do not appear in :attr:`~Mock.method_calls`, but they -are recorded in :attr:`~Mock.mock_calls`. - -.. note:: - - If you use the *spec* keyword argument to create a mock then attempting to - set a magic method that isn't in the spec will raise an :exc:`AttributeError`. - -The full list of supported magic methods is: - -* ``__hash__``, ``__sizeof__``, ``__repr__`` and ``__str__`` -* ``__dir__``, ``__format__`` and ``__subclasses__`` -* ``__round__``, ``__floor__``, ``__trunc__`` and ``__ceil__`` -* Comparisons: ``__lt__``, ``__gt__``, ``__le__``, ``__ge__``, - ``__eq__`` and ``__ne__`` -* Container methods: ``__getitem__``, ``__setitem__``, ``__delitem__``, - ``__contains__``, ``__len__``, ``__iter__``, ``__reversed__`` - and ``__missing__`` -* Context manager: ``__enter__``, ``__exit__``, ``__aenter__`` and ``__aexit__`` -* Unary numeric methods: ``__neg__``, ``__pos__`` and ``__invert__`` -* The numeric methods (including right hand and in-place variants): - ``__add__``, ``__sub__``, ``__mul__``, ``__matmul__``, ``__truediv__``, - ``__floordiv__``, ``__mod__``, ``__divmod__``, ``__lshift__``, - ``__rshift__``, ``__and__``, ``__xor__``, ``__or__``, and ``__pow__`` -* Numeric conversion methods: ``__complex__``, ``__int__``, ``__float__`` - and ``__index__`` -* Descriptor methods: ``__get__``, ``__set__`` and ``__delete__`` -* Pickling: ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, - ``__getnewargs__``, ``__getstate__`` and ``__setstate__`` -* File system path representation: ``__fspath__`` -* Asynchronous iteration methods: ``__aiter__`` and ``__anext__`` - -.. versionchanged:: 3.8 - Added support for :func:`os.PathLike.__fspath__`. - -.. versionchanged:: 3.8 - Added support for ``__aenter__``, ``__aexit__``, ``__aiter__`` and ``__anext__``. - - -The following methods exist but are *not* supported as they are either in use -by mock, can't be set dynamically, or can cause problems: - -* ``__getattr__``, ``__setattr__``, ``__init__`` and ``__new__`` -* ``__prepare__``, ``__instancecheck__``, ``__subclasscheck__``, ``__del__`` - - - -Magic Mock -~~~~~~~~~~ - -There are two ``MagicMock`` variants: :class:`MagicMock` and :class:`NonCallableMagicMock`. - - -.. class:: MagicMock(*args, **kw) - - ``MagicMock`` is a subclass of :class:`Mock` with default implementations - of most of the magic methods. You can use ``MagicMock`` without having to - configure the magic methods yourself. - - The constructor parameters have the same meaning as for :class:`Mock`. - - If you use the *spec* or *spec_set* arguments then *only* magic methods - that exist in the spec will be created. - - -.. class:: NonCallableMagicMock(*args, **kw) - - A non-callable version of :class:`MagicMock`. - - The constructor parameters have the same meaning as for - :class:`MagicMock`, with the exception of *return_value* and - *side_effect* which have no meaning on a non-callable mock. - -The magic methods are setup with :class:`MagicMock` objects, so you can configure them -and use them in the usual way: - - >>> mock = MagicMock() - >>> mock[3] = 'fish' - >>> mock.__setitem__.assert_called_with(3, 'fish') - >>> mock.__getitem__.return_value = 'result' - >>> mock[2] - 'result' - -By default many of the protocol methods are required to return objects of a -specific type. These methods are preconfigured with a default return value, so -that they can be used without you having to do anything if you aren't interested -in the return value. You can still *set* the return value manually if you want -to change the default. - -Methods and their defaults: - -* ``__lt__``: ``NotImplemented`` -* ``__gt__``: ``NotImplemented`` -* ``__le__``: ``NotImplemented`` -* ``__ge__``: ``NotImplemented`` -* ``__int__``: ``1`` -* ``__contains__``: ``False`` -* ``__len__``: ``0`` -* ``__iter__``: ``iter([])`` -* ``__exit__``: ``False`` -* ``__aexit__``: ``False`` -* ``__complex__``: ``1j`` -* ``__float__``: ``1.0`` -* ``__bool__``: ``True`` -* ``__index__``: ``1`` -* ``__hash__``: default hash for the mock -* ``__str__``: default str for the mock -* ``__sizeof__``: default sizeof for the mock - -For example: - - >>> mock = MagicMock() - >>> int(mock) - 1 - >>> len(mock) - 0 - >>> list(mock) - [] - >>> object() in mock - False - -The two equality methods, :meth:`__eq__` and :meth:`__ne__`, are special. -They do the default equality comparison on identity, using the -:attr:`~Mock.side_effect` attribute, unless you change their return value to -return something else:: - - >>> MagicMock() == 3 - False - >>> MagicMock() != 3 - True - >>> mock = MagicMock() - >>> mock.__eq__.return_value = True - >>> mock == 3 - True - -The return value of :meth:`MagicMock.__iter__` can be any iterable object and isn't -required to be an iterator: - - >>> mock = MagicMock() - >>> mock.__iter__.return_value = ['a', 'b', 'c'] - >>> list(mock) - ['a', 'b', 'c'] - >>> list(mock) - ['a', 'b', 'c'] - -If the return value *is* an iterator, then iterating over it once will consume -it and subsequent iterations will result in an empty list: - - >>> mock.__iter__.return_value = iter(['a', 'b', 'c']) - >>> list(mock) - ['a', 'b', 'c'] - >>> list(mock) - [] - -``MagicMock`` has all of the supported magic methods configured except for some -of the obscure and obsolete ones. You can still set these up if you want. - -Magic methods that are supported but not setup by default in ``MagicMock`` are: - -* ``__subclasses__`` -* ``__dir__`` -* ``__format__`` -* ``__get__``, ``__set__`` and ``__delete__`` -* ``__reversed__`` and ``__missing__`` -* ``__reduce__``, ``__reduce_ex__``, ``__getinitargs__``, ``__getnewargs__``, - ``__getstate__`` and ``__setstate__`` -* ``__getformat__`` - - - -.. [#] Magic methods *should* be looked up on the class rather than the - instance. Different versions of Python are inconsistent about applying this - rule. The supported protocol methods should work with all supported versions - of Python. -.. [#] The function is basically hooked up to the class, but each ``Mock`` - instance is kept isolated from the others. - - -Helpers -------- - -sentinel -~~~~~~~~ - -.. data:: sentinel - - The ``sentinel`` object provides a convenient way of providing unique - objects for your tests. - - Attributes are created on demand when you access them by name. Accessing - the same attribute will always return the same object. The objects - returned have a sensible repr so that test failure messages are readable. - - .. versionchanged:: 3.7 - The ``sentinel`` attributes now preserve their identity when they are - :mod:`copied ` or :mod:`pickled `. - -Sometimes when testing you need to test that a specific object is passed as an -argument to another method, or returned. It can be common to create named -sentinel objects to test this. :data:`sentinel` provides a convenient way of -creating and testing the identity of objects like this. - -In this example we monkey patch ``method`` to return ``sentinel.some_object``: - - >>> real = ProductionClass() - >>> real.method = Mock(name="method") - >>> real.method.return_value = sentinel.some_object - >>> result = real.method() - >>> assert result is sentinel.some_object - >>> result - sentinel.some_object - - -DEFAULT -~~~~~~~ - - -.. data:: DEFAULT - - The :data:`DEFAULT` object is a pre-created sentinel (actually - ``sentinel.DEFAULT``). It can be used by :attr:`~Mock.side_effect` - functions to indicate that the normal return value should be used. - - -call -~~~~ - -.. function:: call(*args, **kwargs) - - :func:`call` is a helper object for making simpler assertions, for comparing with - :attr:`~Mock.call_args`, :attr:`~Mock.call_args_list`, - :attr:`~Mock.mock_calls` and :attr:`~Mock.method_calls`. :func:`call` can also be - used with :meth:`~Mock.assert_has_calls`. - - >>> m = MagicMock(return_value=None) - >>> m(1, 2, a='foo', b='bar') - >>> m() - >>> m.call_args_list == [call(1, 2, a='foo', b='bar'), call()] - True - -.. method:: call.call_list() - - For a call object that represents multiple calls, :meth:`call_list` - returns a list of all the intermediate calls as well as the - final call. - -``call_list`` is particularly useful for making assertions on "chained calls". A -chained call is multiple calls on a single line of code. This results in -multiple entries in :attr:`~Mock.mock_calls` on a mock. Manually constructing -the sequence of calls can be tedious. - -:meth:`~call.call_list` can construct the sequence of calls from the same -chained call: - - >>> m = MagicMock() - >>> m(1).method(arg='foo').other('bar')(2.0) - - >>> kall = call(1).method(arg='foo').other('bar')(2.0) - >>> kall.call_list() - [call(1), - call().method(arg='foo'), - call().method().other('bar'), - call().method().other()(2.0)] - >>> m.mock_calls == kall.call_list() - True - -.. _calls-as-tuples: - -A ``call`` object is either a tuple of (positional args, keyword args) or -(name, positional args, keyword args) depending on how it was constructed. When -you construct them yourself this isn't particularly interesting, but the ``call`` -objects that are in the :attr:`Mock.call_args`, :attr:`Mock.call_args_list` and -:attr:`Mock.mock_calls` attributes can be introspected to get at the individual -arguments they contain. - -The ``call`` objects in :attr:`Mock.call_args` and :attr:`Mock.call_args_list` -are two-tuples of (positional args, keyword args) whereas the ``call`` objects -in :attr:`Mock.mock_calls`, along with ones you construct yourself, are -three-tuples of (name, positional args, keyword args). - -You can use their "tupleness" to pull out the individual arguments for more -complex introspection and assertions. The positional arguments are a tuple -(an empty tuple if there are no positional arguments) and the keyword -arguments are a dictionary: - - >>> m = MagicMock(return_value=None) - >>> m(1, 2, 3, arg='one', arg2='two') - >>> kall = m.call_args - >>> kall.args - (1, 2, 3) - >>> kall.kwargs - {'arg': 'one', 'arg2': 'two'} - >>> kall.args is kall[0] - True - >>> kall.kwargs is kall[1] - True - - >>> m = MagicMock() - >>> m.foo(4, 5, 6, arg='two', arg2='three') - - >>> kall = m.mock_calls[0] - >>> name, args, kwargs = kall - >>> name - 'foo' - >>> args - (4, 5, 6) - >>> kwargs - {'arg': 'two', 'arg2': 'three'} - >>> name is m.mock_calls[0][0] - True - - -create_autospec -~~~~~~~~~~~~~~~ - -.. function:: create_autospec(spec, spec_set=False, instance=False, **kwargs) - - Create a mock object using another object as a spec. Attributes on the - mock will use the corresponding attribute on the *spec* object as their - spec. - - Functions or methods being mocked will have their arguments checked to - ensure that they are called with the correct signature. - - If *spec_set* is ``True`` then attempting to set attributes that don't exist - on the spec object will raise an :exc:`AttributeError`. - - If a class is used as a spec then the return value of the mock (the - instance of the class) will have the same spec. You can use a class as the - spec for an instance object by passing ``instance=True``. The returned mock - will only be callable if instances of the mock are callable. - - :func:`create_autospec` also takes arbitrary keyword arguments that are passed to - the constructor of the created mock. - -See :ref:`auto-speccing` for examples of how to use auto-speccing with -:func:`create_autospec` and the *autospec* argument to :func:`patch`. - - -.. versionchanged:: 3.8 - - :func:`create_autospec` now returns an :class:`AsyncMock` if the target is - an async function. - - -ANY -~~~ - -.. data:: ANY - -Sometimes you may need to make assertions about *some* of the arguments in a -call to mock, but either not care about some of the arguments or want to pull -them individually out of :attr:`~Mock.call_args` and make more complex -assertions on them. - -To ignore certain arguments you can pass in objects that compare equal to -*everything*. Calls to :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with` will then succeed no matter what was -passed in. - - >>> mock = Mock(return_value=None) - >>> mock('foo', bar=object()) - >>> mock.assert_called_once_with('foo', bar=ANY) - -:data:`ANY` can also be used in comparisons with call lists like -:attr:`~Mock.mock_calls`: - - >>> m = MagicMock(return_value=None) - >>> m(1) - >>> m(1, 2) - >>> m(object()) - >>> m.mock_calls == [call(1), call(1, 2), ANY] - True - - - -FILTER_DIR -~~~~~~~~~~ - -.. data:: FILTER_DIR - -:data:`FILTER_DIR` is a module level variable that controls the way mock objects -respond to :func:`dir`. The default is ``True``, -which uses the filtering described below, to only show useful members. If you -dislike this filtering, or need to switch it off for diagnostic purposes, then -set ``mock.FILTER_DIR = False``. - -With filtering on, ``dir(some_mock)`` shows only useful attributes and will -include any dynamically created attributes that wouldn't normally be shown. -If the mock was created with a *spec* (or *autospec* of course) then all the -attributes from the original are shown, even if they haven't been accessed -yet: - -.. doctest:: - :options: +ELLIPSIS,+NORMALIZE_WHITESPACE - - >>> dir(Mock()) - ['assert_any_call', - 'assert_called', - 'assert_called_once', - 'assert_called_once_with', - 'assert_called_with', - 'assert_has_calls', - 'assert_not_called', - 'attach_mock', - ... - >>> from urllib import request - >>> dir(Mock(spec=request)) - ['AbstractBasicAuthHandler', - 'AbstractDigestAuthHandler', - 'AbstractHTTPHandler', - 'BaseHandler', - ... - -Many of the not-very-useful (private to :class:`Mock` rather than the thing being -mocked) underscore and double underscore prefixed attributes have been -filtered from the result of calling :func:`dir` on a :class:`Mock`. If you dislike this -behaviour you can switch it off by setting the module level switch -:data:`FILTER_DIR`: - -.. doctest:: - :options: +ELLIPSIS,+NORMALIZE_WHITESPACE - - >>> from unittest import mock - >>> mock.FILTER_DIR = False - >>> dir(mock.Mock()) - ['_NonCallableMock__get_return_value', - '_NonCallableMock__get_side_effect', - '_NonCallableMock__return_value_doc', - '_NonCallableMock__set_return_value', - '_NonCallableMock__set_side_effect', - '__call__', - '__class__', - ... - -Alternatively you can just use ``vars(my_mock)`` (instance members) and -``dir(type(my_mock))`` (type members) to bypass the filtering irrespective of -:const:`mock.FILTER_DIR`. - - -mock_open -~~~~~~~~~ - -.. function:: mock_open(mock=None, read_data=None) - - A helper function to create a mock to replace the use of :func:`open`. It works - for :func:`open` called directly or used as a context manager. - - The *mock* argument is the mock object to configure. If ``None`` (the - default) then a :class:`MagicMock` will be created for you, with the API limited - to methods or attributes available on standard file handles. - - *read_data* is a string for the :meth:`~io.IOBase.read`, - :meth:`~io.IOBase.readline`, and :meth:`~io.IOBase.readlines` methods - of the file handle to return. Calls to those methods will take data from - *read_data* until it is depleted. The mock of these methods is pretty - simplistic: every time the *mock* is called, the *read_data* is rewound to - the start. If you need more control over the data that you are feeding to - the tested code you will need to customize this mock for yourself. When that - is insufficient, one of the in-memory filesystem packages on `PyPI - `_ can offer a realistic filesystem for testing. - - .. versionchanged:: 3.4 - Added :meth:`~io.IOBase.readline` and :meth:`~io.IOBase.readlines` support. - The mock of :meth:`~io.IOBase.read` changed to consume *read_data* rather - than returning it on each call. - - .. versionchanged:: 3.5 - *read_data* is now reset on each call to the *mock*. - - .. versionchanged:: 3.8 - Added :meth:`__iter__` to implementation so that iteration (such as in for - loops) correctly consumes *read_data*. - -Using :func:`open` as a context manager is a great way to ensure your file handles -are closed properly and is becoming common:: - - with open('/some/path', 'w') as f: - f.write('something') - -The issue is that even if you mock out the call to :func:`open` it is the -*returned object* that is used as a context manager (and has :meth:`~object.__enter__` and -:meth:`~object.__exit__` called). - -Mocking context managers with a :class:`MagicMock` is common enough and fiddly -enough that a helper function is useful. :: - - >>> m = mock_open() - >>> with patch('__main__.open', m): - ... with open('foo', 'w') as h: - ... h.write('some stuff') - ... - >>> m.mock_calls - [call('foo', 'w'), - call().__enter__(), - call().write('some stuff'), - call().__exit__(None, None, None)] - >>> m.assert_called_once_with('foo', 'w') - >>> handle = m() - >>> handle.write.assert_called_once_with('some stuff') - -And for reading files:: - - >>> with patch('__main__.open', mock_open(read_data='bibble')) as m: - ... with open('foo') as h: - ... result = h.read() - ... - >>> m.assert_called_once_with('foo') - >>> assert result == 'bibble' - - -.. _auto-speccing: - -Autospeccing -~~~~~~~~~~~~ - -Autospeccing is based on the existing :attr:`spec` feature of mock. It limits the -api of mocks to the api of an original object (the spec), but it is recursive -(implemented lazily) so that attributes of mocks only have the same api as -the attributes of the spec. In addition mocked functions / methods have the -same call signature as the original so they raise a :exc:`TypeError` if they are -called incorrectly. - -Before I explain how auto-speccing works, here's why it is needed. - -:class:`Mock` is a very powerful and flexible object, but it suffers from two flaws -when used to mock out objects from a system under test. One of these flaws is -specific to the :class:`Mock` api and the other is a more general problem with using -mock objects. - -First the problem specific to :class:`Mock`. :class:`Mock` has two assert methods that are -extremely handy: :meth:`~Mock.assert_called_with` and -:meth:`~Mock.assert_called_once_with`. - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - >>> mock(1, 2, 3) - >>> mock.assert_called_once_with(1, 2, 3) - Traceback (most recent call last): - ... - AssertionError: Expected 'mock' to be called once. Called 2 times. - -Because mocks auto-create attributes on demand, and allow you to call them -with arbitrary arguments, if you misspell one of these assert methods then -your assertion is gone: - -.. code-block:: pycon - - >>> mock = Mock(name='Thing', return_value=None) - >>> mock(1, 2, 3) - >>> mock.assret_called_once_with(4, 5, 6) # Intentional typo! - -Your tests can pass silently and incorrectly because of the typo. - -The second issue is more general to mocking. If you refactor some of your -code, rename members and so on, any tests for code that is still using the -*old api* but uses mocks instead of the real objects will still pass. This -means your tests can all pass even though your code is broken. - -Note that this is another reason why you need integration tests as well as -unit tests. Testing everything in isolation is all fine and dandy, but if you -don't test how your units are "wired together" there is still lots of room -for bugs that tests might have caught. - -:mod:`mock` already provides a feature to help with this, called speccing. If you -use a class or instance as the :attr:`spec` for a mock then you can only access -attributes on the mock that exist on the real class: - - >>> from urllib import request - >>> mock = Mock(spec=request.Request) - >>> mock.assret_called_with # Intentional typo! - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'assret_called_with' - -The spec only applies to the mock itself, so we still have the same issue -with any methods on the mock: - -.. code-block:: pycon - - >>> mock.has_data() - - >>> mock.has_data.assret_called_with() # Intentional typo! - -Auto-speccing solves this problem. You can either pass ``autospec=True`` to -:func:`patch` / :func:`patch.object` or use the :func:`create_autospec` function to create a -mock with a spec. If you use the ``autospec=True`` argument to :func:`patch` then the -object that is being replaced will be used as the spec object. Because the -speccing is done "lazily" (the spec is created as attributes on the mock are -accessed) you can use it with very complex or deeply nested objects (like -modules that import modules that import modules) without a big performance -hit. - -Here's an example of it in use:: - - >>> from urllib import request - >>> patcher = patch('__main__.request', autospec=True) - >>> mock_request = patcher.start() - >>> request is mock_request - True - >>> mock_request.Request - - -You can see that :class:`request.Request` has a spec. :class:`request.Request` takes two -arguments in the constructor (one of which is *self*). Here's what happens if -we try to call it incorrectly:: - - >>> req = request.Request() - Traceback (most recent call last): - ... - TypeError: () takes at least 2 arguments (1 given) - -The spec also applies to instantiated classes (i.e. the return value of -specced mocks):: - - >>> req = request.Request('foo') - >>> req - - -:class:`Request` objects are not callable, so the return value of instantiating our -mocked out :class:`request.Request` is a non-callable mock. With the spec in place -any typos in our asserts will raise the correct error:: - - >>> req.add_header('spam', 'eggs') - - >>> req.add_header.assret_called_with # Intentional typo! - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'assret_called_with' - >>> req.add_header.assert_called_with('spam', 'eggs') - -In many cases you will just be able to add ``autospec=True`` to your existing -:func:`patch` calls and then be protected against bugs due to typos and api -changes. - -As well as using *autospec* through :func:`patch` there is a -:func:`create_autospec` for creating autospecced mocks directly: - - >>> from urllib import request - >>> mock_request = create_autospec(request) - >>> mock_request.Request('foo', 'bar') - - -This isn't without caveats and limitations however, which is why it is not -the default behaviour. In order to know what attributes are available on the -spec object, autospec has to introspect (access attributes) the spec. As you -traverse attributes on the mock a corresponding traversal of the original -object is happening under the hood. If any of your specced objects have -properties or descriptors that can trigger code execution then you may not be -able to use autospec. On the other hand it is much better to design your -objects so that introspection is safe [#]_. - -A more serious problem is that it is common for instance attributes to be -created in the :meth:`__init__` method and not to exist on the class at all. -*autospec* can't know about any dynamically created attributes and restricts -the api to visible attributes. :: - - >>> class Something: - ... def __init__(self): - ... self.a = 33 - ... - >>> with patch('__main__.Something', autospec=True): - ... thing = Something() - ... thing.a - ... - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'a' - -There are a few different ways of resolving this problem. The easiest, but -not necessarily the least annoying, way is to simply set the required -attributes on the mock after creation. Just because *autospec* doesn't allow -you to fetch attributes that don't exist on the spec it doesn't prevent you -setting them:: - - >>> with patch('__main__.Something', autospec=True): - ... thing = Something() - ... thing.a = 33 - ... - -There is a more aggressive version of both *spec* and *autospec* that *does* -prevent you setting non-existent attributes. This is useful if you want to -ensure your code only *sets* valid attributes too, but obviously it prevents -this particular scenario: - - >>> with patch('__main__.Something', autospec=True, spec_set=True): - ... thing = Something() - ... thing.a = 33 - ... - Traceback (most recent call last): - ... - AttributeError: Mock object has no attribute 'a' - -Probably the best way of solving the problem is to add class attributes as -default values for instance members initialised in :meth:`__init__`. Note that if -you are only setting default attributes in :meth:`__init__` then providing them via -class attributes (shared between instances of course) is faster too. e.g. - -.. code-block:: python - - class Something: - a = 33 - -This brings up another issue. It is relatively common to provide a default -value of ``None`` for members that will later be an object of a different type. -``None`` would be useless as a spec because it wouldn't let you access *any* -attributes or methods on it. As ``None`` is *never* going to be useful as a -spec, and probably indicates a member that will normally of some other type, -autospec doesn't use a spec for members that are set to ``None``. These will -just be ordinary mocks (well - MagicMocks): - - >>> class Something: - ... member = None - ... - >>> mock = create_autospec(Something) - >>> mock.member.foo.bar.baz() - - -If modifying your production classes to add defaults isn't to your liking -then there are more options. One of these is simply to use an instance as the -spec rather than the class. The other is to create a subclass of the -production class and add the defaults to the subclass without affecting the -production class. Both of these require you to use an alternative object as -the spec. Thankfully :func:`patch` supports this - you can simply pass the -alternative object as the *autospec* argument:: - - >>> class Something: - ... def __init__(self): - ... self.a = 33 - ... - >>> class SomethingForTest(Something): - ... a = 33 - ... - >>> p = patch('__main__.Something', autospec=SomethingForTest) - >>> mock = p.start() - >>> mock.a - - - -.. [#] This only applies to classes or already instantiated objects. Calling - a mocked class to create a mock instance *does not* create a real instance. - It is only attribute lookups - along with calls to :func:`dir` - that are done. - -Sealing mocks -~~~~~~~~~~~~~ - - -.. testsetup:: - - from unittest.mock import seal - -.. function:: seal(mock) - - Seal will disable the automatic creation of mocks when accessing an attribute of - the mock being sealed or any of its attributes that are already mocks recursively. - - If a mock instance with a name or a spec is assigned to an attribute - it won't be considered in the sealing chain. This allows one to prevent seal from - fixing part of the mock object. :: - - >>> mock = Mock() - >>> mock.submock.attribute1 = 2 - >>> mock.not_submock = mock.Mock(name="sample_name") - >>> seal(mock) - >>> mock.new_attribute # This will raise AttributeError. - >>> mock.submock.attribute2 # This will raise AttributeError. - >>> mock.not_submock.attribute2 # This won't raise. - - .. versionadded:: 3.7 diff --git a/Doc/library/unittest.rst b/Doc/library/unittest.rst deleted file mode 100644 index c5826698bd35f5..00000000000000 --- a/Doc/library/unittest.rst +++ /dev/null @@ -1,2582 +0,0 @@ -:mod:`unittest` --- Unit testing framework -========================================== - -.. module:: unittest - :synopsis: Unit testing framework for Python. - -.. moduleauthor:: Steve Purcell -.. sectionauthor:: Steve Purcell -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Raymond Hettinger - -**Source code:** :source:`Lib/unittest/__init__.py` - --------------- - -(If you are already familiar with the basic concepts of testing, you might want -to skip to :ref:`the list of assert methods `.) - -The :mod:`unittest` unit testing framework was originally inspired by JUnit -and has a similar flavor as major unit testing frameworks in other -languages. It supports test automation, sharing of setup and shutdown code -for tests, aggregation of tests into collections, and independence of the -tests from the reporting framework. - -To achieve this, :mod:`unittest` supports some important concepts in an -object-oriented way: - -test fixture - A :dfn:`test fixture` represents the preparation needed to perform one or more - tests, and any associated cleanup actions. This may involve, for example, - creating temporary or proxy databases, directories, or starting a server - process. - -test case - A :dfn:`test case` is the individual unit of testing. It checks for a specific - response to a particular set of inputs. :mod:`unittest` provides a base class, - :class:`TestCase`, which may be used to create new test cases. - -test suite - A :dfn:`test suite` is a collection of test cases, test suites, or both. It is - used to aggregate tests that should be executed together. - -test runner - A :dfn:`test runner` is a component which orchestrates the execution of tests - and provides the outcome to the user. The runner may use a graphical interface, - a textual interface, or return a special value to indicate the results of - executing the tests. - - -.. seealso:: - - Module :mod:`doctest` - Another test-support module with a very different flavor. - - `Simple Smalltalk Testing: With Patterns `_ - Kent Beck's original paper on testing frameworks using the pattern shared - by :mod:`unittest`. - - `pytest `_ - Third-party unittest framework with a lighter-weight syntax for writing - tests. For example, ``assert func(10) == 42``. - - `The Python Testing Tools Taxonomy `_ - An extensive list of Python testing tools including functional testing - frameworks and mock object libraries. - - `Testing in Python Mailing List `_ - A special-interest-group for discussion of testing, and testing tools, - in Python. - - The script :file:`Tools/unittestgui/unittestgui.py` in the Python source distribution is - a GUI tool for test discovery and execution. This is intended largely for ease of use - for those new to unit testing. For production environments it is - recommended that tests be driven by a continuous integration system such as - `Buildbot `_, `Jenkins `_, - `GitHub Actions `_, or - `AppVeyor `_. - - -.. _unittest-minimal-example: - -Basic example -------------- - -The :mod:`unittest` module provides a rich set of tools for constructing and -running tests. This section demonstrates that a small subset of the tools -suffice to meet the needs of most users. - -Here is a short script to test three string methods:: - - import unittest - - class TestStringMethods(unittest.TestCase): - - def test_upper(self): - self.assertEqual('foo'.upper(), 'FOO') - - def test_isupper(self): - self.assertTrue('FOO'.isupper()) - self.assertFalse('Foo'.isupper()) - - def test_split(self): - s = 'hello world' - self.assertEqual(s.split(), ['hello', 'world']) - # check that s.split fails when the separator is not a string - with self.assertRaises(TypeError): - s.split(2) - - if __name__ == '__main__': - unittest.main() - - -A testcase is created by subclassing :class:`unittest.TestCase`. The three -individual tests are defined with methods whose names start with the letters -``test``. This naming convention informs the test runner about which methods -represent tests. - -The crux of each test is a call to :meth:`~TestCase.assertEqual` to check for an -expected result; :meth:`~TestCase.assertTrue` or :meth:`~TestCase.assertFalse` -to verify a condition; or :meth:`~TestCase.assertRaises` to verify that a -specific exception gets raised. These methods are used instead of the -:keyword:`assert` statement so the test runner can accumulate all test results -and produce a report. - -The :meth:`~TestCase.setUp` and :meth:`~TestCase.tearDown` methods allow you -to define instructions that will be executed before and after each test method. -They are covered in more detail in the section :ref:`organizing-tests`. - -The final block shows a simple way to run the tests. :func:`unittest.main` -provides a command-line interface to the test script. When run from the command -line, the above script produces an output that looks like this:: - - ... - ---------------------------------------------------------------------- - Ran 3 tests in 0.000s - - OK - -Passing the ``-v`` option to your test script will instruct :func:`unittest.main` -to enable a higher level of verbosity, and produce the following output:: - - test_isupper (__main__.TestStringMethods.test_isupper) ... ok - test_split (__main__.TestStringMethods.test_split) ... ok - test_upper (__main__.TestStringMethods.test_upper) ... ok - - ---------------------------------------------------------------------- - Ran 3 tests in 0.001s - - OK - -The above examples show the most commonly used :mod:`unittest` features which -are sufficient to meet many everyday testing needs. The remainder of the -documentation explores the full feature set from first principles. - -.. versionchanged:: 3.11 - The behavior of returning a value from a test method (other than the default - ``None`` value), is now deprecated. - - -.. _unittest-command-line-interface: - -Command-Line Interface ----------------------- - -The unittest module can be used from the command line to run tests from -modules, classes or even individual test methods:: - - python -m unittest test_module1 test_module2 - python -m unittest test_module.TestClass - python -m unittest test_module.TestClass.test_method - -You can pass in a list with any combination of module names, and fully -qualified class or method names. - -Test modules can be specified by file path as well:: - - python -m unittest tests/test_something.py - -This allows you to use the shell filename completion to specify the test module. -The file specified must still be importable as a module. The path is converted -to a module name by removing the '.py' and converting path separators into '.'. -If you want to execute a test file that isn't importable as a module you should -execute the file directly instead. - -You can run tests with more detail (higher verbosity) by passing in the -v flag:: - - python -m unittest -v test_module - -When executed without arguments :ref:`unittest-test-discovery` is started:: - - python -m unittest - -For a list of all the command-line options:: - - python -m unittest -h - -.. versionchanged:: 3.2 - In earlier versions it was only possible to run individual test methods and - not modules or classes. - - -Command-line options -~~~~~~~~~~~~~~~~~~~~ - -:program:`unittest` supports these command-line options: - -.. program:: unittest - -.. option:: -b, --buffer - - The standard output and standard error streams are buffered during the test - run. Output during a passing test is discarded. Output is echoed normally - on test fail or error and is added to the failure messages. - -.. option:: -c, --catch - - :kbd:`Control-C` during the test run waits for the current test to end and then - reports all the results so far. A second :kbd:`Control-C` raises the normal - :exc:`KeyboardInterrupt` exception. - - See `Signal Handling`_ for the functions that provide this functionality. - -.. option:: -f, --failfast - - Stop the test run on the first error or failure. - -.. option:: -k - - Only run test methods and classes that match the pattern or substring. - This option may be used multiple times, in which case all test cases that - match any of the given patterns are included. - - Patterns that contain a wildcard character (``*``) are matched against the - test name using :meth:`fnmatch.fnmatchcase`; otherwise simple case-sensitive - substring matching is used. - - Patterns are matched against the fully qualified test method name as - imported by the test loader. - - For example, ``-k foo`` matches ``foo_tests.SomeTest.test_something``, - ``bar_tests.SomeTest.test_foo``, but not ``bar_tests.FooTest.test_something``. - -.. option:: --locals - - Show local variables in tracebacks. - -.. versionadded:: 3.2 - The command-line options ``-b``, ``-c`` and ``-f`` were added. - -.. versionadded:: 3.5 - The command-line option ``--locals``. - -.. versionadded:: 3.7 - The command-line option ``-k``. - -The command line can also be used for test discovery, for running all of the -tests in a project or just a subset. - - -.. _unittest-test-discovery: - -Test Discovery --------------- - -.. versionadded:: 3.2 - -Unittest supports simple test discovery. In order to be compatible with test -discovery, all of the test files must be :ref:`modules ` or -:ref:`packages ` importable from the top-level directory of -the project (this means that their filenames must be valid :ref:`identifiers -`). - -Test discovery is implemented in :meth:`TestLoader.discover`, but can also be -used from the command line. The basic command-line usage is:: - - cd project_directory - python -m unittest discover - -.. note:: - - As a shortcut, ``python -m unittest`` is the equivalent of - ``python -m unittest discover``. If you want to pass arguments to test - discovery the ``discover`` sub-command must be used explicitly. - -The ``discover`` sub-command has the following options: - -.. program:: unittest discover - -.. option:: -v, --verbose - - Verbose output - -.. option:: -s, --start-directory directory - - Directory to start discovery (``.`` default) - -.. option:: -p, --pattern pattern - - Pattern to match test files (``test*.py`` default) - -.. option:: -t, --top-level-directory directory - - Top level directory of project (defaults to start directory) - -The :option:`-s`, :option:`-p`, and :option:`-t` options can be passed in -as positional arguments in that order. The following two command lines -are equivalent:: - - python -m unittest discover -s project_directory -p "*_test.py" - python -m unittest discover project_directory "*_test.py" - -As well as being a path it is possible to pass a package name, for example -``myproject.subpackage.test``, as the start directory. The package name you -supply will then be imported and its location on the filesystem will be used -as the start directory. - -.. caution:: - - Test discovery loads tests by importing them. Once test discovery has found - all the test files from the start directory you specify it turns the paths - into package names to import. For example :file:`foo/bar/baz.py` will be - imported as ``foo.bar.baz``. - - If you have a package installed globally and attempt test discovery on - a different copy of the package then the import *could* happen from the - wrong place. If this happens test discovery will warn you and exit. - - If you supply the start directory as a package name rather than a - path to a directory then discover assumes that whichever location it - imports from is the location you intended, so you will not get the - warning. - -Test modules and packages can customize test loading and discovery by through -the `load_tests protocol`_. - -.. versionchanged:: 3.4 - Test discovery supports :term:`namespace packages ` - for the start directory. Note that you need to specify the top level - directory too (e.g. - ``python -m unittest discover -s root/namespace -t root``). - -.. versionchanged:: 3.11 - Python 3.11 dropped the :term:`namespace packages ` - support. It has been broken since Python 3.7. Start directory and - subdirectories containing tests must be regular package that have - ``__init__.py`` file. - - Directories containing start directory still can be a namespace package. - In this case, you need to specify start directory as dotted package name, - and target directory explicitly. For example:: - - # proj/ <-- current directory - # namespace/ - # mypkg/ - # __init__.py - # test_mypkg.py - - python -m unittest discover -s namespace.mypkg -t . - - -.. _organizing-tests: - -Organizing test code --------------------- - -The basic building blocks of unit testing are :dfn:`test cases` --- single -scenarios that must be set up and checked for correctness. In :mod:`unittest`, -test cases are represented by :class:`unittest.TestCase` instances. -To make your own test cases you must write subclasses of -:class:`TestCase` or use :class:`FunctionTestCase`. - -The testing code of a :class:`TestCase` instance should be entirely self -contained, such that it can be run either in isolation or in arbitrary -combination with any number of other test cases. - -The simplest :class:`TestCase` subclass will simply implement a test method -(i.e. a method whose name starts with ``test``) in order to perform specific -testing code:: - - import unittest - - class DefaultWidgetSizeTestCase(unittest.TestCase): - def test_default_widget_size(self): - widget = Widget('The widget') - self.assertEqual(widget.size(), (50, 50)) - -Note that in order to test something, we use one of the :meth:`assert\*` -methods provided by the :class:`TestCase` base class. If the test fails, an -exception will be raised with an explanatory message, and :mod:`unittest` -will identify the test case as a :dfn:`failure`. Any other exceptions will be -treated as :dfn:`errors`. - -Tests can be numerous, and their set-up can be repetitive. Luckily, we -can factor out set-up code by implementing a method called -:meth:`~TestCase.setUp`, which the testing framework will automatically -call for every single test we run:: - - import unittest - - class WidgetTestCase(unittest.TestCase): - def setUp(self): - self.widget = Widget('The widget') - - def test_default_widget_size(self): - self.assertEqual(self.widget.size(), (50,50), - 'incorrect default size') - - def test_widget_resize(self): - self.widget.resize(100,150) - self.assertEqual(self.widget.size(), (100,150), - 'wrong size after resize') - -.. note:: - The order in which the various tests will be run is determined - by sorting the test method names with respect to the built-in - ordering for strings. - -If the :meth:`~TestCase.setUp` method raises an exception while the test is -running, the framework will consider the test to have suffered an error, and -the test method will not be executed. - -Similarly, we can provide a :meth:`~TestCase.tearDown` method that tidies up -after the test method has been run:: - - import unittest - - class WidgetTestCase(unittest.TestCase): - def setUp(self): - self.widget = Widget('The widget') - - def tearDown(self): - self.widget.dispose() - -If :meth:`~TestCase.setUp` succeeded, :meth:`~TestCase.tearDown` will be -run whether the test method succeeded or not. - -Such a working environment for the testing code is called a -:dfn:`test fixture`. A new TestCase instance is created as a unique -test fixture used to execute each individual test method. Thus -:meth:`~TestCase.setUp`, :meth:`~TestCase.tearDown`, and :meth:`~TestCase.__init__` -will be called once per test. - -It is recommended that you use TestCase implementations to group tests together -according to the features they test. :mod:`unittest` provides a mechanism for -this: the :dfn:`test suite`, represented by :mod:`unittest`'s -:class:`TestSuite` class. In most cases, calling :func:`unittest.main` will do -the right thing and collect all the module's test cases for you and execute -them. - -However, should you want to customize the building of your test suite, -you can do it yourself:: - - def suite(): - suite = unittest.TestSuite() - suite.addTest(WidgetTestCase('test_default_widget_size')) - suite.addTest(WidgetTestCase('test_widget_resize')) - return suite - - if __name__ == '__main__': - runner = unittest.TextTestRunner() - runner.run(suite()) - -You can place the definitions of test cases and test suites in the same modules -as the code they are to test (such as :file:`widget.py`), but there are several -advantages to placing the test code in a separate module, such as -:file:`test_widget.py`: - -* The test module can be run standalone from the command line. - -* The test code can more easily be separated from shipped code. - -* There is less temptation to change test code to fit the code it tests without - a good reason. - -* Test code should be modified much less frequently than the code it tests. - -* Tested code can be refactored more easily. - -* Tests for modules written in C must be in separate modules anyway, so why not - be consistent? - -* If the testing strategy changes, there is no need to change the source code. - - -.. _legacy-unit-tests: - -Re-using old test code ----------------------- - -Some users will find that they have existing test code that they would like to -run from :mod:`unittest`, without converting every old test function to a -:class:`TestCase` subclass. - -For this reason, :mod:`unittest` provides a :class:`FunctionTestCase` class. -This subclass of :class:`TestCase` can be used to wrap an existing test -function. Set-up and tear-down functions can also be provided. - -Given the following test function:: - - def testSomething(): - something = makeSomething() - assert something.name is not None - # ... - -one can create an equivalent test case instance as follows, with optional -set-up and tear-down methods:: - - testcase = unittest.FunctionTestCase(testSomething, - setUp=makeSomethingDB, - tearDown=deleteSomethingDB) - -.. note:: - - Even though :class:`FunctionTestCase` can be used to quickly convert an - existing test base over to a :mod:`unittest`\ -based system, this approach is - not recommended. Taking the time to set up proper :class:`TestCase` - subclasses will make future test refactorings infinitely easier. - -In some cases, the existing tests may have been written using the :mod:`doctest` -module. If so, :mod:`doctest` provides a :class:`DocTestSuite` class that can -automatically build :class:`unittest.TestSuite` instances from the existing -:mod:`doctest`\ -based tests. - - -.. _unittest-skipping: - -Skipping tests and expected failures ------------------------------------- - -.. versionadded:: 3.1 - -Unittest supports skipping individual test methods and even whole classes of -tests. In addition, it supports marking a test as an "expected failure," a test -that is broken and will fail, but shouldn't be counted as a failure on a -:class:`TestResult`. - -Skipping a test is simply a matter of using the :func:`skip` :term:`decorator` -or one of its conditional variants, calling :meth:`TestCase.skipTest` within a -:meth:`~TestCase.setUp` or test method, or raising :exc:`SkipTest` directly. - -Basic skipping looks like this:: - - class MyTestCase(unittest.TestCase): - - @unittest.skip("demonstrating skipping") - def test_nothing(self): - self.fail("shouldn't happen") - - @unittest.skipIf(mylib.__version__ < (1, 3), - "not supported in this library version") - def test_format(self): - # Tests that work for only a certain version of the library. - pass - - @unittest.skipUnless(sys.platform.startswith("win"), "requires Windows") - def test_windows_support(self): - # windows specific testing code - pass - - def test_maybe_skipped(self): - if not external_resource_available(): - self.skipTest("external resource not available") - # test code that depends on the external resource - pass - -This is the output of running the example above in verbose mode:: - - test_format (__main__.MyTestCase.test_format) ... skipped 'not supported in this library version' - test_nothing (__main__.MyTestCase.test_nothing) ... skipped 'demonstrating skipping' - test_maybe_skipped (__main__.MyTestCase.test_maybe_skipped) ... skipped 'external resource not available' - test_windows_support (__main__.MyTestCase.test_windows_support) ... skipped 'requires Windows' - - ---------------------------------------------------------------------- - Ran 4 tests in 0.005s - - OK (skipped=4) - -Classes can be skipped just like methods:: - - @unittest.skip("showing class skipping") - class MySkippedTestCase(unittest.TestCase): - def test_not_run(self): - pass - -:meth:`TestCase.setUp` can also skip the test. This is useful when a resource -that needs to be set up is not available. - -Expected failures use the :func:`expectedFailure` decorator. :: - - class ExpectedFailureTestCase(unittest.TestCase): - @unittest.expectedFailure - def test_fail(self): - self.assertEqual(1, 0, "broken") - -It's easy to roll your own skipping decorators by making a decorator that calls -:func:`skip` on the test when it wants it to be skipped. This decorator skips -the test unless the passed object has a certain attribute:: - - def skipUnlessHasattr(obj, attr): - if hasattr(obj, attr): - return lambda func: func - return unittest.skip("{!r} doesn't have {!r}".format(obj, attr)) - -The following decorators and exception implement test skipping and expected failures: - -.. decorator:: skip(reason) - - Unconditionally skip the decorated test. *reason* should describe why the - test is being skipped. - -.. decorator:: skipIf(condition, reason) - - Skip the decorated test if *condition* is true. - -.. decorator:: skipUnless(condition, reason) - - Skip the decorated test unless *condition* is true. - -.. decorator:: expectedFailure - - Mark the test as an expected failure or error. If the test fails or errors - in the test function itself (rather than in one of the :dfn:`test fixture` - methods) then it will be considered a success. If the test passes, it will - be considered a failure. - -.. exception:: SkipTest(reason) - - This exception is raised to skip a test. - - Usually you can use :meth:`TestCase.skipTest` or one of the skipping - decorators instead of raising this directly. - -Skipped tests will not have :meth:`~TestCase.setUp` or :meth:`~TestCase.tearDown` run around them. -Skipped classes will not have :meth:`~TestCase.setUpClass` or :meth:`~TestCase.tearDownClass` run. -Skipped modules will not have :func:`setUpModule` or :func:`tearDownModule` run. - - -.. _subtests: - -Distinguishing test iterations using subtests ---------------------------------------------- - -.. versionadded:: 3.4 - -When there are very small differences among your tests, for -instance some parameters, unittest allows you to distinguish them inside -the body of a test method using the :meth:`~TestCase.subTest` context manager. - -For example, the following test:: - - class NumbersTest(unittest.TestCase): - - def test_even(self): - """ - Test that numbers between 0 and 5 are all even. - """ - for i in range(0, 6): - with self.subTest(i=i): - self.assertEqual(i % 2, 0) - -will produce the following output:: - - ====================================================================== - FAIL: test_even (__main__.NumbersTest.test_even) (i=1) - Test that numbers between 0 and 5 are all even. - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 11, in test_even - self.assertEqual(i % 2, 0) - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - AssertionError: 1 != 0 - - ====================================================================== - FAIL: test_even (__main__.NumbersTest.test_even) (i=3) - Test that numbers between 0 and 5 are all even. - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 11, in test_even - self.assertEqual(i % 2, 0) - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - AssertionError: 1 != 0 - - ====================================================================== - FAIL: test_even (__main__.NumbersTest.test_even) (i=5) - Test that numbers between 0 and 5 are all even. - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 11, in test_even - self.assertEqual(i % 2, 0) - ^^^^^^^^^^^^^^^^^^^^^^^^^^ - AssertionError: 1 != 0 - -Without using a subtest, execution would stop after the first failure, -and the error would be less easy to diagnose because the value of ``i`` -wouldn't be displayed:: - - ====================================================================== - FAIL: test_even (__main__.NumbersTest.test_even) - ---------------------------------------------------------------------- - Traceback (most recent call last): - File "subtests.py", line 32, in test_even - self.assertEqual(i % 2, 0) - AssertionError: 1 != 0 - - -.. _unittest-contents: - -Classes and functions ---------------------- - -This section describes in depth the API of :mod:`unittest`. - - -.. _testcase-objects: - -Test cases -~~~~~~~~~~ - -.. class:: TestCase(methodName='runTest') - - Instances of the :class:`TestCase` class represent the logical test units - in the :mod:`unittest` universe. This class is intended to be used as a base - class, with specific tests being implemented by concrete subclasses. This class - implements the interface needed by the test runner to allow it to drive the - tests, and methods that the test code can use to check for and report various - kinds of failure. - - Each instance of :class:`TestCase` will run a single base method: the method - named *methodName*. - In most uses of :class:`TestCase`, you will neither change - the *methodName* nor reimplement the default ``runTest()`` method. - - .. versionchanged:: 3.2 - :class:`TestCase` can be instantiated successfully without providing a - *methodName*. This makes it easier to experiment with :class:`TestCase` - from the interactive interpreter. - - :class:`TestCase` instances provide three groups of methods: one group used - to run the test, another used by the test implementation to check conditions - and report failures, and some inquiry methods allowing information about the - test itself to be gathered. - - Methods in the first group (running the test) are: - - .. method:: setUp() - - Method called to prepare the test fixture. This is called immediately - before calling the test method; other than :exc:`AssertionError` or :exc:`SkipTest`, - any exception raised by this method will be considered an error rather than - a test failure. The default implementation does nothing. - - - .. method:: tearDown() - - Method called immediately after the test method has been called and the - result recorded. This is called even if the test method raised an - exception, so the implementation in subclasses may need to be particularly - careful about checking internal state. Any exception, other than - :exc:`AssertionError` or :exc:`SkipTest`, raised by this method will be - considered an additional error rather than a test failure (thus increasing - the total number of reported errors). This method will only be called if - the :meth:`setUp` succeeds, regardless of the outcome of the test method. - The default implementation does nothing. - - - .. method:: setUpClass() - - A class method called before tests in an individual class are run. - ``setUpClass`` is called with the class as the only argument - and must be decorated as a :func:`classmethod`:: - - @classmethod - def setUpClass(cls): - ... - - See `Class and Module Fixtures`_ for more details. - - .. versionadded:: 3.2 - - - .. method:: tearDownClass() - - A class method called after tests in an individual class have run. - ``tearDownClass`` is called with the class as the only argument - and must be decorated as a :meth:`classmethod`:: - - @classmethod - def tearDownClass(cls): - ... - - See `Class and Module Fixtures`_ for more details. - - .. versionadded:: 3.2 - - - .. method:: run(result=None) - - Run the test, collecting the result into the :class:`TestResult` object - passed as *result*. If *result* is omitted or ``None``, a temporary - result object is created (by calling the :meth:`defaultTestResult` - method) and used. The result object is returned to :meth:`run`'s - caller. - - The same effect may be had by simply calling the :class:`TestCase` - instance. - - .. versionchanged:: 3.3 - Previous versions of ``run`` did not return the result. Neither did - calling an instance. - - .. method:: skipTest(reason) - - Calling this during a test method or :meth:`setUp` skips the current - test. See :ref:`unittest-skipping` for more information. - - .. versionadded:: 3.1 - - - .. method:: subTest(msg=None, **params) - - Return a context manager which executes the enclosed code block as a - subtest. *msg* and *params* are optional, arbitrary values which are - displayed whenever a subtest fails, allowing you to identify them - clearly. - - A test case can contain any number of subtest declarations, and - they can be arbitrarily nested. - - See :ref:`subtests` for more information. - - .. versionadded:: 3.4 - - - .. method:: debug() - - Run the test without collecting the result. This allows exceptions raised - by the test to be propagated to the caller, and can be used to support - running tests under a debugger. - - .. _assert-methods: - - The :class:`TestCase` class provides several assert methods to check for and - report failures. The following table lists the most commonly used methods - (see the tables below for more assert methods): - - +-----------------------------------------+-----------------------------+---------------+ - | Method | Checks that | New in | - +=========================================+=============================+===============+ - | :meth:`assertEqual(a, b) | ``a == b`` | | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertNotEqual(a, b) | ``a != b`` | | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertTrue(x) | ``bool(x) is True`` | | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertFalse(x) | ``bool(x) is False`` | | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIs(a, b) | ``a is b`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIsNot(a, b) | ``a is not b`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIsNone(x) | ``x is None`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIsNotNone(x) | ``x is not None`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIn(a, b) | ``a in b`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertNotIn(a, b) | ``a not in b`` | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertIsInstance(a, b) | ``isinstance(a, b)`` | 3.2 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - | :meth:`assertNotIsInstance(a, b) | ``not isinstance(a, b)`` | 3.2 | - | ` | | | - +-----------------------------------------+-----------------------------+---------------+ - - All the assert methods accept a *msg* argument that, if specified, is used - as the error message on failure (see also :data:`longMessage`). - Note that the *msg* keyword argument can be passed to :meth:`assertRaises`, - :meth:`assertRaisesRegex`, :meth:`assertWarns`, :meth:`assertWarnsRegex` - only when they are used as a context manager. - - .. method:: assertEqual(first, second, msg=None) - - Test that *first* and *second* are equal. If the values do not - compare equal, the test will fail. - - In addition, if *first* and *second* are the exact same type and one of - list, tuple, dict, set, frozenset or str or any type that a subclass - registers with :meth:`addTypeEqualityFunc` the type-specific equality - function will be called in order to generate a more useful default - error message (see also the :ref:`list of type-specific methods - `). - - .. versionchanged:: 3.1 - Added the automatic calling of type-specific equality function. - - .. versionchanged:: 3.2 - :meth:`assertMultiLineEqual` added as the default type equality - function for comparing strings. - - - .. method:: assertNotEqual(first, second, msg=None) - - Test that *first* and *second* are not equal. If the values do - compare equal, the test will fail. - - .. method:: assertTrue(expr, msg=None) - assertFalse(expr, msg=None) - - Test that *expr* is true (or false). - - Note that this is equivalent to ``bool(expr) is True`` and not to ``expr - is True`` (use ``assertIs(expr, True)`` for the latter). This method - should also be avoided when more specific methods are available (e.g. - ``assertEqual(a, b)`` instead of ``assertTrue(a == b)``), because they - provide a better error message in case of failure. - - - .. method:: assertIs(first, second, msg=None) - assertIsNot(first, second, msg=None) - - Test that *first* and *second* are (or are not) the same object. - - .. versionadded:: 3.1 - - - .. method:: assertIsNone(expr, msg=None) - assertIsNotNone(expr, msg=None) - - Test that *expr* is (or is not) ``None``. - - .. versionadded:: 3.1 - - - .. method:: assertIn(member, container, msg=None) - assertNotIn(member, container, msg=None) - - Test that *member* is (or is not) in *container*. - - .. versionadded:: 3.1 - - - .. method:: assertIsInstance(obj, cls, msg=None) - assertNotIsInstance(obj, cls, msg=None) - - Test that *obj* is (or is not) an instance of *cls* (which can be a - class or a tuple of classes, as supported by :func:`isinstance`). - To check for the exact type, use :func:`assertIs(type(obj), cls) `. - - .. versionadded:: 3.2 - - - - It is also possible to check the production of exceptions, warnings, and - log messages using the following methods: - - +---------------------------------------------------------+--------------------------------------+------------+ - | Method | Checks that | New in | - +=========================================================+======================================+============+ - | :meth:`assertRaises(exc, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | | - | ` | | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertRaisesRegex(exc, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *exc* | 3.1 | - | ` | and the message matches regex *r* | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertWarns(warn, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | - | ` | | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertWarnsRegex(warn, r, fun, *args, **kwds) | ``fun(*args, **kwds)`` raises *warn* | 3.2 | - | ` | and the message matches regex *r* | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertLogs(logger, level) | The ``with`` block logs on *logger* | 3.4 | - | ` | with minimum *level* | | - +---------------------------------------------------------+--------------------------------------+------------+ - | :meth:`assertNoLogs(logger, level) | The ``with`` block does not log on | 3.10 | - | ` | *logger* with minimum *level* | | - +---------------------------------------------------------+--------------------------------------+------------+ - - .. method:: assertRaises(exception, callable, *args, **kwds) - assertRaises(exception, *, msg=None) - - Test that an exception is raised when *callable* is called with any - positional or keyword arguments that are also passed to - :meth:`assertRaises`. The test passes if *exception* is raised, is an - error if another exception is raised, or fails if no exception is raised. - To catch any of a group of exceptions, a tuple containing the exception - classes may be passed as *exception*. - - If only the *exception* and possibly the *msg* arguments are given, - return a context manager so that the code under test can be written - inline rather than as a function:: - - with self.assertRaises(SomeException): - do_something() - - When used as a context manager, :meth:`assertRaises` accepts the - additional keyword argument *msg*. - - The context manager will store the caught exception object in its - :attr:`exception` attribute. This can be useful if the intention - is to perform additional checks on the exception raised:: - - with self.assertRaises(SomeException) as cm: - do_something() - - the_exception = cm.exception - self.assertEqual(the_exception.error_code, 3) - - .. versionchanged:: 3.1 - Added the ability to use :meth:`assertRaises` as a context manager. - - .. versionchanged:: 3.2 - Added the :attr:`exception` attribute. - - .. versionchanged:: 3.3 - Added the *msg* keyword argument when used as a context manager. - - - .. method:: assertRaisesRegex(exception, regex, callable, *args, **kwds) - assertRaisesRegex(exception, regex, *, msg=None) - - Like :meth:`assertRaises` but also tests that *regex* matches - on the string representation of the raised exception. *regex* may be - a regular expression object or a string containing a regular expression - suitable for use by :func:`re.search`. Examples:: - - self.assertRaisesRegex(ValueError, "invalid literal for.*XYZ'$", - int, 'XYZ') - - or:: - - with self.assertRaisesRegex(ValueError, 'literal'): - int('XYZ') - - .. versionadded:: 3.1 - Added under the name ``assertRaisesRegexp``. - - .. versionchanged:: 3.2 - Renamed to :meth:`assertRaisesRegex`. - - .. versionchanged:: 3.3 - Added the *msg* keyword argument when used as a context manager. - - - .. method:: assertWarns(warning, callable, *args, **kwds) - assertWarns(warning, *, msg=None) - - Test that a warning is triggered when *callable* is called with any - positional or keyword arguments that are also passed to - :meth:`assertWarns`. The test passes if *warning* is triggered and - fails if it isn't. Any exception is an error. - To catch any of a group of warnings, a tuple containing the warning - classes may be passed as *warnings*. - - If only the *warning* and possibly the *msg* arguments are given, - return a context manager so that the code under test can be written - inline rather than as a function:: - - with self.assertWarns(SomeWarning): - do_something() - - When used as a context manager, :meth:`assertWarns` accepts the - additional keyword argument *msg*. - - The context manager will store the caught warning object in its - :attr:`warning` attribute, and the source line which triggered the - warnings in the :attr:`filename` and :attr:`lineno` attributes. - This can be useful if the intention is to perform additional checks - on the warning caught:: - - with self.assertWarns(SomeWarning) as cm: - do_something() - - self.assertIn('myfile.py', cm.filename) - self.assertEqual(320, cm.lineno) - - This method works regardless of the warning filters in place when it - is called. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - Added the *msg* keyword argument when used as a context manager. - - - .. method:: assertWarnsRegex(warning, regex, callable, *args, **kwds) - assertWarnsRegex(warning, regex, *, msg=None) - - Like :meth:`assertWarns` but also tests that *regex* matches on the - message of the triggered warning. *regex* may be a regular expression - object or a string containing a regular expression suitable for use - by :func:`re.search`. Example:: - - self.assertWarnsRegex(DeprecationWarning, - r'legacy_function\(\) is deprecated', - legacy_function, 'XYZ') - - or:: - - with self.assertWarnsRegex(RuntimeWarning, 'unsafe frobnicating'): - frobnicate('/etc/passwd') - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - Added the *msg* keyword argument when used as a context manager. - - .. method:: assertLogs(logger=None, level=None) - - A context manager to test that at least one message is logged on - the *logger* or one of its children, with at least the given - *level*. - - If given, *logger* should be a :class:`logging.Logger` object or a - :class:`str` giving the name of a logger. The default is the root - logger, which will catch all messages that were not blocked by a - non-propagating descendent logger. - - If given, *level* should be either a numeric logging level or - its string equivalent (for example either ``"ERROR"`` or - :const:`logging.ERROR`). The default is :const:`logging.INFO`. - - The test passes if at least one message emitted inside the ``with`` - block matches the *logger* and *level* conditions, otherwise it fails. - - The object returned by the context manager is a recording helper - which keeps tracks of the matching log messages. It has two - attributes: - - .. attribute:: records - - A list of :class:`logging.LogRecord` objects of the matching - log messages. - - .. attribute:: output - - A list of :class:`str` objects with the formatted output of - matching messages. - - Example:: - - with self.assertLogs('foo', level='INFO') as cm: - logging.getLogger('foo').info('first message') - logging.getLogger('foo.bar').error('second message') - self.assertEqual(cm.output, ['INFO:foo:first message', - 'ERROR:foo.bar:second message']) - - .. versionadded:: 3.4 - - .. method:: assertNoLogs(logger=None, level=None) - - A context manager to test that no messages are logged on - the *logger* or one of its children, with at least the given - *level*. - - If given, *logger* should be a :class:`logging.Logger` object or a - :class:`str` giving the name of a logger. The default is the root - logger, which will catch all messages. - - If given, *level* should be either a numeric logging level or - its string equivalent (for example either ``"ERROR"`` or - :const:`logging.ERROR`). The default is :const:`logging.INFO`. - - Unlike :meth:`assertLogs`, nothing will be returned by the context - manager. - - .. versionadded:: 3.10 - - There are also other methods used to perform more specific checks, such as: - - +---------------------------------------+--------------------------------+--------------+ - | Method | Checks that | New in | - +=======================================+================================+==============+ - | :meth:`assertAlmostEqual(a, b) | ``round(a-b, 7) == 0`` | | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotAlmostEqual(a, b) | ``round(a-b, 7) != 0`` | | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertGreater(a, b) | ``a > b`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertGreaterEqual(a, b) | ``a >= b`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertLess(a, b) | ``a < b`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertLessEqual(a, b) | ``a <= b`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertRegex(s, r) | ``r.search(s)`` | 3.1 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertNotRegex(s, r) | ``not r.search(s)`` | 3.2 | - | ` | | | - +---------------------------------------+--------------------------------+--------------+ - | :meth:`assertCountEqual(a, b) | *a* and *b* have the same | 3.2 | - | ` | elements in the same number, | | - | | regardless of their order. | | - +---------------------------------------+--------------------------------+--------------+ - - - .. method:: assertAlmostEqual(first, second, places=7, msg=None, delta=None) - assertNotAlmostEqual(first, second, places=7, msg=None, delta=None) - - Test that *first* and *second* are approximately (or not approximately) - equal by computing the difference, rounding to the given number of - decimal *places* (default 7), and comparing to zero. Note that these - methods round the values to the given number of *decimal places* (i.e. - like the :func:`round` function) and not *significant digits*. - - If *delta* is supplied instead of *places* then the difference - between *first* and *second* must be less or equal to (or greater than) *delta*. - - Supplying both *delta* and *places* raises a :exc:`TypeError`. - - .. versionchanged:: 3.2 - :meth:`assertAlmostEqual` automatically considers almost equal objects - that compare equal. :meth:`assertNotAlmostEqual` automatically fails - if the objects compare equal. Added the *delta* keyword argument. - - - .. method:: assertGreater(first, second, msg=None) - assertGreaterEqual(first, second, msg=None) - assertLess(first, second, msg=None) - assertLessEqual(first, second, msg=None) - - Test that *first* is respectively >, >=, < or <= than *second* depending - on the method name. If not, the test will fail:: - - >>> self.assertGreaterEqual(3, 4) - AssertionError: "3" unexpectedly not greater than or equal to "4" - - .. versionadded:: 3.1 - - - .. method:: assertRegex(text, regex, msg=None) - assertNotRegex(text, regex, msg=None) - - Test that a *regex* search matches (or does not match) *text*. In case - of failure, the error message will include the pattern and the *text* (or - the pattern and the part of *text* that unexpectedly matched). *regex* - may be a regular expression object or a string containing a regular - expression suitable for use by :func:`re.search`. - - .. versionadded:: 3.1 - Added under the name ``assertRegexpMatches``. - .. versionchanged:: 3.2 - The method ``assertRegexpMatches()`` has been renamed to - :meth:`.assertRegex`. - .. versionadded:: 3.2 - :meth:`.assertNotRegex`. - .. versionadded:: 3.5 - The name ``assertNotRegexpMatches`` is a deprecated alias - for :meth:`.assertNotRegex`. - - - .. method:: assertCountEqual(first, second, msg=None) - - Test that sequence *first* contains the same elements as *second*, - regardless of their order. When they don't, an error message listing the - differences between the sequences will be generated. - - Duplicate elements are *not* ignored when comparing *first* and - *second*. It verifies whether each element has the same count in both - sequences. Equivalent to: - ``assertEqual(Counter(list(first)), Counter(list(second)))`` - but works with sequences of unhashable objects as well. - - .. versionadded:: 3.2 - - - .. _type-specific-methods: - - The :meth:`assertEqual` method dispatches the equality check for objects of - the same type to different type-specific methods. These methods are already - implemented for most of the built-in types, but it's also possible to - register new methods using :meth:`addTypeEqualityFunc`: - - .. method:: addTypeEqualityFunc(typeobj, function) - - Registers a type-specific method called by :meth:`assertEqual` to check - if two objects of exactly the same *typeobj* (not subclasses) compare - equal. *function* must take two positional arguments and a third msg=None - keyword argument just as :meth:`assertEqual` does. It must raise - :data:`self.failureException(msg) ` when inequality - between the first two parameters is detected -- possibly providing useful - information and explaining the inequalities in details in the error - message. - - .. versionadded:: 3.1 - - The list of type-specific methods automatically used by - :meth:`~TestCase.assertEqual` are summarized in the following table. Note - that it's usually not necessary to invoke these methods directly. - - +-----------------------------------------+-----------------------------+--------------+ - | Method | Used to compare | New in | - +=========================================+=============================+==============+ - | :meth:`assertMultiLineEqual(a, b) | strings | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertSequenceEqual(a, b) | sequences | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertListEqual(a, b) | lists | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertTupleEqual(a, b) | tuples | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertSetEqual(a, b) | sets or frozensets | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - | :meth:`assertDictEqual(a, b) | dicts | 3.1 | - | ` | | | - +-----------------------------------------+-----------------------------+--------------+ - - - - .. method:: assertMultiLineEqual(first, second, msg=None) - - Test that the multiline string *first* is equal to the string *second*. - When not equal a diff of the two strings highlighting the differences - will be included in the error message. This method is used by default - when comparing strings with :meth:`assertEqual`. - - .. versionadded:: 3.1 - - - .. method:: assertSequenceEqual(first, second, msg=None, seq_type=None) - - Tests that two sequences are equal. If a *seq_type* is supplied, both - *first* and *second* must be instances of *seq_type* or a failure will - be raised. If the sequences are different an error message is - constructed that shows the difference between the two. - - This method is not called directly by :meth:`assertEqual`, but - it's used to implement :meth:`assertListEqual` and - :meth:`assertTupleEqual`. - - .. versionadded:: 3.1 - - - .. method:: assertListEqual(first, second, msg=None) - assertTupleEqual(first, second, msg=None) - - Tests that two lists or tuples are equal. If not, an error message is - constructed that shows only the differences between the two. An error - is also raised if either of the parameters are of the wrong type. - These methods are used by default when comparing lists or tuples with - :meth:`assertEqual`. - - .. versionadded:: 3.1 - - - .. method:: assertSetEqual(first, second, msg=None) - - Tests that two sets are equal. If not, an error message is constructed - that lists the differences between the sets. This method is used by - default when comparing sets or frozensets with :meth:`assertEqual`. - - Fails if either of *first* or *second* does not have a :meth:`set.difference` - method. - - .. versionadded:: 3.1 - - - .. method:: assertDictEqual(first, second, msg=None) - - Test that two dictionaries are equal. If not, an error message is - constructed that shows the differences in the dictionaries. This - method will be used by default to compare dictionaries in - calls to :meth:`assertEqual`. - - .. versionadded:: 3.1 - - - - .. _other-methods-and-attrs: - - Finally the :class:`TestCase` provides the following methods and attributes: - - - .. method:: fail(msg=None) - - Signals a test failure unconditionally, with *msg* or ``None`` for - the error message. - - - .. attribute:: failureException - - This class attribute gives the exception raised by the test method. If a - test framework needs to use a specialized exception, possibly to carry - additional information, it must subclass this exception in order to "play - fair" with the framework. The initial value of this attribute is - :exc:`AssertionError`. - - - .. attribute:: longMessage - - This class attribute determines what happens when a custom failure message - is passed as the msg argument to an assertXYY call that fails. - ``True`` is the default value. In this case, the custom message is appended - to the end of the standard failure message. - When set to ``False``, the custom message replaces the standard message. - - The class setting can be overridden in individual test methods by assigning - an instance attribute, self.longMessage, to ``True`` or ``False`` before - calling the assert methods. - - The class setting gets reset before each test call. - - .. versionadded:: 3.1 - - - .. attribute:: maxDiff - - This attribute controls the maximum length of diffs output by assert - methods that report diffs on failure. It defaults to 80*8 characters. - Assert methods affected by this attribute are - :meth:`assertSequenceEqual` (including all the sequence comparison - methods that delegate to it), :meth:`assertDictEqual` and - :meth:`assertMultiLineEqual`. - - Setting ``maxDiff`` to ``None`` means that there is no maximum length of - diffs. - - .. versionadded:: 3.2 - - - Testing frameworks can use the following methods to collect information on - the test: - - - .. method:: countTestCases() - - Return the number of tests represented by this test object. For - :class:`TestCase` instances, this will always be ``1``. - - - .. method:: defaultTestResult() - - Return an instance of the test result class that should be used for this - test case class (if no other result instance is provided to the - :meth:`run` method). - - For :class:`TestCase` instances, this will always be an instance of - :class:`TestResult`; subclasses of :class:`TestCase` should override this - as necessary. - - - .. method:: id() - - Return a string identifying the specific test case. This is usually the - full name of the test method, including the module and class name. - - - .. method:: shortDescription() - - Returns a description of the test, or ``None`` if no description - has been provided. The default implementation of this method - returns the first line of the test method's docstring, if available, - or ``None``. - - .. versionchanged:: 3.1 - In 3.1 this was changed to add the test name to the short description - even in the presence of a docstring. This caused compatibility issues - with unittest extensions and adding the test name was moved to the - :class:`TextTestResult` in Python 3.2. - - - .. method:: addCleanup(function, /, *args, **kwargs) - - Add a function to be called after :meth:`tearDown` to cleanup resources - used during the test. Functions will be called in reverse order to the - order they are added (:abbr:`LIFO (last-in, first-out)`). They - are called with any arguments and keyword arguments passed into - :meth:`addCleanup` when they are added. - - If :meth:`setUp` fails, meaning that :meth:`tearDown` is not called, - then any cleanup functions added will still be called. - - .. versionadded:: 3.1 - - - .. method:: enterContext(cm) - - Enter the supplied :term:`context manager`. If successful, also - add its :meth:`~object.__exit__` method as a cleanup function by - :meth:`addCleanup` and return the result of the - :meth:`~object.__enter__` method. - - .. versionadded:: 3.11 - - - .. method:: doCleanups() - - This method is called unconditionally after :meth:`tearDown`, or - after :meth:`setUp` if :meth:`setUp` raises an exception. - - It is responsible for calling all the cleanup functions added by - :meth:`addCleanup`. If you need cleanup functions to be called - *prior* to :meth:`tearDown` then you can call :meth:`doCleanups` - yourself. - - :meth:`doCleanups` pops methods off the stack of cleanup - functions one at a time, so it can be called at any time. - - .. versionadded:: 3.1 - - - .. classmethod:: addClassCleanup(function, /, *args, **kwargs) - - Add a function to be called after :meth:`tearDownClass` to cleanup - resources used during the test class. Functions will be called in reverse - order to the order they are added (:abbr:`LIFO (last-in, first-out)`). - They are called with any arguments and keyword arguments passed into - :meth:`addClassCleanup` when they are added. - - If :meth:`setUpClass` fails, meaning that :meth:`tearDownClass` is not - called, then any cleanup functions added will still be called. - - .. versionadded:: 3.8 - - - .. classmethod:: enterClassContext(cm) - - Enter the supplied :term:`context manager`. If successful, also - add its :meth:`~object.__exit__` method as a cleanup function by - :meth:`addClassCleanup` and return the result of the - :meth:`~object.__enter__` method. - - .. versionadded:: 3.11 - - - .. classmethod:: doClassCleanups() - - This method is called unconditionally after :meth:`tearDownClass`, or - after :meth:`setUpClass` if :meth:`setUpClass` raises an exception. - - It is responsible for calling all the cleanup functions added by - :meth:`addClassCleanup`. If you need cleanup functions to be called - *prior* to :meth:`tearDownClass` then you can call - :meth:`doClassCleanups` yourself. - - :meth:`doClassCleanups` pops methods off the stack of cleanup - functions one at a time, so it can be called at any time. - - .. versionadded:: 3.8 - - -.. class:: IsolatedAsyncioTestCase(methodName='runTest') - - This class provides an API similar to :class:`TestCase` and also accepts - coroutines as test functions. - - .. versionadded:: 3.8 - - .. coroutinemethod:: asyncSetUp() - - Method called to prepare the test fixture. This is called after :meth:`setUp`. - This is called immediately before calling the test method; other than - :exc:`AssertionError` or :exc:`SkipTest`, any exception raised by this method - will be considered an error rather than a test failure. The default implementation - does nothing. - - .. coroutinemethod:: asyncTearDown() - - Method called immediately after the test method has been called and the - result recorded. This is called before :meth:`tearDown`. This is called even if - the test method raised an exception, so the implementation in subclasses may need - to be particularly careful about checking internal state. Any exception, other than - :exc:`AssertionError` or :exc:`SkipTest`, raised by this method will be - considered an additional error rather than a test failure (thus increasing - the total number of reported errors). This method will only be called if - the :meth:`asyncSetUp` succeeds, regardless of the outcome of the test method. - The default implementation does nothing. - - .. method:: addAsyncCleanup(function, /, *args, **kwargs) - - This method accepts a coroutine that can be used as a cleanup function. - - .. coroutinemethod:: enterAsyncContext(cm) - - Enter the supplied :term:`asynchronous context manager`. If successful, - also add its :meth:`~object.__aexit__` method as a cleanup function by - :meth:`addAsyncCleanup` and return the result of the - :meth:`~object.__aenter__` method. - - .. versionadded:: 3.11 - - - .. method:: run(result=None) - - Sets up a new event loop to run the test, collecting the result into - the :class:`TestResult` object passed as *result*. If *result* is - omitted or ``None``, a temporary result object is created (by calling - the :meth:`defaultTestResult` method) and used. The result object is - returned to :meth:`run`'s caller. At the end of the test all the tasks - in the event loop are cancelled. - - - An example illustrating the order:: - - from unittest import IsolatedAsyncioTestCase - - events = [] - - - class Test(IsolatedAsyncioTestCase): - - - def setUp(self): - events.append("setUp") - - async def asyncSetUp(self): - self._async_connection = await AsyncConnection() - events.append("asyncSetUp") - - async def test_response(self): - events.append("test_response") - response = await self._async_connection.get("https://example.com") - self.assertEqual(response.status_code, 200) - self.addAsyncCleanup(self.on_cleanup) - - def tearDown(self): - events.append("tearDown") - - async def asyncTearDown(self): - await self._async_connection.close() - events.append("asyncTearDown") - - async def on_cleanup(self): - events.append("cleanup") - - if __name__ == "__main__": - unittest.main() - - After running the test, ``events`` would contain ``["setUp", "asyncSetUp", "test_response", "asyncTearDown", "tearDown", "cleanup"]``. - - -.. class:: FunctionTestCase(testFunc, setUp=None, tearDown=None, description=None) - - This class implements the portion of the :class:`TestCase` interface which - allows the test runner to drive the test, but does not provide the methods - which test code can use to check and report errors. This is used to create - test cases using legacy test code, allowing it to be integrated into a - :mod:`unittest`-based test framework. - - -.. _deprecated-aliases: - -Deprecated aliases -################## - -For historical reasons, some of the :class:`TestCase` methods had one or more -aliases that are now deprecated. The following table lists the correct names -along with their deprecated aliases: - - ============================== ====================== ======================= - Method Name Deprecated alias Deprecated alias - ============================== ====================== ======================= - :meth:`.assertEqual` failUnlessEqual assertEquals - :meth:`.assertNotEqual` failIfEqual assertNotEquals - :meth:`.assertTrue` failUnless assert\_ - :meth:`.assertFalse` failIf - :meth:`.assertRaises` failUnlessRaises - :meth:`.assertAlmostEqual` failUnlessAlmostEqual assertAlmostEquals - :meth:`.assertNotAlmostEqual` failIfAlmostEqual assertNotAlmostEquals - :meth:`.assertRegex` assertRegexpMatches - :meth:`.assertNotRegex` assertNotRegexpMatches - :meth:`.assertRaisesRegex` assertRaisesRegexp - ============================== ====================== ======================= - - .. deprecated:: 3.1 - The fail* aliases listed in the second column have been deprecated. - .. deprecated:: 3.2 - The assert* aliases listed in the third column have been deprecated. - .. deprecated:: 3.2 - ``assertRegexpMatches`` and ``assertRaisesRegexp`` have been renamed to - :meth:`.assertRegex` and :meth:`.assertRaisesRegex`. - .. deprecated:: 3.5 - The ``assertNotRegexpMatches`` name is deprecated in favor of :meth:`.assertNotRegex`. - -.. _testsuite-objects: - -Grouping tests -~~~~~~~~~~~~~~ - -.. class:: TestSuite(tests=()) - - This class represents an aggregation of individual test cases and test suites. - The class presents the interface needed by the test runner to allow it to be run - as any other test case. Running a :class:`TestSuite` instance is the same as - iterating over the suite, running each test individually. - - If *tests* is given, it must be an iterable of individual test cases or other - test suites that will be used to build the suite initially. Additional methods - are provided to add test cases and suites to the collection later on. - - :class:`TestSuite` objects behave much like :class:`TestCase` objects, except - they do not actually implement a test. Instead, they are used to aggregate - tests into groups of tests that should be run together. Some additional - methods are available to add tests to :class:`TestSuite` instances: - - - .. method:: TestSuite.addTest(test) - - Add a :class:`TestCase` or :class:`TestSuite` to the suite. - - - .. method:: TestSuite.addTests(tests) - - Add all the tests from an iterable of :class:`TestCase` and :class:`TestSuite` - instances to this test suite. - - This is equivalent to iterating over *tests*, calling :meth:`addTest` for - each element. - - :class:`TestSuite` shares the following methods with :class:`TestCase`: - - - .. method:: run(result) - - Run the tests associated with this suite, collecting the result into the - test result object passed as *result*. Note that unlike - :meth:`TestCase.run`, :meth:`TestSuite.run` requires the result object to - be passed in. - - - .. method:: debug() - - Run the tests associated with this suite without collecting the - result. This allows exceptions raised by the test to be propagated to the - caller and can be used to support running tests under a debugger. - - - .. method:: countTestCases() - - Return the number of tests represented by this test object, including all - individual tests and sub-suites. - - - .. method:: __iter__() - - Tests grouped by a :class:`TestSuite` are always accessed by iteration. - Subclasses can lazily provide tests by overriding :meth:`__iter__`. Note - that this method may be called several times on a single suite (for - example when counting tests or comparing for equality) so the tests - returned by repeated iterations before :meth:`TestSuite.run` must be the - same for each call iteration. After :meth:`TestSuite.run`, callers should - not rely on the tests returned by this method unless the caller uses a - subclass that overrides :meth:`TestSuite._removeTestAtIndex` to preserve - test references. - - .. versionchanged:: 3.2 - In earlier versions the :class:`TestSuite` accessed tests directly rather - than through iteration, so overriding :meth:`__iter__` wasn't sufficient - for providing tests. - - .. versionchanged:: 3.4 - In earlier versions the :class:`TestSuite` held references to each - :class:`TestCase` after :meth:`TestSuite.run`. Subclasses can restore - that behavior by overriding :meth:`TestSuite._removeTestAtIndex`. - - In the typical usage of a :class:`TestSuite` object, the :meth:`run` method - is invoked by a :class:`TestRunner` rather than by the end-user test harness. - - -Loading and running tests -~~~~~~~~~~~~~~~~~~~~~~~~~ - -.. class:: TestLoader() - - The :class:`TestLoader` class is used to create test suites from classes and - modules. Normally, there is no need to create an instance of this class; the - :mod:`unittest` module provides an instance that can be shared as - :data:`unittest.defaultTestLoader`. Using a subclass or instance, however, - allows customization of some configurable properties. - - :class:`TestLoader` objects have the following attributes: - - - .. attribute:: errors - - A list of the non-fatal errors encountered while loading tests. Not reset - by the loader at any point. Fatal errors are signalled by the relevant - method raising an exception to the caller. Non-fatal errors are also - indicated by a synthetic test that will raise the original error when - run. - - .. versionadded:: 3.5 - - - :class:`TestLoader` objects have the following methods: - - - .. method:: loadTestsFromTestCase(testCaseClass) - - Return a suite of all test cases contained in the :class:`TestCase`\ -derived - :class:`testCaseClass`. - - A test case instance is created for each method named by - :meth:`getTestCaseNames`. By default these are the method names - beginning with ``test``. If :meth:`getTestCaseNames` returns no - methods, but the :meth:`runTest` method is implemented, a single test - case is created for that method instead. - - - .. method:: loadTestsFromModule(module, pattern=None) - - Return a suite of all test cases contained in the given module. This - method searches *module* for classes derived from :class:`TestCase` and - creates an instance of the class for each test method defined for the - class. - - .. note:: - - While using a hierarchy of :class:`TestCase`\ -derived classes can be - convenient in sharing fixtures and helper functions, defining test - methods on base classes that are not intended to be instantiated - directly does not play well with this method. Doing so, however, can - be useful when the fixtures are different and defined in subclasses. - - If a module provides a ``load_tests`` function it will be called to - load the tests. This allows modules to customize test loading. - This is the `load_tests protocol`_. The *pattern* argument is passed as - the third argument to ``load_tests``. - - .. versionchanged:: 3.2 - Support for ``load_tests`` added. - - .. versionchanged:: 3.5 - The undocumented and unofficial *use_load_tests* default argument is - deprecated and ignored, although it is still accepted for backward - compatibility. The method also now accepts a keyword-only argument - *pattern* which is passed to ``load_tests`` as the third argument. - - - .. method:: loadTestsFromName(name, module=None) - - Return a suite of all test cases given a string specifier. - - The specifier *name* is a "dotted name" that may resolve either to a - module, a test case class, a test method within a test case class, a - :class:`TestSuite` instance, or a callable object which returns a - :class:`TestCase` or :class:`TestSuite` instance. These checks are - applied in the order listed here; that is, a method on a possible test - case class will be picked up as "a test method within a test case class", - rather than "a callable object". - - For example, if you have a module :mod:`SampleTests` containing a - :class:`TestCase`\ -derived class :class:`SampleTestCase` with three test - methods (:meth:`test_one`, :meth:`test_two`, and :meth:`test_three`), the - specifier ``'SampleTests.SampleTestCase'`` would cause this method to - return a suite which will run all three test methods. Using the specifier - ``'SampleTests.SampleTestCase.test_two'`` would cause it to return a test - suite which will run only the :meth:`test_two` test method. The specifier - can refer to modules and packages which have not been imported; they will - be imported as a side-effect. - - The method optionally resolves *name* relative to the given *module*. - - .. versionchanged:: 3.5 - If an :exc:`ImportError` or :exc:`AttributeError` occurs while traversing - *name* then a synthetic test that raises that error when run will be - returned. These errors are included in the errors accumulated by - self.errors. - - - .. method:: loadTestsFromNames(names, module=None) - - Similar to :meth:`loadTestsFromName`, but takes a sequence of names rather - than a single name. The return value is a test suite which supports all - the tests defined for each name. - - - .. method:: getTestCaseNames(testCaseClass) - - Return a sorted sequence of method names found within *testCaseClass*; - this should be a subclass of :class:`TestCase`. - - - .. method:: discover(start_dir, pattern='test*.py', top_level_dir=None) - - Find all the test modules by recursing into subdirectories from the - specified start directory, and return a TestSuite object containing them. - Only test files that match *pattern* will be loaded. (Using shell style - pattern matching.) Only module names that are importable (i.e. are valid - Python identifiers) will be loaded. - - All test modules must be importable from the top level of the project. If - the start directory is not the top level directory then the top level - directory must be specified separately. - - If importing a module fails, for example due to a syntax error, then - this will be recorded as a single error and discovery will continue. If - the import failure is due to :exc:`SkipTest` being raised, it will be - recorded as a skip instead of an error. - - If a package (a directory containing a file named :file:`__init__.py`) is - found, the package will be checked for a ``load_tests`` function. If this - exists then it will be called - ``package.load_tests(loader, tests, pattern)``. Test discovery takes care - to ensure that a package is only checked for tests once during an - invocation, even if the load_tests function itself calls - ``loader.discover``. - - If ``load_tests`` exists then discovery does *not* recurse into the - package, ``load_tests`` is responsible for loading all tests in the - package. - - The pattern is deliberately not stored as a loader attribute so that - packages can continue discovery themselves. *top_level_dir* is stored so - ``load_tests`` does not need to pass this argument in to - ``loader.discover()``. - - *start_dir* can be a dotted module name as well as a directory. - - .. versionadded:: 3.2 - - .. versionchanged:: 3.4 - Modules that raise :exc:`SkipTest` on import are recorded as skips, - not errors. - - .. versionchanged:: 3.4 - *start_dir* can be a :term:`namespace packages `. - - .. versionchanged:: 3.4 - Paths are sorted before being imported so that execution order is the - same even if the underlying file system's ordering is not dependent - on file name. - - .. versionchanged:: 3.5 - Found packages are now checked for ``load_tests`` regardless of - whether their path matches *pattern*, because it is impossible for - a package name to match the default pattern. - - .. versionchanged:: 3.11 - *start_dir* can not be a :term:`namespace packages `. - It has been broken since Python 3.7 and Python 3.11 officially remove it. - - - The following attributes of a :class:`TestLoader` can be configured either by - subclassing or assignment on an instance: - - - .. attribute:: testMethodPrefix - - String giving the prefix of method names which will be interpreted as test - methods. The default value is ``'test'``. - - This affects :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` - methods. - - - .. attribute:: sortTestMethodsUsing - - Function to be used to compare method names when sorting them in - :meth:`getTestCaseNames` and all the :meth:`loadTestsFrom\*` methods. - - - .. attribute:: suiteClass - - Callable object that constructs a test suite from a list of tests. No - methods on the resulting object are needed. The default value is the - :class:`TestSuite` class. - - This affects all the :meth:`loadTestsFrom\*` methods. - - .. attribute:: testNamePatterns - - List of Unix shell-style wildcard test name patterns that test methods - have to match to be included in test suites (see ``-k`` option). - - If this attribute is not ``None`` (the default), all test methods to be - included in test suites must match one of the patterns in this list. - Note that matches are always performed using :meth:`fnmatch.fnmatchcase`, - so unlike patterns passed to the ``-k`` option, simple substring patterns - will have to be converted using ``*`` wildcards. - - This affects all the :meth:`loadTestsFrom\*` methods. - - .. versionadded:: 3.7 - - -.. class:: TestResult - - This class is used to compile information about which tests have succeeded - and which have failed. - - A :class:`TestResult` object stores the results of a set of tests. The - :class:`TestCase` and :class:`TestSuite` classes ensure that results are - properly recorded; test authors do not need to worry about recording the - outcome of tests. - - Testing frameworks built on top of :mod:`unittest` may want access to the - :class:`TestResult` object generated by running a set of tests for reporting - purposes; a :class:`TestResult` instance is returned by the - :meth:`TestRunner.run` method for this purpose. - - :class:`TestResult` instances have the following attributes that will be of - interest when inspecting the results of running a set of tests: - - - .. attribute:: errors - - A list containing 2-tuples of :class:`TestCase` instances and strings - holding formatted tracebacks. Each tuple represents a test which raised an - unexpected exception. - - .. attribute:: failures - - A list containing 2-tuples of :class:`TestCase` instances and strings - holding formatted tracebacks. Each tuple represents a test where a failure - was explicitly signalled using the :meth:`TestCase.assert\*` methods. - - .. attribute:: skipped - - A list containing 2-tuples of :class:`TestCase` instances and strings - holding the reason for skipping the test. - - .. versionadded:: 3.1 - - .. attribute:: expectedFailures - - A list containing 2-tuples of :class:`TestCase` instances and strings - holding formatted tracebacks. Each tuple represents an expected failure - or error of the test case. - - .. attribute:: unexpectedSuccesses - - A list containing :class:`TestCase` instances that were marked as expected - failures, but succeeded. - - .. attribute:: shouldStop - - Set to ``True`` when the execution of tests should stop by :meth:`stop`. - - .. attribute:: testsRun - - The total number of tests run so far. - - .. attribute:: buffer - - If set to true, ``sys.stdout`` and ``sys.stderr`` will be buffered in between - :meth:`startTest` and :meth:`stopTest` being called. Collected output will - only be echoed onto the real ``sys.stdout`` and ``sys.stderr`` if the test - fails or errors. Any output is also attached to the failure / error message. - - .. versionadded:: 3.2 - - .. attribute:: failfast - - If set to true :meth:`stop` will be called on the first failure or error, - halting the test run. - - .. versionadded:: 3.2 - - .. attribute:: tb_locals - - If set to true then local variables will be shown in tracebacks. - - .. versionadded:: 3.5 - - .. method:: wasSuccessful() - - Return ``True`` if all tests run so far have passed, otherwise returns - ``False``. - - .. versionchanged:: 3.4 - Returns ``False`` if there were any :attr:`unexpectedSuccesses` - from tests marked with the :func:`expectedFailure` decorator. - - .. method:: stop() - - This method can be called to signal that the set of tests being run should - be aborted by setting the :attr:`shouldStop` attribute to ``True``. - :class:`TestRunner` objects should respect this flag and return without - running any additional tests. - - For example, this feature is used by the :class:`TextTestRunner` class to - stop the test framework when the user signals an interrupt from the - keyboard. Interactive tools which provide :class:`TestRunner` - implementations can use this in a similar manner. - - The following methods of the :class:`TestResult` class are used to maintain - the internal data structures, and may be extended in subclasses to support - additional reporting requirements. This is particularly useful in building - tools which support interactive reporting while tests are being run. - - - .. method:: startTest(test) - - Called when the test case *test* is about to be run. - - .. method:: stopTest(test) - - Called after the test case *test* has been executed, regardless of the - outcome. - - .. method:: startTestRun() - - Called once before any tests are executed. - - .. versionadded:: 3.1 - - - .. method:: stopTestRun() - - Called once after all tests are executed. - - .. versionadded:: 3.1 - - - .. method:: addError(test, err) - - Called when the test case *test* raises an unexpected exception. *err* is a - tuple of the form returned by :func:`sys.exc_info`: ``(type, value, - traceback)``. - - The default implementation appends a tuple ``(test, formatted_err)`` to - the instance's :attr:`errors` attribute, where *formatted_err* is a - formatted traceback derived from *err*. - - - .. method:: addFailure(test, err) - - Called when the test case *test* signals a failure. *err* is a tuple of - the form returned by :func:`sys.exc_info`: ``(type, value, traceback)``. - - The default implementation appends a tuple ``(test, formatted_err)`` to - the instance's :attr:`failures` attribute, where *formatted_err* is a - formatted traceback derived from *err*. - - - .. method:: addSuccess(test) - - Called when the test case *test* succeeds. - - The default implementation does nothing. - - - .. method:: addSkip(test, reason) - - Called when the test case *test* is skipped. *reason* is the reason the - test gave for skipping. - - The default implementation appends a tuple ``(test, reason)`` to the - instance's :attr:`skipped` attribute. - - - .. method:: addExpectedFailure(test, err) - - Called when the test case *test* fails or errors, but was marked with - the :func:`expectedFailure` decorator. - - The default implementation appends a tuple ``(test, formatted_err)`` to - the instance's :attr:`expectedFailures` attribute, where *formatted_err* - is a formatted traceback derived from *err*. - - - .. method:: addUnexpectedSuccess(test) - - Called when the test case *test* was marked with the - :func:`expectedFailure` decorator, but succeeded. - - The default implementation appends the test to the instance's - :attr:`unexpectedSuccesses` attribute. - - - .. method:: addSubTest(test, subtest, outcome) - - Called when a subtest finishes. *test* is the test case - corresponding to the test method. *subtest* is a custom - :class:`TestCase` instance describing the subtest. - - If *outcome* is :const:`None`, the subtest succeeded. Otherwise, - it failed with an exception where *outcome* is a tuple of the form - returned by :func:`sys.exc_info`: ``(type, value, traceback)``. - - The default implementation does nothing when the outcome is a - success, and records subtest failures as normal failures. - - .. versionadded:: 3.4 - - -.. class:: TextTestResult(stream, descriptions, verbosity) - - A concrete implementation of :class:`TestResult` used by the - :class:`TextTestRunner`. - - .. versionadded:: 3.2 - This class was previously named ``_TextTestResult``. The old name still - exists as an alias but is deprecated. - - -.. data:: defaultTestLoader - - Instance of the :class:`TestLoader` class intended to be shared. If no - customization of the :class:`TestLoader` is needed, this instance can be used - instead of repeatedly creating new instances. - - -.. class:: TextTestRunner(stream=None, descriptions=True, verbosity=1, failfast=False, \ - buffer=False, resultclass=None, warnings=None, *, tb_locals=False) - - A basic test runner implementation that outputs results to a stream. If *stream* - is ``None``, the default, :data:`sys.stderr` is used as the output stream. This class - has a few configurable parameters, but is essentially very simple. Graphical - applications which run test suites should provide alternate implementations. Such - implementations should accept ``**kwargs`` as the interface to construct runners - changes when features are added to unittest. - - By default this runner shows :exc:`DeprecationWarning`, - :exc:`PendingDeprecationWarning`, :exc:`ResourceWarning` and - :exc:`ImportWarning` even if they are :ref:`ignored by default - `. Deprecation warnings caused by :ref:`deprecated unittest - methods ` are also special-cased and, when the warning - filters are ``'default'`` or ``'always'``, they will appear only once - per-module, in order to avoid too many warning messages. This behavior can - be overridden using Python's :option:`!-Wd` or :option:`!-Wa` options - (see :ref:`Warning control `) and leaving - *warnings* to ``None``. - - .. versionchanged:: 3.2 - Added the ``warnings`` argument. - - .. versionchanged:: 3.2 - The default stream is set to :data:`sys.stderr` at instantiation time rather - than import time. - - .. versionchanged:: 3.5 - Added the tb_locals parameter. - - .. method:: _makeResult() - - This method returns the instance of ``TestResult`` used by :meth:`run`. - It is not intended to be called directly, but can be overridden in - subclasses to provide a custom ``TestResult``. - - ``_makeResult()`` instantiates the class or callable passed in the - ``TextTestRunner`` constructor as the ``resultclass`` argument. It - defaults to :class:`TextTestResult` if no ``resultclass`` is provided. - The result class is instantiated with the following arguments:: - - stream, descriptions, verbosity - - .. method:: run(test) - - This method is the main public interface to the ``TextTestRunner``. This - method takes a :class:`TestSuite` or :class:`TestCase` instance. A - :class:`TestResult` is created by calling - :func:`_makeResult` and the test(s) are run and the - results printed to stdout. - - -.. function:: main(module='__main__', defaultTest=None, argv=None, testRunner=None, \ - testLoader=unittest.defaultTestLoader, exit=True, verbosity=1, \ - failfast=None, catchbreak=None, buffer=None, warnings=None) - - A command-line program that loads a set of tests from *module* and runs them; - this is primarily for making test modules conveniently executable. - The simplest use for this function is to include the following line at the - end of a test script:: - - if __name__ == '__main__': - unittest.main() - - You can run tests with more detailed information by passing in the verbosity - argument:: - - if __name__ == '__main__': - unittest.main(verbosity=2) - - The *defaultTest* argument is either the name of a single test or an - iterable of test names to run if no test names are specified via *argv*. If - not specified or ``None`` and no test names are provided via *argv*, all - tests found in *module* are run. - - The *argv* argument can be a list of options passed to the program, with the - first element being the program name. If not specified or ``None``, - the values of :data:`sys.argv` are used. - - The *testRunner* argument can either be a test runner class or an already - created instance of it. By default ``main`` calls :func:`sys.exit` with - an exit code indicating success or failure of the tests run. - - The *testLoader* argument has to be a :class:`TestLoader` instance, - and defaults to :data:`defaultTestLoader`. - - ``main`` supports being used from the interactive interpreter by passing in the - argument ``exit=False``. This displays the result on standard output without - calling :func:`sys.exit`:: - - >>> from unittest import main - >>> main(module='test_module', exit=False) - - The *failfast*, *catchbreak* and *buffer* parameters have the same - effect as the same-name `command-line options`_. - - The *warnings* argument specifies the :ref:`warning filter ` - that should be used while running the tests. If it's not specified, it will - remain ``None`` if a :option:`!-W` option is passed to :program:`python` - (see :ref:`Warning control `), - otherwise it will be set to ``'default'``. - - Calling ``main`` actually returns an instance of the ``TestProgram`` class. - This stores the result of the tests run as the ``result`` attribute. - - .. versionchanged:: 3.1 - The *exit* parameter was added. - - .. versionchanged:: 3.2 - The *verbosity*, *failfast*, *catchbreak*, *buffer* - and *warnings* parameters were added. - - .. versionchanged:: 3.4 - The *defaultTest* parameter was changed to also accept an iterable of - test names. - - -load_tests Protocol -################### - -.. versionadded:: 3.2 - -Modules or packages can customize how tests are loaded from them during normal -test runs or test discovery by implementing a function called ``load_tests``. - -If a test module defines ``load_tests`` it will be called by -:meth:`TestLoader.loadTestsFromModule` with the following arguments:: - - load_tests(loader, standard_tests, pattern) - -where *pattern* is passed straight through from ``loadTestsFromModule``. It -defaults to ``None``. - -It should return a :class:`TestSuite`. - -*loader* is the instance of :class:`TestLoader` doing the loading. -*standard_tests* are the tests that would be loaded by default from the -module. It is common for test modules to only want to add or remove tests -from the standard set of tests. -The third argument is used when loading packages as part of test discovery. - -A typical ``load_tests`` function that loads tests from a specific set of -:class:`TestCase` classes may look like:: - - test_cases = (TestCase1, TestCase2, TestCase3) - - def load_tests(loader, tests, pattern): - suite = TestSuite() - for test_class in test_cases: - tests = loader.loadTestsFromTestCase(test_class) - suite.addTests(tests) - return suite - -If discovery is started in a directory containing a package, either from the -command line or by calling :meth:`TestLoader.discover`, then the package -:file:`__init__.py` will be checked for ``load_tests``. If that function does -not exist, discovery will recurse into the package as though it were just -another directory. Otherwise, discovery of the package's tests will be left up -to ``load_tests`` which is called with the following arguments:: - - load_tests(loader, standard_tests, pattern) - -This should return a :class:`TestSuite` representing all the tests -from the package. (``standard_tests`` will only contain tests -collected from :file:`__init__.py`.) - -Because the pattern is passed into ``load_tests`` the package is free to -continue (and potentially modify) test discovery. A 'do nothing' -``load_tests`` function for a test package would look like:: - - def load_tests(loader, standard_tests, pattern): - # top level directory cached on loader instance - this_dir = os.path.dirname(__file__) - package_tests = loader.discover(start_dir=this_dir, pattern=pattern) - standard_tests.addTests(package_tests) - return standard_tests - -.. versionchanged:: 3.5 - Discovery no longer checks package names for matching *pattern* due to the - impossibility of package names matching the default pattern. - - - -Class and Module Fixtures -------------------------- - -Class and module level fixtures are implemented in :class:`TestSuite`. When -the test suite encounters a test from a new class then :meth:`tearDownClass` -from the previous class (if there is one) is called, followed by -:meth:`setUpClass` from the new class. - -Similarly if a test is from a different module from the previous test then -``tearDownModule`` from the previous module is run, followed by -``setUpModule`` from the new module. - -After all the tests have run the final ``tearDownClass`` and -``tearDownModule`` are run. - -Note that shared fixtures do not play well with [potential] features like test -parallelization and they break test isolation. They should be used with care. - -The default ordering of tests created by the unittest test loaders is to group -all tests from the same modules and classes together. This will lead to -``setUpClass`` / ``setUpModule`` (etc) being called exactly once per class and -module. If you randomize the order, so that tests from different modules and -classes are adjacent to each other, then these shared fixture functions may be -called multiple times in a single test run. - -Shared fixtures are not intended to work with suites with non-standard -ordering. A ``BaseTestSuite`` still exists for frameworks that don't want to -support shared fixtures. - -If there are any exceptions raised during one of the shared fixture functions -the test is reported as an error. Because there is no corresponding test -instance an ``_ErrorHolder`` object (that has the same interface as a -:class:`TestCase`) is created to represent the error. If you are just using -the standard unittest test runner then this detail doesn't matter, but if you -are a framework author it may be relevant. - - -setUpClass and tearDownClass -~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -These must be implemented as class methods:: - - import unittest - - class Test(unittest.TestCase): - @classmethod - def setUpClass(cls): - cls._connection = createExpensiveConnectionObject() - - @classmethod - def tearDownClass(cls): - cls._connection.destroy() - -If you want the ``setUpClass`` and ``tearDownClass`` on base classes called -then you must call up to them yourself. The implementations in -:class:`TestCase` are empty. - -If an exception is raised during a ``setUpClass`` then the tests in the class -are not run and the ``tearDownClass`` is not run. Skipped classes will not -have ``setUpClass`` or ``tearDownClass`` run. If the exception is a -:exc:`SkipTest` exception then the class will be reported as having been skipped -instead of as an error. - - -setUpModule and tearDownModule -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -These should be implemented as functions:: - - def setUpModule(): - createConnection() - - def tearDownModule(): - closeConnection() - -If an exception is raised in a ``setUpModule`` then none of the tests in the -module will be run and the ``tearDownModule`` will not be run. If the exception is a -:exc:`SkipTest` exception then the module will be reported as having been skipped -instead of as an error. - -To add cleanup code that must be run even in the case of an exception, use -``addModuleCleanup``: - - -.. function:: addModuleCleanup(function, /, *args, **kwargs) - - Add a function to be called after :func:`tearDownModule` to cleanup - resources used during the test class. Functions will be called in reverse - order to the order they are added (:abbr:`LIFO (last-in, first-out)`). - They are called with any arguments and keyword arguments passed into - :meth:`addModuleCleanup` when they are added. - - If :meth:`setUpModule` fails, meaning that :func:`tearDownModule` is not - called, then any cleanup functions added will still be called. - - .. versionadded:: 3.8 - - -.. classmethod:: enterModuleContext(cm) - - Enter the supplied :term:`context manager`. If successful, also - add its :meth:`~object.__exit__` method as a cleanup function by - :func:`addModuleCleanup` and return the result of the - :meth:`~object.__enter__` method. - - .. versionadded:: 3.11 - - -.. function:: doModuleCleanups() - - This function is called unconditionally after :func:`tearDownModule`, or - after :func:`setUpModule` if :func:`setUpModule` raises an exception. - - It is responsible for calling all the cleanup functions added by - :func:`addModuleCleanup`. If you need cleanup functions to be called - *prior* to :func:`tearDownModule` then you can call - :func:`doModuleCleanups` yourself. - - :func:`doModuleCleanups` pops methods off the stack of cleanup - functions one at a time, so it can be called at any time. - - .. versionadded:: 3.8 - - -Signal Handling ---------------- - -.. versionadded:: 3.2 - -The :option:`-c/--catch ` command-line option to unittest, -along with the ``catchbreak`` parameter to :func:`unittest.main()`, provide -more friendly handling of control-C during a test run. With catch break -behavior enabled control-C will allow the currently running test to complete, -and the test run will then end and report all the results so far. A second -control-c will raise a :exc:`KeyboardInterrupt` in the usual way. - -The control-c handling signal handler attempts to remain compatible with code or -tests that install their own :const:`signal.SIGINT` handler. If the ``unittest`` -handler is called but *isn't* the installed :const:`signal.SIGINT` handler, -i.e. it has been replaced by the system under test and delegated to, then it -calls the default handler. This will normally be the expected behavior by code -that replaces an installed handler and delegates to it. For individual tests -that need ``unittest`` control-c handling disabled the :func:`removeHandler` -decorator can be used. - -There are a few utility functions for framework authors to enable control-c -handling functionality within test frameworks. - -.. function:: installHandler() - - Install the control-c handler. When a :const:`signal.SIGINT` is received - (usually in response to the user pressing control-c) all registered results - have :meth:`~TestResult.stop` called. - - -.. function:: registerResult(result) - - Register a :class:`TestResult` object for control-c handling. Registering a - result stores a weak reference to it, so it doesn't prevent the result from - being garbage collected. - - Registering a :class:`TestResult` object has no side-effects if control-c - handling is not enabled, so test frameworks can unconditionally register - all results they create independently of whether or not handling is enabled. - - -.. function:: removeResult(result) - - Remove a registered result. Once a result has been removed then - :meth:`~TestResult.stop` will no longer be called on that result object in - response to a control-c. - - -.. function:: removeHandler(function=None) - - When called without arguments this function removes the control-c handler - if it has been installed. This function can also be used as a test decorator - to temporarily remove the handler while the test is being executed:: - - @unittest.removeHandler - def test_signal_handling(self): - ... diff --git a/Doc/library/unix.rst b/Doc/library/unix.rst deleted file mode 100644 index 4553a104d15a24..00000000000000 --- a/Doc/library/unix.rst +++ /dev/null @@ -1,22 +0,0 @@ -.. _unix: - -********************** -Unix Specific Services -********************** - -The modules described in this chapter provide interfaces to features that are -unique to the Unix operating system, or in some cases to some or many variants -of it. Here's an overview: - - -.. toctree:: - - posix.rst - pwd.rst - grp.rst - termios.rst - tty.rst - pty.rst - fcntl.rst - resource.rst - syslog.rst diff --git a/Doc/library/urllib.error.rst b/Doc/library/urllib.error.rst deleted file mode 100644 index 8af07c5ec0474c..00000000000000 --- a/Doc/library/urllib.error.rst +++ /dev/null @@ -1,68 +0,0 @@ -:mod:`urllib.error` --- Exception classes raised by urllib.request -================================================================== - -.. module:: urllib.error - :synopsis: Exception classes raised by urllib.request. - -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Senthil Kumaran - -**Source code:** :source:`Lib/urllib/error.py` - --------------- - -The :mod:`urllib.error` module defines the exception classes for exceptions -raised by :mod:`urllib.request`. The base exception class is :exc:`URLError`. - -The following exceptions are raised by :mod:`urllib.error` as appropriate: - -.. exception:: URLError - - The handlers raise this exception (or derived exceptions) when they run into - a problem. It is a subclass of :exc:`OSError`. - - .. attribute:: reason - - The reason for this error. It can be a message string or another - exception instance. - - .. versionchanged:: 3.3 - :exc:`URLError` used to be a subtype of :exc:`IOError`, which is now an - alias of :exc:`OSError`. - - -.. exception:: HTTPError - - Though being an exception (a subclass of :exc:`URLError`), an - :exc:`HTTPError` can also function as a non-exceptional file-like return - value (the same thing that :func:`~urllib.request.urlopen` returns). This - is useful when handling exotic HTTP errors, such as requests for - authentication. - - .. attribute:: code - - An HTTP status code as defined in :rfc:`2616`. This numeric value corresponds - to a value found in the dictionary of codes as found in - :attr:`http.server.BaseHTTPRequestHandler.responses`. - - .. attribute:: reason - - This is usually a string explaining the reason for this error. - - .. attribute:: headers - - The HTTP response headers for the HTTP request that caused the - :exc:`HTTPError`. - - .. versionadded:: 3.4 - -.. exception:: ContentTooShortError(msg, content) - - This exception is raised when the :func:`~urllib.request.urlretrieve` - function detects that - the amount of the downloaded data is less than the expected amount (given by - the *Content-Length* header). - - .. attribute:: content - - The downloaded (and supposedly truncated) data. diff --git a/Doc/library/urllib.parse.rst b/Doc/library/urllib.parse.rst deleted file mode 100644 index 3e04131a222394..00000000000000 --- a/Doc/library/urllib.parse.rst +++ /dev/null @@ -1,766 +0,0 @@ -:mod:`urllib.parse` --- Parse URLs into components -================================================== - -.. module:: urllib.parse - :synopsis: Parse URLs into or assemble them from components. - -**Source code:** :source:`Lib/urllib/parse.py` - -.. index:: - single: WWW - single: World Wide Web - single: URL - pair: URL; parsing - pair: relative; URL - --------------- - -This module defines a standard interface to break Uniform Resource Locator (URL) -strings up in components (addressing scheme, network location, path etc.), to -combine the components back into a URL string, and to convert a "relative URL" -to an absolute URL given a "base URL." - -The module has been designed to match the internet RFC on Relative Uniform -Resource Locators. It supports the following URL schemes: ``file``, ``ftp``, -``gopher``, ``hdl``, ``http``, ``https``, ``imap``, ``mailto``, ``mms``, -``news``, ``nntp``, ``prospero``, ``rsync``, ``rtsp``, ``rtsps``, ``rtspu``, -``sftp``, ``shttp``, ``sip``, ``sips``, ``snews``, ``svn``, ``svn+ssh``, -``telnet``, ``wais``, ``ws``, ``wss``. - -The :mod:`urllib.parse` module defines functions that fall into two broad -categories: URL parsing and URL quoting. These are covered in detail in -the following sections. - -URL Parsing ------------ - -The URL parsing functions focus on splitting a URL string into its components, -or on combining URL components into a URL string. - -.. function:: urlparse(urlstring, scheme='', allow_fragments=True) - - Parse a URL into six components, returning a 6-item :term:`named tuple`. This - corresponds to the general structure of a URL: - ``scheme://netloc/path;parameters?query#fragment``. - Each tuple item is a string, possibly empty. The components are not broken up - into smaller parts (for example, the network location is a single string), and % - escapes are not expanded. The delimiters as shown above are not part of the - result, except for a leading slash in the *path* component, which is retained if - present. For example: - - .. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> from urllib.parse import urlparse - >>> urlparse("scheme://netloc/path;parameters?query#fragment") - ParseResult(scheme='scheme', netloc='netloc', path='/path;parameters', params='', - query='query', fragment='fragment') - >>> o = urlparse("http://docs.python.org:80/3/library/urllib.parse.html?" - ... "highlight=params#url-parsing") - >>> o - ParseResult(scheme='http', netloc='docs.python.org:80', - path='/3/library/urllib.parse.html', params='', - query='highlight=params', fragment='url-parsing') - >>> o.scheme - 'http' - >>> o.netloc - 'docs.python.org:80' - >>> o.hostname - 'docs.python.org' - >>> o.port - 80 - >>> o._replace(fragment="").geturl() - 'http://docs.python.org:80/3/library/urllib.parse.html?highlight=params' - - Following the syntax specifications in :rfc:`1808`, urlparse recognizes - a netloc only if it is properly introduced by '//'. Otherwise the - input is presumed to be a relative URL and thus to start with - a path component. - - .. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> from urllib.parse import urlparse - >>> urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - >>> urlparse('www.cwi.nl/%7Eguido/Python.html') - ParseResult(scheme='', netloc='', path='www.cwi.nl/%7Eguido/Python.html', - params='', query='', fragment='') - >>> urlparse('help/Python.html') - ParseResult(scheme='', netloc='', path='help/Python.html', params='', - query='', fragment='') - - The *scheme* argument gives the default addressing scheme, to be - used only if the URL does not specify one. It should be the same type - (text or bytes) as *urlstring*, except that the default value ``''`` is - always allowed, and is automatically converted to ``b''`` if appropriate. - - If the *allow_fragments* argument is false, fragment identifiers are not - recognized. Instead, they are parsed as part of the path, parameters - or query component, and :attr:`fragment` is set to the empty string in - the return value. - - The return value is a :term:`named tuple`, which means that its items can - be accessed by index or as named attributes, which are: - - +------------------+-------+-------------------------+------------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+=========================+========================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | - +------------------+-------+-------------------------+------------------------+ - | :attr:`netloc` | 1 | Network location part | empty string | - +------------------+-------+-------------------------+------------------------+ - | :attr:`path` | 2 | Hierarchical path | empty string | - +------------------+-------+-------------------------+------------------------+ - | :attr:`params` | 3 | Parameters for last | empty string | - | | | path element | | - +------------------+-------+-------------------------+------------------------+ - | :attr:`query` | 4 | Query component | empty string | - +------------------+-------+-------------------------+------------------------+ - | :attr:`fragment` | 5 | Fragment identifier | empty string | - +------------------+-------+-------------------------+------------------------+ - | :attr:`username` | | User name | :const:`None` | - +------------------+-------+-------------------------+------------------------+ - | :attr:`password` | | Password | :const:`None` | - +------------------+-------+-------------------------+------------------------+ - | :attr:`hostname` | | Host name (lower case) | :const:`None` | - +------------------+-------+-------------------------+------------------------+ - | :attr:`port` | | Port number as integer, | :const:`None` | - | | | if present | | - +------------------+-------+-------------------------+------------------------+ - - Reading the :attr:`port` attribute will raise a :exc:`ValueError` if - an invalid port is specified in the URL. See section - :ref:`urlparse-result-object` for more information on the result object. - - Unmatched square brackets in the :attr:`netloc` attribute will raise a - :exc:`ValueError`. - - Characters in the :attr:`netloc` attribute that decompose under NFKC - normalization (as used by the IDNA encoding) into any of ``/``, ``?``, - ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is - decomposed before parsing, no error will be raised. - - As is the case with all named tuples, the subclass has a few additional methods - and attributes that are particularly useful. One such method is :meth:`_replace`. - The :meth:`_replace` method will return a new ParseResult object replacing specified - fields with new values. - - .. doctest:: - :options: +NORMALIZE_WHITESPACE - - >>> from urllib.parse import urlparse - >>> u = urlparse('//www.cwi.nl:80/%7Eguido/Python.html') - >>> u - ParseResult(scheme='', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - >>> u._replace(scheme='http') - ParseResult(scheme='http', netloc='www.cwi.nl:80', path='/%7Eguido/Python.html', - params='', query='', fragment='') - - .. warning:: - - :func:`urlparse` does not perform validation. See :ref:`URL parsing - security ` for details. - - .. versionchanged:: 3.2 - Added IPv6 URL parsing capabilities. - - .. versionchanged:: 3.3 - The fragment is now parsed for all URL schemes (unless *allow_fragment* is - false), in accordance with :rfc:`3986`. Previously, an allowlist of - schemes that support fragments existed. - - .. versionchanged:: 3.6 - Out-of-range port numbers now raise :exc:`ValueError`, instead of - returning :const:`None`. - - .. versionchanged:: 3.8 - Characters that affect netloc parsing under NFKC normalization will - now raise :exc:`ValueError`. - - -.. function:: parse_qs(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') - - Parse a query string given as a string argument (data of type - :mimetype:`application/x-www-form-urlencoded`). Data are returned as a - dictionary. The dictionary keys are the unique query variable names and the - values are lists of values for each name. - - The optional argument *keep_blank_values* is a flag indicating whether blank - values in percent-encoded queries should be treated as blank strings. A true value - indicates that blanks should be retained as blank strings. The default false - value indicates that blank values are to be ignored and treated as if they were - not included. - - The optional argument *strict_parsing* is a flag indicating what to do with - parsing errors. If false (the default), errors are silently ignored. If true, - errors raise a :exc:`ValueError` exception. - - The optional *encoding* and *errors* parameters specify how to decode - percent-encoded sequences into Unicode characters, as accepted by the - :meth:`bytes.decode` method. - - The optional argument *max_num_fields* is the maximum number of fields to - read. If set, then throws a :exc:`ValueError` if there are more than - *max_num_fields* fields read. - - The optional argument *separator* is the symbol to use for separating the - query arguments. It defaults to ``&``. - - Use the :func:`urllib.parse.urlencode` function (with the ``doseq`` - parameter set to ``True``) to convert such dictionaries into query - strings. - - - .. versionchanged:: 3.2 - Add *encoding* and *errors* parameters. - - .. versionchanged:: 3.8 - Added *max_num_fields* parameter. - - .. versionchanged:: 3.10 - Added *separator* parameter with the default value of ``&``. Python - versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as - query parameter separator. This has been changed to allow only a single - separator key, with ``&`` as the default separator. - - -.. function:: parse_qsl(qs, keep_blank_values=False, strict_parsing=False, encoding='utf-8', errors='replace', max_num_fields=None, separator='&') - - Parse a query string given as a string argument (data of type - :mimetype:`application/x-www-form-urlencoded`). Data are returned as a list of - name, value pairs. - - The optional argument *keep_blank_values* is a flag indicating whether blank - values in percent-encoded queries should be treated as blank strings. A true value - indicates that blanks should be retained as blank strings. The default false - value indicates that blank values are to be ignored and treated as if they were - not included. - - The optional argument *strict_parsing* is a flag indicating what to do with - parsing errors. If false (the default), errors are silently ignored. If true, - errors raise a :exc:`ValueError` exception. - - The optional *encoding* and *errors* parameters specify how to decode - percent-encoded sequences into Unicode characters, as accepted by the - :meth:`bytes.decode` method. - - The optional argument *max_num_fields* is the maximum number of fields to - read. If set, then throws a :exc:`ValueError` if there are more than - *max_num_fields* fields read. - - The optional argument *separator* is the symbol to use for separating the - query arguments. It defaults to ``&``. - - Use the :func:`urllib.parse.urlencode` function to convert such lists of pairs into - query strings. - - .. versionchanged:: 3.2 - Add *encoding* and *errors* parameters. - - .. versionchanged:: 3.8 - Added *max_num_fields* parameter. - - .. versionchanged:: 3.10 - Added *separator* parameter with the default value of ``&``. Python - versions earlier than Python 3.10 allowed using both ``;`` and ``&`` as - query parameter separator. This has been changed to allow only a single - separator key, with ``&`` as the default separator. - - -.. function:: urlunparse(parts) - - Construct a URL from a tuple as returned by ``urlparse()``. The *parts* - argument can be any six-item iterable. This may result in a slightly - different, but equivalent URL, if the URL that was parsed originally had - unnecessary delimiters (for example, a ``?`` with an empty query; the RFC - states that these are equivalent). - - -.. function:: urlsplit(urlstring, scheme='', allow_fragments=True) - - This is similar to :func:`urlparse`, but does not split the params from the URL. - This should generally be used instead of :func:`urlparse` if the more recent URL - syntax allowing parameters to be applied to each segment of the *path* portion - of the URL (see :rfc:`2396`) is wanted. A separate function is needed to - separate the path segments and parameters. This function returns a 5-item - :term:`named tuple`:: - - (addressing scheme, network location, path, query, fragment identifier). - - The return value is a :term:`named tuple`, its items can be accessed by index - or as named attributes: - - +------------------+-------+-------------------------+----------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+=========================+======================+ - | :attr:`scheme` | 0 | URL scheme specifier | *scheme* parameter | - +------------------+-------+-------------------------+----------------------+ - | :attr:`netloc` | 1 | Network location part | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`path` | 2 | Hierarchical path | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`query` | 3 | Query component | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`fragment` | 4 | Fragment identifier | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`username` | | User name | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`password` | | Password | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`hostname` | | Host name (lower case) | :const:`None` | - +------------------+-------+-------------------------+----------------------+ - | :attr:`port` | | Port number as integer, | :const:`None` | - | | | if present | | - +------------------+-------+-------------------------+----------------------+ - - Reading the :attr:`port` attribute will raise a :exc:`ValueError` if - an invalid port is specified in the URL. See section - :ref:`urlparse-result-object` for more information on the result object. - - Unmatched square brackets in the :attr:`netloc` attribute will raise a - :exc:`ValueError`. - - Characters in the :attr:`netloc` attribute that decompose under NFKC - normalization (as used by the IDNA encoding) into any of ``/``, ``?``, - ``#``, ``@``, or ``:`` will raise a :exc:`ValueError`. If the URL is - decomposed before parsing, no error will be raised. - - Following some of the `WHATWG spec`_ that updates RFC 3986, leading C0 - control and space characters are stripped from the URL. ``\n``, - ``\r`` and tab ``\t`` characters are removed from the URL at any position. - - .. warning:: - - :func:`urlsplit` does not perform validation. See :ref:`URL parsing - security ` for details. - - .. versionchanged:: 3.6 - Out-of-range port numbers now raise :exc:`ValueError`, instead of - returning :const:`None`. - - .. versionchanged:: 3.8 - Characters that affect netloc parsing under NFKC normalization will - now raise :exc:`ValueError`. - - .. versionchanged:: 3.10 - ASCII newline and tab characters are stripped from the URL. - - .. versionchanged:: 3.11.4 - Leading WHATWG C0 control and space characters are stripped from the URL. - -.. _WHATWG spec: https://url.spec.whatwg.org/#concept-basic-url-parser - -.. function:: urlunsplit(parts) - - Combine the elements of a tuple as returned by :func:`urlsplit` into a - complete URL as a string. The *parts* argument can be any five-item - iterable. This may result in a slightly different, but equivalent URL, if the - URL that was parsed originally had unnecessary delimiters (for example, a ? - with an empty query; the RFC states that these are equivalent). - - -.. function:: urljoin(base, url, allow_fragments=True) - - Construct a full ("absolute") URL by combining a "base URL" (*base*) with - another URL (*url*). Informally, this uses components of the base URL, in - particular the addressing scheme, the network location and (part of) the - path, to provide missing components in the relative URL. For example: - - >>> from urllib.parse import urljoin - >>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', 'FAQ.html') - 'http://www.cwi.nl/%7Eguido/FAQ.html' - - The *allow_fragments* argument has the same meaning and default as for - :func:`urlparse`. - - .. note:: - - If *url* is an absolute URL (that is, it starts with ``//`` or ``scheme://``), - the *url*'s hostname and/or scheme will be present in the result. For example: - - .. doctest:: - - >>> urljoin('http://www.cwi.nl/%7Eguido/Python.html', - ... '//www.python.org/%7Eguido') - 'http://www.python.org/%7Eguido' - - If you do not want that behavior, preprocess the *url* with :func:`urlsplit` and - :func:`urlunsplit`, removing possible *scheme* and *netloc* parts. - - - .. versionchanged:: 3.5 - - Behavior updated to match the semantics defined in :rfc:`3986`. - - -.. function:: urldefrag(url) - - If *url* contains a fragment identifier, return a modified version of *url* - with no fragment identifier, and the fragment identifier as a separate - string. If there is no fragment identifier in *url*, return *url* unmodified - and an empty string. - - The return value is a :term:`named tuple`, its items can be accessed by index - or as named attributes: - - +------------------+-------+-------------------------+----------------------+ - | Attribute | Index | Value | Value if not present | - +==================+=======+=========================+======================+ - | :attr:`url` | 0 | URL with no fragment | empty string | - +------------------+-------+-------------------------+----------------------+ - | :attr:`fragment` | 1 | Fragment identifier | empty string | - +------------------+-------+-------------------------+----------------------+ - - See section :ref:`urlparse-result-object` for more information on the result - object. - - .. versionchanged:: 3.2 - Result is a structured object rather than a simple 2-tuple. - -.. function:: unwrap(url) - - Extract the url from a wrapped URL (that is, a string formatted as - ````, ````, ``URL:scheme://host/path`` - or ``scheme://host/path``). If *url* is not a wrapped URL, it is returned - without changes. - -.. _url-parsing-security: - -URL parsing security --------------------- - -The :func:`urlsplit` and :func:`urlparse` APIs do not perform **validation** of -inputs. They may not raise errors on inputs that other applications consider -invalid. They may also succeed on some inputs that might not be considered -URLs elsewhere. Their purpose is for practical functionality rather than -purity. - -Instead of raising an exception on unusual input, they may instead return some -component parts as empty strings. Or components may contain more than perhaps -they should. - -We recommend that users of these APIs where the values may be used anywhere -with security implications code defensively. Do some verification within your -code before trusting a returned component part. Does that ``scheme`` make -sense? Is that a sensible ``path``? Is there anything strange about that -``hostname``? etc. - -What constitutes a URL is not universally well defined. Different applications -have different needs and desired constraints. For instance the living `WHATWG -spec`_ describes what user facing web clients such as a web browser require. -While :rfc:`3986` is more general. These functions incorporate some aspects of -both, but cannot be claimed compliant with either. The APIs and existing user -code with expectations on specific behaviors predate both standards leading us -to be very cautious about making API behavior changes. - -.. _parsing-ascii-encoded-bytes: - -Parsing ASCII Encoded Bytes ---------------------------- - -The URL parsing functions were originally designed to operate on character -strings only. In practice, it is useful to be able to manipulate properly -quoted and encoded URLs as sequences of ASCII bytes. Accordingly, the -URL parsing functions in this module all operate on :class:`bytes` and -:class:`bytearray` objects in addition to :class:`str` objects. - -If :class:`str` data is passed in, the result will also contain only -:class:`str` data. If :class:`bytes` or :class:`bytearray` data is -passed in, the result will contain only :class:`bytes` data. - -Attempting to mix :class:`str` data with :class:`bytes` or -:class:`bytearray` in a single function call will result in a -:exc:`TypeError` being raised, while attempting to pass in non-ASCII -byte values will trigger :exc:`UnicodeDecodeError`. - -To support easier conversion of result objects between :class:`str` and -:class:`bytes`, all return values from URL parsing functions provide -either an :meth:`encode` method (when the result contains :class:`str` -data) or a :meth:`decode` method (when the result contains :class:`bytes` -data). The signatures of these methods match those of the corresponding -:class:`str` and :class:`bytes` methods (except that the default encoding -is ``'ascii'`` rather than ``'utf-8'``). Each produces a value of a -corresponding type that contains either :class:`bytes` data (for -:meth:`encode` methods) or :class:`str` data (for -:meth:`decode` methods). - -Applications that need to operate on potentially improperly quoted URLs -that may contain non-ASCII data will need to do their own decoding from -bytes to characters before invoking the URL parsing methods. - -The behaviour described in this section applies only to the URL parsing -functions. The URL quoting functions use their own rules when producing -or consuming byte sequences as detailed in the documentation of the -individual URL quoting functions. - -.. versionchanged:: 3.2 - URL parsing functions now accept ASCII encoded byte sequences - - -.. _urlparse-result-object: - -Structured Parse Results ------------------------- - -The result objects from the :func:`urlparse`, :func:`urlsplit` and -:func:`urldefrag` functions are subclasses of the :class:`tuple` type. -These subclasses add the attributes listed in the documentation for -those functions, the encoding and decoding support described in the -previous section, as well as an additional method: - -.. method:: urllib.parse.SplitResult.geturl() - - Return the re-combined version of the original URL as a string. This may - differ from the original URL in that the scheme may be normalized to lower - case and empty components may be dropped. Specifically, empty parameters, - queries, and fragment identifiers will be removed. - - For :func:`urldefrag` results, only empty fragment identifiers will be removed. - For :func:`urlsplit` and :func:`urlparse` results, all noted changes will be - made to the URL returned by this method. - - The result of this method remains unchanged if passed back through the original - parsing function: - - >>> from urllib.parse import urlsplit - >>> url = 'HTTP://www.Python.org/doc/#' - >>> r1 = urlsplit(url) - >>> r1.geturl() - 'http://www.Python.org/doc/' - >>> r2 = urlsplit(r1.geturl()) - >>> r2.geturl() - 'http://www.Python.org/doc/' - - -The following classes provide the implementations of the structured parse -results when operating on :class:`str` objects: - -.. class:: DefragResult(url, fragment) - - Concrete class for :func:`urldefrag` results containing :class:`str` - data. The :meth:`encode` method returns a :class:`DefragResultBytes` - instance. - - .. versionadded:: 3.2 - -.. class:: ParseResult(scheme, netloc, path, params, query, fragment) - - Concrete class for :func:`urlparse` results containing :class:`str` - data. The :meth:`encode` method returns a :class:`ParseResultBytes` - instance. - -.. class:: SplitResult(scheme, netloc, path, query, fragment) - - Concrete class for :func:`urlsplit` results containing :class:`str` - data. The :meth:`encode` method returns a :class:`SplitResultBytes` - instance. - - -The following classes provide the implementations of the parse results when -operating on :class:`bytes` or :class:`bytearray` objects: - -.. class:: DefragResultBytes(url, fragment) - - Concrete class for :func:`urldefrag` results containing :class:`bytes` - data. The :meth:`decode` method returns a :class:`DefragResult` - instance. - - .. versionadded:: 3.2 - -.. class:: ParseResultBytes(scheme, netloc, path, params, query, fragment) - - Concrete class for :func:`urlparse` results containing :class:`bytes` - data. The :meth:`decode` method returns a :class:`ParseResult` - instance. - - .. versionadded:: 3.2 - -.. class:: SplitResultBytes(scheme, netloc, path, query, fragment) - - Concrete class for :func:`urlsplit` results containing :class:`bytes` - data. The :meth:`decode` method returns a :class:`SplitResult` - instance. - - .. versionadded:: 3.2 - - -URL Quoting ------------ - -The URL quoting functions focus on taking program data and making it safe -for use as URL components by quoting special characters and appropriately -encoding non-ASCII text. They also support reversing these operations to -recreate the original data from the contents of a URL component if that -task isn't already covered by the URL parsing functions above. - -.. function:: quote(string, safe='/', encoding=None, errors=None) - - Replace special characters in *string* using the :samp:`%{xx}` escape. Letters, - digits, and the characters ``'_.-~'`` are never quoted. By default, this - function is intended for quoting the path section of a URL. The optional - *safe* parameter specifies additional ASCII characters that should not be - quoted --- its default value is ``'/'``. - - *string* may be either a :class:`str` or a :class:`bytes` object. - - .. versionchanged:: 3.7 - Moved from :rfc:`2396` to :rfc:`3986` for quoting URL strings. "~" is now - included in the set of unreserved characters. - - The optional *encoding* and *errors* parameters specify how to deal with - non-ASCII characters, as accepted by the :meth:`str.encode` method. - *encoding* defaults to ``'utf-8'``. - *errors* defaults to ``'strict'``, meaning unsupported characters raise a - :class:`UnicodeEncodeError`. - *encoding* and *errors* must not be supplied if *string* is a - :class:`bytes`, or a :class:`TypeError` is raised. - - Note that ``quote(string, safe, encoding, errors)`` is equivalent to - ``quote_from_bytes(string.encode(encoding, errors), safe)``. - - Example: ``quote('/El Niño/')`` yields ``'/El%20Ni%C3%B1o/'``. - - -.. function:: quote_plus(string, safe='', encoding=None, errors=None) - - Like :func:`quote`, but also replace spaces with plus signs, as required for - quoting HTML form values when building up a query string to go into a URL. - Plus signs in the original string are escaped unless they are included in - *safe*. It also does not have *safe* default to ``'/'``. - - Example: ``quote_plus('/El Niño/')`` yields ``'%2FEl+Ni%C3%B1o%2F'``. - - -.. function:: quote_from_bytes(bytes, safe='/') - - Like :func:`quote`, but accepts a :class:`bytes` object rather than a - :class:`str`, and does not perform string-to-bytes encoding. - - Example: ``quote_from_bytes(b'a&\xef')`` yields - ``'a%26%EF'``. - - -.. function:: unquote(string, encoding='utf-8', errors='replace') - - Replace :samp:`%{xx}` escapes with their single-character equivalent. - The optional *encoding* and *errors* parameters specify how to decode - percent-encoded sequences into Unicode characters, as accepted by the - :meth:`bytes.decode` method. - - *string* may be either a :class:`str` or a :class:`bytes` object. - - *encoding* defaults to ``'utf-8'``. - *errors* defaults to ``'replace'``, meaning invalid sequences are replaced - by a placeholder character. - - Example: ``unquote('/El%20Ni%C3%B1o/')`` yields ``'/El Niño/'``. - - .. versionchanged:: 3.9 - *string* parameter supports bytes and str objects (previously only str). - - - - -.. function:: unquote_plus(string, encoding='utf-8', errors='replace') - - Like :func:`unquote`, but also replace plus signs with spaces, as required - for unquoting HTML form values. - - *string* must be a :class:`str`. - - Example: ``unquote_plus('/El+Ni%C3%B1o/')`` yields ``'/El Niño/'``. - - -.. function:: unquote_to_bytes(string) - - Replace :samp:`%{xx}` escapes with their single-octet equivalent, and return a - :class:`bytes` object. - - *string* may be either a :class:`str` or a :class:`bytes` object. - - If it is a :class:`str`, unescaped non-ASCII characters in *string* - are encoded into UTF-8 bytes. - - Example: ``unquote_to_bytes('a%26%EF')`` yields ``b'a&\xef'``. - - -.. function:: urlencode(query, doseq=False, safe='', encoding=None, \ - errors=None, quote_via=quote_plus) - - Convert a mapping object or a sequence of two-element tuples, which may - contain :class:`str` or :class:`bytes` objects, to a percent-encoded ASCII - text string. If the resultant string is to be used as a *data* for POST - operation with the :func:`~urllib.request.urlopen` function, then - it should be encoded to bytes, otherwise it would result in a - :exc:`TypeError`. - - The resulting string is a series of ``key=value`` pairs separated by ``'&'`` - characters, where both *key* and *value* are quoted using the *quote_via* - function. By default, :func:`quote_plus` is used to quote the values, which - means spaces are quoted as a ``'+'`` character and '/' characters are - encoded as ``%2F``, which follows the standard for GET requests - (``application/x-www-form-urlencoded``). An alternate function that can be - passed as *quote_via* is :func:`quote`, which will encode spaces as ``%20`` - and not encode '/' characters. For maximum control of what is quoted, use - ``quote`` and specify a value for *safe*. - - When a sequence of two-element tuples is used as the *query* - argument, the first element of each tuple is a key and the second is a - value. The value element in itself can be a sequence and in that case, if - the optional parameter *doseq* evaluates to ``True``, individual - ``key=value`` pairs separated by ``'&'`` are generated for each element of - the value sequence for the key. The order of parameters in the encoded - string will match the order of parameter tuples in the sequence. - - The *safe*, *encoding*, and *errors* parameters are passed down to - *quote_via* (the *encoding* and *errors* parameters are only passed - when a query element is a :class:`str`). - - To reverse this encoding process, :func:`parse_qs` and :func:`parse_qsl` are - provided in this module to parse query strings into Python data structures. - - Refer to :ref:`urllib examples ` to find out how the - :func:`urllib.parse.urlencode` method can be used for generating the query - string of a URL or data for a POST request. - - .. versionchanged:: 3.2 - *query* supports bytes and string objects. - - .. versionadded:: 3.5 - *quote_via* parameter. - - -.. seealso:: - - `WHATWG`_ - URL Living standard - Working Group for the URL Standard that defines URLs, domains, IP addresses, the - application/x-www-form-urlencoded format, and their API. - - :rfc:`3986` - Uniform Resource Identifiers - This is the current standard (STD66). Any changes to urllib.parse module - should conform to this. Certain deviations could be observed, which are - mostly for backward compatibility purposes and for certain de-facto - parsing requirements as commonly observed in major browsers. - - :rfc:`2732` - Format for Literal IPv6 Addresses in URL's. - This specifies the parsing requirements of IPv6 URLs. - - :rfc:`2396` - Uniform Resource Identifiers (URI): Generic Syntax - Document describing the generic syntactic requirements for both Uniform Resource - Names (URNs) and Uniform Resource Locators (URLs). - - :rfc:`2368` - The mailto URL scheme. - Parsing requirements for mailto URL schemes. - - :rfc:`1808` - Relative Uniform Resource Locators - This Request For Comments includes the rules for joining an absolute and a - relative URL, including a fair number of "Abnormal Examples" which govern the - treatment of border cases. - - :rfc:`1738` - Uniform Resource Locators (URL) - This specifies the formal syntax and semantics of absolute URLs. - -.. _WHATWG: https://url.spec.whatwg.org/ diff --git a/Doc/library/urllib.request.rst b/Doc/library/urllib.request.rst deleted file mode 100644 index 002dab8a65195c..00000000000000 --- a/Doc/library/urllib.request.rst +++ /dev/null @@ -1,1636 +0,0 @@ -:mod:`urllib.request` --- Extensible library for opening URLs -============================================================= - -.. module:: urllib.request - :synopsis: Extensible library for opening URLs. - -.. moduleauthor:: Jeremy Hylton -.. sectionauthor:: Moshe Zadka -.. sectionauthor:: Senthil Kumaran - -**Source code:** :source:`Lib/urllib/request.py` - --------------- - -The :mod:`urllib.request` module defines functions and classes which help in -opening URLs (mostly HTTP) in a complex world --- basic and digest -authentication, redirections, cookies and more. - -.. seealso:: - - The `Requests package `_ - is recommended for a higher-level HTTP client interface. - -.. include:: ../includes/wasm-notavail.rst - -The :mod:`urllib.request` module defines the following functions: - - -.. function:: urlopen(url, data=None[, timeout], *, cafile=None, capath=None, cadefault=False, context=None) - - Open *url*, which can be either a string containing a valid, properly - encoded URL, or a :class:`Request` object. - - *data* must be an object specifying additional data to be sent to the - server, or ``None`` if no such data is needed. See :class:`Request` - for details. - - urllib.request module uses HTTP/1.1 and includes ``Connection:close`` header - in its HTTP requests. - - The optional *timeout* parameter specifies a timeout in seconds for - blocking operations like the connection attempt (if not specified, - the global default timeout setting will be used). This actually - only works for HTTP, HTTPS and FTP connections. - - If *context* is specified, it must be a :class:`ssl.SSLContext` instance - describing the various SSL options. See :class:`~http.client.HTTPSConnection` - for more details. - - The optional *cafile* and *capath* parameters specify a set of trusted - CA certificates for HTTPS requests. *cafile* should point to a single - file containing a bundle of CA certificates, whereas *capath* should - point to a directory of hashed certificate files. More information can - be found in :meth:`ssl.SSLContext.load_verify_locations`. - - The *cadefault* parameter is ignored. - - This function always returns an object which can work as a - :term:`context manager` and has the properties *url*, *headers*, and *status*. - See :class:`urllib.response.addinfourl` for more detail on these properties. - - For HTTP and HTTPS URLs, this function returns a - :class:`http.client.HTTPResponse` object slightly modified. In addition - to the three new methods above, the msg attribute contains the - same information as the :attr:`~http.client.HTTPResponse.reason` - attribute --- the reason phrase returned by server --- instead of - the response headers as it is specified in the documentation for - :class:`~http.client.HTTPResponse`. - - For FTP, file, and data URLs and requests explicitly handled by legacy - :class:`URLopener` and :class:`FancyURLopener` classes, this function - returns a :class:`urllib.response.addinfourl` object. - - Raises :exc:`~urllib.error.URLError` on protocol errors. - - Note that ``None`` may be returned if no handler handles the request (though - the default installed global :class:`OpenerDirector` uses - :class:`UnknownHandler` to ensure this never happens). - - In addition, if proxy settings are detected (for example, when a ``*_proxy`` - environment variable like :envvar:`http_proxy` is set), - :class:`ProxyHandler` is default installed and makes sure the requests are - handled through the proxy. - - The legacy ``urllib.urlopen`` function from Python 2.6 and earlier has been - discontinued; :func:`urllib.request.urlopen` corresponds to the old - ``urllib2.urlopen``. Proxy handling, which was done by passing a dictionary - parameter to ``urllib.urlopen``, can be obtained by using - :class:`ProxyHandler` objects. - - .. audit-event:: urllib.Request fullurl,data,headers,method urllib.request.urlopen - - The default opener raises an :ref:`auditing event ` - ``urllib.Request`` with arguments ``fullurl``, ``data``, ``headers``, - ``method`` taken from the request object. - - .. versionchanged:: 3.2 - *cafile* and *capath* were added. - - .. versionchanged:: 3.2 - HTTPS virtual hosts are now supported if possible (that is, if - :const:`ssl.HAS_SNI` is true). - - .. versionadded:: 3.2 - *data* can be an iterable object. - - .. versionchanged:: 3.3 - *cadefault* was added. - - .. versionchanged:: 3.4.3 - *context* was added. - - .. versionchanged:: 3.10 - HTTPS connection now send an ALPN extension with protocol indicator - ``http/1.1`` when no *context* is given. Custom *context* should set - ALPN protocols with :meth:`~ssl.SSLContext.set_alpn_protocol`. - - .. deprecated:: 3.6 - - *cafile*, *capath* and *cadefault* are deprecated in favor of *context*. - Please use :meth:`ssl.SSLContext.load_cert_chain` instead, or let - :func:`ssl.create_default_context` select the system's trusted CA - certificates for you. - - -.. function:: install_opener(opener) - - Install an :class:`OpenerDirector` instance as the default global opener. - Installing an opener is only necessary if you want urlopen to use that - opener; otherwise, simply call :meth:`OpenerDirector.open` instead of - :func:`~urllib.request.urlopen`. The code does not check for a real - :class:`OpenerDirector`, and any class with the appropriate interface will - work. - - -.. function:: build_opener([handler, ...]) - - Return an :class:`OpenerDirector` instance, which chains the handlers in the - order given. *handler*\s can be either instances of :class:`BaseHandler`, or - subclasses of :class:`BaseHandler` (in which case it must be possible to call - the constructor without any parameters). Instances of the following classes - will be in front of the *handler*\s, unless the *handler*\s contain them, - instances of them or subclasses of them: :class:`ProxyHandler` (if proxy - settings are detected), :class:`UnknownHandler`, :class:`HTTPHandler`, - :class:`HTTPDefaultErrorHandler`, :class:`HTTPRedirectHandler`, - :class:`FTPHandler`, :class:`FileHandler`, :class:`HTTPErrorProcessor`. - - If the Python installation has SSL support (i.e., if the :mod:`ssl` module - can be imported), :class:`HTTPSHandler` will also be added. - - A :class:`BaseHandler` subclass may also change its :attr:`handler_order` - attribute to modify its position in the handlers list. - - -.. function:: pathname2url(path) - - Convert the pathname *path* from the local syntax for a path to the form used in - the path component of a URL. This does not produce a complete URL. The return - value will already be quoted using the :func:`~urllib.parse.quote` function. - - -.. function:: url2pathname(path) - - Convert the path component *path* from a percent-encoded URL to the local syntax for a - path. This does not accept a complete URL. This function uses - :func:`~urllib.parse.unquote` to decode *path*. - -.. function:: getproxies() - - This helper function returns a dictionary of scheme to proxy server URL - mappings. It scans the environment for variables named ``_proxy``, - in a case insensitive approach, for all operating systems first, and when it - cannot find it, looks for proxy information from System - Configuration for macOS and Windows Systems Registry for Windows. - If both lowercase and uppercase environment variables exist (and disagree), - lowercase is preferred. - - .. note:: - - If the environment variable ``REQUEST_METHOD`` is set, which usually - indicates your script is running in a CGI environment, the environment - variable ``HTTP_PROXY`` (uppercase ``_PROXY``) will be ignored. This is - because that variable can be injected by a client using the "Proxy:" HTTP - header. If you need to use an HTTP proxy in a CGI environment, either use - ``ProxyHandler`` explicitly, or make sure the variable name is in - lowercase (or at least the ``_proxy`` suffix). - - -The following classes are provided: - -.. class:: Request(url, data=None, headers={}, origin_req_host=None, unverifiable=False, method=None) - - This class is an abstraction of a URL request. - - *url* should be a string containing a valid, properly encoded URL. - - *data* must be an object specifying additional data to send to the - server, or ``None`` if no such data is needed. Currently HTTP - requests are the only ones that use *data*. The supported object - types include bytes, file-like objects, and iterables of bytes-like objects. - If no ``Content-Length`` nor ``Transfer-Encoding`` header field - has been provided, :class:`HTTPHandler` will set these headers according - to the type of *data*. ``Content-Length`` will be used to send - bytes objects, while ``Transfer-Encoding: chunked`` as specified in - :rfc:`7230`, Section 3.3.1 will be used to send files and other iterables. - - For an HTTP POST request method, *data* should be a buffer in the - standard :mimetype:`application/x-www-form-urlencoded` format. The - :func:`urllib.parse.urlencode` function takes a mapping or sequence - of 2-tuples and returns an ASCII string in this format. It should - be encoded to bytes before being used as the *data* parameter. - - *headers* should be a dictionary, and will be treated as if - :meth:`add_header` was called with each key and value as arguments. - This is often used to "spoof" the ``User-Agent`` header value, which is - used by a browser to identify itself -- some HTTP servers only - allow requests coming from common browsers as opposed to scripts. - For example, Mozilla Firefox may identify itself as ``"Mozilla/5.0 - (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11"``, while - :mod:`urllib`'s default user agent string is - ``"Python-urllib/2.6"`` (on Python 2.6). - All header keys are sent in camel case. - - An appropriate ``Content-Type`` header should be included if the *data* - argument is present. If this header has not been provided and *data* - is not None, ``Content-Type: application/x-www-form-urlencoded`` will - be added as a default. - - The next two arguments are only of interest for correct handling - of third-party HTTP cookies: - - *origin_req_host* should be the request-host of the origin - transaction, as defined by :rfc:`2965`. It defaults to - ``http.cookiejar.request_host(self)``. This is the host name or IP - address of the original request that was initiated by the user. - For example, if the request is for an image in an HTML document, - this should be the request-host of the request for the page - containing the image. - - *unverifiable* should indicate whether the request is unverifiable, - as defined by :rfc:`2965`. It defaults to ``False``. An unverifiable - request is one whose URL the user did not have the option to - approve. For example, if the request is for an image in an HTML - document, and the user had no option to approve the automatic - fetching of the image, this should be true. - - *method* should be a string that indicates the HTTP request method that - will be used (e.g. ``'HEAD'``). If provided, its value is stored in the - :attr:`~Request.method` attribute and is used by :meth:`get_method()`. - The default is ``'GET'`` if *data* is ``None`` or ``'POST'`` otherwise. - Subclasses may indicate a different default method by setting the - :attr:`~Request.method` attribute in the class itself. - - .. note:: - The request will not work as expected if the data object is unable - to deliver its content more than once (e.g. a file or an iterable - that can produce the content only once) and the request is retried - for HTTP redirects or authentication. The *data* is sent to the - HTTP server right away after the headers. There is no support for - a 100-continue expectation in the library. - - .. versionchanged:: 3.3 - :attr:`Request.method` argument is added to the Request class. - - .. versionchanged:: 3.4 - Default :attr:`Request.method` may be indicated at the class level. - - .. versionchanged:: 3.6 - Do not raise an error if the ``Content-Length`` has not been - provided and *data* is neither ``None`` nor a bytes object. - Fall back to use chunked transfer encoding instead. - -.. class:: OpenerDirector() - - The :class:`OpenerDirector` class opens URLs via :class:`BaseHandler`\ s chained - together. It manages the chaining of handlers, and recovery from errors. - - -.. class:: BaseHandler() - - This is the base class for all registered handlers --- and handles only the - simple mechanics of registration. - - -.. class:: HTTPDefaultErrorHandler() - - A class which defines a default handler for HTTP error responses; all responses - are turned into :exc:`~urllib.error.HTTPError` exceptions. - - -.. class:: HTTPRedirectHandler() - - A class to handle redirections. - - -.. class:: HTTPCookieProcessor(cookiejar=None) - - A class to handle HTTP Cookies. - - -.. class:: ProxyHandler(proxies=None) - - Cause requests to go through a proxy. If *proxies* is given, it must be a - dictionary mapping protocol names to URLs of proxies. The default is to read - the list of proxies from the environment variables - ``_proxy``. If no proxy environment variables are set, then - in a Windows environment proxy settings are obtained from the registry's - Internet Settings section, and in a macOS environment proxy information - is retrieved from the System Configuration Framework. - - To disable autodetected proxy pass an empty dictionary. - - The :envvar:`no_proxy` environment variable can be used to specify hosts - which shouldn't be reached via proxy; if set, it should be a comma-separated - list of hostname suffixes, optionally with ``:port`` appended, for example - ``cern.ch,ncsa.uiuc.edu,some.host:8080``. - - .. note:: - - ``HTTP_PROXY`` will be ignored if a variable ``REQUEST_METHOD`` is set; - see the documentation on :func:`~urllib.request.getproxies`. - - -.. class:: HTTPPasswordMgr() - - Keep a database of ``(realm, uri) -> (user, password)`` mappings. - - -.. class:: HTTPPasswordMgrWithDefaultRealm() - - Keep a database of ``(realm, uri) -> (user, password)`` mappings. A realm of - ``None`` is considered a catch-all realm, which is searched if no other realm - fits. - - -.. class:: HTTPPasswordMgrWithPriorAuth() - - A variant of :class:`HTTPPasswordMgrWithDefaultRealm` that also has a - database of ``uri -> is_authenticated`` mappings. Can be used by a - BasicAuth handler to determine when to send authentication credentials - immediately instead of waiting for a ``401`` response first. - - .. versionadded:: 3.5 - - -.. class:: AbstractBasicAuthHandler(password_mgr=None) - - This is a mixin class that helps with HTTP authentication, both to the remote - host and to a proxy. *password_mgr*, if given, should be something that is - compatible with :class:`HTTPPasswordMgr`; refer to section - :ref:`http-password-mgr` for information on the interface that must be - supported. If *passwd_mgr* also provides ``is_authenticated`` and - ``update_authenticated`` methods (see - :ref:`http-password-mgr-with-prior-auth`), then the handler will use the - ``is_authenticated`` result for a given URI to determine whether or not to - send authentication credentials with the request. If ``is_authenticated`` - returns ``True`` for the URI, credentials are sent. If ``is_authenticated`` - is ``False``, credentials are not sent, and then if a ``401`` response is - received the request is re-sent with the authentication credentials. If - authentication succeeds, ``update_authenticated`` is called to set - ``is_authenticated`` ``True`` for the URI, so that subsequent requests to - the URI or any of its super-URIs will automatically include the - authentication credentials. - - .. versionadded:: 3.5 - Added ``is_authenticated`` support. - - -.. class:: HTTPBasicAuthHandler(password_mgr=None) - - Handle authentication with the remote host. *password_mgr*, if given, should - be something that is compatible with :class:`HTTPPasswordMgr`; refer to - section :ref:`http-password-mgr` for information on the interface that must - be supported. HTTPBasicAuthHandler will raise a :exc:`ValueError` when - presented with a wrong Authentication scheme. - - -.. class:: ProxyBasicAuthHandler(password_mgr=None) - - Handle authentication with the proxy. *password_mgr*, if given, should be - something that is compatible with :class:`HTTPPasswordMgr`; refer to section - :ref:`http-password-mgr` for information on the interface that must be - supported. - - -.. class:: AbstractDigestAuthHandler(password_mgr=None) - - This is a mixin class that helps with HTTP authentication, both to the remote - host and to a proxy. *password_mgr*, if given, should be something that is - compatible with :class:`HTTPPasswordMgr`; refer to section - :ref:`http-password-mgr` for information on the interface that must be - supported. - - -.. class:: HTTPDigestAuthHandler(password_mgr=None) - - Handle authentication with the remote host. *password_mgr*, if given, should - be something that is compatible with :class:`HTTPPasswordMgr`; refer to - section :ref:`http-password-mgr` for information on the interface that must - be supported. When both Digest Authentication Handler and Basic - Authentication Handler are both added, Digest Authentication is always tried - first. If the Digest Authentication returns a 40x response again, it is sent - to Basic Authentication handler to Handle. This Handler method will raise a - :exc:`ValueError` when presented with an authentication scheme other than - Digest or Basic. - - .. versionchanged:: 3.3 - Raise :exc:`ValueError` on unsupported Authentication Scheme. - - - -.. class:: ProxyDigestAuthHandler(password_mgr=None) - - Handle authentication with the proxy. *password_mgr*, if given, should be - something that is compatible with :class:`HTTPPasswordMgr`; refer to section - :ref:`http-password-mgr` for information on the interface that must be - supported. - - -.. class:: HTTPHandler() - - A class to handle opening of HTTP URLs. - - -.. class:: HTTPSHandler(debuglevel=0, context=None, check_hostname=None) - - A class to handle opening of HTTPS URLs. *context* and *check_hostname* - have the same meaning as in :class:`http.client.HTTPSConnection`. - - .. versionchanged:: 3.2 - *context* and *check_hostname* were added. - - -.. class:: FileHandler() - - Open local files. - -.. class:: DataHandler() - - Open data URLs. - - .. versionadded:: 3.4 - -.. class:: FTPHandler() - - Open FTP URLs. - - -.. class:: CacheFTPHandler() - - Open FTP URLs, keeping a cache of open FTP connections to minimize delays. - - -.. class:: UnknownHandler() - - A catch-all class to handle unknown URLs. - - -.. class:: HTTPErrorProcessor() - - Process HTTP error responses. - - -.. _request-objects: - -Request Objects ---------------- - -The following methods describe :class:`Request`'s public interface, -and so all may be overridden in subclasses. It also defines several -public attributes that can be used by clients to inspect the parsed -request. - -.. attribute:: Request.full_url - - The original URL passed to the constructor. - - .. versionchanged:: 3.4 - - Request.full_url is a property with setter, getter and a deleter. Getting - :attr:`~Request.full_url` returns the original request URL with the - fragment, if it was present. - -.. attribute:: Request.type - - The URI scheme. - -.. attribute:: Request.host - - The URI authority, typically a host, but may also contain a port - separated by a colon. - -.. attribute:: Request.origin_req_host - - The original host for the request, without port. - -.. attribute:: Request.selector - - The URI path. If the :class:`Request` uses a proxy, then selector - will be the full URL that is passed to the proxy. - -.. attribute:: Request.data - - The entity body for the request, or ``None`` if not specified. - - .. versionchanged:: 3.4 - Changing value of :attr:`Request.data` now deletes "Content-Length" - header if it was previously set or calculated. - -.. attribute:: Request.unverifiable - - boolean, indicates whether the request is unverifiable as defined - by :rfc:`2965`. - -.. attribute:: Request.method - - The HTTP request method to use. By default its value is :const:`None`, - which means that :meth:`~Request.get_method` will do its normal computation - of the method to be used. Its value can be set (thus overriding the default - computation in :meth:`~Request.get_method`) either by providing a default - value by setting it at the class level in a :class:`Request` subclass, or by - passing a value in to the :class:`Request` constructor via the *method* - argument. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - A default value can now be set in subclasses; previously it could only - be set via the constructor argument. - - -.. method:: Request.get_method() - - Return a string indicating the HTTP request method. If - :attr:`Request.method` is not ``None``, return its value, otherwise return - ``'GET'`` if :attr:`Request.data` is ``None``, or ``'POST'`` if it's not. - This is only meaningful for HTTP requests. - - .. versionchanged:: 3.3 - get_method now looks at the value of :attr:`Request.method`. - - -.. method:: Request.add_header(key, val) - - Add another header to the request. Headers are currently ignored by all - handlers except HTTP handlers, where they are added to the list of headers sent - to the server. Note that there cannot be more than one header with the same - name, and later calls will overwrite previous calls in case the *key* collides. - Currently, this is no loss of HTTP functionality, since all headers which have - meaning when used more than once have a (header-specific) way of gaining the - same functionality using only one header. Note that headers added using - this method are also added to redirected requests. - - -.. method:: Request.add_unredirected_header(key, header) - - Add a header that will not be added to a redirected request. - - -.. method:: Request.has_header(header) - - Return whether the instance has the named header (checks both regular and - unredirected). - - -.. method:: Request.remove_header(header) - - Remove named header from the request instance (both from regular and - unredirected headers). - - .. versionadded:: 3.4 - - -.. method:: Request.get_full_url() - - Return the URL given in the constructor. - - .. versionchanged:: 3.4 - - Returns :attr:`Request.full_url` - - -.. method:: Request.set_proxy(host, type) - - Prepare the request by connecting to a proxy server. The *host* and *type* will - replace those of the instance, and the instance's selector will be the original - URL given in the constructor. - - -.. method:: Request.get_header(header_name, default=None) - - Return the value of the given header. If the header is not present, return - the default value. - - -.. method:: Request.header_items() - - Return a list of tuples (header_name, header_value) of the Request headers. - -.. versionchanged:: 3.4 - The request methods add_data, has_data, get_data, get_type, get_host, - get_selector, get_origin_req_host and is_unverifiable that were deprecated - since 3.3 have been removed. - - -.. _opener-director-objects: - -OpenerDirector Objects ----------------------- - -:class:`OpenerDirector` instances have the following methods: - - -.. method:: OpenerDirector.add_handler(handler) - - *handler* should be an instance of :class:`BaseHandler`. The following methods - are searched, and added to the possible chains (note that HTTP errors are a - special case). Note that, in the following, *protocol* should be replaced - with the actual protocol to handle, for example :meth:`http_response` would - be the HTTP protocol response handler. Also *type* should be replaced with - the actual HTTP code, for example :meth:`http_error_404` would handle HTTP - 404 errors. - - * :meth:`_open` --- signal that the handler knows how to open *protocol* - URLs. - - See |protocol_open|_ for more information. - - * :meth:`http_error_\` --- signal that the handler knows how to handle HTTP - errors with HTTP error code *type*. - - See |http_error_nnn|_ for more information. - - * :meth:`_error` --- signal that the handler knows how to handle errors - from (non-\ ``http``) *protocol*. - - * :meth:`_request` --- signal that the handler knows how to pre-process - *protocol* requests. - - See |protocol_request|_ for more information. - - * :meth:`_response` --- signal that the handler knows how to - post-process *protocol* responses. - - See |protocol_response|_ for more information. - -.. |protocol_open| replace:: :meth:`BaseHandler._open` -.. |http_error_nnn| replace:: :meth:`BaseHandler.http_error_\` -.. |protocol_request| replace:: :meth:`BaseHandler._request` -.. |protocol_response| replace:: :meth:`BaseHandler._response` - -.. method:: OpenerDirector.open(url, data=None[, timeout]) - - Open the given *url* (which can be a request object or a string), optionally - passing the given *data*. Arguments, return values and exceptions raised are - the same as those of :func:`urlopen` (which simply calls the :meth:`open` - method on the currently installed global :class:`OpenerDirector`). The - optional *timeout* parameter specifies a timeout in seconds for blocking - operations like the connection attempt (if not specified, the global default - timeout setting will be used). The timeout feature actually works only for - HTTP, HTTPS and FTP connections. - - -.. method:: OpenerDirector.error(proto, *args) - - Handle an error of the given protocol. This will call the registered error - handlers for the given protocol with the given arguments (which are protocol - specific). The HTTP protocol is a special case which uses the HTTP response - code to determine the specific error handler; refer to the :meth:`http_error_\` - methods of the handler classes. - - Return values and exceptions raised are the same as those of :func:`urlopen`. - -OpenerDirector objects open URLs in three stages: - -The order in which these methods are called within each stage is determined by -sorting the handler instances. - -#. Every handler with a method named like :meth:`_request` has that - method called to pre-process the request. - -#. Handlers with a method named like :meth:`_open` are called to handle - the request. This stage ends when a handler either returns a non-\ :const:`None` - value (ie. a response), or raises an exception (usually - :exc:`~urllib.error.URLError`). Exceptions are allowed to propagate. - - In fact, the above algorithm is first tried for methods named - :meth:`default_open`. If all such methods return :const:`None`, the algorithm - is repeated for methods named like :meth:`_open`. If all such methods - return :const:`None`, the algorithm is repeated for methods named - :meth:`unknown_open`. - - Note that the implementation of these methods may involve calls of the parent - :class:`OpenerDirector` instance's :meth:`~OpenerDirector.open` and - :meth:`~OpenerDirector.error` methods. - -#. Every handler with a method named like :meth:`_response` has that - method called to post-process the response. - - -.. _base-handler-objects: - -BaseHandler Objects -------------------- - -:class:`BaseHandler` objects provide a couple of methods that are directly -useful, and others that are meant to be used by derived classes. These are -intended for direct use: - - -.. method:: BaseHandler.add_parent(director) - - Add a director as parent. - - -.. method:: BaseHandler.close() - - Remove any parents. - -The following attribute and methods should only be used by classes derived from -:class:`BaseHandler`. - -.. note:: - - The convention has been adopted that subclasses defining - :meth:`_request` or :meth:`_response` methods are named - :class:`\*Processor`; all others are named :class:`\*Handler`. - - -.. attribute:: BaseHandler.parent - - A valid :class:`OpenerDirector`, which can be used to open using a different - protocol, or handle errors. - - -.. method:: BaseHandler.default_open(req) - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to catch all URLs. - - This method, if implemented, will be called by the parent - :class:`OpenerDirector`. It should return a file-like object as described in - the return value of the :meth:`~OpenerDirector.open` method of :class:`OpenerDirector`, or ``None``. - It should raise :exc:`~urllib.error.URLError`, unless a truly exceptional - thing happens (for example, :exc:`MemoryError` should not be mapped to - :exc:`URLError`). - - This method will be called before any protocol-specific open method. - - -.. _protocol_open: -.. method:: BaseHandler._open(req) - :noindex: - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to handle URLs with the given protocol. - - This method, if defined, will be called by the parent :class:`OpenerDirector`. - Return values should be the same as for :meth:`default_open`. - - -.. method:: BaseHandler.unknown_open(req) - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to catch all URLs with no specific registered handler to - open it. - - This method, if implemented, will be called by the :attr:`parent` - :class:`OpenerDirector`. Return values should be the same as for - :meth:`default_open`. - - -.. method:: BaseHandler.http_error_default(req, fp, code, msg, hdrs) - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - override it if they intend to provide a catch-all for otherwise unhandled HTTP - errors. It will be called automatically by the :class:`OpenerDirector` getting - the error, and should not normally be called in other circumstances. - - *req* will be a :class:`Request` object, *fp* will be a file-like object with - the HTTP error body, *code* will be the three-digit code of the error, *msg* - will be the user-visible explanation of the code and *hdrs* will be a mapping - object with the headers of the error. - - Return values and exceptions raised should be the same as those of - :func:`urlopen`. - - -.. _http_error_nnn: -.. method:: BaseHandler.http_error_(req, fp, code, msg, hdrs) - - *nnn* should be a three-digit HTTP error code. This method is also not defined - in :class:`BaseHandler`, but will be called, if it exists, on an instance of a - subclass, when an HTTP error with code *nnn* occurs. - - Subclasses should override this method to handle specific HTTP errors. - - Arguments, return values and exceptions raised should be the same as for - :meth:`http_error_default`. - - -.. _protocol_request: -.. method:: BaseHandler._request(req) - :noindex: - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to pre-process requests of the given protocol. - - This method, if defined, will be called by the parent :class:`OpenerDirector`. - *req* will be a :class:`Request` object. The return value should be a - :class:`Request` object. - - -.. _protocol_response: -.. method:: BaseHandler._response(req, response) - :noindex: - - This method is *not* defined in :class:`BaseHandler`, but subclasses should - define it if they want to post-process responses of the given protocol. - - This method, if defined, will be called by the parent :class:`OpenerDirector`. - *req* will be a :class:`Request` object. *response* will be an object - implementing the same interface as the return value of :func:`urlopen`. The - return value should implement the same interface as the return value of - :func:`urlopen`. - - -.. _http-redirect-handler: - -HTTPRedirectHandler Objects ---------------------------- - -.. note:: - - Some HTTP redirections require action from this module's client code. If this - is the case, :exc:`~urllib.error.HTTPError` is raised. See :rfc:`2616` for - details of the precise meanings of the various redirection codes. - - An :class:`HTTPError` exception raised as a security consideration if the - HTTPRedirectHandler is presented with a redirected URL which is not an HTTP, - HTTPS or FTP URL. - - -.. method:: HTTPRedirectHandler.redirect_request(req, fp, code, msg, hdrs, newurl) - - Return a :class:`Request` or ``None`` in response to a redirect. This is called - by the default implementations of the :meth:`http_error_30\*` methods when a - redirection is received from the server. If a redirection should take place, - return a new :class:`Request` to allow :meth:`http_error_30\*` to perform the - redirect to *newurl*. Otherwise, raise :exc:`~urllib.error.HTTPError` if - no other handler should try to handle this URL, or return ``None`` if you - can't but another handler might. - - .. note:: - - The default implementation of this method does not strictly follow :rfc:`2616`, - which says that 301 and 302 responses to ``POST`` requests must not be - automatically redirected without confirmation by the user. In reality, browsers - do allow automatic redirection of these responses, changing the POST to a - ``GET``, and the default implementation reproduces this behavior. - - -.. method:: HTTPRedirectHandler.http_error_301(req, fp, code, msg, hdrs) - - Redirect to the ``Location:`` or ``URI:`` URL. This method is called by the - parent :class:`OpenerDirector` when getting an HTTP 'moved permanently' response. - - -.. method:: HTTPRedirectHandler.http_error_302(req, fp, code, msg, hdrs) - - The same as :meth:`http_error_301`, but called for the 'found' response. - - -.. method:: HTTPRedirectHandler.http_error_303(req, fp, code, msg, hdrs) - - The same as :meth:`http_error_301`, but called for the 'see other' response. - - -.. method:: HTTPRedirectHandler.http_error_307(req, fp, code, msg, hdrs) - - The same as :meth:`http_error_301`, but called for the 'temporary redirect' - response. It does not allow changing the request method from ``POST`` - to ``GET``. - - -.. method:: HTTPRedirectHandler.http_error_308(req, fp, code, msg, hdrs) - - The same as :meth:`http_error_301`, but called for the 'permanent redirect' - response. It does not allow changing the request method from ``POST`` - to ``GET``. - - .. versionadded:: 3.11 - - -.. _http-cookie-processor: - -HTTPCookieProcessor Objects ---------------------------- - -:class:`HTTPCookieProcessor` instances have one attribute: - -.. attribute:: HTTPCookieProcessor.cookiejar - - The :class:`http.cookiejar.CookieJar` in which cookies are stored. - - -.. _proxy-handler: - -ProxyHandler Objects --------------------- - - -.. method:: ProxyHandler._open(request) - :noindex: - - The :class:`ProxyHandler` will have a method :meth:`_open` for every - *protocol* which has a proxy in the *proxies* dictionary given in the - constructor. The method will modify requests to go through the proxy, by - calling ``request.set_proxy()``, and call the next handler in the chain to - actually execute the protocol. - - -.. _http-password-mgr: - -HTTPPasswordMgr Objects ------------------------ - -These methods are available on :class:`HTTPPasswordMgr` and -:class:`HTTPPasswordMgrWithDefaultRealm` objects. - - -.. method:: HTTPPasswordMgr.add_password(realm, uri, user, passwd) - - *uri* can be either a single URI, or a sequence of URIs. *realm*, *user* and - *passwd* must be strings. This causes ``(user, passwd)`` to be used as - authentication tokens when authentication for *realm* and a super-URI of any of - the given URIs is given. - - -.. method:: HTTPPasswordMgr.find_user_password(realm, authuri) - - Get user/password for given realm and URI, if any. This method will return - ``(None, None)`` if there is no matching user/password. - - For :class:`HTTPPasswordMgrWithDefaultRealm` objects, the realm ``None`` will be - searched if the given *realm* has no matching user/password. - - -.. _http-password-mgr-with-prior-auth: - -HTTPPasswordMgrWithPriorAuth Objects ------------------------------------- - -This password manager extends :class:`HTTPPasswordMgrWithDefaultRealm` to support -tracking URIs for which authentication credentials should always be sent. - - -.. method:: HTTPPasswordMgrWithPriorAuth.add_password(realm, uri, user, \ - passwd, is_authenticated=False) - - *realm*, *uri*, *user*, *passwd* are as for - :meth:`HTTPPasswordMgr.add_password`. *is_authenticated* sets the initial - value of the ``is_authenticated`` flag for the given URI or list of URIs. - If *is_authenticated* is specified as ``True``, *realm* is ignored. - - -.. method:: HTTPPasswordMgrWithPriorAuth.find_user_password(realm, authuri) - - Same as for :class:`HTTPPasswordMgrWithDefaultRealm` objects - - -.. method:: HTTPPasswordMgrWithPriorAuth.update_authenticated(self, uri, \ - is_authenticated=False) - - Update the ``is_authenticated`` flag for the given *uri* or list - of URIs. - - -.. method:: HTTPPasswordMgrWithPriorAuth.is_authenticated(self, authuri) - - Returns the current state of the ``is_authenticated`` flag for - the given URI. - - -.. _abstract-basic-auth-handler: - -AbstractBasicAuthHandler Objects --------------------------------- - - -.. method:: AbstractBasicAuthHandler.http_error_auth_reqed(authreq, host, req, headers) - - Handle an authentication request by getting a user/password pair, and re-trying - the request. *authreq* should be the name of the header where the information - about the realm is included in the request, *host* specifies the URL and path to - authenticate for, *req* should be the (failed) :class:`Request` object, and - *headers* should be the error headers. - - *host* is either an authority (e.g. ``"python.org"``) or a URL containing an - authority component (e.g. ``"http://python.org/"``). In either case, the - authority must not contain a userinfo component (so, ``"python.org"`` and - ``"python.org:80"`` are fine, ``"joe:password@python.org"`` is not). - - -.. _http-basic-auth-handler: - -HTTPBasicAuthHandler Objects ----------------------------- - - -.. method:: HTTPBasicAuthHandler.http_error_401(req, fp, code, msg, hdrs) - - Retry the request with authentication information, if available. - - -.. _proxy-basic-auth-handler: - -ProxyBasicAuthHandler Objects ------------------------------ - - -.. method:: ProxyBasicAuthHandler.http_error_407(req, fp, code, msg, hdrs) - - Retry the request with authentication information, if available. - - -.. _abstract-digest-auth-handler: - -AbstractDigestAuthHandler Objects ---------------------------------- - - -.. method:: AbstractDigestAuthHandler.http_error_auth_reqed(authreq, host, req, headers) - - *authreq* should be the name of the header where the information about the realm - is included in the request, *host* should be the host to authenticate to, *req* - should be the (failed) :class:`Request` object, and *headers* should be the - error headers. - - -.. _http-digest-auth-handler: - -HTTPDigestAuthHandler Objects ------------------------------ - - -.. method:: HTTPDigestAuthHandler.http_error_401(req, fp, code, msg, hdrs) - - Retry the request with authentication information, if available. - - -.. _proxy-digest-auth-handler: - -ProxyDigestAuthHandler Objects ------------------------------- - - -.. method:: ProxyDigestAuthHandler.http_error_407(req, fp, code, msg, hdrs) - - Retry the request with authentication information, if available. - - -.. _http-handler-objects: - -HTTPHandler Objects -------------------- - - -.. method:: HTTPHandler.http_open(req) - - Send an HTTP request, which can be either GET or POST, depending on - ``req.has_data()``. - - -.. _https-handler-objects: - -HTTPSHandler Objects --------------------- - - -.. method:: HTTPSHandler.https_open(req) - - Send an HTTPS request, which can be either GET or POST, depending on - ``req.has_data()``. - - -.. _file-handler-objects: - -FileHandler Objects -------------------- - - -.. method:: FileHandler.file_open(req) - - Open the file locally, if there is no host name, or the host name is - ``'localhost'``. - - .. versionchanged:: 3.2 - This method is applicable only for local hostnames. When a remote - hostname is given, an :exc:`~urllib.error.URLError` is raised. - - -.. _data-handler-objects: - -DataHandler Objects -------------------- - -.. method:: DataHandler.data_open(req) - - Read a data URL. This kind of URL contains the content encoded in the URL - itself. The data URL syntax is specified in :rfc:`2397`. This implementation - ignores white spaces in base64 encoded data URLs so the URL may be wrapped - in whatever source file it comes from. But even though some browsers don't - mind about a missing padding at the end of a base64 encoded data URL, this - implementation will raise an :exc:`ValueError` in that case. - - -.. _ftp-handler-objects: - -FTPHandler Objects ------------------- - - -.. method:: FTPHandler.ftp_open(req) - - Open the FTP file indicated by *req*. The login is always done with empty - username and password. - - -.. _cacheftp-handler-objects: - -CacheFTPHandler Objects ------------------------ - -:class:`CacheFTPHandler` objects are :class:`FTPHandler` objects with the -following additional methods: - - -.. method:: CacheFTPHandler.setTimeout(t) - - Set timeout of connections to *t* seconds. - - -.. method:: CacheFTPHandler.setMaxConns(m) - - Set maximum number of cached connections to *m*. - - -.. _unknown-handler-objects: - -UnknownHandler Objects ----------------------- - - -.. method:: UnknownHandler.unknown_open() - - Raise a :exc:`~urllib.error.URLError` exception. - - -.. _http-error-processor-objects: - -HTTPErrorProcessor Objects --------------------------- - -.. method:: HTTPErrorProcessor.http_response(request, response) - - Process HTTP error responses. - - For 200 error codes, the response object is returned immediately. - - For non-200 error codes, this simply passes the job on to the - :meth:`http_error_\` handler methods, via :meth:`OpenerDirector.error`. - Eventually, :class:`HTTPDefaultErrorHandler` will raise an - :exc:`~urllib.error.HTTPError` if no other handler handles the error. - - -.. method:: HTTPErrorProcessor.https_response(request, response) - - Process HTTPS error responses. - - The behavior is same as :meth:`http_response`. - - -.. _urllib-request-examples: - -Examples --------- - -In addition to the examples below, more examples are given in -:ref:`urllib-howto`. - -This example gets the python.org main page and displays the first 300 bytes of -it. :: - - >>> import urllib.request - >>> with urllib.request.urlopen('http://www.python.org/') as f: - ... print(f.read(300)) - ... - b'\n\n\n\n\n\n - \n - Python Programming ' - -Note that urlopen returns a bytes object. This is because there is no way -for urlopen to automatically determine the encoding of the byte stream -it receives from the HTTP server. In general, a program will decode -the returned bytes object to string once it determines or guesses -the appropriate encoding. - -The following W3C document, https://www.w3.org/International/O-charset\ , lists -the various ways in which an (X)HTML or an XML document could have specified its -encoding information. - -As the python.org website uses *utf-8* encoding as specified in its meta tag, we -will use the same for decoding the bytes object. :: - - >>> with urllib.request.urlopen('http://www.python.org/') as f: - ... print(f.read(100).decode('utf-8')) - ... - <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtm - -It is also possible to achieve the same result without using the -:term:`context manager` approach. :: - - >>> import urllib.request - >>> f = urllib.request.urlopen('http://www.python.org/') - >>> print(f.read(100).decode('utf-8')) - <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" - "http://www.w3.org/TR/xhtml1/DTD/xhtm - -In the following example, we are sending a data-stream to the stdin of a CGI -and reading the data it returns to us. Note that this example will only work -when the Python installation supports SSL. :: - - >>> import urllib.request - >>> req = urllib.request.Request(url='https://localhost/cgi-bin/test.cgi', - ... data=b'This data is passed to stdin of the CGI') - >>> with urllib.request.urlopen(req) as f: - ... print(f.read().decode('utf-8')) - ... - Got Data: "This data is passed to stdin of the CGI" - -The code for the sample CGI used in the above example is:: - - #!/usr/bin/env python - import sys - data = sys.stdin.read() - print('Content-type: text/plain\n\nGot Data: "%s"' % data) - -Here is an example of doing a ``PUT`` request using :class:`Request`:: - - import urllib.request - DATA = b'some data' - req = urllib.request.Request(url='http://localhost:8080', data=DATA, method='PUT') - with urllib.request.urlopen(req) as f: - pass - print(f.status) - print(f.reason) - -Use of Basic HTTP Authentication:: - - import urllib.request - # Create an OpenerDirector with support for Basic HTTP Authentication... - auth_handler = urllib.request.HTTPBasicAuthHandler() - auth_handler.add_password(realm='PDQ Application', - uri='https://mahler:8092/site-updates.py', - user='klem', - passwd='kadidd!ehopper') - opener = urllib.request.build_opener(auth_handler) - # ...and install it globally so it can be used with urlopen. - urllib.request.install_opener(opener) - urllib.request.urlopen('http://www.example.com/login.html') - -:func:`build_opener` provides many handlers by default, including a -:class:`ProxyHandler`. By default, :class:`ProxyHandler` uses the environment -variables named ``<scheme>_proxy``, where ``<scheme>`` is the URL scheme -involved. For example, the :envvar:`http_proxy` environment variable is read to -obtain the HTTP proxy's URL. - -This example replaces the default :class:`ProxyHandler` with one that uses -programmatically supplied proxy URLs, and adds proxy authorization support with -:class:`ProxyBasicAuthHandler`. :: - - proxy_handler = urllib.request.ProxyHandler({'http': 'http://www.example.com:3128/'}) - proxy_auth_handler = urllib.request.ProxyBasicAuthHandler() - proxy_auth_handler.add_password('realm', 'host', 'username', 'password') - - opener = urllib.request.build_opener(proxy_handler, proxy_auth_handler) - # This time, rather than install the OpenerDirector, we use it directly: - opener.open('http://www.example.com/login.html') - -Adding HTTP headers: - -Use the *headers* argument to the :class:`Request` constructor, or:: - - import urllib.request - req = urllib.request.Request('http://www.example.com/') - req.add_header('Referer', 'http://www.python.org/') - # Customize the default User-Agent header value: - req.add_header('User-Agent', 'urllib-example/0.1 (Contact: . . .)') - r = urllib.request.urlopen(req) - -:class:`OpenerDirector` automatically adds a :mailheader:`User-Agent` header to -every :class:`Request`. To change this:: - - import urllib.request - opener = urllib.request.build_opener() - opener.addheaders = [('User-agent', 'Mozilla/5.0')] - opener.open('http://www.example.com/') - -Also, remember that a few standard headers (:mailheader:`Content-Length`, -:mailheader:`Content-Type` and :mailheader:`Host`) -are added when the :class:`Request` is passed to :func:`urlopen` (or -:meth:`OpenerDirector.open`). - -.. _urllib-examples: - -Here is an example session that uses the ``GET`` method to retrieve a URL -containing parameters:: - - >>> import urllib.request - >>> import urllib.parse - >>> params = urllib.parse.urlencode({'spam': 1, 'eggs': 2, 'bacon': 0}) - >>> url = "http://www.musi-cal.com/cgi-bin/query?%s" % params - >>> with urllib.request.urlopen(url) as f: - ... print(f.read().decode('utf-8')) - ... - -The following example uses the ``POST`` method instead. Note that params output -from urlencode is encoded to bytes before it is sent to urlopen as data:: - - >>> import urllib.request - >>> import urllib.parse - >>> data = urllib.parse.urlencode({'spam': 1, 'eggs': 2, 'bacon': 0}) - >>> data = data.encode('ascii') - >>> with urllib.request.urlopen("http://requestb.in/xrbl82xr", data) as f: - ... print(f.read().decode('utf-8')) - ... - -The following example uses an explicitly specified HTTP proxy, overriding -environment settings:: - - >>> import urllib.request - >>> proxies = {'http': 'http://proxy.example.com:8080/'} - >>> opener = urllib.request.FancyURLopener(proxies) - >>> with opener.open("http://www.python.org") as f: - ... f.read().decode('utf-8') - ... - -The following example uses no proxies at all, overriding environment settings:: - - >>> import urllib.request - >>> opener = urllib.request.FancyURLopener({}) - >>> with opener.open("http://www.python.org/") as f: - ... f.read().decode('utf-8') - ... - - -Legacy interface ----------------- - -The following functions and classes are ported from the Python 2 module -``urllib`` (as opposed to ``urllib2``). They might become deprecated at -some point in the future. - -.. function:: urlretrieve(url, filename=None, reporthook=None, data=None) - - Copy a network object denoted by a URL to a local file. If the URL - points to a local file, the object will not be copied unless filename is supplied. - Return a tuple ``(filename, headers)`` where *filename* is the - local file name under which the object can be found, and *headers* is whatever - the :meth:`info` method of the object returned by :func:`urlopen` returned (for - a remote object). Exceptions are the same as for :func:`urlopen`. - - The second argument, if present, specifies the file location to copy to (if - absent, the location will be a tempfile with a generated name). The third - argument, if present, is a callable that will be called once on - establishment of the network connection and once after each block read - thereafter. The callable will be passed three arguments; a count of blocks - transferred so far, a block size in bytes, and the total size of the file. The - third argument may be ``-1`` on older FTP servers which do not return a file - size in response to a retrieval request. - - The following example illustrates the most common usage scenario:: - - >>> import urllib.request - >>> local_filename, headers = urllib.request.urlretrieve('http://python.org/') - >>> html = open(local_filename) - >>> html.close() - - If the *url* uses the :file:`http:` scheme identifier, the optional *data* - argument may be given to specify a ``POST`` request (normally the request - type is ``GET``). The *data* argument must be a bytes object in standard - :mimetype:`application/x-www-form-urlencoded` format; see the - :func:`urllib.parse.urlencode` function. - - :func:`urlretrieve` will raise :exc:`ContentTooShortError` when it detects that - the amount of data available was less than the expected amount (which is the - size reported by a *Content-Length* header). This can occur, for example, when - the download is interrupted. - - The *Content-Length* is treated as a lower bound: if there's more data to read, - urlretrieve reads more data, but if less data is available, it raises the - exception. - - You can still retrieve the downloaded data in this case, it is stored in the - :attr:`content` attribute of the exception instance. - - If no *Content-Length* header was supplied, urlretrieve can not check the size - of the data it has downloaded, and just returns it. In this case you just have - to assume that the download was successful. - -.. function:: urlcleanup() - - Cleans up temporary files that may have been left behind by previous - calls to :func:`urlretrieve`. - -.. class:: URLopener(proxies=None, **x509) - - .. deprecated:: 3.3 - - Base class for opening and reading URLs. Unless you need to support opening - objects using schemes other than :file:`http:`, :file:`ftp:`, or :file:`file:`, - you probably want to use :class:`FancyURLopener`. - - By default, the :class:`URLopener` class sends a :mailheader:`User-Agent` header - of ``urllib/VVV``, where *VVV* is the :mod:`urllib` version number. - Applications can define their own :mailheader:`User-Agent` header by subclassing - :class:`URLopener` or :class:`FancyURLopener` and setting the class attribute - :attr:`version` to an appropriate string value in the subclass definition. - - The optional *proxies* parameter should be a dictionary mapping scheme names to - proxy URLs, where an empty dictionary turns proxies off completely. Its default - value is ``None``, in which case environmental proxy settings will be used if - present, as discussed in the definition of :func:`urlopen`, above. - - Additional keyword parameters, collected in *x509*, may be used for - authentication of the client when using the :file:`https:` scheme. The keywords - *key_file* and *cert_file* are supported to provide an SSL key and certificate; - both are needed to support client authentication. - - :class:`URLopener` objects will raise an :exc:`OSError` exception if the server - returns an error code. - - .. method:: open(fullurl, data=None) - - Open *fullurl* using the appropriate protocol. This method sets up cache and - proxy information, then calls the appropriate open method with its input - arguments. If the scheme is not recognized, :meth:`open_unknown` is called. - The *data* argument has the same meaning as the *data* argument of - :func:`urlopen`. - - This method always quotes *fullurl* using :func:`~urllib.parse.quote`. - - .. method:: open_unknown(fullurl, data=None) - - Overridable interface to open unknown URL types. - - - .. method:: retrieve(url, filename=None, reporthook=None, data=None) - - Retrieves the contents of *url* and places it in *filename*. The return value - is a tuple consisting of a local filename and either an - :class:`email.message.Message` object containing the response headers (for remote - URLs) or ``None`` (for local URLs). The caller must then open and read the - contents of *filename*. If *filename* is not given and the URL refers to a - local file, the input filename is returned. If the URL is non-local and - *filename* is not given, the filename is the output of :func:`tempfile.mktemp` - with a suffix that matches the suffix of the last path component of the input - URL. If *reporthook* is given, it must be a function accepting three numeric - parameters: A chunk number, the maximum size chunks are read in and the total size of the download - (-1 if unknown). It will be called once at the start and after each chunk of data is read from the - network. *reporthook* is ignored for local URLs. - - If the *url* uses the :file:`http:` scheme identifier, the optional *data* - argument may be given to specify a ``POST`` request (normally the request type - is ``GET``). The *data* argument must in standard - :mimetype:`application/x-www-form-urlencoded` format; see the - :func:`urllib.parse.urlencode` function. - - - .. attribute:: version - - Variable that specifies the user agent of the opener object. To get - :mod:`urllib` to tell servers that it is a particular user agent, set this in a - subclass as a class variable or in the constructor before calling the base - constructor. - - -.. class:: FancyURLopener(...) - - .. deprecated:: 3.3 - - :class:`FancyURLopener` subclasses :class:`URLopener` providing default handling - for the following HTTP response codes: 301, 302, 303, 307 and 401. For the 30x - response codes listed above, the :mailheader:`Location` header is used to fetch - the actual URL. For 401 response codes (authentication required), basic HTTP - authentication is performed. For the 30x response codes, recursion is bounded - by the value of the *maxtries* attribute, which defaults to 10. - - For all other response codes, the method :meth:`http_error_default` is called - which you can override in subclasses to handle the error appropriately. - - .. note:: - - According to the letter of :rfc:`2616`, 301 and 302 responses to POST requests - must not be automatically redirected without confirmation by the user. In - reality, browsers do allow automatic redirection of these responses, changing - the POST to a GET, and :mod:`urllib` reproduces this behaviour. - - The parameters to the constructor are the same as those for :class:`URLopener`. - - .. note:: - - When performing basic authentication, a :class:`FancyURLopener` instance calls - its :meth:`prompt_user_passwd` method. The default implementation asks the - users for the required information on the controlling terminal. A subclass may - override this method to support more appropriate behavior if needed. - - The :class:`FancyURLopener` class offers one additional method that should be - overloaded to provide the appropriate behavior: - - .. method:: prompt_user_passwd(host, realm) - - Return information needed to authenticate the user at the given host in the - specified security realm. The return value should be a tuple, ``(user, - password)``, which can be used for basic authentication. - - The implementation prompts for this information on the terminal; an application - should override this method to use an appropriate interaction model in the local - environment. - - -:mod:`urllib.request` Restrictions ----------------------------------- - -.. index:: - pair: HTTP; protocol - pair: FTP; protocol - -* Currently, only the following protocols are supported: HTTP (versions 0.9 and - 1.0), FTP, local files, and data URLs. - - .. versionchanged:: 3.4 Added support for data URLs. - -* The caching feature of :func:`urlretrieve` has been disabled until someone - finds the time to hack proper processing of Expiration time headers. - -* There should be a function to query whether a particular URL is in the cache. - -* For backward compatibility, if a URL appears to point to a local file but the - file can't be opened, the URL is re-interpreted using the FTP protocol. This - can sometimes cause confusing error messages. - -* The :func:`urlopen` and :func:`urlretrieve` functions can cause arbitrarily - long delays while waiting for a network connection to be set up. This means - that it is difficult to build an interactive web client using these functions - without using threads. - - .. index:: - single: HTML - pair: HTTP; protocol - -* The data returned by :func:`urlopen` or :func:`urlretrieve` is the raw data - returned by the server. This may be binary data (such as an image), plain text - or (for example) HTML. The HTTP protocol provides type information in the reply - header, which can be inspected by looking at the :mailheader:`Content-Type` - header. If the returned data is HTML, you can use the module - :mod:`html.parser` to parse it. - - .. index:: single: FTP - -* The code handling the FTP protocol cannot differentiate between a file and a - directory. This can lead to unexpected behavior when attempting to read a URL - that points to a file that is not accessible. If the URL ends in a ``/``, it is - assumed to refer to a directory and will be handled accordingly. But if an - attempt to read a file leads to a 550 error (meaning the URL cannot be found or - is not accessible, often for permission reasons), then the path is treated as a - directory in order to handle the case when a directory is specified by a URL but - the trailing ``/`` has been left off. This can cause misleading results when - you try to fetch a file whose read permissions make it inaccessible; the FTP - code will try to read it, fail with a 550 error, and then perform a directory - listing for the unreadable file. If fine-grained control is needed, consider - using the :mod:`ftplib` module, subclassing :class:`FancyURLopener`, or changing - *_urlopener* to meet your needs. - - - -:mod:`urllib.response` --- Response classes used by urllib -========================================================== - -.. module:: urllib.response - :synopsis: Response classes used by urllib. - -The :mod:`urllib.response` module defines functions and classes which define a -minimal file-like interface, including ``read()`` and ``readline()``. -Functions defined by this module are used internally by the :mod:`urllib.request` module. -The typical response object is a :class:`urllib.response.addinfourl` instance: - -.. class:: addinfourl - - .. attribute:: url - - URL of the resource retrieved, commonly used to determine if a redirect was followed. - - .. attribute:: headers - - Returns the headers of the response in the form of an :class:`~email.message.EmailMessage` instance. - - .. attribute:: status - - .. versionadded:: 3.9 - - Status code returned by server. - - .. method:: geturl() - - .. deprecated:: 3.9 - Deprecated in favor of :attr:`~addinfourl.url`. - - .. method:: info() - - .. deprecated:: 3.9 - Deprecated in favor of :attr:`~addinfourl.headers`. - - .. attribute:: code - - .. deprecated:: 3.9 - Deprecated in favor of :attr:`~addinfourl.status`. - - .. method:: getcode() - - .. deprecated:: 3.9 - Deprecated in favor of :attr:`~addinfourl.status`. diff --git a/Doc/library/urllib.robotparser.rst b/Doc/library/urllib.robotparser.rst deleted file mode 100644 index f063e463753e0b..00000000000000 --- a/Doc/library/urllib.robotparser.rst +++ /dev/null @@ -1,106 +0,0 @@ -:mod:`urllib.robotparser` --- Parser for robots.txt -==================================================== - -.. module:: urllib.robotparser - :synopsis: Load a robots.txt file and answer questions about - fetchability of other URLs. - -.. sectionauthor:: Skip Montanaro <skip@pobox.com> - -**Source code:** :source:`Lib/urllib/robotparser.py` - -.. index:: - single: WWW - single: World Wide Web - single: URL - single: robots.txt - --------------- - -This module provides a single class, :class:`RobotFileParser`, which answers -questions about whether or not a particular user agent can fetch a URL on the -web site that published the :file:`robots.txt` file. For more details on the -structure of :file:`robots.txt` files, see http://www.robotstxt.org/orig.html. - - -.. class:: RobotFileParser(url='') - - This class provides methods to read, parse and answer questions about the - :file:`robots.txt` file at *url*. - - .. method:: set_url(url) - - Sets the URL referring to a :file:`robots.txt` file. - - .. method:: read() - - Reads the :file:`robots.txt` URL and feeds it to the parser. - - .. method:: parse(lines) - - Parses the lines argument. - - .. method:: can_fetch(useragent, url) - - Returns ``True`` if the *useragent* is allowed to fetch the *url* - according to the rules contained in the parsed :file:`robots.txt` - file. - - .. method:: mtime() - - Returns the time the ``robots.txt`` file was last fetched. This is - useful for long-running web spiders that need to check for new - ``robots.txt`` files periodically. - - .. method:: modified() - - Sets the time the ``robots.txt`` file was last fetched to the current - time. - - .. method:: crawl_delay(useragent) - - Returns the value of the ``Crawl-delay`` parameter from ``robots.txt`` - for the *useragent* in question. If there is no such parameter or it - doesn't apply to the *useragent* specified or the ``robots.txt`` entry - for this parameter has invalid syntax, return ``None``. - - .. versionadded:: 3.6 - - .. method:: request_rate(useragent) - - Returns the contents of the ``Request-rate`` parameter from - ``robots.txt`` as a :term:`named tuple` ``RequestRate(requests, seconds)``. - If there is no such parameter or it doesn't apply to the *useragent* - specified or the ``robots.txt`` entry for this parameter has invalid - syntax, return ``None``. - - .. versionadded:: 3.6 - - .. method:: site_maps() - - Returns the contents of the ``Sitemap`` parameter from - ``robots.txt`` in the form of a :func:`list`. If there is no such - parameter or the ``robots.txt`` entry for this parameter has - invalid syntax, return ``None``. - - .. versionadded:: 3.8 - - -The following example demonstrates basic use of the :class:`RobotFileParser` -class:: - - >>> import urllib.robotparser - >>> rp = urllib.robotparser.RobotFileParser() - >>> rp.set_url("http://www.musi-cal.com/robots.txt") - >>> rp.read() - >>> rrate = rp.request_rate("*") - >>> rrate.requests - 3 - >>> rrate.seconds - 20 - >>> rp.crawl_delay("*") - 6 - >>> rp.can_fetch("*", "http://www.musi-cal.com/cgi-bin/search?city=San+Francisco") - False - >>> rp.can_fetch("*", "http://www.musi-cal.com/") - True diff --git a/Doc/library/urllib.rst b/Doc/library/urllib.rst deleted file mode 100644 index 624e164625556a..00000000000000 --- a/Doc/library/urllib.rst +++ /dev/null @@ -1,15 +0,0 @@ -:mod:`urllib` --- URL handling modules -====================================== - -.. module:: urllib - -**Source code:** :source:`Lib/urllib/` - --------------- - -``urllib`` is a package that collects several modules for working with URLs: - -* :mod:`urllib.request` for opening and reading URLs -* :mod:`urllib.error` containing the exceptions raised by :mod:`urllib.request` -* :mod:`urllib.parse` for parsing URLs -* :mod:`urllib.robotparser` for parsing ``robots.txt`` files diff --git a/Doc/library/uu.rst b/Doc/library/uu.rst deleted file mode 100644 index 83c4aec47bbefc..00000000000000 --- a/Doc/library/uu.rst +++ /dev/null @@ -1,72 +0,0 @@ -:mod:`uu` --- Encode and decode uuencode files -============================================== - -.. module:: uu - :synopsis: Encode and decode files in uuencode format. - :deprecated: - -.. moduleauthor:: Lance Ellinghouse - -**Source code:** :source:`Lib/uu.py` - -.. deprecated-removed:: 3.11 3.13 - The :mod:`uu` module is deprecated - (see :pep:`PEP 594 <594#uu-and-the-uu-encoding>` for details). - :mod:`base64` is a modern alternative. - --------------- - -This module encodes and decodes files in uuencode format, allowing arbitrary -binary data to be transferred over ASCII-only connections. Wherever a file -argument is expected, the methods accept a file-like object. For backwards -compatibility, a string containing a pathname is also accepted, and the -corresponding file will be opened for reading and writing; the pathname ``'-'`` -is understood to mean the standard input or output. However, this interface is -deprecated; it's better for the caller to open the file itself, and be sure -that, when required, the mode is ``'rb'`` or ``'wb'`` on Windows. - -.. index:: - single: Jansen, Jack - single: Ellinghouse, Lance - -This code was contributed by Lance Ellinghouse, and modified by Jack Jansen. - -The :mod:`uu` module defines the following functions: - - -.. function:: encode(in_file, out_file, name=None, mode=None, *, backtick=False) - - Uuencode file *in_file* into file *out_file*. The uuencoded file will have - the header specifying *name* and *mode* as the defaults for the results of - decoding the file. The default defaults are taken from *in_file*, or ``'-'`` - and ``0o666`` respectively. If *backtick* is true, zeros are represented by - ``'`'`` instead of spaces. - - .. versionchanged:: 3.7 - Added the *backtick* parameter. - - -.. function:: decode(in_file, out_file=None, mode=None, quiet=False) - - This call decodes uuencoded file *in_file* placing the result on file - *out_file*. If *out_file* is a pathname, *mode* is used to set the permission - bits if the file must be created. Defaults for *out_file* and *mode* are taken - from the uuencode header. However, if the file specified in the header already - exists, a :exc:`uu.Error` is raised. - - :func:`decode` may print a warning to standard error if the input was produced - by an incorrect uuencoder and Python could recover from that error. Setting - *quiet* to a true value silences this warning. - - -.. exception:: Error() - - Subclass of :exc:`Exception`, this can be raised by :func:`uu.decode` under - various situations, such as described above, but also including a badly - formatted header, or truncated input file. - - -.. seealso:: - - Module :mod:`binascii` - Support module containing ASCII-to-binary and binary-to-ASCII conversions. diff --git a/Doc/library/uuid.rst b/Doc/library/uuid.rst deleted file mode 100644 index c1e2288071320e..00000000000000 --- a/Doc/library/uuid.rst +++ /dev/null @@ -1,312 +0,0 @@ -:mod:`uuid` --- UUID objects according to :rfc:`4122` -===================================================== - -.. module:: uuid - :synopsis: UUID objects (universally unique identifiers) according to RFC 4122 -.. moduleauthor:: Ka-Ping Yee <ping@zesty.ca> -.. sectionauthor:: George Yoshida <quiver@users.sourceforge.net> - -**Source code:** :source:`Lib/uuid.py` - --------------- - -This module provides immutable :class:`UUID` objects (the :class:`UUID` class) -and the functions :func:`uuid1`, :func:`uuid3`, :func:`uuid4`, :func:`uuid5` for -generating version 1, 3, 4, and 5 UUIDs as specified in :rfc:`4122`. - -If all you want is a unique ID, you should probably call :func:`uuid1` or -:func:`uuid4`. Note that :func:`uuid1` may compromise privacy since it creates -a UUID containing the computer's network address. :func:`uuid4` creates a -random UUID. - -Depending on support from the underlying platform, :func:`uuid1` may or may -not return a "safe" UUID. A safe UUID is one which is generated using -synchronization methods that ensure no two processes can obtain the same -UUID. All instances of :class:`UUID` have an :attr:`~UUID.is_safe` attribute -which relays any information about the UUID's safety, using this enumeration: - -.. class:: SafeUUID - - .. versionadded:: 3.7 - - .. attribute:: SafeUUID.safe - - The UUID was generated by the platform in a multiprocessing-safe way. - - .. attribute:: SafeUUID.unsafe - - The UUID was not generated in a multiprocessing-safe way. - - .. attribute:: SafeUUID.unknown - - The platform does not provide information on whether the UUID was - generated safely or not. - -.. class:: UUID(hex=None, bytes=None, bytes_le=None, fields=None, int=None, version=None, *, is_safe=SafeUUID.unknown) - - Create a UUID from either a string of 32 hexadecimal digits, a string of 16 - bytes in big-endian order as the *bytes* argument, a string of 16 bytes in - little-endian order as the *bytes_le* argument, a tuple of six integers - (32-bit *time_low*, 16-bit *time_mid*, 16-bit *time_hi_version*, - 8-bit *clock_seq_hi_variant*, 8-bit *clock_seq_low*, 48-bit *node*) as the - *fields* argument, or a single 128-bit integer as the *int* argument. - When a string of hex digits is given, curly braces, hyphens, - and a URN prefix are all optional. For example, these - expressions all yield the same UUID:: - - UUID('{12345678-1234-5678-1234-567812345678}') - UUID('12345678123456781234567812345678') - UUID('urn:uuid:12345678-1234-5678-1234-567812345678') - UUID(bytes=b'\x12\x34\x56\x78'*4) - UUID(bytes_le=b'\x78\x56\x34\x12\x34\x12\x78\x56' + - b'\x12\x34\x56\x78\x12\x34\x56\x78') - UUID(fields=(0x12345678, 0x1234, 0x5678, 0x12, 0x34, 0x567812345678)) - UUID(int=0x12345678123456781234567812345678) - - Exactly one of *hex*, *bytes*, *bytes_le*, *fields*, or *int* must be given. - The *version* argument is optional; if given, the resulting UUID will have its - variant and version number set according to :rfc:`4122`, overriding bits in the - given *hex*, *bytes*, *bytes_le*, *fields*, or *int*. - - Comparison of UUID objects are made by way of comparing their - :attr:`UUID.int` attributes. Comparison with a non-UUID object - raises a :exc:`TypeError`. - - ``str(uuid)`` returns a string in the form - ``12345678-1234-5678-1234-567812345678`` where the 32 hexadecimal digits - represent the UUID. - -:class:`UUID` instances have these read-only attributes: - -.. attribute:: UUID.bytes - - The UUID as a 16-byte string (containing the six integer fields in big-endian - byte order). - - -.. attribute:: UUID.bytes_le - - The UUID as a 16-byte string (with *time_low*, *time_mid*, and *time_hi_version* - in little-endian byte order). - - -.. attribute:: UUID.fields - - A tuple of the six integer fields of the UUID, which are also available as six - individual attributes and two derived attributes: - -.. list-table:: - - * - Field - - Meaning - - * - .. attribute:: UUID.time_low - - The first 32 bits of the UUID. - - * - .. attribute:: UUID.time_mid - - The next 16 bits of the UUID. - - * - .. attribute:: UUID.time_hi_version - - The next 16 bits of the UUID. - - * - .. attribute:: UUID.clock_seq_hi_variant - - The next 8 bits of the UUID. - - * - .. attribute:: UUID.clock_seq_low - - The next 8 bits of the UUID. - - * - .. attribute:: UUID.node - - The last 48 bits of the UUID. - - * - .. attribute:: UUID.time - - The 60-bit timestamp. - - * - .. attribute:: UUID.clock_seq - - The 14-bit sequence number. - - -.. attribute:: UUID.hex - - The UUID as a 32-character lowercase hexadecimal string. - - -.. attribute:: UUID.int - - The UUID as a 128-bit integer. - - -.. attribute:: UUID.urn - - The UUID as a URN as specified in :rfc:`4122`. - - -.. attribute:: UUID.variant - - The UUID variant, which determines the internal layout of the UUID. This will be - one of the constants :const:`RESERVED_NCS`, :const:`RFC_4122`, - :const:`RESERVED_MICROSOFT`, or :const:`RESERVED_FUTURE`. - - -.. attribute:: UUID.version - - The UUID version number (1 through 5, meaningful only when the variant is - :const:`RFC_4122`). - -.. attribute:: UUID.is_safe - - An enumeration of :class:`SafeUUID` which indicates whether the platform - generated the UUID in a multiprocessing-safe way. - - .. versionadded:: 3.7 - -The :mod:`uuid` module defines the following functions: - - -.. function:: getnode() - - Get the hardware address as a 48-bit positive integer. The first time this - runs, it may launch a separate program, which could be quite slow. If all - attempts to obtain the hardware address fail, we choose a random 48-bit - number with the multicast bit (least significant bit of the first octet) - set to 1 as recommended in :rfc:`4122`. "Hardware address" means the MAC - address of a network interface. On a machine with multiple network - interfaces, universally administered MAC addresses (i.e. where the second - least significant bit of the first octet is *unset*) will be preferred over - locally administered MAC addresses, but with no other ordering guarantees. - - .. versionchanged:: 3.7 - Universally administered MAC addresses are preferred over locally - administered MAC addresses, since the former are guaranteed to be - globally unique, while the latter are not. - -.. index:: single: getnode - - -.. function:: uuid1(node=None, clock_seq=None) - - Generate a UUID from a host ID, sequence number, and the current time. If *node* - is not given, :func:`getnode` is used to obtain the hardware address. If - *clock_seq* is given, it is used as the sequence number; otherwise a random - 14-bit sequence number is chosen. - -.. index:: single: uuid1 - - -.. function:: uuid3(namespace, name) - - Generate a UUID based on the MD5 hash of a namespace identifier (which is a - UUID) and a name (which is a string). - -.. index:: single: uuid3 - - -.. function:: uuid4() - - Generate a random UUID. - -.. index:: single: uuid4 - - -.. function:: uuid5(namespace, name) - - Generate a UUID based on the SHA-1 hash of a namespace identifier (which is a - UUID) and a name (which is a string). - -.. index:: single: uuid5 - -The :mod:`uuid` module defines the following namespace identifiers for use with -:func:`uuid3` or :func:`uuid5`. - - -.. data:: NAMESPACE_DNS - - When this namespace is specified, the *name* string is a fully qualified domain - name. - - -.. data:: NAMESPACE_URL - - When this namespace is specified, the *name* string is a URL. - - -.. data:: NAMESPACE_OID - - When this namespace is specified, the *name* string is an ISO OID. - - -.. data:: NAMESPACE_X500 - - When this namespace is specified, the *name* string is an X.500 DN in DER or a - text output format. - -The :mod:`uuid` module defines the following constants for the possible values -of the :attr:`~UUID.variant` attribute: - - -.. data:: RESERVED_NCS - - Reserved for NCS compatibility. - - -.. data:: RFC_4122 - - Specifies the UUID layout given in :rfc:`4122`. - - -.. data:: RESERVED_MICROSOFT - - Reserved for Microsoft compatibility. - - -.. data:: RESERVED_FUTURE - - Reserved for future definition. - - -.. seealso:: - - :rfc:`4122` - A Universally Unique IDentifier (UUID) URN Namespace - This specification defines a Uniform Resource Name namespace for UUIDs, the - internal format of UUIDs, and methods of generating UUIDs. - - -.. _uuid-example: - -Example -------- - -Here are some examples of typical usage of the :mod:`uuid` module:: - - >>> import uuid - - >>> # make a UUID based on the host ID and current time - >>> uuid.uuid1() - UUID('a8098c1a-f86e-11da-bd1a-00112444be1e') - - >>> # make a UUID using an MD5 hash of a namespace UUID and a name - >>> uuid.uuid3(uuid.NAMESPACE_DNS, 'python.org') - UUID('6fa459ea-ee8a-3ca4-894e-db77e160355e') - - >>> # make a random UUID - >>> uuid.uuid4() - UUID('16fd2706-8baf-433b-82eb-8c7fada847da') - - >>> # make a UUID using a SHA-1 hash of a namespace UUID and a name - >>> uuid.uuid5(uuid.NAMESPACE_DNS, 'python.org') - UUID('886313e1-3b8a-5372-9b90-0c9aee199e5d') - - >>> # make a UUID from a string of hex digits (braces and hyphens ignored) - >>> x = uuid.UUID('{00010203-0405-0607-0809-0a0b0c0d0e0f}') - - >>> # convert a UUID to a string of hex digits in standard form - >>> str(x) - '00010203-0405-0607-0809-0a0b0c0d0e0f' - - >>> # get the raw 16 bytes of the UUID - >>> x.bytes - b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f' - - >>> # make a UUID from a 16-byte string - >>> uuid.UUID(bytes=x.bytes) - UUID('00010203-0405-0607-0809-0a0b0c0d0e0f') - diff --git a/Doc/library/venv.rst b/Doc/library/venv.rst deleted file mode 100644 index f66e874d09f31d..00000000000000 --- a/Doc/library/venv.rst +++ /dev/null @@ -1,605 +0,0 @@ -:mod:`venv` --- Creation of virtual environments -================================================ - -.. module:: venv - :synopsis: Creation of virtual environments. - -.. moduleauthor:: Vinay Sajip <vinay_sajip@yahoo.co.uk> -.. sectionauthor:: Vinay Sajip <vinay_sajip@yahoo.co.uk> - -.. versionadded:: 3.3 - -**Source code:** :source:`Lib/venv/` - -.. index:: pair: Environments; virtual - --------------- - -.. _venv-def: -.. _venv-intro: - -The :mod:`!venv` module supports creating lightweight "virtual environments", -each with their own independent set of Python packages installed in -their :mod:`site` directories. -A virtual environment is created on top of an existing -Python installation, known as the virtual environment's "base" Python, and may -optionally be isolated from the packages in the base environment, -so only those explicitly installed in the virtual environment are available. - -When used from within a virtual environment, common installation tools such as -`pip`_ will install Python packages into a virtual environment -without needing to be told to do so explicitly. - -A virtual environment is (amongst other things): - -* Used to contain a specific Python interpreter and software libraries and - binaries which are needed to support a project (library or application). These - are by default isolated from software in other virtual environments and Python - interpreters and libraries installed in the operating system. - -* Contained in a directory, conventionally either named ``venv`` or ``.venv`` in - the project directory, or under a container directory for lots of virtual - environments, such as ``~/.virtualenvs``. - -* Not checked into source control systems such as Git. - -* Considered as disposable -- it should be simple to delete and recreate it from - scratch. You don't place any project code in the environment - -* Not considered as movable or copyable -- you just recreate the same - environment in the target location. - -See :pep:`405` for more background on Python virtual environments. - -.. seealso:: - - `Python Packaging User Guide: Creating and using virtual environments - <https://packaging.python.org/guides/installing-using-pip-and-virtual-environments/#creating-a-virtual-environment>`__ - -.. include:: ../includes/wasm-notavail.rst - -Creating virtual environments ------------------------------ - -.. include:: /using/venv-create.inc - -.. _venv-explanation: - -How venvs work --------------- - -When a Python interpreter is running from a virtual environment, -:data:`sys.prefix` and :data:`sys.exec_prefix` -point to the directories of the virtual environment, -whereas :data:`sys.base_prefix` and :data:`sys.base_exec_prefix` -point to those of the base Python used to create the environment. -It is sufficient to check -``sys.prefix != sys.base_prefix`` to determine if the current interpreter is -running from a virtual environment. - -A virtual environment may be "activated" using a script in its binary directory -(``bin`` on POSIX; ``Scripts`` on Windows). -This will prepend that directory to your :envvar:`PATH`, so that running -:program:`python` will invoke the environment's Python interpreter -and you can run installed scripts without having to use their full path. -The invocation of the activation script is platform-specific -(:samp:`{<venv>}` must be replaced by the path to the directory -containing the virtual environment): - -+-------------+------------+--------------------------------------------------+ -| Platform | Shell | Command to activate virtual environment | -+=============+============+==================================================+ -| POSIX | bash/zsh | :samp:`$ source {<venv>}/bin/activate` | -| +------------+--------------------------------------------------+ -| | fish | :samp:`$ source {<venv>}/bin/activate.fish` | -| +------------+--------------------------------------------------+ -| | csh/tcsh | :samp:`$ source {<venv>}/bin/activate.csh` | -| +------------+--------------------------------------------------+ -| | PowerShell | :samp:`$ {<venv>}/bin/Activate.ps1` | -+-------------+------------+--------------------------------------------------+ -| Windows | cmd.exe | :samp:`C:\\> {<venv>}\\Scripts\\activate.bat` | -| +------------+--------------------------------------------------+ -| | PowerShell | :samp:`PS C:\\> {<venv>}\\Scripts\\Activate.ps1` | -+-------------+------------+--------------------------------------------------+ - -.. versionadded:: 3.4 - :program:`fish` and :program:`csh` activation scripts. - -.. versionadded:: 3.8 - PowerShell activation scripts installed under POSIX for PowerShell Core - support. - -You don't specifically *need* to activate a virtual environment, -as you can just specify the full path to that environment's -Python interpreter when invoking Python. -Furthermore, all scripts installed in the environment -should be runnable without activating it. - -In order to achieve this, scripts installed into virtual environments have -a "shebang" line which points to the environment's Python interpreter, -i.e. :samp:`#!/{<path-to-venv>}/bin/python`. -This means that the script will run with that interpreter regardless of the -value of :envvar:`PATH`. On Windows, "shebang" line processing is supported if -you have the :ref:`launcher` installed. Thus, double-clicking an installed -script in a Windows Explorer window should run it with the correct interpreter -without the environment needing to be activated or on the :envvar:`PATH`. - -When a virtual environment has been activated, the :envvar:`!VIRTUAL_ENV` -environment variable is set to the path of the environment. -Since explicitly activating a virtual environment is not required to use it, -:envvar:`!VIRTUAL_ENV` cannot be relied upon to determine -whether a virtual environment is being used. - -.. warning:: Because scripts installed in environments should not expect the - environment to be activated, their shebang lines contain the absolute paths - to their environment's interpreters. Because of this, environments are - inherently non-portable, in the general case. You should always have a - simple means of recreating an environment (for example, if you have a - requirements file ``requirements.txt``, you can invoke ``pip install -r - requirements.txt`` using the environment's ``pip`` to install all of the - packages needed by the environment). If for any reason you need to move the - environment to a new location, you should recreate it at the desired - location and delete the one at the old location. If you move an environment - because you moved a parent directory of it, you should recreate the - environment in its new location. Otherwise, software installed into the - environment may not work as expected. - -You can deactivate a virtual environment by typing ``deactivate`` in your shell. -The exact mechanism is platform-specific and is an internal implementation -detail (typically, a script or shell function will be used). - - -.. _venv-api: - -API ---- - -.. highlight:: python - -The high-level method described above makes use of a simple API which provides -mechanisms for third-party virtual environment creators to customize environment -creation according to their needs, the :class:`EnvBuilder` class. - -.. class:: EnvBuilder(system_site_packages=False, clear=False, \ - symlinks=False, upgrade=False, with_pip=False, \ - prompt=None, upgrade_deps=False) - - The :class:`EnvBuilder` class accepts the following keyword arguments on - instantiation: - - * ``system_site_packages`` -- a Boolean value indicating that the system Python - site-packages should be available to the environment (defaults to ``False``). - - * ``clear`` -- a Boolean value which, if true, will delete the contents of - any existing target directory, before creating the environment. - - * ``symlinks`` -- a Boolean value indicating whether to attempt to symlink the - Python binary rather than copying. - - * ``upgrade`` -- a Boolean value which, if true, will upgrade an existing - environment with the running Python - for use when that Python has been - upgraded in-place (defaults to ``False``). - - * ``with_pip`` -- a Boolean value which, if true, ensures pip is - installed in the virtual environment. This uses :mod:`ensurepip` with - the ``--default-pip`` option. - - * ``prompt`` -- a String to be used after virtual environment is activated - (defaults to ``None`` which means directory name of the environment would - be used). If the special string ``"."`` is provided, the basename of the - current directory is used as the prompt. - - * ``upgrade_deps`` -- Update the base venv modules to the latest on PyPI - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - .. versionadded:: 3.6 - Added the ``prompt`` parameter - - .. versionadded:: 3.9 - Added the ``upgrade_deps`` parameter - - Creators of third-party virtual environment tools will be free to use the - provided :class:`EnvBuilder` class as a base class. - - The returned env-builder is an object which has a method, ``create``: - - .. method:: create(env_dir) - - Create a virtual environment by specifying the target directory - (absolute or relative to the current directory) which is to contain the - virtual environment. The ``create`` method will either create the - environment in the specified directory, or raise an appropriate - exception. - - The ``create`` method of the :class:`EnvBuilder` class illustrates the - hooks available for subclass customization:: - - def create(self, env_dir): - """ - Create a virtualized Python environment in a directory. - env_dir is the target directory to create an environment in. - """ - env_dir = os.path.abspath(env_dir) - context = self.ensure_directories(env_dir) - self.create_configuration(context) - self.setup_python(context) - self.setup_scripts(context) - self.post_setup(context) - - Each of the methods :meth:`ensure_directories`, - :meth:`create_configuration`, :meth:`setup_python`, - :meth:`setup_scripts` and :meth:`post_setup` can be overridden. - - .. method:: ensure_directories(env_dir) - - Creates the environment directory and all necessary subdirectories that - don't already exist, and returns a context object. This context object - is just a holder for attributes (such as paths) for use by the other - methods. If the :class:`EnvBuilder` is created with the arg - ``clear=True``, contents of the environment directory will be cleared - and then all necessary subdirectories will be recreated. - - The returned context object is a :class:`types.SimpleNamespace` with the - following attributes: - - * ``env_dir`` - The location of the virtual environment. Used for - ``__VENV_DIR__`` in activation scripts (see :meth:`install_scripts`). - - * ``env_name`` - The name of the virtual environment. Used for - ``__VENV_NAME__`` in activation scripts (see :meth:`install_scripts`). - - * ``prompt`` - The prompt to be used by the activation scripts. Used for - ``__VENV_PROMPT__`` in activation scripts (see :meth:`install_scripts`). - - * ``executable`` - The underlying Python executable used by the virtual - environment. This takes into account the case where a virtual environment - is created from another virtual environment. - - * ``inc_path`` - The include path for the virtual environment. - - * ``lib_path`` - The purelib path for the virtual environment. - - * ``bin_path`` - The script path for the virtual environment. - - * ``bin_name`` - The name of the script path relative to the virtual - environment location. Used for ``__VENV_BIN_NAME__`` in activation - scripts (see :meth:`install_scripts`). - - * ``env_exe`` - The name of the Python interpreter in the virtual - environment. Used for ``__VENV_PYTHON__`` in activation scripts - (see :meth:`install_scripts`). - - * ``env_exec_cmd`` - The name of the Python interpreter, taking into - account filesystem redirections. This can be used to run Python in - the virtual environment. - - - .. versionchanged:: 3.12 - The attribute ``lib_path`` was added to the context, and the context - object was documented. - - .. versionchanged:: 3.11 - The *venv* - :ref:`sysconfig installation scheme <installation_paths>` - is used to construct the paths of the created directories. - - .. method:: create_configuration(context) - - Creates the ``pyvenv.cfg`` configuration file in the environment. - - .. method:: setup_python(context) - - Creates a copy or symlink to the Python executable in the environment. - On POSIX systems, if a specific executable ``python3.x`` was used, - symlinks to ``python`` and ``python3`` will be created pointing to that - executable, unless files with those names already exist. - - .. method:: setup_scripts(context) - - Installs activation scripts appropriate to the platform into the virtual - environment. - - .. method:: upgrade_dependencies(context) - - Upgrades the core venv dependency packages (currently ``pip`` and - ``setuptools``) in the environment. This is done by shelling out to the - ``pip`` executable in the environment. - - .. versionadded:: 3.9 - - .. method:: post_setup(context) - - A placeholder method which can be overridden in third party - implementations to pre-install packages in the virtual environment or - perform other post-creation steps. - - .. versionchanged:: 3.7.2 - Windows now uses redirector scripts for ``python[w].exe`` instead of - copying the actual binaries. In 3.7.2 only :meth:`setup_python` does - nothing unless running from a build in the source tree. - - .. versionchanged:: 3.7.3 - Windows copies the redirector scripts as part of :meth:`setup_python` - instead of :meth:`setup_scripts`. This was not the case in 3.7.2. - When using symlinks, the original executables will be linked. - - In addition, :class:`EnvBuilder` provides this utility method that can be - called from :meth:`setup_scripts` or :meth:`post_setup` in subclasses to - assist in installing custom scripts into the virtual environment. - - .. method:: install_scripts(context, path) - - *path* is the path to a directory that should contain subdirectories - "common", "posix", "nt", each containing scripts destined for the bin - directory in the environment. The contents of "common" and the - directory corresponding to :data:`os.name` are copied after some text - replacement of placeholders: - - * ``__VENV_DIR__`` is replaced with the absolute path of the environment - directory. - - * ``__VENV_NAME__`` is replaced with the environment name (final path - segment of environment directory). - - * ``__VENV_PROMPT__`` is replaced with the prompt (the environment - name surrounded by parentheses and with a following space) - - * ``__VENV_BIN_NAME__`` is replaced with the name of the bin directory - (either ``bin`` or ``Scripts``). - - * ``__VENV_PYTHON__`` is replaced with the absolute path of the - environment's executable. - - The directories are allowed to exist (for when an existing environment - is being upgraded). - -There is also a module-level convenience function: - -.. function:: create(env_dir, system_site_packages=False, clear=False, \ - symlinks=False, with_pip=False, prompt=None, \ - upgrade_deps=False) - - Create an :class:`EnvBuilder` with the given keyword arguments, and call its - :meth:`~EnvBuilder.create` method with the *env_dir* argument. - - .. versionadded:: 3.3 - - .. versionchanged:: 3.4 - Added the ``with_pip`` parameter - - .. versionchanged:: 3.6 - Added the ``prompt`` parameter - - .. versionchanged:: 3.9 - Added the ``upgrade_deps`` parameter - -An example of extending ``EnvBuilder`` --------------------------------------- - -The following script shows how to extend :class:`EnvBuilder` by implementing a -subclass which installs setuptools and pip into a created virtual environment:: - - import os - import os.path - from subprocess import Popen, PIPE - import sys - from threading import Thread - from urllib.parse import urlparse - from urllib.request import urlretrieve - import venv - - class ExtendedEnvBuilder(venv.EnvBuilder): - """ - This builder installs setuptools and pip so that you can pip or - easy_install other packages into the created virtual environment. - - :param nodist: If true, setuptools and pip are not installed into the - created virtual environment. - :param nopip: If true, pip is not installed into the created - virtual environment. - :param progress: If setuptools or pip are installed, the progress of the - installation can be monitored by passing a progress - callable. If specified, it is called with two - arguments: a string indicating some progress, and a - context indicating where the string is coming from. - The context argument can have one of three values: - 'main', indicating that it is called from virtualize() - itself, and 'stdout' and 'stderr', which are obtained - by reading lines from the output streams of a subprocess - which is used to install the app. - - If a callable is not specified, default progress - information is output to sys.stderr. - """ - - def __init__(self, *args, **kwargs): - self.nodist = kwargs.pop('nodist', False) - self.nopip = kwargs.pop('nopip', False) - self.progress = kwargs.pop('progress', None) - self.verbose = kwargs.pop('verbose', False) - super().__init__(*args, **kwargs) - - def post_setup(self, context): - """ - Set up any packages which need to be pre-installed into the - virtual environment being created. - - :param context: The information for the virtual environment - creation request being processed. - """ - os.environ['VIRTUAL_ENV'] = context.env_dir - if not self.nodist: - self.install_setuptools(context) - # Can't install pip without setuptools - if not self.nopip and not self.nodist: - self.install_pip(context) - - def reader(self, stream, context): - """ - Read lines from a subprocess' output stream and either pass to a progress - callable (if specified) or write progress information to sys.stderr. - """ - progress = self.progress - while True: - s = stream.readline() - if not s: - break - if progress is not None: - progress(s, context) - else: - if not self.verbose: - sys.stderr.write('.') - else: - sys.stderr.write(s.decode('utf-8')) - sys.stderr.flush() - stream.close() - - def install_script(self, context, name, url): - _, _, path, _, _, _ = urlparse(url) - fn = os.path.split(path)[-1] - binpath = context.bin_path - distpath = os.path.join(binpath, fn) - # Download script into the virtual environment's binaries folder - urlretrieve(url, distpath) - progress = self.progress - if self.verbose: - term = '\n' - else: - term = '' - if progress is not None: - progress('Installing %s ...%s' % (name, term), 'main') - else: - sys.stderr.write('Installing %s ...%s' % (name, term)) - sys.stderr.flush() - # Install in the virtual environment - args = [context.env_exe, fn] - p = Popen(args, stdout=PIPE, stderr=PIPE, cwd=binpath) - t1 = Thread(target=self.reader, args=(p.stdout, 'stdout')) - t1.start() - t2 = Thread(target=self.reader, args=(p.stderr, 'stderr')) - t2.start() - p.wait() - t1.join() - t2.join() - if progress is not None: - progress('done.', 'main') - else: - sys.stderr.write('done.\n') - # Clean up - no longer needed - os.unlink(distpath) - - def install_setuptools(self, context): - """ - Install setuptools in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://bitbucket.org/pypa/setuptools/downloads/ez_setup.py' - self.install_script(context, 'setuptools', url) - # clear up the setuptools archive which gets downloaded - pred = lambda o: o.startswith('setuptools-') and o.endswith('.tar.gz') - files = filter(pred, os.listdir(context.bin_path)) - for f in files: - f = os.path.join(context.bin_path, f) - os.unlink(f) - - def install_pip(self, context): - """ - Install pip in the virtual environment. - - :param context: The information for the virtual environment - creation request being processed. - """ - url = 'https://bootstrap.pypa.io/get-pip.py' - self.install_script(context, 'pip', url) - - def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with ' - 'Python 3.3 or later') - else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory in which to create the ' - 'virtual environment.') - parser.add_argument('--no-setuptools', default=False, - action='store_true', dest='nodist', - help="Don't install setuptools or pip in the " - "virtual environment.") - parser.add_argument('--no-pip', default=False, - action='store_true', dest='nopip', - help="Don't install pip in the virtual " - "environment.") - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - parser.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'virtual environment ' - 'directory if it already ' - 'exists, before virtual ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the virtual ' - 'environment directory to ' - 'use this version of ' - 'Python, assuming Python ' - 'has been upgraded ' - 'in-place.') - parser.add_argument('--verbose', default=False, action='store_true', - dest='verbose', help='Display the output ' - 'from the scripts which ' - 'install setuptools and pip.') - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = ExtendedEnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - nodist=options.nodist, - nopip=options.nopip, - verbose=options.verbose) - for d in options.dirs: - builder.create(d) - - if __name__ == '__main__': - rc = 1 - try: - main() - rc = 0 - except Exception as e: - print('Error: %s' % e, file=sys.stderr) - sys.exit(rc) - - -This script is also available for download `online -<https://gist.github.com/vsajip/4673395>`_. - - -.. _setuptools: https://pypi.org/project/setuptools/ -.. _pip: https://pypi.org/project/pip/ diff --git a/Doc/library/warnings.rst b/Doc/library/warnings.rst deleted file mode 100644 index 28579ce8df4a62..00000000000000 --- a/Doc/library/warnings.rst +++ /dev/null @@ -1,528 +0,0 @@ -:mod:`warnings` --- Warning control -=================================== - -.. module:: warnings - :synopsis: Issue warning messages and control their disposition. - -**Source code:** :source:`Lib/warnings.py` - -.. index:: single: warnings - --------------- - -Warning messages are typically issued in situations where it is useful to alert -the user of some condition in a program, where that condition (normally) doesn't -warrant raising an exception and terminating the program. For example, one -might want to issue a warning when a program uses an obsolete module. - -Python programmers issue warnings by calling the :func:`warn` function defined -in this module. (C programmers use :c:func:`PyErr_WarnEx`; see -:ref:`exceptionhandling` for details). - -Warning messages are normally written to :data:`sys.stderr`, but their disposition -can be changed flexibly, from ignoring all warnings to turning them into -exceptions. The disposition of warnings can vary based on the :ref:`warning category -<warning-categories>`, the text of the warning message, and the source location where it -is issued. Repetitions of a particular warning for the same source location are -typically suppressed. - -There are two stages in warning control: first, each time a warning is issued, a -determination is made whether a message should be issued or not; next, if a -message is to be issued, it is formatted and printed using a user-settable hook. - -The determination whether to issue a warning message is controlled by the -:ref:`warning filter <warning-filter>`, which is a sequence of matching rules and actions. Rules can be -added to the filter by calling :func:`filterwarnings` and reset to its default -state by calling :func:`resetwarnings`. - -The printing of warning messages is done by calling :func:`showwarning`, which -may be overridden; the default implementation of this function formats the -message by calling :func:`formatwarning`, which is also available for use by -custom implementations. - -.. seealso:: - :func:`logging.captureWarnings` allows you to handle all warnings with - the standard logging infrastructure. - - -.. _warning-categories: - -Warning Categories ------------------- - -There are a number of built-in exceptions that represent warning categories. -This categorization is useful to be able to filter out groups of warnings. - -While these are technically -:ref:`built-in exceptions <warning-categories-as-exceptions>`, they are -documented here, because conceptually they belong to the warnings mechanism. - -User code can define additional warning categories by subclassing one of the -standard warning categories. A warning category must always be a subclass of -the :exc:`Warning` class. - -The following warnings category classes are currently defined: - -.. tabularcolumns:: |l|p{0.6\linewidth}| - -+----------------------------------+-----------------------------------------------+ -| Class | Description | -+==================================+===============================================+ -| :exc:`Warning` | This is the base class of all warning | -| | category classes. It is a subclass of | -| | :exc:`Exception`. | -+----------------------------------+-----------------------------------------------+ -| :exc:`UserWarning` | The default category for :func:`warn`. | -+----------------------------------+-----------------------------------------------+ -| :exc:`DeprecationWarning` | Base category for warnings about deprecated | -| | features when those warnings are intended for | -| | other Python developers (ignored by default, | -| | unless triggered by code in ``__main__``). | -+----------------------------------+-----------------------------------------------+ -| :exc:`SyntaxWarning` | Base category for warnings about dubious | -| | syntactic features. | -+----------------------------------+-----------------------------------------------+ -| :exc:`RuntimeWarning` | Base category for warnings about dubious | -| | runtime features. | -+----------------------------------+-----------------------------------------------+ -| :exc:`FutureWarning` | Base category for warnings about deprecated | -| | features when those warnings are intended for | -| | end users of applications that are written in | -| | Python. | -+----------------------------------+-----------------------------------------------+ -| :exc:`PendingDeprecationWarning` | Base category for warnings about features | -| | that will be deprecated in the future | -| | (ignored by default). | -+----------------------------------+-----------------------------------------------+ -| :exc:`ImportWarning` | Base category for warnings triggered during | -| | the process of importing a module (ignored by | -| | default). | -+----------------------------------+-----------------------------------------------+ -| :exc:`UnicodeWarning` | Base category for warnings related to | -| | Unicode. | -+----------------------------------+-----------------------------------------------+ -| :exc:`BytesWarning` | Base category for warnings related to | -| | :class:`bytes` and :class:`bytearray`. | -+----------------------------------+-----------------------------------------------+ -| :exc:`ResourceWarning` | Base category for warnings related to | -| | resource usage (ignored by default). | -+----------------------------------+-----------------------------------------------+ - -.. versionchanged:: 3.7 - Previously :exc:`DeprecationWarning` and :exc:`FutureWarning` were - distinguished based on whether a feature was being removed entirely or - changing its behaviour. They are now distinguished based on their - intended audience and the way they're handled by the default warnings - filters. - - -.. _warning-filter: - -The Warnings Filter -------------------- - -The warnings filter controls whether warnings are ignored, displayed, or turned -into errors (raising an exception). - -Conceptually, the warnings filter maintains an ordered list of filter -specifications; any specific warning is matched against each filter -specification in the list in turn until a match is found; the filter determines -the disposition of the match. Each entry is a tuple of the form (*action*, -*message*, *category*, *module*, *lineno*), where: - -* *action* is one of the following strings: - - +---------------+----------------------------------------------+ - | Value | Disposition | - +===============+==============================================+ - | ``"default"`` | print the first occurrence of matching | - | | warnings for each location (module + | - | | line number) where the warning is issued | - +---------------+----------------------------------------------+ - | ``"error"`` | turn matching warnings into exceptions | - +---------------+----------------------------------------------+ - | ``"ignore"`` | never print matching warnings | - +---------------+----------------------------------------------+ - | ``"always"`` | always print matching warnings | - +---------------+----------------------------------------------+ - | ``"module"`` | print the first occurrence of matching | - | | warnings for each module where the warning | - | | is issued (regardless of line number) | - +---------------+----------------------------------------------+ - | ``"once"`` | print only the first occurrence of matching | - | | warnings, regardless of location | - +---------------+----------------------------------------------+ - -* *message* is a string containing a regular expression that the start of - the warning message must match, case-insensitively. In :option:`-W` and - :envvar:`PYTHONWARNINGS`, *message* is a literal string that the start of the - warning message must contain (case-insensitively), ignoring any whitespace at - the start or end of *message*. - -* *category* is a class (a subclass of :exc:`Warning`) of which the warning - category must be a subclass in order to match. - -* *module* is a string containing a regular expression that the start of the - fully qualified module name must match, case-sensitively. In :option:`-W` and - :envvar:`PYTHONWARNINGS`, *module* is a literal string that the - fully qualified module name must be equal to (case-sensitively), ignoring any - whitespace at the start or end of *module*. - -* *lineno* is an integer that the line number where the warning occurred must - match, or ``0`` to match all line numbers. - -Since the :exc:`Warning` class is derived from the built-in :exc:`Exception` -class, to turn a warning into an error we simply raise ``category(message)``. - -If a warning is reported and doesn't match any registered filter then the -"default" action is applied (hence its name). - - -.. _describing-warning-filters: - -Describing Warning Filters -~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The warnings filter is initialized by :option:`-W` options passed to the Python -interpreter command line and the :envvar:`PYTHONWARNINGS` environment variable. -The interpreter saves the arguments for all supplied entries without -interpretation in :data:`sys.warnoptions`; the :mod:`warnings` module parses these -when it is first imported (invalid options are ignored, after printing a -message to :data:`sys.stderr`). - -Individual warnings filters are specified as a sequence of fields separated by -colons:: - - action:message:category:module:line - -The meaning of each of these fields is as described in :ref:`warning-filter`. -When listing multiple filters on a single line (as for -:envvar:`PYTHONWARNINGS`), the individual filters are separated by commas and -the filters listed later take precedence over those listed before them (as -they're applied left-to-right, and the most recently applied filters take -precedence over earlier ones). - -Commonly used warning filters apply to either all warnings, warnings in a -particular category, or warnings raised by particular modules or packages. -Some examples:: - - default # Show all warnings (even those ignored by default) - ignore # Ignore all warnings - error # Convert all warnings to errors - error::ResourceWarning # Treat ResourceWarning messages as errors - default::DeprecationWarning # Show DeprecationWarning messages - ignore,default:::mymodule # Only report warnings triggered by "mymodule" - error:::mymodule # Convert warnings to errors in "mymodule" - - -.. _default-warning-filter: - -Default Warning Filter -~~~~~~~~~~~~~~~~~~~~~~ - -By default, Python installs several warning filters, which can be overridden by -the :option:`-W` command-line option, the :envvar:`PYTHONWARNINGS` environment -variable and calls to :func:`filterwarnings`. - -In regular release builds, the default warning filter has the following entries -(in order of precedence):: - - default::DeprecationWarning:__main__ - ignore::DeprecationWarning - ignore::PendingDeprecationWarning - ignore::ImportWarning - ignore::ResourceWarning - -In a :ref:`debug build <debug-build>`, the list of default warning filters is empty. - -.. versionchanged:: 3.2 - :exc:`DeprecationWarning` is now ignored by default in addition to - :exc:`PendingDeprecationWarning`. - -.. versionchanged:: 3.7 - :exc:`DeprecationWarning` is once again shown by default when triggered - directly by code in ``__main__``. - -.. versionchanged:: 3.7 - :exc:`BytesWarning` no longer appears in the default filter list and is - instead configured via :data:`sys.warnoptions` when :option:`-b` is specified - twice. - - -.. _warning-disable: - -Overriding the default filter -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Developers of applications written in Python may wish to hide *all* Python level -warnings from their users by default, and only display them when running tests -or otherwise working on the application. The :data:`sys.warnoptions` attribute -used to pass filter configurations to the interpreter can be used as a marker to -indicate whether or not warnings should be disabled:: - - import sys - - if not sys.warnoptions: - import warnings - warnings.simplefilter("ignore") - -Developers of test runners for Python code are advised to instead ensure that -*all* warnings are displayed by default for the code under test, using code -like:: - - import sys - - if not sys.warnoptions: - import os, warnings - warnings.simplefilter("default") # Change the filter in this process - os.environ["PYTHONWARNINGS"] = "default" # Also affect subprocesses - -Finally, developers of interactive shells that run user code in a namespace -other than ``__main__`` are advised to ensure that :exc:`DeprecationWarning` -messages are made visible by default, using code like the following (where -``user_ns`` is the module used to execute code entered interactively):: - - import warnings - warnings.filterwarnings("default", category=DeprecationWarning, - module=user_ns.get("__name__")) - - -.. _warning-suppress: - -Temporarily Suppressing Warnings --------------------------------- - -If you are using code that you know will raise a warning, such as a deprecated -function, but do not want to see the warning (even when warnings have been -explicitly configured via the command line), then it is possible to suppress -the warning using the :class:`catch_warnings` context manager:: - - import warnings - - def fxn(): - warnings.warn("deprecated", DeprecationWarning) - - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - fxn() - -While within the context manager all warnings will simply be ignored. This -allows you to use known-deprecated code without having to see the warning while -not suppressing the warning for other code that might not be aware of its use -of deprecated code. Note: this can only be guaranteed in a single-threaded -application. If two or more threads use the :class:`catch_warnings` context -manager at the same time, the behavior is undefined. - - - -.. _warning-testing: - -Testing Warnings ----------------- - -To test warnings raised by code, use the :class:`catch_warnings` context -manager. With it you can temporarily mutate the warnings filter to facilitate -your testing. For instance, do the following to capture all raised warnings to -check:: - - import warnings - - def fxn(): - warnings.warn("deprecated", DeprecationWarning) - - with warnings.catch_warnings(record=True) as w: - # Cause all warnings to always be triggered. - warnings.simplefilter("always") - # Trigger a warning. - fxn() - # Verify some things - assert len(w) == 1 - assert issubclass(w[-1].category, DeprecationWarning) - assert "deprecated" in str(w[-1].message) - -One can also cause all warnings to be exceptions by using ``error`` instead of -``always``. One thing to be aware of is that if a warning has already been -raised because of a ``once``/``default`` rule, then no matter what filters are -set the warning will not be seen again unless the warnings registry related to -the warning has been cleared. - -Once the context manager exits, the warnings filter is restored to its state -when the context was entered. This prevents tests from changing the warnings -filter in unexpected ways between tests and leading to indeterminate test -results. The :func:`showwarning` function in the module is also restored to -its original value. Note: this can only be guaranteed in a single-threaded -application. If two or more threads use the :class:`catch_warnings` context -manager at the same time, the behavior is undefined. - -When testing multiple operations that raise the same kind of warning, it -is important to test them in a manner that confirms each operation is raising -a new warning (e.g. set warnings to be raised as exceptions and check the -operations raise exceptions, check that the length of the warning list -continues to increase after each operation, or else delete the previous -entries from the warnings list before each new operation). - - -.. _warning-ignored: - -Updating Code For New Versions of Dependencies ----------------------------------------------- - -Warning categories that are primarily of interest to Python developers (rather -than end users of applications written in Python) are ignored by default. - -Notably, this "ignored by default" list includes :exc:`DeprecationWarning` -(for every module except ``__main__``), which means developers should make sure -to test their code with typically ignored warnings made visible in order to -receive timely notifications of future breaking API changes (whether in the -standard library or third party packages). - -In the ideal case, the code will have a suitable test suite, and the test runner -will take care of implicitly enabling all warnings when running tests -(the test runner provided by the :mod:`unittest` module does this). - -In less ideal cases, applications can be checked for use of deprecated -interfaces by passing :option:`-Wd <-W>` to the Python interpreter (this is -shorthand for :option:`!-W default`) or setting ``PYTHONWARNINGS=default`` in -the environment. This enables default handling for all warnings, including those -that are ignored by default. To change what action is taken for encountered -warnings you can change what argument is passed to :option:`-W` (e.g. -:option:`!-W error`). See the :option:`-W` flag for more details on what is -possible. - - -.. _warning-functions: - -Available Functions -------------------- - - -.. function:: warn(message, category=None, stacklevel=1, source=None) - - Issue a warning, or maybe ignore it or raise an exception. The *category* - argument, if given, must be a :ref:`warning category class <warning-categories>`; it - defaults to :exc:`UserWarning`. Alternatively, *message* can be a :exc:`Warning` instance, - in which case *category* will be ignored and ``message.__class__`` will be used. - In this case, the message text will be ``str(message)``. This function raises an - exception if the particular warning issued is changed into an error by the - :ref:`warnings filter <warning-filter>`. The *stacklevel* argument can be used by wrapper - functions written in Python, like this:: - - def deprecation(message): - warnings.warn(message, DeprecationWarning, stacklevel=2) - - This makes the warning refer to :func:`deprecation`'s caller, rather than to the - source of :func:`deprecation` itself (since the latter would defeat the purpose - of the warning message). - - *source*, if supplied, is the destroyed object which emitted a - :exc:`ResourceWarning`. - - .. versionchanged:: 3.6 - Added *source* parameter. - - -.. function:: warn_explicit(message, category, filename, lineno, module=None, registry=None, module_globals=None, source=None) - - This is a low-level interface to the functionality of :func:`warn`, passing in - explicitly the message, category, filename and line number, and optionally the - module name and the registry (which should be the ``__warningregistry__`` - dictionary of the module). The module name defaults to the filename with - ``.py`` stripped; if no registry is passed, the warning is never suppressed. - *message* must be a string and *category* a subclass of :exc:`Warning` or - *message* may be a :exc:`Warning` instance, in which case *category* will be - ignored. - - *module_globals*, if supplied, should be the global namespace in use by the code - for which the warning is issued. (This argument is used to support displaying - source for modules found in zipfiles or other non-filesystem import - sources). - - *source*, if supplied, is the destroyed object which emitted a - :exc:`ResourceWarning`. - - .. versionchanged:: 3.6 - Add the *source* parameter. - - -.. function:: showwarning(message, category, filename, lineno, file=None, line=None) - - Write a warning to a file. The default implementation calls - ``formatwarning(message, category, filename, lineno, line)`` and writes the - resulting string to *file*, which defaults to :data:`sys.stderr`. You may replace - this function with any callable by assigning to ``warnings.showwarning``. - *line* is a line of source code to be included in the warning - message; if *line* is not supplied, :func:`showwarning` will - try to read the line specified by *filename* and *lineno*. - - -.. function:: formatwarning(message, category, filename, lineno, line=None) - - Format a warning the standard way. This returns a string which may contain - embedded newlines and ends in a newline. *line* is a line of source code to - be included in the warning message; if *line* is not supplied, - :func:`formatwarning` will try to read the line specified by *filename* and - *lineno*. - - -.. function:: filterwarnings(action, message='', category=Warning, module='', lineno=0, append=False) - - Insert an entry into the list of :ref:`warnings filter specifications - <warning-filter>`. The entry is inserted at the front by default; if - *append* is true, it is inserted at the end. This checks the types of the - arguments, compiles the *message* and *module* regular expressions, and - inserts them as a tuple in the list of warnings filters. Entries closer to - the front of the list override entries later in the list, if both match a - particular warning. Omitted arguments default to a value that matches - everything. - - -.. function:: simplefilter(action, category=Warning, lineno=0, append=False) - - Insert a simple entry into the list of :ref:`warnings filter specifications - <warning-filter>`. The meaning of the function parameters is as for - :func:`filterwarnings`, but regular expressions are not needed as the filter - inserted always matches any message in any module as long as the category and - line number match. - - -.. function:: resetwarnings() - - Reset the warnings filter. This discards the effect of all previous calls to - :func:`filterwarnings`, including that of the :option:`-W` command line options - and calls to :func:`simplefilter`. - - -Available Context Managers --------------------------- - -.. class:: catch_warnings(*, record=False, module=None, action=None, category=Warning, lineno=0, append=False) - - A context manager that copies and, upon exit, restores the warnings filter - and the :func:`showwarning` function. - If the *record* argument is :const:`False` (the default) the context manager - returns :class:`None` on entry. If *record* is :const:`True`, a list is - returned that is progressively populated with objects as seen by a custom - :func:`showwarning` function (which also suppresses output to ``sys.stdout``). - Each object in the list has attributes with the same names as the arguments to - :func:`showwarning`. - - The *module* argument takes a module that will be used instead of the - module returned when you import :mod:`warnings` whose filter will be - protected. This argument exists primarily for testing the :mod:`warnings` - module itself. - - If the *action* argument is not ``None``, the remaining arguments are - passed to :func:`simplefilter` as if it were called immediately on - entering the context. - - .. note:: - - The :class:`catch_warnings` manager works by replacing and - then later restoring the module's - :func:`showwarning` function and internal list of filter - specifications. This means the context manager is modifying - global state and therefore is not thread-safe. - - .. versionchanged:: 3.11 - - Added the *action*, *category*, *lineno*, and *append* parameters. diff --git a/Doc/library/wave.rst b/Doc/library/wave.rst deleted file mode 100644 index 4f3d8540651d4f..00000000000000 --- a/Doc/library/wave.rst +++ /dev/null @@ -1,250 +0,0 @@ -:mod:`wave` --- Read and write WAV files -======================================== - -.. module:: wave - :synopsis: Provide an interface to the WAV sound format. - -.. sectionauthor:: Moshe Zadka <moshez@zadka.site.co.il> -.. Documentations stolen from comments in file. - -**Source code:** :source:`Lib/wave.py` - --------------- - -The :mod:`wave` module provides a convenient interface to the Waveform Audio -"WAVE" (or "WAV") file format. -Only files using ``WAVE_FORMAT_PCM`` are supported. Note that this does not -include files using ``WAVE_FORMAT_EXTENSIBLE`` even if the subformat is PCM. - -The :mod:`wave` module defines the following function and exception: - - -.. function:: open(file, mode=None) - - If *file* is a string, open the file by that name, otherwise treat it as a - file-like object. *mode* can be: - - ``'rb'`` - Read only mode. - - ``'wb'`` - Write only mode. - - Note that it does not allow read/write WAV files. - - A *mode* of ``'rb'`` returns a :class:`Wave_read` object, while a *mode* of - ``'wb'`` returns a :class:`Wave_write` object. If *mode* is omitted and a - file-like object is passed as *file*, ``file.mode`` is used as the default - value for *mode*. - - If you pass in a file-like object, the wave object will not close it when its - ``close()`` method is called; it is the caller's responsibility to close - the file object. - - The :func:`.open` function may be used in a :keyword:`with` statement. When - the :keyword:`!with` block completes, the :meth:`Wave_read.close()` or - :meth:`Wave_write.close()` method is called. - - .. versionchanged:: 3.4 - Added support for unseekable files. - -.. exception:: Error - - An error raised when something is impossible because it violates the WAV - specification or hits an implementation deficiency. - - -.. _wave-read-objects: - -Wave_read Objects ------------------ - -.. class:: Wave_read - - Read a WAV file. - - Wave_read objects, as returned by :func:`.open`, have the following methods: - - - .. method:: close() - - Close the stream if it was opened by :mod:`wave`, and make the instance - unusable. This is called automatically on object collection. - - - .. method:: getnchannels() - - Returns number of audio channels (``1`` for mono, ``2`` for stereo). - - - .. method:: getsampwidth() - - Returns sample width in bytes. - - - .. method:: getframerate() - - Returns sampling frequency. - - - .. method:: getnframes() - - Returns number of audio frames. - - - .. method:: getcomptype() - - Returns compression type (``'NONE'`` is the only supported type). - - - .. method:: getcompname() - - Human-readable version of :meth:`getcomptype`. Usually ``'not compressed'`` - parallels ``'NONE'``. - - - .. method:: getparams() - - Returns a :func:`~collections.namedtuple` ``(nchannels, sampwidth, - framerate, nframes, comptype, compname)``, equivalent to output of the - ``get*()`` methods. - - - .. method:: readframes(n) - - Reads and returns at most *n* frames of audio, as a :class:`bytes` object. - - - .. method:: rewind() - - Rewind the file pointer to the beginning of the audio stream. - - The following two methods are defined for compatibility with the :mod:`aifc` - module, and don't do anything interesting. - - - .. method:: getmarkers() - - Returns ``None``. - - - .. method:: getmark(id) - - Raise an error. - - The following two methods define a term "position" which is compatible between - them, and is otherwise implementation dependent. - - - .. method:: setpos(pos) - - Set the file pointer to the specified position. - - - .. method:: tell() - - Return current file pointer position. - - -.. _wave-write-objects: - -Wave_write Objects ------------------- - -.. class:: Wave_write - - Write a WAV file. - - Wave_write objects, as returned by :func:`.open`. - - For seekable output streams, the ``wave`` header will automatically be updated - to reflect the number of frames actually written. For unseekable streams, the - *nframes* value must be accurate when the first frame data is written. An - accurate *nframes* value can be achieved either by calling - :meth:`setnframes` or :meth:`setparams` with the number - of frames that will be written before :meth:`close` is called and - then using :meth:`writeframesraw` to write the frame data, or by - calling :meth:`writeframes` with all of the frame data to be - written. In the latter case :meth:`writeframes` will calculate - the number of frames in the data and set *nframes* accordingly before writing - the frame data. - - .. versionchanged:: 3.4 - Added support for unseekable files. - - Wave_write objects have the following methods: - - .. method:: close() - - Make sure *nframes* is correct, and close the file if it was opened by - :mod:`wave`. This method is called upon object collection. It will raise - an exception if the output stream is not seekable and *nframes* does not - match the number of frames actually written. - - - .. method:: setnchannels(n) - - Set the number of channels. - - - .. method:: setsampwidth(n) - - Set the sample width to *n* bytes. - - - .. method:: setframerate(n) - - Set the frame rate to *n*. - - .. versionchanged:: 3.2 - A non-integral input to this method is rounded to the nearest - integer. - - - .. method:: setnframes(n) - - Set the number of frames to *n*. This will be changed later if the number - of frames actually written is different (this update attempt will - raise an error if the output stream is not seekable). - - - .. method:: setcomptype(type, name) - - Set the compression type and description. At the moment, only compression type - ``NONE`` is supported, meaning no compression. - - - .. method:: setparams(tuple) - - The *tuple* should be ``(nchannels, sampwidth, framerate, nframes, comptype, - compname)``, with values valid for the ``set*()`` methods. Sets all - parameters. - - - .. method:: tell() - - Return current position in the file, with the same disclaimer for the - :meth:`Wave_read.tell` and :meth:`Wave_read.setpos` methods. - - - .. method:: writeframesraw(data) - - Write audio frames, without correcting *nframes*. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - - .. method:: writeframes(data) - - Write audio frames and make sure *nframes* is correct. It will raise an - error if the output stream is not seekable and the total number of frames - that have been written after *data* has been written does not match the - previously set value for *nframes*. - - .. versionchanged:: 3.4 - Any :term:`bytes-like object` is now accepted. - - Note that it is invalid to set any parameters after calling :meth:`writeframes` - or :meth:`writeframesraw`, and any attempt to do so will raise - :exc:`wave.Error`. diff --git a/Doc/library/weakref.rst b/Doc/library/weakref.rst deleted file mode 100644 index d6e062df945c64..00000000000000 --- a/Doc/library/weakref.rst +++ /dev/null @@ -1,600 +0,0 @@ -.. _mod-weakref: - -:mod:`weakref` --- Weak references -================================== - -.. module:: weakref - :synopsis: Support for weak references and weak dictionaries. - -.. moduleauthor:: Fred L. Drake, Jr. <fdrake@acm.org> -.. moduleauthor:: Neil Schemenauer <nas@arctrix.com> -.. moduleauthor:: Martin von Löwis <martin@loewis.home.cs.tu-berlin.de> -.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org> - -**Source code:** :source:`Lib/weakref.py` - --------------- - -The :mod:`weakref` module allows the Python programmer to create :dfn:`weak -references` to objects. - -.. When making changes to the examples in this file, be sure to update - Lib/test/test_weakref.py::libreftest too! - -In the following, the term :dfn:`referent` means the object which is referred to -by a weak reference. - -A weak reference to an object is not enough to keep the object alive: when the -only remaining references to a referent are weak references, -:term:`garbage collection` is free to destroy the referent and reuse its memory -for something else. However, until the object is actually destroyed the weak -reference may return the object even if there are no strong references to it. - -A primary use for weak references is to implement caches or -mappings holding large objects, where it's desired that a large object not be -kept alive solely because it appears in a cache or mapping. - -For example, if you have a number of large binary image objects, you may wish to -associate a name with each. If you used a Python dictionary to map names to -images, or images to names, the image objects would remain alive just because -they appeared as values or keys in the dictionaries. The -:class:`WeakKeyDictionary` and :class:`WeakValueDictionary` classes supplied by -the :mod:`weakref` module are an alternative, using weak references to construct -mappings that don't keep objects alive solely because they appear in the mapping -objects. If, for example, an image object is a value in a -:class:`WeakValueDictionary`, then when the last remaining references to that -image object are the weak references held by weak mappings, garbage collection -can reclaim the object, and its corresponding entries in weak mappings are -simply deleted. - -:class:`WeakKeyDictionary` and :class:`WeakValueDictionary` use weak references -in their implementation, setting up callback functions on the weak references -that notify the weak dictionaries when a key or value has been reclaimed by -garbage collection. :class:`WeakSet` implements the :class:`set` interface, -but keeps weak references to its elements, just like a -:class:`WeakKeyDictionary` does. - -:class:`finalize` provides a straight forward way to register a -cleanup function to be called when an object is garbage collected. -This is simpler to use than setting up a callback function on a raw -weak reference, since the module automatically ensures that the finalizer -remains alive until the object is collected. - -Most programs should find that using one of these weak container types -or :class:`finalize` is all they need -- it's not usually necessary to -create your own weak references directly. The low-level machinery is -exposed by the :mod:`weakref` module for the benefit of advanced uses. - -Not all objects can be weakly referenced. Objects which support weak references -include class instances, functions written in Python (but not in C), instance methods, -sets, frozensets, some :term:`file objects <file object>`, :term:`generators <generator>`, -type objects, sockets, arrays, deques, regular expression pattern objects, and code -objects. - -.. versionchanged:: 3.2 - Added support for thread.lock, threading.Lock, and code objects. - -Several built-in types such as :class:`list` and :class:`dict` do not directly -support weak references but can add support through subclassing:: - - class Dict(dict): - pass - - obj = Dict(red=1, green=2, blue=3) # this object is weak referenceable - -.. impl-detail:: - - Other built-in types such as :class:`tuple` and :class:`int` do not support weak - references even when subclassed. - -Extension types can easily be made to support weak references; see -:ref:`weakref-support`. - -When ``__slots__`` are defined for a given type, weak reference support is -disabled unless a ``'__weakref__'`` string is also present in the sequence of -strings in the ``__slots__`` declaration. -See :ref:`__slots__ documentation <slots>` for details. - -.. class:: ref(object[, callback]) - - Return a weak reference to *object*. The original object can be retrieved by - calling the reference object if the referent is still alive; if the referent is - no longer alive, calling the reference object will cause :const:`None` to be - returned. If *callback* is provided and not :const:`None`, and the returned - weakref object is still alive, the callback will be called when the object is - about to be finalized; the weak reference object will be passed as the only - parameter to the callback; the referent will no longer be available. - - It is allowable for many weak references to be constructed for the same object. - Callbacks registered for each weak reference will be called from the most - recently registered callback to the oldest registered callback. - - Exceptions raised by the callback will be noted on the standard error output, - but cannot be propagated; they are handled in exactly the same way as exceptions - raised from an object's :meth:`~object.__del__` method. - - Weak references are :term:`hashable` if the *object* is hashable. They will - maintain their hash value even after the *object* was deleted. If - :func:`hash` is called the first time only after the *object* was deleted, - the call will raise :exc:`TypeError`. - - Weak references support tests for equality, but not ordering. If the referents - are still alive, two references have the same equality relationship as their - referents (regardless of the *callback*). If either referent has been deleted, - the references are equal only if the reference objects are the same object. - - This is a subclassable type rather than a factory function. - - .. attribute:: __callback__ - - This read-only attribute returns the callback currently associated to the - weakref. If there is no callback or if the referent of the weakref is - no longer alive then this attribute will have value ``None``. - - .. versionchanged:: 3.4 - Added the :attr:`__callback__` attribute. - - -.. function:: proxy(object[, callback]) - - Return a proxy to *object* which uses a weak reference. This supports use of - the proxy in most contexts instead of requiring the explicit dereferencing used - with weak reference objects. The returned object will have a type of either - ``ProxyType`` or ``CallableProxyType``, depending on whether *object* is - callable. Proxy objects are not :term:`hashable` regardless of the referent; this - avoids a number of problems related to their fundamentally mutable nature, and - prevents their use as dictionary keys. *callback* is the same as the parameter - of the same name to the :func:`ref` function. - - Accessing an attribute of the proxy object after the referent is - garbage collected raises :exc:`ReferenceError`. - - .. versionchanged:: 3.8 - Extended the operator support on proxy objects to include the matrix - multiplication operators ``@`` and ``@=``. - - -.. function:: getweakrefcount(object) - - Return the number of weak references and proxies which refer to *object*. - - -.. function:: getweakrefs(object) - - Return a list of all weak reference and proxy objects which refer to *object*. - - -.. class:: WeakKeyDictionary([dict]) - - Mapping class that references keys weakly. Entries in the dictionary will be - discarded when there is no longer a strong reference to the key. This can be - used to associate additional data with an object owned by other parts of an - application without adding attributes to those objects. This can be especially - useful with objects that override attribute accesses. - - Note that when a key with equal value to an existing key (but not equal identity) - is inserted into the dictionary, it replaces the value but does not replace the - existing key. Due to this, when the reference to the original key is deleted, it - also deletes the entry in the dictionary:: - - >>> class T(str): pass - ... - >>> k1, k2 = T(), T() - >>> d = weakref.WeakKeyDictionary() - >>> d[k1] = 1 # d = {k1: 1} - >>> d[k2] = 2 # d = {k1: 2} - >>> del k1 # d = {} - - A workaround would be to remove the key prior to reassignment:: - - >>> class T(str): pass - ... - >>> k1, k2 = T(), T() - >>> d = weakref.WeakKeyDictionary() - >>> d[k1] = 1 # d = {k1: 1} - >>> del d[k1] - >>> d[k2] = 2 # d = {k2: 2} - >>> del k1 # d = {k2: 2} - - .. versionchanged:: 3.9 - Added support for ``|`` and ``|=`` operators, specified in :pep:`584`. - -:class:`WeakKeyDictionary` objects have an additional method that -exposes the internal references directly. The references are not guaranteed to -be "live" at the time they are used, so the result of calling the references -needs to be checked before being used. This can be used to avoid creating -references that will cause the garbage collector to keep the keys around longer -than needed. - - -.. method:: WeakKeyDictionary.keyrefs() - - Return an iterable of the weak references to the keys. - - -.. class:: WeakValueDictionary([dict]) - - Mapping class that references values weakly. Entries in the dictionary will be - discarded when no strong reference to the value exists any more. - - .. versionchanged:: 3.9 - Added support for ``|`` and ``|=`` operators, as specified in :pep:`584`. - -:class:`WeakValueDictionary` objects have an additional method that has the -same issues as the :meth:`WeakKeyDictionary.keyrefs` method. - - -.. method:: WeakValueDictionary.valuerefs() - - Return an iterable of the weak references to the values. - - -.. class:: WeakSet([elements]) - - Set class that keeps weak references to its elements. An element will be - discarded when no strong reference to it exists any more. - - -.. class:: WeakMethod(method[, callback]) - - A custom :class:`ref` subclass which simulates a weak reference to a bound - method (i.e., a method defined on a class and looked up on an instance). - Since a bound method is ephemeral, a standard weak reference cannot keep - hold of it. :class:`WeakMethod` has special code to recreate the bound - method until either the object or the original function dies:: - - >>> class C: - ... def method(self): - ... print("method called!") - ... - >>> c = C() - >>> r = weakref.ref(c.method) - >>> r() - >>> r = weakref.WeakMethod(c.method) - >>> r() - <bound method C.method of <__main__.C object at 0x7fc859830220>> - >>> r()() - method called! - >>> del c - >>> gc.collect() - 0 - >>> r() - >>> - - *callback* is the same as the parameter of the same name to the :func:`ref` function. - - .. versionadded:: 3.4 - -.. class:: finalize(obj, func, /, *args, **kwargs) - - Return a callable finalizer object which will be called when *obj* - is garbage collected. Unlike an ordinary weak reference, a finalizer - will always survive until the reference object is collected, greatly - simplifying lifecycle management. - - A finalizer is considered *alive* until it is called (either explicitly - or at garbage collection), and after that it is *dead*. Calling a live - finalizer returns the result of evaluating ``func(*arg, **kwargs)``, - whereas calling a dead finalizer returns :const:`None`. - - Exceptions raised by finalizer callbacks during garbage collection - will be shown on the standard error output, but cannot be - propagated. They are handled in the same way as exceptions raised - from an object's :meth:`~object.__del__` method or a weak reference's - callback. - - When the program exits, each remaining live finalizer is called - unless its :attr:`atexit` attribute has been set to false. They - are called in reverse order of creation. - - A finalizer will never invoke its callback during the later part of - the :term:`interpreter shutdown` when module globals are liable to have - been replaced by :const:`None`. - - .. method:: __call__() - - If *self* is alive then mark it as dead and return the result of - calling ``func(*args, **kwargs)``. If *self* is dead then return - :const:`None`. - - .. method:: detach() - - If *self* is alive then mark it as dead and return the tuple - ``(obj, func, args, kwargs)``. If *self* is dead then return - :const:`None`. - - .. method:: peek() - - If *self* is alive then return the tuple ``(obj, func, args, - kwargs)``. If *self* is dead then return :const:`None`. - - .. attribute:: alive - - Property which is true if the finalizer is alive, false otherwise. - - .. attribute:: atexit - - A writable boolean property which by default is true. When the - program exits, it calls all remaining live finalizers for which - :attr:`.atexit` is true. They are called in reverse order of - creation. - - .. note:: - - It is important to ensure that *func*, *args* and *kwargs* do - not own any references to *obj*, either directly or indirectly, - since otherwise *obj* will never be garbage collected. In - particular, *func* should not be a bound method of *obj*. - - .. versionadded:: 3.4 - - -.. data:: ReferenceType - - The type object for weak references objects. - - -.. data:: ProxyType - - The type object for proxies of objects which are not callable. - - -.. data:: CallableProxyType - - The type object for proxies of callable objects. - - -.. data:: ProxyTypes - - Sequence containing all the type objects for proxies. This can make it simpler - to test if an object is a proxy without being dependent on naming both proxy - types. - - -.. seealso:: - - :pep:`205` - Weak References - The proposal and rationale for this feature, including links to earlier - implementations and information about similar features in other languages. - - -.. _weakref-objects: - -Weak Reference Objects ----------------------- - -Weak reference objects have no methods and no attributes besides -:attr:`ref.__callback__`. A weak reference object allows the referent to be -obtained, if it still exists, by calling it: - - >>> import weakref - >>> class Object: - ... pass - ... - >>> o = Object() - >>> r = weakref.ref(o) - >>> o2 = r() - >>> o is o2 - True - -If the referent no longer exists, calling the reference object returns -:const:`None`: - - >>> del o, o2 - >>> print(r()) - None - -Testing that a weak reference object is still live should be done using the -expression ``ref() is not None``. Normally, application code that needs to use -a reference object should follow this pattern:: - - # r is a weak reference object - o = r() - if o is None: - # referent has been garbage collected - print("Object has been deallocated; can't frobnicate.") - else: - print("Object is still live!") - o.do_something_useful() - -Using a separate test for "liveness" creates race conditions in threaded -applications; another thread can cause a weak reference to become invalidated -before the weak reference is called; the idiom shown above is safe in threaded -applications as well as single-threaded applications. - -Specialized versions of :class:`ref` objects can be created through subclassing. -This is used in the implementation of the :class:`WeakValueDictionary` to reduce -the memory overhead for each entry in the mapping. This may be most useful to -associate additional information with a reference, but could also be used to -insert additional processing on calls to retrieve the referent. - -This example shows how a subclass of :class:`ref` can be used to store -additional information about an object and affect the value that's returned when -the referent is accessed:: - - import weakref - - class ExtendedRef(weakref.ref): - def __init__(self, ob, callback=None, /, **annotations): - super().__init__(ob, callback) - self.__counter = 0 - for k, v in annotations.items(): - setattr(self, k, v) - - def __call__(self): - """Return a pair containing the referent and the number of - times the reference has been called. - """ - ob = super().__call__() - if ob is not None: - self.__counter += 1 - ob = (ob, self.__counter) - return ob - - -.. _weakref-example: - -Example -------- - -This simple example shows how an application can use object IDs to retrieve -objects that it has seen before. The IDs of the objects can then be used in -other data structures without forcing the objects to remain alive, but the -objects can still be retrieved by ID if they do. - -.. Example contributed by Tim Peters. - -:: - - import weakref - - _id2obj_dict = weakref.WeakValueDictionary() - - def remember(obj): - oid = id(obj) - _id2obj_dict[oid] = obj - return oid - - def id2obj(oid): - return _id2obj_dict[oid] - - -.. _finalize-examples: - -Finalizer Objects ------------------ - -The main benefit of using :class:`finalize` is that it makes it simple -to register a callback without needing to preserve the returned finalizer -object. For instance - - >>> import weakref - >>> class Object: - ... pass - ... - >>> kenny = Object() - >>> weakref.finalize(kenny, print, "You killed Kenny!") #doctest:+ELLIPSIS - <finalize object at ...; for 'Object' at ...> - >>> del kenny - You killed Kenny! - -The finalizer can be called directly as well. However the finalizer -will invoke the callback at most once. - - >>> def callback(x, y, z): - ... print("CALLBACK") - ... return x + y + z - ... - >>> obj = Object() - >>> f = weakref.finalize(obj, callback, 1, 2, z=3) - >>> assert f.alive - >>> assert f() == 6 - CALLBACK - >>> assert not f.alive - >>> f() # callback not called because finalizer dead - >>> del obj # callback not called because finalizer dead - -You can unregister a finalizer using its :meth:`~finalize.detach` -method. This kills the finalizer and returns the arguments passed to -the constructor when it was created. - - >>> obj = Object() - >>> f = weakref.finalize(obj, callback, 1, 2, z=3) - >>> f.detach() #doctest:+ELLIPSIS - (<...Object object ...>, <function callback ...>, (1, 2), {'z': 3}) - >>> newobj, func, args, kwargs = _ - >>> assert not f.alive - >>> assert newobj is obj - >>> assert func(*args, **kwargs) == 6 - CALLBACK - -Unless you set the :attr:`~finalize.atexit` attribute to -:const:`False`, a finalizer will be called when the program exits if it -is still alive. For instance - -.. doctest:: - :options: +SKIP - - >>> obj = Object() - >>> weakref.finalize(obj, print, "obj dead or exiting") - <finalize object at ...; for 'Object' at ...> - >>> exit() - obj dead or exiting - - -Comparing finalizers with :meth:`~object.__del__` methods ---------------------------------------------------------- - -Suppose we want to create a class whose instances represent temporary -directories. The directories should be deleted with their contents -when the first of the following events occurs: - -* the object is garbage collected, -* the object's :meth:`!remove` method is called, or -* the program exits. - -We might try to implement the class using a :meth:`~object.__del__` method as -follows:: - - class TempDir: - def __init__(self): - self.name = tempfile.mkdtemp() - - def remove(self): - if self.name is not None: - shutil.rmtree(self.name) - self.name = None - - @property - def removed(self): - return self.name is None - - def __del__(self): - self.remove() - -Starting with Python 3.4, :meth:`~object.__del__` methods no longer prevent -reference cycles from being garbage collected, and module globals are -no longer forced to :const:`None` during :term:`interpreter shutdown`. -So this code should work without any issues on CPython. - -However, handling of :meth:`~object.__del__` methods is notoriously implementation -specific, since it depends on internal details of the interpreter's garbage -collector implementation. - -A more robust alternative can be to define a finalizer which only references -the specific functions and objects that it needs, rather than having access -to the full state of the object:: - - class TempDir: - def __init__(self): - self.name = tempfile.mkdtemp() - self._finalizer = weakref.finalize(self, shutil.rmtree, self.name) - - def remove(self): - self._finalizer() - - @property - def removed(self): - return not self._finalizer.alive - -Defined like this, our finalizer only receives a reference to the details -it needs to clean up the directory appropriately. If the object never gets -garbage collected the finalizer will still be called at exit. - -The other advantage of weakref based finalizers is that they can be used to -register finalizers for classes where the definition is controlled by a -third party, such as running code when a module is unloaded:: - - import weakref, sys - def unloading_module(): - # implicit reference to the module globals from the function body - weakref.finalize(sys.modules[__name__], unloading_module) - - -.. note:: - - If you create a finalizer object in a daemonic thread just as the program - exits then there is the possibility that the finalizer - does not get called at exit. However, in a daemonic thread - :func:`atexit.register`, ``try: ... finally: ...`` and ``with: ...`` - do not guarantee that cleanup occurs either. diff --git a/Doc/library/webbrowser.rst b/Doc/library/webbrowser.rst deleted file mode 100644 index 734b6321e5a7e7..00000000000000 --- a/Doc/library/webbrowser.rst +++ /dev/null @@ -1,230 +0,0 @@ -:mod:`webbrowser` --- Convenient web-browser controller -======================================================= - -.. module:: webbrowser - :synopsis: Easy-to-use controller for web browsers. - -.. moduleauthor:: Fred L. Drake, Jr. <fdrake@acm.org> -.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org> - -**Source code:** :source:`Lib/webbrowser.py` - --------------- - -The :mod:`webbrowser` module provides a high-level interface to allow displaying -web-based documents to users. Under most circumstances, simply calling the -:func:`.open` function from this module will do the right thing. - -Under Unix, graphical browsers are preferred under X11, but text-mode browsers -will be used if graphical browsers are not available or an X11 display isn't -available. If text-mode browsers are used, the calling process will block until -the user exits the browser. - -If the environment variable :envvar:`BROWSER` exists, it is interpreted as the -:data:`os.pathsep`-separated list of browsers to try ahead of the platform -defaults. When the value of a list part contains the string ``%s``, then it is -interpreted as a literal browser command line to be used with the argument URL -substituted for ``%s``; if the part does not contain ``%s``, it is simply -interpreted as the name of the browser to launch. [1]_ - -For non-Unix platforms, or when a remote browser is available on Unix, the -controlling process will not wait for the user to finish with the browser, but -allow the remote browser to maintain its own windows on the display. If remote -browsers are not available on Unix, the controlling process will launch a new -browser and wait. - -The script :program:`webbrowser` can be used as a command-line interface for the -module. It accepts a URL as the argument. It accepts the following optional -parameters: ``-n`` opens the URL in a new browser window, if possible; -``-t`` opens the URL in a new browser page ("tab"). The options are, -naturally, mutually exclusive. Usage example:: - - python -m webbrowser -t "https://www.python.org" - -.. include:: ../includes/wasm-notavail.rst - -The following exception is defined: - - -.. exception:: Error - - Exception raised when a browser control error occurs. - -The following functions are defined: - - -.. function:: open(url, new=0, autoraise=True) - - Display *url* using the default browser. If *new* is 0, the *url* is opened - in the same browser window if possible. If *new* is 1, a new browser window - is opened if possible. If *new* is 2, a new browser page ("tab") is opened - if possible. If *autoraise* is ``True``, the window is raised if possible - (note that under many window managers this will occur regardless of the - setting of this variable). - - Note that on some platforms, trying to open a filename using this function, - may work and start the operating system's associated program. However, this - is neither supported nor portable. - - .. audit-event:: webbrowser.open url webbrowser.open - - -.. function:: open_new(url) - - Open *url* in a new window of the default browser, if possible, otherwise, open - *url* in the only browser window. - -.. function:: open_new_tab(url) - - Open *url* in a new page ("tab") of the default browser, if possible, otherwise - equivalent to :func:`open_new`. - - -.. function:: get(using=None) - - Return a controller object for the browser type *using*. If *using* is - ``None``, return a controller for a default browser appropriate to the - caller's environment. - - -.. function:: register(name, constructor, instance=None, *, preferred=False) - - Register the browser type *name*. Once a browser type is registered, the - :func:`get` function can return a controller for that browser type. If - *instance* is not provided, or is ``None``, *constructor* will be called without - parameters to create an instance when needed. If *instance* is provided, - *constructor* will never be called, and may be ``None``. - - Setting *preferred* to ``True`` makes this browser a preferred result for - a :func:`get` call with no argument. Otherwise, this entry point is only - useful if you plan to either set the :envvar:`BROWSER` variable or call - :func:`get` with a nonempty argument matching the name of a handler you - declare. - - .. versionchanged:: 3.7 - *preferred* keyword-only parameter was added. - -A number of browser types are predefined. This table gives the type names that -may be passed to the :func:`get` function and the corresponding instantiations -for the controller classes, all defined in this module. - -+------------------------+-----------------------------------------+-------+ -| Type Name | Class Name | Notes | -+========================+=========================================+=======+ -| ``'mozilla'`` | :class:`Mozilla('mozilla')` | | -+------------------------+-----------------------------------------+-------+ -| ``'firefox'`` | :class:`Mozilla('mozilla')` | | -+------------------------+-----------------------------------------+-------+ -| ``'netscape'`` | :class:`Mozilla('netscape')` | | -+------------------------+-----------------------------------------+-------+ -| ``'galeon'`` | :class:`Galeon('galeon')` | | -+------------------------+-----------------------------------------+-------+ -| ``'epiphany'`` | :class:`Galeon('epiphany')` | | -+------------------------+-----------------------------------------+-------+ -| ``'skipstone'`` | :class:`BackgroundBrowser('skipstone')` | | -+------------------------+-----------------------------------------+-------+ -| ``'kfmclient'`` | :class:`Konqueror()` | \(1) | -+------------------------+-----------------------------------------+-------+ -| ``'konqueror'`` | :class:`Konqueror()` | \(1) | -+------------------------+-----------------------------------------+-------+ -| ``'kfm'`` | :class:`Konqueror()` | \(1) | -+------------------------+-----------------------------------------+-------+ -| ``'mosaic'`` | :class:`BackgroundBrowser('mosaic')` | | -+------------------------+-----------------------------------------+-------+ -| ``'opera'`` | :class:`Opera()` | | -+------------------------+-----------------------------------------+-------+ -| ``'grail'`` | :class:`Grail()` | | -+------------------------+-----------------------------------------+-------+ -| ``'links'`` | :class:`GenericBrowser('links')` | | -+------------------------+-----------------------------------------+-------+ -| ``'elinks'`` | :class:`Elinks('elinks')` | | -+------------------------+-----------------------------------------+-------+ -| ``'lynx'`` | :class:`GenericBrowser('lynx')` | | -+------------------------+-----------------------------------------+-------+ -| ``'w3m'`` | :class:`GenericBrowser('w3m')` | | -+------------------------+-----------------------------------------+-------+ -| ``'windows-default'`` | :class:`WindowsDefault` | \(2) | -+------------------------+-----------------------------------------+-------+ -| ``'macosx'`` | :class:`MacOSXOSAScript('default')` | \(3) | -+------------------------+-----------------------------------------+-------+ -| ``'safari'`` | :class:`MacOSXOSAScript('safari')` | \(3) | -+------------------------+-----------------------------------------+-------+ -| ``'google-chrome'`` | :class:`Chrome('google-chrome')` | | -+------------------------+-----------------------------------------+-------+ -| ``'chrome'`` | :class:`Chrome('chrome')` | | -+------------------------+-----------------------------------------+-------+ -| ``'chromium'`` | :class:`Chromium('chromium')` | | -+------------------------+-----------------------------------------+-------+ -| ``'chromium-browser'`` | :class:`Chromium('chromium-browser')` | | -+------------------------+-----------------------------------------+-------+ - -Notes: - -(1) - "Konqueror" is the file manager for the KDE desktop environment for Unix, and - only makes sense to use if KDE is running. Some way of reliably detecting KDE - would be nice; the :envvar:`KDEDIR` variable is not sufficient. Note also that - the name "kfm" is used even when using the :program:`konqueror` command with KDE - 2 --- the implementation selects the best strategy for running Konqueror. - -(2) - Only on Windows platforms. - -(3) - Only on macOS platform. - -.. versionadded:: 3.3 - Support for Chrome/Chromium has been added. - -.. deprecated-removed:: 3.11 3.13 - :class:`MacOSX` is deprecated, use :class:`MacOSXOSAScript` instead. - -Here are some simple examples:: - - url = 'https://docs.python.org/' - - # Open URL in a new tab, if a browser window is already open. - webbrowser.open_new_tab(url) - - # Open URL in new window, raising the window if possible. - webbrowser.open_new(url) - - -.. _browser-controllers: - -Browser Controller Objects --------------------------- - -Browser controllers provide these methods which parallel three of the -module-level convenience functions: - - -.. attribute:: name - - System-dependent name for the browser. - - -.. method:: controller.open(url, new=0, autoraise=True) - - Display *url* using the browser handled by this controller. If *new* is 1, a new - browser window is opened if possible. If *new* is 2, a new browser page ("tab") - is opened if possible. - - -.. method:: controller.open_new(url) - - Open *url* in a new window of the browser handled by this controller, if - possible, otherwise, open *url* in the only browser window. Alias - :func:`open_new`. - - -.. method:: controller.open_new_tab(url) - - Open *url* in a new page ("tab") of the browser handled by this controller, if - possible, otherwise equivalent to :func:`open_new`. - - -.. rubric:: Footnotes - -.. [1] Executables named here without a full path will be searched in the - directories given in the :envvar:`PATH` environment variable. diff --git a/Doc/library/windows.rst b/Doc/library/windows.rst deleted file mode 100644 index 4d72ead12aadc7..00000000000000 --- a/Doc/library/windows.rst +++ /dev/null @@ -1,14 +0,0 @@ -.. _mswin-specific-services: - -**************************** -MS Windows Specific Services -**************************** - -This chapter describes modules that are only available on MS Windows platforms. - - -.. toctree:: - - msvcrt.rst - winreg.rst - winsound.rst diff --git a/Doc/library/winreg.rst b/Doc/library/winreg.rst deleted file mode 100644 index 06bd4d87eb03c6..00000000000000 --- a/Doc/library/winreg.rst +++ /dev/null @@ -1,799 +0,0 @@ -:mod:`winreg` --- Windows registry access -========================================= - -.. module:: winreg - :platform: Windows - :synopsis: Routines and objects for manipulating the Windows registry. - -.. sectionauthor:: Mark Hammond <MarkH@ActiveState.com> - --------------- - -These functions expose the Windows registry API to Python. Instead of using an -integer as the registry handle, a :ref:`handle object <handle-object>` is used -to ensure that the handles are closed correctly, even if the programmer neglects -to explicitly close them. - -.. _exception-changed: - -.. versionchanged:: 3.3 - Several functions in this module used to raise a - :exc:`WindowsError`, which is now an alias of :exc:`OSError`. - -.. _functions: - -Functions ------------------- - -This module offers the following functions: - - -.. function:: CloseKey(hkey) - - Closes a previously opened registry key. The *hkey* argument specifies a - previously opened key. - - .. note:: - - If *hkey* is not closed using this method (or via :meth:`hkey.Close() - <PyHKEY.Close>`), it is closed when the *hkey* object is destroyed by - Python. - - -.. function:: ConnectRegistry(computer_name, key) - - Establishes a connection to a predefined registry handle on another computer, - and returns a :ref:`handle object <handle-object>`. - - *computer_name* is the name of the remote computer, of the form - ``r"\\computername"``. If ``None``, the local computer is used. - - *key* is the predefined handle to connect to. - - The return value is the handle of the opened key. If the function fails, an - :exc:`OSError` exception is raised. - - .. audit-event:: winreg.ConnectRegistry computer_name,key winreg.ConnectRegistry - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: CreateKey(key, sub_key) - - Creates or opens the specified key, returning a - :ref:`handle object <handle-object>`. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that names the key this method opens or creates. - - If *key* is one of the predefined keys, *sub_key* may be ``None``. In that - case, the handle returned is the same key handle passed in to the function. - - If the key already exists, this function opens the existing key. - - The return value is the handle of the opened key. If the function fails, an - :exc:`OSError` exception is raised. - - .. audit-event:: winreg.CreateKey key,sub_key,access winreg.CreateKey - - .. audit-event:: winreg.OpenKey/result key winreg.CreateKey - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: CreateKeyEx(key, sub_key, reserved=0, access=KEY_WRITE) - - Creates or opens the specified key, returning a - :ref:`handle object <handle-object>`. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that names the key this method opens or creates. - - *reserved* is a reserved integer, and must be zero. The default is zero. - - *access* is an integer that specifies an access mask that describes the desired - security access for the key. Default is :const:`KEY_WRITE`. See - :ref:`Access Rights <access-rights>` for other allowed values. - - If *key* is one of the predefined keys, *sub_key* may be ``None``. In that - case, the handle returned is the same key handle passed in to the function. - - If the key already exists, this function opens the existing key. - - The return value is the handle of the opened key. If the function fails, an - :exc:`OSError` exception is raised. - - .. audit-event:: winreg.CreateKey key,sub_key,access winreg.CreateKeyEx - - .. audit-event:: winreg.OpenKey/result key winreg.CreateKeyEx - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: DeleteKey(key, sub_key) - - Deletes the specified key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that must be a subkey of the key identified by the *key* - parameter. This value must not be ``None``, and the key may not have subkeys. - - *This method can not delete keys with subkeys.* - - If the method succeeds, the entire key, including all of its values, is removed. - If the method fails, an :exc:`OSError` exception is raised. - - .. audit-event:: winreg.DeleteKey key,sub_key,access winreg.DeleteKey - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: DeleteKeyEx(key, sub_key, access=KEY_WOW64_64KEY, reserved=0) - - Deletes the specified key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that must be a subkey of the key identified by the - *key* parameter. This value must not be ``None``, and the key may not have - subkeys. - - *reserved* is a reserved integer, and must be zero. The default is zero. - - *access* is an integer that specifies an access mask that describes the - desired security access for the key. Default is :const:`KEY_WOW64_64KEY`. - On 32-bit Windows, the WOW64 constants are ignored. - See :ref:`Access Rights <access-rights>` for other allowed values. - - *This method can not delete keys with subkeys.* - - If the method succeeds, the entire key, including all of its values, is - removed. If the method fails, an :exc:`OSError` exception is raised. - - On unsupported Windows versions, :exc:`NotImplementedError` is raised. - - .. audit-event:: winreg.DeleteKey key,sub_key,access winreg.DeleteKeyEx - - .. versionadded:: 3.2 - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: DeleteValue(key, value) - - Removes a named value from a registry key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *value* is a string that identifies the value to remove. - - .. audit-event:: winreg.DeleteValue key,value winreg.DeleteValue - - -.. function:: EnumKey(key, index) - - Enumerates subkeys of an open registry key, returning a string. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *index* is an integer that identifies the index of the key to retrieve. - - The function retrieves the name of one subkey each time it is called. It is - typically called repeatedly until an :exc:`OSError` exception is - raised, indicating, no more values are available. - - .. audit-event:: winreg.EnumKey key,index winreg.EnumKey - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: EnumValue(key, index) - - Enumerates values of an open registry key, returning a tuple. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *index* is an integer that identifies the index of the value to retrieve. - - The function retrieves the name of one subkey each time it is called. It is - typically called repeatedly, until an :exc:`OSError` exception is - raised, indicating no more values. - - The result is a tuple of 3 items: - - +-------+--------------------------------------------+ - | Index | Meaning | - +=======+============================================+ - | ``0`` | A string that identifies the value name | - +-------+--------------------------------------------+ - | ``1`` | An object that holds the value data, and | - | | whose type depends on the underlying | - | | registry type | - +-------+--------------------------------------------+ - | ``2`` | An integer that identifies the type of the | - | | value data (see table in docs for | - | | :meth:`SetValueEx`) | - +-------+--------------------------------------------+ - - .. audit-event:: winreg.EnumValue key,index winreg.EnumValue - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. index:: - single: % (percent); environment variables expansion (Windows) - -.. function:: ExpandEnvironmentStrings(str) - - Expands environment variable placeholders ``%NAME%`` in strings like - :const:`REG_EXPAND_SZ`:: - - >>> ExpandEnvironmentStrings('%windir%') - 'C:\\Windows' - - .. audit-event:: winreg.ExpandEnvironmentStrings str winreg.ExpandEnvironmentStrings - - -.. function:: FlushKey(key) - - Writes all the attributes of a key to the registry. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - It is not necessary to call :func:`FlushKey` to change a key. Registry changes are - flushed to disk by the registry using its lazy flusher. Registry changes are - also flushed to disk at system shutdown. Unlike :func:`CloseKey`, the - :func:`FlushKey` method returns only when all the data has been written to the - registry. An application should only call :func:`FlushKey` if it requires - absolute certainty that registry changes are on disk. - - .. note:: - - If you don't know whether a :func:`FlushKey` call is required, it probably - isn't. - - -.. function:: LoadKey(key, sub_key, file_name) - - Creates a subkey under the specified key and stores registration information - from a specified file into that subkey. - - *key* is a handle returned by :func:`ConnectRegistry` or one of the constants - :const:`HKEY_USERS` or :const:`HKEY_LOCAL_MACHINE`. - - *sub_key* is a string that identifies the subkey to load. - - *file_name* is the name of the file to load registry data from. This file must - have been created with the :func:`SaveKey` function. Under the file allocation - table (FAT) file system, the filename may not have an extension. - - A call to :func:`LoadKey` fails if the calling process does not have the - :c:data:`!SE_RESTORE_PRIVILEGE` privilege. Note that privileges are different - from permissions -- see the `RegLoadKey documentation - <https://msdn.microsoft.com/en-us/library/ms724889%28v=VS.85%29.aspx>`__ for - more details. - - If *key* is a handle returned by :func:`ConnectRegistry`, then the path - specified in *file_name* is relative to the remote computer. - - .. audit-event:: winreg.LoadKey key,sub_key,file_name winreg.LoadKey - - -.. function:: OpenKey(key, sub_key, reserved=0, access=KEY_READ) - OpenKeyEx(key, sub_key, reserved=0, access=KEY_READ) - - Opens the specified key, returning a :ref:`handle object <handle-object>`. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that identifies the sub_key to open. - - *reserved* is a reserved integer, and must be zero. The default is zero. - - *access* is an integer that specifies an access mask that describes the desired - security access for the key. Default is :const:`KEY_READ`. See :ref:`Access - Rights <access-rights>` for other allowed values. - - The result is a new handle to the specified key. - - If the function fails, :exc:`OSError` is raised. - - .. audit-event:: winreg.OpenKey key,sub_key,access winreg.OpenKey - - .. audit-event:: winreg.OpenKey/result key winreg.OpenKey - - .. versionchanged:: 3.2 - Allow the use of named arguments. - - .. versionchanged:: 3.3 - See :ref:`above <exception-changed>`. - - -.. function:: QueryInfoKey(key) - - Returns information about a key, as a tuple. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - The result is a tuple of 3 items: - - +-------+---------------------------------------------+ - | Index | Meaning | - +=======+=============================================+ - | ``0`` | An integer giving the number of sub keys | - | | this key has. | - +-------+---------------------------------------------+ - | ``1`` | An integer giving the number of values this | - | | key has. | - +-------+---------------------------------------------+ - | ``2`` | An integer giving when the key was last | - | | modified (if available) as 100's of | - | | nanoseconds since Jan 1, 1601. | - +-------+---------------------------------------------+ - - .. audit-event:: winreg.QueryInfoKey key winreg.QueryInfoKey - - -.. function:: QueryValue(key, sub_key) - - Retrieves the unnamed value for a key, as a string. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that holds the name of the subkey with which the value is - associated. If this parameter is ``None`` or empty, the function retrieves the - value set by the :func:`SetValue` method for the key identified by *key*. - - Values in the registry have name, type, and data components. This method - retrieves the data for a key's first value that has a ``NULL`` name. But the - underlying API call doesn't return the type, so always use - :func:`QueryValueEx` if possible. - - .. audit-event:: winreg.QueryValue key,sub_key,value_name winreg.QueryValue - - -.. function:: QueryValueEx(key, value_name) - - Retrieves the type and data for a specified value name associated with - an open registry key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *value_name* is a string indicating the value to query. - - The result is a tuple of 2 items: - - +-------+-----------------------------------------+ - | Index | Meaning | - +=======+=========================================+ - | ``0`` | The value of the registry item. | - +-------+-----------------------------------------+ - | ``1`` | An integer giving the registry type for | - | | this value (see table in docs for | - | | :meth:`SetValueEx`) | - +-------+-----------------------------------------+ - - .. audit-event:: winreg.QueryValue key,sub_key,value_name winreg.QueryValueEx - - -.. function:: SaveKey(key, file_name) - - Saves the specified key, and all its subkeys to the specified file. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *file_name* is the name of the file to save registry data to. This file - cannot already exist. If this filename includes an extension, it cannot be - used on file allocation table (FAT) file systems by the :meth:`LoadKey` - method. - - If *key* represents a key on a remote computer, the path described by - *file_name* is relative to the remote computer. The caller of this method must - possess the **SeBackupPrivilege** security privilege. Note that - privileges are different than permissions -- see the - `Conflicts Between User Rights and Permissions documentation - <https://msdn.microsoft.com/en-us/library/ms724878%28v=VS.85%29.aspx>`__ - for more details. - - This function passes ``NULL`` for *security_attributes* to the API. - - .. audit-event:: winreg.SaveKey key,file_name winreg.SaveKey - - -.. function:: SetValue(key, sub_key, type, value) - - Associates a value with a specified key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *sub_key* is a string that names the subkey with which the value is associated. - - *type* is an integer that specifies the type of the data. Currently this must be - :const:`REG_SZ`, meaning only strings are supported. Use the :func:`SetValueEx` - function for support for other data types. - - *value* is a string that specifies the new value. - - If the key specified by the *sub_key* parameter does not exist, the SetValue - function creates it. - - Value lengths are limited by available memory. Long values (more than 2048 - bytes) should be stored as files with the filenames stored in the configuration - registry. This helps the registry perform efficiently. - - The key identified by the *key* parameter must have been opened with - :const:`KEY_SET_VALUE` access. - - .. audit-event:: winreg.SetValue key,sub_key,type,value winreg.SetValue - - -.. function:: SetValueEx(key, value_name, reserved, type, value) - - Stores data in the value field of an open registry key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - *value_name* is a string that names the subkey with which the value is - associated. - - *reserved* can be anything -- zero is always passed to the API. - - *type* is an integer that specifies the type of the data. See - :ref:`Value Types <value-types>` for the available types. - - *value* is a string that specifies the new value. - - This method can also set additional value and type information for the specified - key. The key identified by the key parameter must have been opened with - :const:`KEY_SET_VALUE` access. - - To open the key, use the :func:`CreateKey` or :func:`OpenKey` methods. - - Value lengths are limited by available memory. Long values (more than 2048 - bytes) should be stored as files with the filenames stored in the configuration - registry. This helps the registry perform efficiently. - - .. audit-event:: winreg.SetValue key,sub_key,type,value winreg.SetValueEx - - -.. function:: DisableReflectionKey(key) - - Disables registry reflection for 32-bit processes running on a 64-bit - operating system. - - *key* is an already open key, or one of the predefined :ref:`HKEY_* constants - <hkey-constants>`. - - Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating - system. - - If the key is not on the reflection list, the function succeeds but has no - effect. Disabling reflection for a key does not affect reflection of any - subkeys. - - .. audit-event:: winreg.DisableReflectionKey key winreg.DisableReflectionKey - - -.. function:: EnableReflectionKey(key) - - Restores registry reflection for the specified disabled key. - - *key* is an already open key, or one of the predefined :ref:`HKEY_* constants - <hkey-constants>`. - - Will generally raise :exc:`NotImplementedError` if executed on a 32-bit operating - system. - - Restoring reflection for a key does not affect reflection of any subkeys. - - .. audit-event:: winreg.EnableReflectionKey key winreg.EnableReflectionKey - - -.. function:: QueryReflectionKey(key) - - Determines the reflection state for the specified key. - - *key* is an already open key, or one of the predefined - :ref:`HKEY_* constants <hkey-constants>`. - - Returns ``True`` if reflection is disabled. - - Will generally raise :exc:`NotImplementedError` if executed on a 32-bit - operating system. - - .. audit-event:: winreg.QueryReflectionKey key winreg.QueryReflectionKey - - -.. _constants: - -Constants ------------------- - -The following constants are defined for use in many :mod:`winreg` functions. - -.. _hkey-constants: - -HKEY_* Constants -++++++++++++++++ - -.. data:: HKEY_CLASSES_ROOT - - Registry entries subordinate to this key define types (or classes) of - documents and the properties associated with those types. Shell and - COM applications use the information stored under this key. - - -.. data:: HKEY_CURRENT_USER - - Registry entries subordinate to this key define the preferences of - the current user. These preferences include the settings of - environment variables, data about program groups, colors, printers, - network connections, and application preferences. - -.. data:: HKEY_LOCAL_MACHINE - - Registry entries subordinate to this key define the physical state - of the computer, including data about the bus type, system memory, - and installed hardware and software. - -.. data:: HKEY_USERS - - Registry entries subordinate to this key define the default user - configuration for new users on the local computer and the user - configuration for the current user. - -.. data:: HKEY_PERFORMANCE_DATA - - Registry entries subordinate to this key allow you to access - performance data. The data is not actually stored in the registry; - the registry functions cause the system to collect the data from - its source. - - -.. data:: HKEY_CURRENT_CONFIG - - Contains information about the current hardware profile of the - local computer system. - -.. data:: HKEY_DYN_DATA - - This key is not used in versions of Windows after 98. - - -.. _access-rights: - -Access Rights -+++++++++++++ - -For more information, see `Registry Key Security and Access -<https://msdn.microsoft.com/en-us/library/ms724878%28v=VS.85%29.aspx>`__. - -.. data:: KEY_ALL_ACCESS - - Combines the STANDARD_RIGHTS_REQUIRED, :const:`KEY_QUERY_VALUE`, - :const:`KEY_SET_VALUE`, :const:`KEY_CREATE_SUB_KEY`, - :const:`KEY_ENUMERATE_SUB_KEYS`, :const:`KEY_NOTIFY`, - and :const:`KEY_CREATE_LINK` access rights. - -.. data:: KEY_WRITE - - Combines the STANDARD_RIGHTS_WRITE, :const:`KEY_SET_VALUE`, and - :const:`KEY_CREATE_SUB_KEY` access rights. - -.. data:: KEY_READ - - Combines the STANDARD_RIGHTS_READ, :const:`KEY_QUERY_VALUE`, - :const:`KEY_ENUMERATE_SUB_KEYS`, and :const:`KEY_NOTIFY` values. - -.. data:: KEY_EXECUTE - - Equivalent to :const:`KEY_READ`. - -.. data:: KEY_QUERY_VALUE - - Required to query the values of a registry key. - -.. data:: KEY_SET_VALUE - - Required to create, delete, or set a registry value. - -.. data:: KEY_CREATE_SUB_KEY - - Required to create a subkey of a registry key. - -.. data:: KEY_ENUMERATE_SUB_KEYS - - Required to enumerate the subkeys of a registry key. - -.. data:: KEY_NOTIFY - - Required to request change notifications for a registry key or for - subkeys of a registry key. - -.. data:: KEY_CREATE_LINK - - Reserved for system use. - - -.. _64-bit-access-rights: - -64-bit Specific -*************** - -For more information, see `Accessing an Alternate Registry View -<https://msdn.microsoft.com/en-us/library/aa384129(v=VS.85).aspx>`__. - -.. data:: KEY_WOW64_64KEY - - Indicates that an application on 64-bit Windows should operate on - the 64-bit registry view. On 32-bit Windows, this constant is ignored. - -.. data:: KEY_WOW64_32KEY - - Indicates that an application on 64-bit Windows should operate on - the 32-bit registry view. On 32-bit Windows, this constant is ignored. - -.. _value-types: - -Value Types -+++++++++++ - -For more information, see `Registry Value Types -<https://msdn.microsoft.com/en-us/library/ms724884%28v=VS.85%29.aspx>`__. - -.. data:: REG_BINARY - - Binary data in any form. - -.. data:: REG_DWORD - - 32-bit number. - -.. data:: REG_DWORD_LITTLE_ENDIAN - - A 32-bit number in little-endian format. Equivalent to :const:`REG_DWORD`. - -.. data:: REG_DWORD_BIG_ENDIAN - - A 32-bit number in big-endian format. - -.. data:: REG_EXPAND_SZ - - Null-terminated string containing references to environment - variables (``%PATH%``). - -.. data:: REG_LINK - - A Unicode symbolic link. - -.. data:: REG_MULTI_SZ - - A sequence of null-terminated strings, terminated by two null characters. - (Python handles this termination automatically.) - -.. data:: REG_NONE - - No defined value type. - -.. data:: REG_QWORD - - A 64-bit number. - - .. versionadded:: 3.6 - -.. data:: REG_QWORD_LITTLE_ENDIAN - - A 64-bit number in little-endian format. Equivalent to :const:`REG_QWORD`. - - .. versionadded:: 3.6 - -.. data:: REG_RESOURCE_LIST - - A device-driver resource list. - -.. data:: REG_FULL_RESOURCE_DESCRIPTOR - - A hardware setting. - -.. data:: REG_RESOURCE_REQUIREMENTS_LIST - - A hardware resource list. - -.. data:: REG_SZ - - A null-terminated string. - - -.. _handle-object: - -Registry Handle Objects ------------------------ - -This object wraps a Windows HKEY object, automatically closing it when the -object is destroyed. To guarantee cleanup, you can call either the -:meth:`~PyHKEY.Close` method on the object, or the :func:`CloseKey` function. - -All registry functions in this module return one of these objects. - -All registry functions in this module which accept a handle object also accept -an integer, however, use of the handle object is encouraged. - -Handle objects provide semantics for :meth:`~object.__bool__` -- thus :: - - if handle: - print("Yes") - -will print ``Yes`` if the handle is currently valid (has not been closed or -detached). - -The object also support comparison semantics, so handle objects will compare -true if they both reference the same underlying Windows handle value. - -Handle objects can be converted to an integer (e.g., using the built-in -:func:`int` function), in which case the underlying Windows handle value is -returned. You can also use the :meth:`~PyHKEY.Detach` method to return the -integer handle, and also disconnect the Windows handle from the handle object. - - -.. method:: PyHKEY.Close() - - Closes the underlying Windows handle. - - If the handle is already closed, no error is raised. - - -.. method:: PyHKEY.Detach() - - Detaches the Windows handle from the handle object. - - The result is an integer that holds the value of the handle before it is - detached. If the handle is already detached or closed, this will return - zero. - - After calling this function, the handle is effectively invalidated, but the - handle is not closed. You would call this function when you need the - underlying Win32 handle to exist beyond the lifetime of the handle object. - - .. audit-event:: winreg.PyHKEY.Detach key winreg.PyHKEY.Detach - - -.. method:: PyHKEY.__enter__() - PyHKEY.__exit__(*exc_info) - - The HKEY object implements :meth:`~object.__enter__` and - :meth:`~object.__exit__` and thus supports the context protocol for the - :keyword:`with` statement:: - - with OpenKey(HKEY_LOCAL_MACHINE, "foo") as key: - ... # work with key - - will automatically close *key* when control leaves the :keyword:`with` block. - - diff --git a/Doc/library/winsound.rst b/Doc/library/winsound.rst deleted file mode 100644 index 370c5216652ba7..00000000000000 --- a/Doc/library/winsound.rst +++ /dev/null @@ -1,161 +0,0 @@ -:mod:`winsound` --- Sound-playing interface for Windows -======================================================= - -.. module:: winsound - :platform: Windows - :synopsis: Access to the sound-playing machinery for Windows. - -.. moduleauthor:: Toby Dickenson <htrd90@zepler.org> -.. sectionauthor:: Fred L. Drake, Jr. <fdrake@acm.org> - --------------- - -The :mod:`winsound` module provides access to the basic sound-playing machinery -provided by Windows platforms. It includes functions and several constants. - - -.. function:: Beep(frequency, duration) - - Beep the PC's speaker. The *frequency* parameter specifies frequency, in hertz, - of the sound, and must be in the range 37 through 32,767. The *duration* - parameter specifies the number of milliseconds the sound should last. If the - system is not able to beep the speaker, :exc:`RuntimeError` is raised. - - -.. function:: PlaySound(sound, flags) - - Call the underlying :c:func:`!PlaySound` function from the Platform API. The - *sound* parameter may be a filename, a system sound alias, audio data as a - :term:`bytes-like object`, or ``None``. Its - interpretation depends on the value of *flags*, which can be a bitwise ORed - combination of the constants described below. If the *sound* parameter is - ``None``, any currently playing waveform sound is stopped. If the system - indicates an error, :exc:`RuntimeError` is raised. - - -.. function:: MessageBeep(type=MB_OK) - - Call the underlying :c:func:`!MessageBeep` function from the Platform API. This - plays a sound as specified in the registry. The *type* argument specifies which - sound to play; possible values are ``-1``, ``MB_ICONASTERISK``, - ``MB_ICONEXCLAMATION``, ``MB_ICONHAND``, ``MB_ICONQUESTION``, and ``MB_OK``, all - described below. The value ``-1`` produces a "simple beep"; this is the final - fallback if a sound cannot be played otherwise. If the system indicates an - error, :exc:`RuntimeError` is raised. - - -.. data:: SND_FILENAME - - The *sound* parameter is the name of a WAV file. Do not use with - :const:`SND_ALIAS`. - - -.. data:: SND_ALIAS - - The *sound* parameter is a sound association name from the registry. If the - registry contains no such name, play the system default sound unless - :const:`SND_NODEFAULT` is also specified. If no default sound is registered, - raise :exc:`RuntimeError`. Do not use with :const:`SND_FILENAME`. - - All Win32 systems support at least the following; most systems support many - more: - - +--------------------------+----------------------------------------+ - | :func:`PlaySound` *name* | Corresponding Control Panel Sound name | - +==========================+========================================+ - | ``'SystemAsterisk'`` | Asterisk | - +--------------------------+----------------------------------------+ - | ``'SystemExclamation'`` | Exclamation | - +--------------------------+----------------------------------------+ - | ``'SystemExit'`` | Exit Windows | - +--------------------------+----------------------------------------+ - | ``'SystemHand'`` | Critical Stop | - +--------------------------+----------------------------------------+ - | ``'SystemQuestion'`` | Question | - +--------------------------+----------------------------------------+ - - For example:: - - import winsound - # Play Windows exit sound. - winsound.PlaySound("SystemExit", winsound.SND_ALIAS) - - # Probably play Windows default sound, if any is registered (because - # "*" probably isn't the registered name of any sound). - winsound.PlaySound("*", winsound.SND_ALIAS) - - -.. data:: SND_LOOP - - Play the sound repeatedly. The :const:`SND_ASYNC` flag must also be used to - avoid blocking. Cannot be used with :const:`SND_MEMORY`. - - -.. data:: SND_MEMORY - - The *sound* parameter to :func:`PlaySound` is a memory image of a WAV file, as a - :term:`bytes-like object`. - - .. note:: - - This module does not support playing from a memory image asynchronously, so a - combination of this flag and :const:`SND_ASYNC` will raise :exc:`RuntimeError`. - - -.. data:: SND_PURGE - - Stop playing all instances of the specified sound. - - .. note:: - - This flag is not supported on modern Windows platforms. - - -.. data:: SND_ASYNC - - Return immediately, allowing sounds to play asynchronously. - - -.. data:: SND_NODEFAULT - - If the specified sound cannot be found, do not play the system default sound. - - -.. data:: SND_NOSTOP - - Do not interrupt sounds currently playing. - - -.. data:: SND_NOWAIT - - Return immediately if the sound driver is busy. - - .. note:: - - This flag is not supported on modern Windows platforms. - - -.. data:: MB_ICONASTERISK - - Play the ``SystemDefault`` sound. - - -.. data:: MB_ICONEXCLAMATION - - Play the ``SystemExclamation`` sound. - - -.. data:: MB_ICONHAND - - Play the ``SystemHand`` sound. - - -.. data:: MB_ICONQUESTION - - Play the ``SystemQuestion`` sound. - - -.. data:: MB_OK - - Play the ``SystemDefault`` sound. - diff --git a/Doc/library/wsgiref.rst b/Doc/library/wsgiref.rst deleted file mode 100644 index be9e56b04c1fbf..00000000000000 --- a/Doc/library/wsgiref.rst +++ /dev/null @@ -1,890 +0,0 @@ -:mod:`wsgiref` --- WSGI Utilities and Reference Implementation -============================================================== - -.. module:: wsgiref - :synopsis: WSGI Utilities and Reference Implementation. - -.. moduleauthor:: Phillip J. Eby <pje@telecommunity.com> -.. sectionauthor:: Phillip J. Eby <pje@telecommunity.com> - -**Source code:** :source:`Lib/wsgiref` - --------------- - -The Web Server Gateway Interface (WSGI) is a standard interface between web -server software and web applications written in Python. Having a standard -interface makes it easy to use an application that supports WSGI with a number -of different web servers. - -Only authors of web servers and programming frameworks need to know every detail -and corner case of the WSGI design. You don't need to understand every detail -of WSGI just to install a WSGI application or to write a web application using -an existing framework. - -:mod:`wsgiref` is a reference implementation of the WSGI specification that can -be used to add WSGI support to a web server or framework. It provides utilities -for manipulating WSGI environment variables and response headers, base classes -for implementing WSGI servers, a demo HTTP server that serves WSGI applications, -types for static type checking, -and a validation tool that checks WSGI servers and applications for conformance -to the WSGI specification (:pep:`3333`). - -See `wsgi.readthedocs.io <https://wsgi.readthedocs.io/>`_ for more information about WSGI, and links -to tutorials and other resources. - -.. XXX If you're just trying to write a web application... - - -:mod:`wsgiref.util` -- WSGI environment utilities -------------------------------------------------- - -.. module:: wsgiref.util - :synopsis: WSGI environment utilities. - - -This module provides a variety of utility functions for working with WSGI -environments. A WSGI environment is a dictionary containing HTTP request -variables as described in :pep:`3333`. All of the functions taking an *environ* -parameter expect a WSGI-compliant dictionary to be supplied; please see -:pep:`3333` for a detailed specification and -:data:`~wsgiref.types.WSGIEnvironment` for a type alias that can be used -in type annotations. - - -.. function:: guess_scheme(environ) - - Return a guess for whether ``wsgi.url_scheme`` should be "http" or "https", by - checking for a ``HTTPS`` environment variable in the *environ* dictionary. The - return value is a string. - - This function is useful when creating a gateway that wraps CGI or a CGI-like - protocol such as FastCGI. Typically, servers providing such protocols will - include a ``HTTPS`` variable with a value of "1", "yes", or "on" when a request - is received via SSL. So, this function returns "https" if such a value is - found, and "http" otherwise. - - -.. function:: request_uri(environ, include_query=True) - - Return the full request URI, optionally including the query string, using the - algorithm found in the "URL Reconstruction" section of :pep:`3333`. If - *include_query* is false, the query string is not included in the resulting URI. - - -.. function:: application_uri(environ) - - Similar to :func:`request_uri`, except that the ``PATH_INFO`` and - ``QUERY_STRING`` variables are ignored. The result is the base URI of the - application object addressed by the request. - - -.. function:: shift_path_info(environ) - - Shift a single name from ``PATH_INFO`` to ``SCRIPT_NAME`` and return the name. - The *environ* dictionary is *modified* in-place; use a copy if you need to keep - the original ``PATH_INFO`` or ``SCRIPT_NAME`` intact. - - If there are no remaining path segments in ``PATH_INFO``, ``None`` is returned. - - Typically, this routine is used to process each portion of a request URI path, - for example to treat the path as a series of dictionary keys. This routine - modifies the passed-in environment to make it suitable for invoking another WSGI - application that is located at the target URI. For example, if there is a WSGI - application at ``/foo``, and the request URI path is ``/foo/bar/baz``, and the - WSGI application at ``/foo`` calls :func:`shift_path_info`, it will receive the - string "bar", and the environment will be updated to be suitable for passing to - a WSGI application at ``/foo/bar``. That is, ``SCRIPT_NAME`` will change from - ``/foo`` to ``/foo/bar``, and ``PATH_INFO`` will change from ``/bar/baz`` to - ``/baz``. - - When ``PATH_INFO`` is just a "/", this routine returns an empty string and - appends a trailing slash to ``SCRIPT_NAME``, even though empty path segments are - normally ignored, and ``SCRIPT_NAME`` doesn't normally end in a slash. This is - intentional behavior, to ensure that an application can tell the difference - between URIs ending in ``/x`` from ones ending in ``/x/`` when using this - routine to do object traversal. - - -.. function:: setup_testing_defaults(environ) - - Update *environ* with trivial defaults for testing purposes. - - This routine adds various parameters required for WSGI, including ``HTTP_HOST``, - ``SERVER_NAME``, ``SERVER_PORT``, ``REQUEST_METHOD``, ``SCRIPT_NAME``, - ``PATH_INFO``, and all of the :pep:`3333`\ -defined ``wsgi.*`` variables. It - only supplies default values, and does not replace any existing settings for - these variables. - - This routine is intended to make it easier for unit tests of WSGI servers and - applications to set up dummy environments. It should NOT be used by actual WSGI - servers or applications, since the data is fake! - - Example usage:: - - from wsgiref.util import setup_testing_defaults - from wsgiref.simple_server import make_server - - # A relatively simple WSGI application. It's going to print out the - # environment dictionary after being updated by setup_testing_defaults - def simple_app(environ, start_response): - setup_testing_defaults(environ) - - status = '200 OK' - headers = [('Content-type', 'text/plain; charset=utf-8')] - - start_response(status, headers) - - ret = [("%s: %s\n" % (key, value)).encode("utf-8") - for key, value in environ.items()] - return ret - - with make_server('', 8000, simple_app) as httpd: - print("Serving on port 8000...") - httpd.serve_forever() - - -In addition to the environment functions above, the :mod:`wsgiref.util` module -also provides these miscellaneous utilities: - - -.. function:: is_hop_by_hop(header_name) - - Return ``True`` if 'header_name' is an HTTP/1.1 "Hop-by-Hop" header, as defined by - :rfc:`2616`. - - -.. class:: FileWrapper(filelike, blksize=8192) - - A concrete implementation of the :class:`wsgiref.types.FileWrapper` - protocol used to convert a file-like object to an :term:`iterator`. - The resulting objects - are :term:`iterable`\ s. As the object is iterated over, the - optional *blksize* parameter will be repeatedly passed to the *filelike* - object's :meth:`read` method to obtain bytestrings to yield. When :meth:`read` - returns an empty bytestring, iteration is ended and is not resumable. - - If *filelike* has a :meth:`close` method, the returned object will also have a - :meth:`close` method, and it will invoke the *filelike* object's :meth:`close` - method when called. - - Example usage:: - - from io import StringIO - from wsgiref.util import FileWrapper - - # We're using a StringIO-buffer for as the file-like object - filelike = StringIO("This is an example file-like object"*10) - wrapper = FileWrapper(filelike, blksize=5) - - for chunk in wrapper: - print(chunk) - - .. versionchanged:: 3.11 - Support for :meth:`~object.__getitem__` method has been removed. - - -:mod:`wsgiref.headers` -- WSGI response header tools ----------------------------------------------------- - -.. module:: wsgiref.headers - :synopsis: WSGI response header tools. - - -This module provides a single class, :class:`Headers`, for convenient -manipulation of WSGI response headers using a mapping-like interface. - - -.. class:: Headers([headers]) - - Create a mapping-like object wrapping *headers*, which must be a list of header - name/value tuples as described in :pep:`3333`. The default value of *headers* is - an empty list. - - :class:`Headers` objects support typical mapping operations including - :meth:`~object.__getitem__`, :meth:`get`, :meth:`__setitem__`, :meth:`setdefault`, - :meth:`__delitem__` and :meth:`__contains__`. For each of - these methods, the key is the header name (treated case-insensitively), and the - value is the first value associated with that header name. Setting a header - deletes any existing values for that header, then adds a new value at the end of - the wrapped header list. Headers' existing order is generally maintained, with - new headers added to the end of the wrapped list. - - Unlike a dictionary, :class:`Headers` objects do not raise an error when you try - to get or delete a key that isn't in the wrapped header list. Getting a - nonexistent header just returns ``None``, and deleting a nonexistent header does - nothing. - - :class:`Headers` objects also support :meth:`keys`, :meth:`values`, and - :meth:`items` methods. The lists returned by :meth:`keys` and :meth:`items` can - include the same key more than once if there is a multi-valued header. The - ``len()`` of a :class:`Headers` object is the same as the length of its - :meth:`items`, which is the same as the length of the wrapped header list. In - fact, the :meth:`items` method just returns a copy of the wrapped header list. - - Calling ``bytes()`` on a :class:`Headers` object returns a formatted bytestring - suitable for transmission as HTTP response headers. Each header is placed on a - line with its value, separated by a colon and a space. Each line is terminated - by a carriage return and line feed, and the bytestring is terminated with a - blank line. - - In addition to their mapping interface and formatting features, :class:`Headers` - objects also have the following methods for querying and adding multi-valued - headers, and for adding headers with MIME parameters: - - - .. method:: Headers.get_all(name) - - Return a list of all the values for the named header. - - The returned list will be sorted in the order they appeared in the original - header list or were added to this instance, and may contain duplicates. Any - fields deleted and re-inserted are always appended to the header list. If no - fields exist with the given name, returns an empty list. - - - .. method:: Headers.add_header(name, value, **_params) - - Add a (possibly multi-valued) header, with optional MIME parameters specified - via keyword arguments. - - *name* is the header field to add. Keyword arguments can be used to set MIME - parameters for the header field. Each parameter must be a string or ``None``. - Underscores in parameter names are converted to dashes, since dashes are illegal - in Python identifiers, but many MIME parameter names include dashes. If the - parameter value is a string, it is added to the header value parameters in the - form ``name="value"``. If it is ``None``, only the parameter name is added. - (This is used for MIME parameters without a value.) Example usage:: - - h.add_header('content-disposition', 'attachment', filename='bud.gif') - - The above will add a header that looks like this:: - - Content-Disposition: attachment; filename="bud.gif" - - - .. versionchanged:: 3.5 - *headers* parameter is optional. - - -:mod:`wsgiref.simple_server` -- a simple WSGI HTTP server ---------------------------------------------------------- - -.. module:: wsgiref.simple_server - :synopsis: A simple WSGI HTTP server. - - -This module implements a simple HTTP server (based on :mod:`http.server`) -that serves WSGI applications. Each server instance serves a single WSGI -application on a given host and port. If you want to serve multiple -applications on a single host and port, you should create a WSGI application -that parses ``PATH_INFO`` to select which application to invoke for each -request. (E.g., using the :func:`shift_path_info` function from -:mod:`wsgiref.util`.) - - -.. function:: make_server(host, port, app, server_class=WSGIServer, handler_class=WSGIRequestHandler) - - Create a new WSGI server listening on *host* and *port*, accepting connections - for *app*. The return value is an instance of the supplied *server_class*, and - will process requests using the specified *handler_class*. *app* must be a WSGI - application object, as defined by :pep:`3333`. - - Example usage:: - - from wsgiref.simple_server import make_server, demo_app - - with make_server('', 8000, demo_app) as httpd: - print("Serving HTTP on port 8000...") - - # Respond to requests until process is killed - httpd.serve_forever() - - # Alternative: serve one request, then exit - httpd.handle_request() - - -.. function:: demo_app(environ, start_response) - - This function is a small but complete WSGI application that returns a text page - containing the message "Hello world!" and a list of the key/value pairs provided - in the *environ* parameter. It's useful for verifying that a WSGI server (such - as :mod:`wsgiref.simple_server`) is able to run a simple WSGI application - correctly. - - -.. class:: WSGIServer(server_address, RequestHandlerClass) - - Create a :class:`WSGIServer` instance. *server_address* should be a - ``(host,port)`` tuple, and *RequestHandlerClass* should be the subclass of - :class:`http.server.BaseHTTPRequestHandler` that will be used to process - requests. - - You do not normally need to call this constructor, as the :func:`make_server` - function can handle all the details for you. - - :class:`WSGIServer` is a subclass of :class:`http.server.HTTPServer`, so all - of its methods (such as :meth:`serve_forever` and :meth:`handle_request`) are - available. :class:`WSGIServer` also provides these WSGI-specific methods: - - - .. method:: WSGIServer.set_app(application) - - Sets the callable *application* as the WSGI application that will receive - requests. - - - .. method:: WSGIServer.get_app() - - Returns the currently set application callable. - - Normally, however, you do not need to use these additional methods, as - :meth:`set_app` is normally called by :func:`make_server`, and the - :meth:`get_app` exists mainly for the benefit of request handler instances. - - -.. class:: WSGIRequestHandler(request, client_address, server) - - Create an HTTP handler for the given *request* (i.e. a socket), *client_address* - (a ``(host,port)`` tuple), and *server* (:class:`WSGIServer` instance). - - You do not need to create instances of this class directly; they are - automatically created as needed by :class:`WSGIServer` objects. You can, - however, subclass this class and supply it as a *handler_class* to the - :func:`make_server` function. Some possibly relevant methods for overriding in - subclasses: - - - .. method:: WSGIRequestHandler.get_environ() - - Return a :data:`~wsgiref.types.WSGIEnvironment` dictionary for a - request. The default - implementation copies the contents of the :class:`WSGIServer` object's - :attr:`base_environ` dictionary attribute and then adds various headers derived - from the HTTP request. Each call to this method should return a new dictionary - containing all of the relevant CGI environment variables as specified in - :pep:`3333`. - - - .. method:: WSGIRequestHandler.get_stderr() - - Return the object that should be used as the ``wsgi.errors`` stream. The default - implementation just returns ``sys.stderr``. - - - .. method:: WSGIRequestHandler.handle() - - Process the HTTP request. The default implementation creates a handler instance - using a :mod:`wsgiref.handlers` class to implement the actual WSGI application - interface. - - -:mod:`wsgiref.validate` --- WSGI conformance checker ----------------------------------------------------- - -.. module:: wsgiref.validate - :synopsis: WSGI conformance checker. - - -When creating new WSGI application objects, frameworks, servers, or middleware, -it can be useful to validate the new code's conformance using -:mod:`wsgiref.validate`. This module provides a function that creates WSGI -application objects that validate communications between a WSGI server or -gateway and a WSGI application object, to check both sides for protocol -conformance. - -Note that this utility does not guarantee complete :pep:`3333` compliance; an -absence of errors from this module does not necessarily mean that errors do not -exist. However, if this module does produce an error, then it is virtually -certain that either the server or application is not 100% compliant. - -This module is based on the :mod:`paste.lint` module from Ian Bicking's "Python -Paste" library. - - -.. function:: validator(application) - - Wrap *application* and return a new WSGI application object. The returned - application will forward all requests to the original *application*, and will - check that both the *application* and the server invoking it are conforming to - the WSGI specification and to :rfc:`2616`. - - Any detected nonconformance results in an :exc:`AssertionError` being raised; - note, however, that how these errors are handled is server-dependent. For - example, :mod:`wsgiref.simple_server` and other servers based on - :mod:`wsgiref.handlers` (that don't override the error handling methods to do - something else) will simply output a message that an error has occurred, and - dump the traceback to ``sys.stderr`` or some other error stream. - - This wrapper may also generate output using the :mod:`warnings` module to - indicate behaviors that are questionable but which may not actually be - prohibited by :pep:`3333`. Unless they are suppressed using Python command-line - options or the :mod:`warnings` API, any such warnings will be written to - ``sys.stderr`` (*not* ``wsgi.errors``, unless they happen to be the same - object). - - Example usage:: - - from wsgiref.validate import validator - from wsgiref.simple_server import make_server - - # Our callable object which is intentionally not compliant to the - # standard, so the validator is going to break - def simple_app(environ, start_response): - status = '200 OK' # HTTP Status - headers = [('Content-type', 'text/plain')] # HTTP Headers - start_response(status, headers) - - # This is going to break because we need to return a list, and - # the validator is going to inform us - return b"Hello World" - - # This is the application wrapped in a validator - validator_app = validator(simple_app) - - with make_server('', 8000, validator_app) as httpd: - print("Listening on port 8000....") - httpd.serve_forever() - - -:mod:`wsgiref.handlers` -- server/gateway base classes ------------------------------------------------------- - -.. module:: wsgiref.handlers - :synopsis: WSGI server/gateway base classes. - - -This module provides base handler classes for implementing WSGI servers and -gateways. These base classes handle most of the work of communicating with a -WSGI application, as long as they are given a CGI-like environment, along with -input, output, and error streams. - - -.. class:: CGIHandler() - - CGI-based invocation via ``sys.stdin``, ``sys.stdout``, ``sys.stderr`` and - ``os.environ``. This is useful when you have a WSGI application and want to run - it as a CGI script. Simply invoke ``CGIHandler().run(app)``, where ``app`` is - the WSGI application object you wish to invoke. - - This class is a subclass of :class:`BaseCGIHandler` that sets ``wsgi.run_once`` - to true, ``wsgi.multithread`` to false, and ``wsgi.multiprocess`` to true, and - always uses :mod:`sys` and :mod:`os` to obtain the necessary CGI streams and - environment. - - -.. class:: IISCGIHandler() - - A specialized alternative to :class:`CGIHandler`, for use when deploying on - Microsoft's IIS web server, without having set the config allowPathInfo - option (IIS>=7) or metabase allowPathInfoForScriptMappings (IIS<7). - - By default, IIS gives a ``PATH_INFO`` that duplicates the ``SCRIPT_NAME`` at - the front, causing problems for WSGI applications that wish to implement - routing. This handler strips any such duplicated path. - - IIS can be configured to pass the correct ``PATH_INFO``, but this causes - another bug where ``PATH_TRANSLATED`` is wrong. Luckily this variable is - rarely used and is not guaranteed by WSGI. On IIS<7, though, the - setting can only be made on a vhost level, affecting all other script - mappings, many of which break when exposed to the ``PATH_TRANSLATED`` bug. - For this reason IIS<7 is almost never deployed with the fix (Even IIS7 - rarely uses it because there is still no UI for it.). - - There is no way for CGI code to tell whether the option was set, so a - separate handler class is provided. It is used in the same way as - :class:`CGIHandler`, i.e., by calling ``IISCGIHandler().run(app)``, where - ``app`` is the WSGI application object you wish to invoke. - - .. versionadded:: 3.2 - - -.. class:: BaseCGIHandler(stdin, stdout, stderr, environ, multithread=True, multiprocess=False) - - Similar to :class:`CGIHandler`, but instead of using the :mod:`sys` and - :mod:`os` modules, the CGI environment and I/O streams are specified explicitly. - The *multithread* and *multiprocess* values are used to set the - ``wsgi.multithread`` and ``wsgi.multiprocess`` flags for any applications run by - the handler instance. - - This class is a subclass of :class:`SimpleHandler` intended for use with - software other than HTTP "origin servers". If you are writing a gateway - protocol implementation (such as CGI, FastCGI, SCGI, etc.) that uses a - ``Status:`` header to send an HTTP status, you probably want to subclass this - instead of :class:`SimpleHandler`. - - -.. class:: SimpleHandler(stdin, stdout, stderr, environ, multithread=True, multiprocess=False) - - Similar to :class:`BaseCGIHandler`, but designed for use with HTTP origin - servers. If you are writing an HTTP server implementation, you will probably - want to subclass this instead of :class:`BaseCGIHandler`. - - This class is a subclass of :class:`BaseHandler`. It overrides the - :meth:`__init__`, :meth:`get_stdin`, :meth:`get_stderr`, :meth:`add_cgi_vars`, - :meth:`_write`, and :meth:`_flush` methods to support explicitly setting the - environment and streams via the constructor. The supplied environment and - streams are stored in the :attr:`stdin`, :attr:`stdout`, :attr:`stderr`, and - :attr:`environ` attributes. - - The :meth:`~io.BufferedIOBase.write` method of *stdout* should write - each chunk in full, like :class:`io.BufferedIOBase`. - - -.. class:: BaseHandler() - - This is an abstract base class for running WSGI applications. Each instance - will handle a single HTTP request, although in principle you could create a - subclass that was reusable for multiple requests. - - :class:`BaseHandler` instances have only one method intended for external use: - - - .. method:: BaseHandler.run(app) - - Run the specified WSGI application, *app*. - - All of the other :class:`BaseHandler` methods are invoked by this method in the - process of running the application, and thus exist primarily to allow - customizing the process. - - The following methods MUST be overridden in a subclass: - - - .. method:: BaseHandler._write(data) - - Buffer the bytes *data* for transmission to the client. It's okay if this - method actually transmits the data; :class:`BaseHandler` just separates write - and flush operations for greater efficiency when the underlying system actually - has such a distinction. - - - .. method:: BaseHandler._flush() - - Force buffered data to be transmitted to the client. It's okay if this method - is a no-op (i.e., if :meth:`_write` actually sends the data). - - - .. method:: BaseHandler.get_stdin() - - Return an object compatible with :class:`~wsgiref.types.InputStream` - suitable for use as the ``wsgi.input`` of the - request currently being processed. - - - .. method:: BaseHandler.get_stderr() - - Return an object compatible with :class:`~wsgiref.types.ErrorStream` - suitable for use as the ``wsgi.errors`` of the - request currently being processed. - - - .. method:: BaseHandler.add_cgi_vars() - - Insert CGI variables for the current request into the :attr:`environ` attribute. - - Here are some other methods and attributes you may wish to override. This list - is only a summary, however, and does not include every method that can be - overridden. You should consult the docstrings and source code for additional - information before attempting to create a customized :class:`BaseHandler` - subclass. - - Attributes and methods for customizing the WSGI environment: - - - .. attribute:: BaseHandler.wsgi_multithread - - The value to be used for the ``wsgi.multithread`` environment variable. It - defaults to true in :class:`BaseHandler`, but may have a different default (or - be set by the constructor) in the other subclasses. - - - .. attribute:: BaseHandler.wsgi_multiprocess - - The value to be used for the ``wsgi.multiprocess`` environment variable. It - defaults to true in :class:`BaseHandler`, but may have a different default (or - be set by the constructor) in the other subclasses. - - - .. attribute:: BaseHandler.wsgi_run_once - - The value to be used for the ``wsgi.run_once`` environment variable. It - defaults to false in :class:`BaseHandler`, but :class:`CGIHandler` sets it to - true by default. - - - .. attribute:: BaseHandler.os_environ - - The default environment variables to be included in every request's WSGI - environment. By default, this is a copy of ``os.environ`` at the time that - :mod:`wsgiref.handlers` was imported, but subclasses can either create their own - at the class or instance level. Note that the dictionary should be considered - read-only, since the default value is shared between multiple classes and - instances. - - - .. attribute:: BaseHandler.server_software - - If the :attr:`origin_server` attribute is set, this attribute's value is used to - set the default ``SERVER_SOFTWARE`` WSGI environment variable, and also to set a - default ``Server:`` header in HTTP responses. It is ignored for handlers (such - as :class:`BaseCGIHandler` and :class:`CGIHandler`) that are not HTTP origin - servers. - - .. versionchanged:: 3.3 - The term "Python" is replaced with implementation specific term like - "CPython", "Jython" etc. - - .. method:: BaseHandler.get_scheme() - - Return the URL scheme being used for the current request. The default - implementation uses the :func:`guess_scheme` function from :mod:`wsgiref.util` - to guess whether the scheme should be "http" or "https", based on the current - request's :attr:`environ` variables. - - - .. method:: BaseHandler.setup_environ() - - Set the :attr:`environ` attribute to a fully populated WSGI environment. The - default implementation uses all of the above methods and attributes, plus the - :meth:`get_stdin`, :meth:`get_stderr`, and :meth:`add_cgi_vars` methods and the - :attr:`wsgi_file_wrapper` attribute. It also inserts a ``SERVER_SOFTWARE`` key - if not present, as long as the :attr:`origin_server` attribute is a true value - and the :attr:`server_software` attribute is set. - - Methods and attributes for customizing exception handling: - - - .. method:: BaseHandler.log_exception(exc_info) - - Log the *exc_info* tuple in the server log. *exc_info* is a ``(type, value, - traceback)`` tuple. The default implementation simply writes the traceback to - the request's ``wsgi.errors`` stream and flushes it. Subclasses can override - this method to change the format or retarget the output, mail the traceback to - an administrator, or whatever other action may be deemed suitable. - - - .. attribute:: BaseHandler.traceback_limit - - The maximum number of frames to include in tracebacks output by the default - :meth:`log_exception` method. If ``None``, all frames are included. - - - .. method:: BaseHandler.error_output(environ, start_response) - - This method is a WSGI application to generate an error page for the user. It is - only invoked if an error occurs before headers are sent to the client. - - This method can access the current error using ``sys.exception()``, - and should pass that information to *start_response* when calling it (as - described in the "Error Handling" section of :pep:`3333`). - - The default implementation just uses the :attr:`error_status`, - :attr:`error_headers`, and :attr:`error_body` attributes to generate an output - page. Subclasses can override this to produce more dynamic error output. - - Note, however, that it's not recommended from a security perspective to spit out - diagnostics to any old user; ideally, you should have to do something special to - enable diagnostic output, which is why the default implementation doesn't - include any. - - - .. attribute:: BaseHandler.error_status - - The HTTP status used for error responses. This should be a status string as - defined in :pep:`3333`; it defaults to a 500 code and message. - - - .. attribute:: BaseHandler.error_headers - - The HTTP headers used for error responses. This should be a list of WSGI - response headers (``(name, value)`` tuples), as described in :pep:`3333`. The - default list just sets the content type to ``text/plain``. - - - .. attribute:: BaseHandler.error_body - - The error response body. This should be an HTTP response body bytestring. It - defaults to the plain text, "A server error occurred. Please contact the - administrator." - - Methods and attributes for :pep:`3333`'s "Optional Platform-Specific File - Handling" feature: - - - .. attribute:: BaseHandler.wsgi_file_wrapper - - A ``wsgi.file_wrapper`` factory, compatible with - :class:`wsgiref.types.FileWrapper`, or ``None``. The default value - of this attribute is the :class:`wsgiref.util.FileWrapper` class. - - - .. method:: BaseHandler.sendfile() - - Override to implement platform-specific file transmission. This method is - called only if the application's return value is an instance of the class - specified by the :attr:`wsgi_file_wrapper` attribute. It should return a true - value if it was able to successfully transmit the file, so that the default - transmission code will not be executed. The default implementation of this - method just returns a false value. - - Miscellaneous methods and attributes: - - - .. attribute:: BaseHandler.origin_server - - This attribute should be set to a true value if the handler's :meth:`_write` and - :meth:`_flush` are being used to communicate directly to the client, rather than - via a CGI-like gateway protocol that wants the HTTP status in a special - ``Status:`` header. - - This attribute's default value is true in :class:`BaseHandler`, but false in - :class:`BaseCGIHandler` and :class:`CGIHandler`. - - - .. attribute:: BaseHandler.http_version - - If :attr:`origin_server` is true, this string attribute is used to set the HTTP - version of the response set to the client. It defaults to ``"1.0"``. - - -.. function:: read_environ() - - Transcode CGI variables from ``os.environ`` to :pep:`3333` "bytes in unicode" - strings, returning a new dictionary. This function is used by - :class:`CGIHandler` and :class:`IISCGIHandler` in place of directly using - ``os.environ``, which is not necessarily WSGI-compliant on all platforms - and web servers using Python 3 -- specifically, ones where the OS's - actual environment is Unicode (i.e. Windows), or ones where the environment - is bytes, but the system encoding used by Python to decode it is anything - other than ISO-8859-1 (e.g. Unix systems using UTF-8). - - If you are implementing a CGI-based handler of your own, you probably want - to use this routine instead of just copying values out of ``os.environ`` - directly. - - .. versionadded:: 3.2 - - -:mod:`wsgiref.types` -- WSGI types for static type checking ------------------------------------------------------------ - -.. module:: wsgiref.types - :synopsis: WSGI types for static type checking - - -This module provides various types for static type checking as described -in :pep:`3333`. - -.. versionadded:: 3.11 - - -.. class:: StartResponse() - - A :class:`typing.Protocol` describing `start_response() - <https://peps.python.org/pep-3333/#the-start-response-callable>`_ - callables (:pep:`3333`). - -.. data:: WSGIEnvironment - - A type alias describing a WSGI environment dictionary. - -.. data:: WSGIApplication - - A type alias describing a WSGI application callable. - -.. class:: InputStream() - - A :class:`typing.Protocol` describing a `WSGI Input Stream - <https://peps.python.org/pep-3333/#input-and-error-streams>`_. - -.. class:: ErrorStream() - - A :class:`typing.Protocol` describing a `WSGI Error Stream - <https://peps.python.org/pep-3333/#input-and-error-streams>`_. - -.. class:: FileWrapper() - - A :class:`typing.Protocol` describing a `file wrapper - <https://peps.python.org/pep-3333/#optional-platform-specific-file-handling>`_. - See :class:`wsgiref.util.FileWrapper` for a concrete implementation of this - protocol. - - -Examples --------- - -This is a working "Hello World" WSGI application:: - - """ - Every WSGI application must have an application object - a callable - object that accepts two arguments. For that purpose, we're going to - use a function (note that you're not limited to a function, you can - use a class for example). The first argument passed to the function - is a dictionary containing CGI-style environment variables and the - second variable is the callable object. - """ - from wsgiref.simple_server import make_server - - - def hello_world_app(environ, start_response): - status = "200 OK" # HTTP Status - headers = [("Content-type", "text/plain; charset=utf-8")] # HTTP Headers - start_response(status, headers) - - # The returned object is going to be printed - return [b"Hello World"] - - with make_server("", 8000, hello_world_app) as httpd: - print("Serving on port 8000...") - - # Serve until process is killed - httpd.serve_forever() - - - -Example of a WSGI application serving the current directory, accept optional -directory and port number (default: 8000) on the command line:: - - """ - Small wsgiref based web server. Takes a path to serve from and an - optional port number (defaults to 8000), then tries to serve files. - MIME types are guessed from the file names, 404 errors are raised - if the file is not found. - """ - import mimetypes - import os - import sys - from wsgiref import simple_server, util - - - def app(environ, respond): - # Get the file name and MIME type - fn = os.path.join(path, environ["PATH_INFO"][1:]) - if "." not in fn.split(os.path.sep)[-1]: - fn = os.path.join(fn, "index.html") - mime_type = mimetypes.guess_type(fn)[0] - - # Return 200 OK if file exists, otherwise 404 Not Found - if os.path.exists(fn): - respond("200 OK", [("Content-Type", mime_type)]) - return util.FileWrapper(open(fn, "rb")) - else: - respond("404 Not Found", [("Content-Type", "text/plain")]) - return [b"not found"] - - - if __name__ == "__main__": - # Get the path and port from command-line arguments - path = sys.argv[1] if len(sys.argv) > 1 else os.getcwd() - port = int(sys.argv[2]) if len(sys.argv) > 2 else 8000 - - # Make and start the server until control-c - httpd = simple_server.make_server("", port, app) - print(f"Serving {path} on port {port}, control-C to stop") - try: - httpd.serve_forever() - except KeyboardInterrupt: - print("Shutting down.") - httpd.server_close() - - diff --git a/Doc/library/xdrlib.rst b/Doc/library/xdrlib.rst deleted file mode 100644 index 39e75573260c50..00000000000000 --- a/Doc/library/xdrlib.rst +++ /dev/null @@ -1,283 +0,0 @@ -:mod:`xdrlib` --- Encode and decode XDR data -============================================ - -.. module:: xdrlib - :synopsis: Encoders and decoders for the External Data Representation (XDR). - :deprecated: - -**Source code:** :source:`Lib/xdrlib.py` - -.. index:: - single: XDR - single: External Data Representation - -.. deprecated-removed:: 3.11 3.13 - The :mod:`xdrlib` module is deprecated - (see :pep:`PEP 594 <594#xdrlib>` for details). - --------------- - -The :mod:`xdrlib` module supports the External Data Representation Standard as -described in :rfc:`1014`, written by Sun Microsystems, Inc. June 1987. It -supports most of the data types described in the RFC. - -The :mod:`xdrlib` module defines two classes, one for packing variables into XDR -representation, and another for unpacking from XDR representation. There are -also two exception classes. - - -.. class:: Packer() - - :class:`Packer` is the class for packing data into XDR representation. The - :class:`Packer` class is instantiated with no arguments. - - -.. class:: Unpacker(data) - - ``Unpacker`` is the complementary class which unpacks XDR data values from a - string buffer. The input buffer is given as *data*. - - -.. seealso:: - - :rfc:`1014` - XDR: External Data Representation Standard - This RFC defined the encoding of data which was XDR at the time this module was - originally written. It has apparently been obsoleted by :rfc:`1832`. - - :rfc:`1832` - XDR: External Data Representation Standard - Newer RFC that provides a revised definition of XDR. - - -.. _xdr-packer-objects: - -Packer Objects --------------- - -:class:`Packer` instances have the following methods: - - -.. method:: Packer.get_buffer() - - Returns the current pack buffer as a string. - - -.. method:: Packer.reset() - - Resets the pack buffer to the empty string. - -In general, you can pack any of the most common XDR data types by calling the -appropriate ``pack_type()`` method. Each method takes a single argument, the -value to pack. The following simple data type packing methods are supported: -:meth:`pack_uint`, :meth:`pack_int`, :meth:`pack_enum`, :meth:`pack_bool`, -:meth:`pack_uhyper`, and :meth:`pack_hyper`. - - -.. method:: Packer.pack_float(value) - - Packs the single-precision floating point number *value*. - - -.. method:: Packer.pack_double(value) - - Packs the double-precision floating point number *value*. - -The following methods support packing strings, bytes, and opaque data: - - -.. method:: Packer.pack_fstring(n, s) - - Packs a fixed length string, *s*. *n* is the length of the string but it is - *not* packed into the data buffer. The string is padded with null bytes if - necessary to guaranteed 4 byte alignment. - - -.. method:: Packer.pack_fopaque(n, data) - - Packs a fixed length opaque data stream, similarly to :meth:`pack_fstring`. - - -.. method:: Packer.pack_string(s) - - Packs a variable length string, *s*. The length of the string is first packed - as an unsigned integer, then the string data is packed with - :meth:`pack_fstring`. - - -.. method:: Packer.pack_opaque(data) - - Packs a variable length opaque data string, similarly to :meth:`pack_string`. - - -.. method:: Packer.pack_bytes(bytes) - - Packs a variable length byte stream, similarly to :meth:`pack_string`. - -The following methods support packing arrays and lists: - - -.. method:: Packer.pack_list(list, pack_item) - - Packs a *list* of homogeneous items. This method is useful for lists with an - indeterminate size; i.e. the size is not available until the entire list has - been walked. For each item in the list, an unsigned integer ``1`` is packed - first, followed by the data value from the list. *pack_item* is the function - that is called to pack the individual item. At the end of the list, an unsigned - integer ``0`` is packed. - - For example, to pack a list of integers, the code might appear like this:: - - import xdrlib - p = xdrlib.Packer() - p.pack_list([1, 2, 3], p.pack_int) - - -.. method:: Packer.pack_farray(n, array, pack_item) - - Packs a fixed length list (*array*) of homogeneous items. *n* is the length of - the list; it is *not* packed into the buffer, but a :exc:`ValueError` exception - is raised if ``len(array)`` is not equal to *n*. As above, *pack_item* is the - function used to pack each element. - - -.. method:: Packer.pack_array(list, pack_item) - - Packs a variable length *list* of homogeneous items. First, the length of the - list is packed as an unsigned integer, then each element is packed as in - :meth:`pack_farray` above. - - -.. _xdr-unpacker-objects: - -Unpacker Objects ----------------- - -The :class:`Unpacker` class offers the following methods: - - -.. method:: Unpacker.reset(data) - - Resets the string buffer with the given *data*. - - -.. method:: Unpacker.get_position() - - Returns the current unpack position in the data buffer. - - -.. method:: Unpacker.set_position(position) - - Sets the data buffer unpack position to *position*. You should be careful about - using :meth:`get_position` and :meth:`set_position`. - - -.. method:: Unpacker.get_buffer() - - Returns the current unpack data buffer as a string. - - -.. method:: Unpacker.done() - - Indicates unpack completion. Raises an :exc:`Error` exception if all of the - data has not been unpacked. - -In addition, every data type that can be packed with a :class:`Packer`, can be -unpacked with an :class:`Unpacker`. Unpacking methods are of the form -``unpack_type()``, and take no arguments. They return the unpacked object. - - -.. method:: Unpacker.unpack_float() - - Unpacks a single-precision floating point number. - - -.. method:: Unpacker.unpack_double() - - Unpacks a double-precision floating point number, similarly to - :meth:`unpack_float`. - -In addition, the following methods unpack strings, bytes, and opaque data: - - -.. method:: Unpacker.unpack_fstring(n) - - Unpacks and returns a fixed length string. *n* is the number of characters - expected. Padding with null bytes to guaranteed 4 byte alignment is assumed. - - -.. method:: Unpacker.unpack_fopaque(n) - - Unpacks and returns a fixed length opaque data stream, similarly to - :meth:`unpack_fstring`. - - -.. method:: Unpacker.unpack_string() - - Unpacks and returns a variable length string. The length of the string is first - unpacked as an unsigned integer, then the string data is unpacked with - :meth:`unpack_fstring`. - - -.. method:: Unpacker.unpack_opaque() - - Unpacks and returns a variable length opaque data string, similarly to - :meth:`unpack_string`. - - -.. method:: Unpacker.unpack_bytes() - - Unpacks and returns a variable length byte stream, similarly to - :meth:`unpack_string`. - -The following methods support unpacking arrays and lists: - - -.. method:: Unpacker.unpack_list(unpack_item) - - Unpacks and returns a list of homogeneous items. The list is unpacked one - element at a time by first unpacking an unsigned integer flag. If the flag is - ``1``, then the item is unpacked and appended to the list. A flag of ``0`` - indicates the end of the list. *unpack_item* is the function that is called to - unpack the items. - - -.. method:: Unpacker.unpack_farray(n, unpack_item) - - Unpacks and returns (as a list) a fixed length array of homogeneous items. *n* - is number of list elements to expect in the buffer. As above, *unpack_item* is - the function used to unpack each element. - - -.. method:: Unpacker.unpack_array(unpack_item) - - Unpacks and returns a variable length *list* of homogeneous items. First, the - length of the list is unpacked as an unsigned integer, then each element is - unpacked as in :meth:`unpack_farray` above. - - -.. _xdr-exceptions: - -Exceptions ----------- - -Exceptions in this module are coded as class instances: - - -.. exception:: Error - - The base exception class. :exc:`Error` has a single public attribute - :attr:`msg` containing the description of the error. - - -.. exception:: ConversionError - - Class derived from :exc:`Error`. Contains no additional instance variables. - -Here is an example of how you would catch one of these exceptions:: - - import xdrlib - p = xdrlib.Packer() - try: - p.pack_double(8.01) - except xdrlib.ConversionError as instance: - print('packing the double failed:', instance.msg) - diff --git a/Doc/library/xml.dom.minidom.rst b/Doc/library/xml.dom.minidom.rst deleted file mode 100644 index 72a7a98c2ac4f2..00000000000000 --- a/Doc/library/xml.dom.minidom.rst +++ /dev/null @@ -1,275 +0,0 @@ -:mod:`xml.dom.minidom` --- Minimal DOM implementation -===================================================== - -.. module:: xml.dom.minidom - :synopsis: Minimal Document Object Model (DOM) implementation. - -.. moduleauthor:: Paul Prescod <paul@prescod.net> -.. sectionauthor:: Paul Prescod <paul@prescod.net> -.. sectionauthor:: Martin v. Löwis <martin@v.loewis.de> - -**Source code:** :source:`Lib/xml/dom/minidom.py` - --------------- - -:mod:`xml.dom.minidom` is a minimal implementation of the Document Object -Model interface, with an API similar to that in other languages. It is intended -to be simpler than the full DOM and also significantly smaller. Users who are -not already proficient with the DOM should consider using the -:mod:`xml.etree.ElementTree` module for their XML processing instead. - - -.. warning:: - - The :mod:`xml.dom.minidom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. - - -DOM applications typically start by parsing some XML into a DOM. With -:mod:`xml.dom.minidom`, this is done through the parse functions:: - - from xml.dom.minidom import parse, parseString - - dom1 = parse('c:\\temp\\mydata.xml') # parse an XML file by name - - datasource = open('c:\\temp\\mydata.xml') - dom2 = parse(datasource) # parse an open file - - dom3 = parseString('<myxml>Some data<empty/> some more data</myxml>') - -The :func:`parse` function can take either a filename or an open file object. - - -.. function:: parse(filename_or_file, parser=None, bufsize=None) - - Return a :class:`Document` from the given input. *filename_or_file* may be - either a file name, or a file-like object. *parser*, if given, must be a SAX2 - parser object. This function will change the document handler of the parser and - activate namespace support; other parser configuration (like setting an entity - resolver) must have been done in advance. - -If you have XML in a string, you can use the :func:`parseString` function -instead: - - -.. function:: parseString(string, parser=None) - - Return a :class:`Document` that represents the *string*. This method creates an - :class:`io.StringIO` object for the string and passes that on to :func:`parse`. - -Both functions return a :class:`Document` object representing the content of the -document. - -What the :func:`parse` and :func:`parseString` functions do is connect an XML -parser with a "DOM builder" that can accept parse events from any SAX parser and -convert them into a DOM tree. The name of the functions are perhaps misleading, -but are easy to grasp when learning the interfaces. The parsing of the document -will be completed before these functions return; it's simply that these -functions do not provide a parser implementation themselves. - -You can also create a :class:`Document` by calling a method on a "DOM -Implementation" object. You can get this object either by calling the -:func:`getDOMImplementation` function in the :mod:`xml.dom` package or the -:mod:`xml.dom.minidom` module. Once you have a :class:`Document`, you -can add child nodes to it to populate the DOM:: - - from xml.dom.minidom import getDOMImplementation - - impl = getDOMImplementation() - - newdoc = impl.createDocument(None, "some_tag", None) - top_element = newdoc.documentElement - text = newdoc.createTextNode('Some textual content.') - top_element.appendChild(text) - -Once you have a DOM document object, you can access the parts of your XML -document through its properties and methods. These properties are defined in -the DOM specification. The main property of the document object is the -:attr:`documentElement` property. It gives you the main element in the XML -document: the one that holds all others. Here is an example program:: - - dom3 = parseString("<myxml>Some data</myxml>") - assert dom3.documentElement.tagName == "myxml" - -When you are finished with a DOM tree, you may optionally call the -:meth:`unlink` method to encourage early cleanup of the now-unneeded -objects. :meth:`unlink` is an :mod:`xml.dom.minidom`\ -specific -extension to the DOM API that renders the node and its descendants -essentially useless. Otherwise, Python's garbage collector will -eventually take care of the objects in the tree. - -.. seealso:: - - `Document Object Model (DOM) Level 1 Specification <https://www.w3.org/TR/REC-DOM-Level-1/>`_ - The W3C recommendation for the DOM supported by :mod:`xml.dom.minidom`. - - -.. _minidom-objects: - -DOM Objects ------------ - -The definition of the DOM API for Python is given as part of the :mod:`xml.dom` -module documentation. This section lists the differences between the API and -:mod:`xml.dom.minidom`. - - -.. method:: Node.unlink() - - Break internal references within the DOM so that it will be garbage collected on - versions of Python without cyclic GC. Even when cyclic GC is available, using - this can make large amounts of memory available sooner, so calling this on DOM - objects as soon as they are no longer needed is good practice. This only needs - to be called on the :class:`Document` object, but may be called on child nodes - to discard children of that node. - - You can avoid calling this method explicitly by using the :keyword:`with` - statement. The following code will automatically unlink *dom* when the - :keyword:`!with` block is exited:: - - with xml.dom.minidom.parse(datasource) as dom: - ... # Work with dom. - - -.. method:: Node.writexml(writer, indent="", addindent="", newl="", \ - encoding=None, standalone=None) - - Write XML to the writer object. The writer receives texts but not bytes as input, - it should have a :meth:`write` method which matches that of the file object - interface. The *indent* parameter is the indentation of the current node. - The *addindent* parameter is the incremental indentation to use for subnodes - of the current one. The *newl* parameter specifies the string to use to - terminate newlines. - - For the :class:`Document` node, an additional keyword argument *encoding* can - be used to specify the encoding field of the XML header. - - Similarly, explicitly stating the *standalone* argument causes the - standalone document declarations to be added to the prologue of the XML - document. - If the value is set to ``True``, ``standalone="yes"`` is added, - otherwise it is set to ``"no"``. - Not stating the argument will omit the declaration from the document. - - .. versionchanged:: 3.8 - The :meth:`writexml` method now preserves the attribute order specified - by the user. - - .. versionchanged:: 3.9 - The *standalone* parameter was added. - -.. method:: Node.toxml(encoding=None, standalone=None) - - Return a string or byte string containing the XML represented by - the DOM node. - - With an explicit *encoding* [1]_ argument, the result is a byte - string in the specified encoding. - With no *encoding* argument, the result is a Unicode string, and the - XML declaration in the resulting string does not specify an - encoding. Encoding this string in an encoding other than UTF-8 is - likely incorrect, since UTF-8 is the default encoding of XML. - - The *standalone* argument behaves exactly as in :meth:`writexml`. - - .. versionchanged:: 3.8 - The :meth:`toxml` method now preserves the attribute order specified - by the user. - - .. versionchanged:: 3.9 - The *standalone* parameter was added. - -.. method:: Node.toprettyxml(indent="\t", newl="\n", encoding=None, \ - standalone=None) - - Return a pretty-printed version of the document. *indent* specifies the - indentation string and defaults to a tabulator; *newl* specifies the string - emitted at the end of each line and defaults to ``\n``. - - The *encoding* argument behaves like the corresponding argument of - :meth:`toxml`. - - The *standalone* argument behaves exactly as in :meth:`writexml`. - - .. versionchanged:: 3.8 - The :meth:`toprettyxml` method now preserves the attribute order specified - by the user. - - .. versionchanged:: 3.9 - The *standalone* parameter was added. - -.. _dom-example: - -DOM Example ------------ - -This example program is a fairly realistic example of a simple program. In this -particular case, we do not take much advantage of the flexibility of the DOM. - -.. literalinclude:: ../includes/minidom-example.py - - -.. _minidom-and-dom: - -minidom and the DOM standard ----------------------------- - -The :mod:`xml.dom.minidom` module is essentially a DOM 1.0-compatible DOM with -some DOM 2 features (primarily namespace features). - -Usage of the DOM interface in Python is straight-forward. The following mapping -rules apply: - -* Interfaces are accessed through instance objects. Applications should not - instantiate the classes themselves; they should use the creator functions - available on the :class:`Document` object. Derived interfaces support all - operations (and attributes) from the base interfaces, plus any new operations. - -* Operations are used as methods. Since the DOM uses only :keyword:`in` - parameters, the arguments are passed in normal order (from left to right). - There are no optional arguments. ``void`` operations return ``None``. - -* IDL attributes map to instance attributes. For compatibility with the OMG IDL - language mapping for Python, an attribute ``foo`` can also be accessed through - accessor methods :meth:`_get_foo` and :meth:`_set_foo`. ``readonly`` - attributes must not be changed; this is not enforced at runtime. - -* The types ``short int``, ``unsigned int``, ``unsigned long long``, and - ``boolean`` all map to Python integer objects. - -* The type ``DOMString`` maps to Python strings. :mod:`xml.dom.minidom` supports - either bytes or strings, but will normally produce strings. - Values of type ``DOMString`` may also be ``None`` where allowed to have the IDL - ``null`` value by the DOM specification from the W3C. - -* ``const`` declarations map to variables in their respective scope (e.g. - ``xml.dom.minidom.Node.PROCESSING_INSTRUCTION_NODE``); they must not be changed. - -* ``DOMException`` is currently not supported in :mod:`xml.dom.minidom`. - Instead, :mod:`xml.dom.minidom` uses standard Python exceptions such as - :exc:`TypeError` and :exc:`AttributeError`. - -* :class:`NodeList` objects are implemented using Python's built-in list type. - These objects provide the interface defined in the DOM specification, but with - earlier versions of Python they do not support the official API. They are, - however, much more "Pythonic" than the interface defined in the W3C - recommendations. - -The following interfaces have no implementation in :mod:`xml.dom.minidom`: - -* :class:`DOMTimeStamp` - -* :class:`EntityReference` - -Most of these reflect information in the XML document that is not of general -utility to most DOM users. - -.. rubric:: Footnotes - -.. [1] The encoding name included in the XML output should conform to - the appropriate standards. For example, "UTF-8" is valid, but - "UTF8" is not valid in an XML document's declaration, even though - Python accepts it as an encoding name. - See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl - and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/xml.dom.pulldom.rst b/Doc/library/xml.dom.pulldom.rst deleted file mode 100644 index 843c2fd7fdb937..00000000000000 --- a/Doc/library/xml.dom.pulldom.rst +++ /dev/null @@ -1,146 +0,0 @@ -:mod:`xml.dom.pulldom` --- Support for building partial DOM trees -================================================================= - -.. module:: xml.dom.pulldom - :synopsis: Support for building partial DOM trees from SAX events. - -.. moduleauthor:: Paul Prescod <paul@prescod.net> - -**Source code:** :source:`Lib/xml/dom/pulldom.py` - --------------- - -The :mod:`xml.dom.pulldom` module provides a "pull parser" which can also be -asked to produce DOM-accessible fragments of the document where necessary. The -basic concept involves pulling "events" from a stream of incoming XML and -processing them. In contrast to SAX which also employs an event-driven -processing model together with callbacks, the user of a pull parser is -responsible for explicitly pulling events from the stream, looping over those -events until either processing is finished or an error condition occurs. - - -.. warning:: - - The :mod:`xml.dom.pulldom` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. - -.. versionchanged:: 3.7.1 - - The SAX parser no longer processes general external entities by default to - increase security by default. To enable processing of external entities, - pass a custom parser instance in:: - - from xml.dom.pulldom import parse - from xml.sax import make_parser - from xml.sax.handler import feature_external_ges - - parser = make_parser() - parser.setFeature(feature_external_ges, True) - parse(filename, parser=parser) - - -Example:: - - from xml.dom import pulldom - - doc = pulldom.parse('sales_items.xml') - for event, node in doc: - if event == pulldom.START_ELEMENT and node.tagName == 'item': - if int(node.getAttribute('price')) > 50: - doc.expandNode(node) - print(node.toxml()) - -``event`` is a constant and can be one of: - -* :data:`START_ELEMENT` -* :data:`END_ELEMENT` -* :data:`COMMENT` -* :data:`START_DOCUMENT` -* :data:`END_DOCUMENT` -* :data:`CHARACTERS` -* :data:`PROCESSING_INSTRUCTION` -* :data:`IGNORABLE_WHITESPACE` - -``node`` is an object of type :class:`xml.dom.minidom.Document`, -:class:`xml.dom.minidom.Element` or :class:`xml.dom.minidom.Text`. - -Since the document is treated as a "flat" stream of events, the document "tree" -is implicitly traversed and the desired elements are found regardless of their -depth in the tree. In other words, one does not need to consider hierarchical -issues such as recursive searching of the document nodes, although if the -context of elements were important, one would either need to maintain some -context-related state (i.e. remembering where one is in the document at any -given point) or to make use of the :func:`DOMEventStream.expandNode` method -and switch to DOM-related processing. - - -.. class:: PullDom(documentFactory=None) - - Subclass of :class:`xml.sax.handler.ContentHandler`. - - -.. class:: SAX2DOM(documentFactory=None) - - Subclass of :class:`xml.sax.handler.ContentHandler`. - - -.. function:: parse(stream_or_string, parser=None, bufsize=None) - - Return a :class:`DOMEventStream` from the given input. *stream_or_string* may be - either a file name, or a file-like object. *parser*, if given, must be an - :class:`~xml.sax.xmlreader.XMLReader` object. This function will change the - document handler of the - parser and activate namespace support; other parser configuration (like - setting an entity resolver) must have been done in advance. - -If you have XML in a string, you can use the :func:`parseString` function instead: - -.. function:: parseString(string, parser=None) - - Return a :class:`DOMEventStream` that represents the (Unicode) *string*. - -.. data:: default_bufsize - - Default value for the *bufsize* parameter to :func:`parse`. - - The value of this variable can be changed before calling :func:`parse` and - the new value will take effect. - -.. _domeventstream-objects: - -DOMEventStream Objects ----------------------- - -.. class:: DOMEventStream(stream, parser, bufsize) - - .. versionchanged:: 3.11 - Support for :meth:`~object.__getitem__` method has been removed. - - .. method:: getEvent() - - Return a tuple containing *event* and the current *node* as - :class:`xml.dom.minidom.Document` if event equals :data:`START_DOCUMENT`, - :class:`xml.dom.minidom.Element` if event equals :data:`START_ELEMENT` or - :data:`END_ELEMENT` or :class:`xml.dom.minidom.Text` if event equals - :data:`CHARACTERS`. - The current node does not contain information about its children, unless - :func:`expandNode` is called. - - .. method:: expandNode(node) - - Expands all children of *node* into *node*. Example:: - - from xml.dom import pulldom - - xml = '<html><title>Foo

    Some text

    and more

    ' - doc = pulldom.parseString(xml) - for event, node in doc: - if event == pulldom.START_ELEMENT and node.tagName == 'p': - # Following statement only prints '

    ' - print(node.toxml()) - doc.expandNode(node) - # Following statement prints node with all its children '

    Some text

    and more

    ' - print(node.toxml()) - - .. method:: DOMEventStream.reset() diff --git a/Doc/library/xml.dom.rst b/Doc/library/xml.dom.rst deleted file mode 100644 index b387240a3716cc..00000000000000 --- a/Doc/library/xml.dom.rst +++ /dev/null @@ -1,1033 +0,0 @@ -:mod:`xml.dom` --- The Document Object Model API -================================================ - -.. module:: xml.dom - :synopsis: Document Object Model API for Python. - -.. sectionauthor:: Paul Prescod -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/dom/__init__.py` - --------------- - -The Document Object Model, or "DOM," is a cross-language API from the World Wide -Web Consortium (W3C) for accessing and modifying XML documents. A DOM -implementation presents an XML document as a tree structure, or allows client -code to build such a structure from scratch. It then gives access to the -structure through a set of objects which provided well-known interfaces. - -The DOM is extremely useful for random-access applications. SAX only allows you -a view of one bit of the document at a time. If you are looking at one SAX -element, you have no access to another. If you are looking at a text node, you -have no access to a containing element. When you write a SAX application, you -need to keep track of your program's position in the document somewhere in your -own code. SAX does not do it for you. Also, if you need to look ahead in the -XML document, you are just out of luck. - -Some applications are simply impossible in an event driven model with no access -to a tree. Of course you could build some sort of tree yourself in SAX events, -but the DOM allows you to avoid writing that code. The DOM is a standard tree -representation for XML data. - -The Document Object Model is being defined by the W3C in stages, or "levels" in -their terminology. The Python mapping of the API is substantially based on the -DOM Level 2 recommendation. - -.. What if your needs are somewhere between SAX and the DOM? Perhaps - you cannot afford to load the entire tree in memory but you find the - SAX model somewhat cumbersome and low-level. There is also a module - called xml.dom.pulldom that allows you to build trees of only the - parts of a document that you need structured access to. It also has - features that allow you to find your way around the DOM. - See http://www.prescod.net/python/pulldom - -DOM applications typically start by parsing some XML into a DOM. How this is -accomplished is not covered at all by DOM Level 1, and Level 2 provides only -limited improvements: There is a :class:`DOMImplementation` object class which -provides access to :class:`Document` creation methods, but no way to access an -XML reader/parser/Document builder in an implementation-independent way. There -is also no well-defined way to access these methods without an existing -:class:`Document` object. In Python, each DOM implementation will provide a -function :func:`getDOMImplementation`. DOM Level 3 adds a Load/Store -specification, which defines an interface to the reader, but this is not yet -available in the Python standard library. - -Once you have a DOM document object, you can access the parts of your XML -document through its properties and methods. These properties are defined in -the DOM specification; this portion of the reference manual describes the -interpretation of the specification in Python. - -The specification provided by the W3C defines the DOM API for Java, ECMAScript, -and OMG IDL. The Python mapping defined here is based in large part on the IDL -version of the specification, but strict compliance is not required (though -implementations are free to support the strict mapping from IDL). See section -:ref:`dom-conformance` for a detailed discussion of mapping requirements. - - -.. seealso:: - - `Document Object Model (DOM) Level 2 Specification `_ - The W3C recommendation upon which the Python DOM API is based. - - `Document Object Model (DOM) Level 1 Specification `_ - The W3C recommendation for the DOM supported by :mod:`xml.dom.minidom`. - - `Python Language Mapping Specification `_ - This specifies the mapping from OMG IDL to Python. - - -Module Contents ---------------- - -The :mod:`xml.dom` contains the following functions: - - -.. function:: registerDOMImplementation(name, factory) - - Register the *factory* function with the name *name*. The factory function - should return an object which implements the :class:`DOMImplementation` - interface. The factory function can return the same object every time, or a new - one for each call, as appropriate for the specific implementation (e.g. if that - implementation supports some customization). - - -.. function:: getDOMImplementation(name=None, features=()) - - Return a suitable DOM implementation. The *name* is either well-known, the - module name of a DOM implementation, or ``None``. If it is not ``None``, imports - the corresponding module and returns a :class:`DOMImplementation` object if the - import succeeds. If no name is given, and if the environment variable - :envvar:`PYTHON_DOM` is set, this variable is used to find the implementation. - - If name is not given, this examines the available implementations to find one - with the required feature set. If no implementation can be found, raise an - :exc:`ImportError`. The features list must be a sequence of ``(feature, - version)`` pairs which are passed to the :meth:`hasFeature` method on available - :class:`DOMImplementation` objects. - -Some convenience constants are also provided: - - -.. data:: EMPTY_NAMESPACE - - The value used to indicate that no namespace is associated with a node in the - DOM. This is typically found as the :attr:`namespaceURI` of a node, or used as - the *namespaceURI* parameter to a namespaces-specific method. - - -.. data:: XML_NAMESPACE - - The namespace URI associated with the reserved prefix ``xml``, as defined by - `Namespaces in XML `_ (section 4). - - -.. data:: XMLNS_NAMESPACE - - The namespace URI for namespace declarations, as defined by `Document Object - Model (DOM) Level 2 Core Specification - `_ (section 1.1.8). - - -.. data:: XHTML_NAMESPACE - - The URI of the XHTML namespace as defined by `XHTML 1.0: The Extensible - HyperText Markup Language `_ (section 3.1.1). - - -In addition, :mod:`xml.dom` contains a base :class:`Node` class and the DOM -exception classes. The :class:`Node` class provided by this module does not -implement any of the methods or attributes defined by the DOM specification; -concrete DOM implementations must provide those. The :class:`Node` class -provided as part of this module does provide the constants used for the -:attr:`nodeType` attribute on concrete :class:`Node` objects; they are located -within the class rather than at the module level to conform with the DOM -specifications. - -.. Should the Node documentation go here? - - -.. _dom-objects: - -Objects in the DOM ------------------- - -The definitive documentation for the DOM is the DOM specification from the W3C. - -Note that DOM attributes may also be manipulated as nodes instead of as simple -strings. It is fairly rare that you must do this, however, so this usage is not -yet documented. - -+--------------------------------+-----------------------------------+---------------------------------+ -| Interface | Section | Purpose | -+================================+===================================+=================================+ -| :class:`DOMImplementation` | :ref:`dom-implementation-objects` | Interface to the underlying | -| | | implementation. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Node` | :ref:`dom-node-objects` | Base interface for most objects | -| | | in a document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`NodeList` | :ref:`dom-nodelist-objects` | Interface for a sequence of | -| | | nodes. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`DocumentType` | :ref:`dom-documenttype-objects` | Information about the | -| | | declarations needed to process | -| | | a document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Document` | :ref:`dom-document-objects` | Object which represents an | -| | | entire document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Element` | :ref:`dom-element-objects` | Element nodes in the document | -| | | hierarchy. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Attr` | :ref:`dom-attr-objects` | Attribute value nodes on | -| | | element nodes. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Comment` | :ref:`dom-comment-objects` | Representation of comments in | -| | | the source document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`Text` | :ref:`dom-text-objects` | Nodes containing textual | -| | | content from the document. | -+--------------------------------+-----------------------------------+---------------------------------+ -| :class:`ProcessingInstruction` | :ref:`dom-pi-objects` | Processing instruction | -| | | representation. | -+--------------------------------+-----------------------------------+---------------------------------+ - -An additional section describes the exceptions defined for working with the DOM -in Python. - - -.. _dom-implementation-objects: - -DOMImplementation Objects -^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :class:`DOMImplementation` interface provides a way for applications to -determine the availability of particular features in the DOM they are using. -DOM Level 2 added the ability to create new :class:`Document` and -:class:`DocumentType` objects using the :class:`DOMImplementation` as well. - - -.. method:: DOMImplementation.hasFeature(feature, version) - - Return ``True`` if the feature identified by the pair of strings *feature* and - *version* is implemented. - - -.. method:: DOMImplementation.createDocument(namespaceUri, qualifiedName, doctype) - - Return a new :class:`Document` object (the root of the DOM), with a child - :class:`Element` object having the given *namespaceUri* and *qualifiedName*. The - *doctype* must be a :class:`DocumentType` object created by - :meth:`createDocumentType`, or ``None``. In the Python DOM API, the first two - arguments can also be ``None`` in order to indicate that no :class:`Element` - child is to be created. - - -.. method:: DOMImplementation.createDocumentType(qualifiedName, publicId, systemId) - - Return a new :class:`DocumentType` object that encapsulates the given - *qualifiedName*, *publicId*, and *systemId* strings, representing the - information contained in an XML document type declaration. - - -.. _dom-node-objects: - -Node Objects -^^^^^^^^^^^^ - -All of the components of an XML document are subclasses of :class:`Node`. - - -.. attribute:: Node.nodeType - - An integer representing the node type. Symbolic constants for the types are on - the :class:`Node` object: :const:`ELEMENT_NODE`, :const:`ATTRIBUTE_NODE`, - :const:`TEXT_NODE`, :const:`CDATA_SECTION_NODE`, :const:`ENTITY_NODE`, - :const:`PROCESSING_INSTRUCTION_NODE`, :const:`COMMENT_NODE`, - :const:`DOCUMENT_NODE`, :const:`DOCUMENT_TYPE_NODE`, :const:`NOTATION_NODE`. - This is a read-only attribute. - - -.. attribute:: Node.parentNode - - The parent of the current node, or ``None`` for the document node. The value is - always a :class:`Node` object or ``None``. For :class:`Element` nodes, this - will be the parent element, except for the root element, in which case it will - be the :class:`Document` object. For :class:`Attr` nodes, this is always - ``None``. This is a read-only attribute. - - -.. attribute:: Node.attributes - - A :class:`NamedNodeMap` of attribute objects. Only elements have actual values - for this; others provide ``None`` for this attribute. This is a read-only - attribute. - - -.. attribute:: Node.previousSibling - - The node that immediately precedes this one with the same parent. For - instance the element with an end-tag that comes just before the *self* - element's start-tag. Of course, XML documents are made up of more than just - elements so the previous sibling could be text, a comment, or something else. - If this node is the first child of the parent, this attribute will be - ``None``. This is a read-only attribute. - - -.. attribute:: Node.nextSibling - - The node that immediately follows this one with the same parent. See also - :attr:`previousSibling`. If this is the last child of the parent, this - attribute will be ``None``. This is a read-only attribute. - - -.. attribute:: Node.childNodes - - A list of nodes contained within this node. This is a read-only attribute. - - -.. attribute:: Node.firstChild - - The first child of the node, if there are any, or ``None``. This is a read-only - attribute. - - -.. attribute:: Node.lastChild - - The last child of the node, if there are any, or ``None``. This is a read-only - attribute. - - -.. attribute:: Node.localName - - The part of the :attr:`tagName` following the colon if there is one, else the - entire :attr:`tagName`. The value is a string. - - -.. attribute:: Node.prefix - - The part of the :attr:`tagName` preceding the colon if there is one, else the - empty string. The value is a string, or ``None``. - - -.. attribute:: Node.namespaceURI - - The namespace associated with the element name. This will be a string or - ``None``. This is a read-only attribute. - - -.. attribute:: Node.nodeName - - This has a different meaning for each node type; see the DOM specification for - details. You can always get the information you would get here from another - property such as the :attr:`tagName` property for elements or the :attr:`name` - property for attributes. For all node types, the value of this attribute will be - either a string or ``None``. This is a read-only attribute. - - -.. attribute:: Node.nodeValue - - This has a different meaning for each node type; see the DOM specification for - details. The situation is similar to that with :attr:`nodeName`. The value is - a string or ``None``. - - -.. method:: Node.hasAttributes() - - Return ``True`` if the node has any attributes. - - -.. method:: Node.hasChildNodes() - - Return ``True`` if the node has any child nodes. - - -.. method:: Node.isSameNode(other) - - Return ``True`` if *other* refers to the same node as this node. This is especially - useful for DOM implementations which use any sort of proxy architecture (because - more than one object can refer to the same node). - - .. note:: - - This is based on a proposed DOM Level 3 API which is still in the "working - draft" stage, but this particular interface appears uncontroversial. Changes - from the W3C will not necessarily affect this method in the Python DOM interface - (though any new W3C API for this would also be supported). - - -.. method:: Node.appendChild(newChild) - - Add a new child node to this node at the end of the list of - children, returning *newChild*. If the node was already in - the tree, it is removed first. - - -.. method:: Node.insertBefore(newChild, refChild) - - Insert a new child node before an existing child. It must be the case that - *refChild* is a child of this node; if not, :exc:`ValueError` is raised. - *newChild* is returned. If *refChild* is ``None``, it inserts *newChild* at the - end of the children's list. - - -.. method:: Node.removeChild(oldChild) - - Remove a child node. *oldChild* must be a child of this node; if not, - :exc:`ValueError` is raised. *oldChild* is returned on success. If *oldChild* - will not be used further, its :meth:`unlink` method should be called. - - -.. method:: Node.replaceChild(newChild, oldChild) - - Replace an existing node with a new node. It must be the case that *oldChild* - is a child of this node; if not, :exc:`ValueError` is raised. - - -.. method:: Node.normalize() - - Join adjacent text nodes so that all stretches of text are stored as single - :class:`Text` instances. This simplifies processing text from a DOM tree for - many applications. - - -.. method:: Node.cloneNode(deep) - - Clone this node. Setting *deep* means to clone all child nodes as well. This - returns the clone. - - -.. _dom-nodelist-objects: - -NodeList Objects -^^^^^^^^^^^^^^^^ - -A :class:`NodeList` represents a sequence of nodes. These objects are used in -two ways in the DOM Core recommendation: an :class:`Element` object provides -one as its list of child nodes, and the :meth:`getElementsByTagName` and -:meth:`getElementsByTagNameNS` methods of :class:`Node` return objects with this -interface to represent query results. - -The DOM Level 2 recommendation defines one method and one attribute for these -objects: - - -.. method:: NodeList.item(i) - - Return the *i*'th item from the sequence, if there is one, or ``None``. The - index *i* is not allowed to be less than zero or greater than or equal to the - length of the sequence. - - -.. attribute:: NodeList.length - - The number of nodes in the sequence. - -In addition, the Python DOM interface requires that some additional support is -provided to allow :class:`NodeList` objects to be used as Python sequences. All -:class:`NodeList` implementations must include support for -:meth:`~object.__len__` and -:meth:`~object.__getitem__`; this allows iteration over the :class:`NodeList` in -:keyword:`for` statements and proper support for the :func:`len` built-in -function. - -If a DOM implementation supports modification of the document, the -:class:`NodeList` implementation must also support the -:meth:`~object.__setitem__` and :meth:`~object.__delitem__` methods. - - -.. _dom-documenttype-objects: - -DocumentType Objects -^^^^^^^^^^^^^^^^^^^^ - -Information about the notations and entities declared by a document (including -the external subset if the parser uses it and can provide the information) is -available from a :class:`DocumentType` object. The :class:`DocumentType` for a -document is available from the :class:`Document` object's :attr:`doctype` -attribute; if there is no ``DOCTYPE`` declaration for the document, the -document's :attr:`doctype` attribute will be set to ``None`` instead of an -instance of this interface. - -:class:`DocumentType` is a specialization of :class:`Node`, and adds the -following attributes: - - -.. attribute:: DocumentType.publicId - - The public identifier for the external subset of the document type definition. - This will be a string or ``None``. - - -.. attribute:: DocumentType.systemId - - The system identifier for the external subset of the document type definition. - This will be a URI as a string, or ``None``. - - -.. attribute:: DocumentType.internalSubset - - A string giving the complete internal subset from the document. This does not - include the brackets which enclose the subset. If the document has no internal - subset, this should be ``None``. - - -.. attribute:: DocumentType.name - - The name of the root element as given in the ``DOCTYPE`` declaration, if - present. - - -.. attribute:: DocumentType.entities - - This is a :class:`NamedNodeMap` giving the definitions of external entities. - For entity names defined more than once, only the first definition is provided - (others are ignored as required by the XML recommendation). This may be - ``None`` if the information is not provided by the parser, or if no entities are - defined. - - -.. attribute:: DocumentType.notations - - This is a :class:`NamedNodeMap` giving the definitions of notations. For - notation names defined more than once, only the first definition is provided - (others are ignored as required by the XML recommendation). This may be - ``None`` if the information is not provided by the parser, or if no notations - are defined. - - -.. _dom-document-objects: - -Document Objects -^^^^^^^^^^^^^^^^ - -A :class:`Document` represents an entire XML document, including its constituent -elements, attributes, processing instructions, comments etc. Remember that it -inherits properties from :class:`Node`. - - -.. attribute:: Document.documentElement - - The one and only root element of the document. - - -.. method:: Document.createElement(tagName) - - Create and return a new element node. The element is not inserted into the - document when it is created. You need to explicitly insert it with one of the - other methods such as :meth:`insertBefore` or :meth:`appendChild`. - - -.. method:: Document.createElementNS(namespaceURI, tagName) - - Create and return a new element with a namespace. The *tagName* may have a - prefix. The element is not inserted into the document when it is created. You - need to explicitly insert it with one of the other methods such as - :meth:`insertBefore` or :meth:`appendChild`. - - -.. method:: Document.createTextNode(data) - - Create and return a text node containing the data passed as a parameter. As - with the other creation methods, this one does not insert the node into the - tree. - - -.. method:: Document.createComment(data) - - Create and return a comment node containing the data passed as a parameter. As - with the other creation methods, this one does not insert the node into the - tree. - - -.. method:: Document.createProcessingInstruction(target, data) - - Create and return a processing instruction node containing the *target* and - *data* passed as parameters. As with the other creation methods, this one does - not insert the node into the tree. - - -.. method:: Document.createAttribute(name) - - Create and return an attribute node. This method does not associate the - attribute node with any particular element. You must use - :meth:`setAttributeNode` on the appropriate :class:`Element` object to use the - newly created attribute instance. - - -.. method:: Document.createAttributeNS(namespaceURI, qualifiedName) - - Create and return an attribute node with a namespace. The *tagName* may have a - prefix. This method does not associate the attribute node with any particular - element. You must use :meth:`setAttributeNode` on the appropriate - :class:`Element` object to use the newly created attribute instance. - - -.. method:: Document.getElementsByTagName(tagName) - - Search for all descendants (direct children, children's children, etc.) with a - particular element type name. - - -.. method:: Document.getElementsByTagNameNS(namespaceURI, localName) - - Search for all descendants (direct children, children's children, etc.) with a - particular namespace URI and localname. The localname is the part of the - namespace after the prefix. - - -.. _dom-element-objects: - -Element Objects -^^^^^^^^^^^^^^^ - -:class:`Element` is a subclass of :class:`Node`, so inherits all the attributes -of that class. - - -.. attribute:: Element.tagName - - The element type name. In a namespace-using document it may have colons in it. - The value is a string. - - -.. method:: Element.getElementsByTagName(tagName) - - Same as equivalent method in the :class:`Document` class. - - -.. method:: Element.getElementsByTagNameNS(namespaceURI, localName) - - Same as equivalent method in the :class:`Document` class. - - -.. method:: Element.hasAttribute(name) - - Return ``True`` if the element has an attribute named by *name*. - - -.. method:: Element.hasAttributeNS(namespaceURI, localName) - - Return ``True`` if the element has an attribute named by *namespaceURI* and - *localName*. - - -.. method:: Element.getAttribute(name) - - Return the value of the attribute named by *name* as a string. If no such - attribute exists, an empty string is returned, as if the attribute had no value. - - -.. method:: Element.getAttributeNode(attrname) - - Return the :class:`Attr` node for the attribute named by *attrname*. - - -.. method:: Element.getAttributeNS(namespaceURI, localName) - - Return the value of the attribute named by *namespaceURI* and *localName* as a - string. If no such attribute exists, an empty string is returned, as if the - attribute had no value. - - -.. method:: Element.getAttributeNodeNS(namespaceURI, localName) - - Return an attribute value as a node, given a *namespaceURI* and *localName*. - - -.. method:: Element.removeAttribute(name) - - Remove an attribute by name. If there is no matching attribute, a - :exc:`NotFoundErr` is raised. - - -.. method:: Element.removeAttributeNode(oldAttr) - - Remove and return *oldAttr* from the attribute list, if present. If *oldAttr* is - not present, :exc:`NotFoundErr` is raised. - - -.. method:: Element.removeAttributeNS(namespaceURI, localName) - - Remove an attribute by name. Note that it uses a localName, not a qname. No - exception is raised if there is no matching attribute. - - -.. method:: Element.setAttribute(name, value) - - Set an attribute value from a string. - - -.. method:: Element.setAttributeNode(newAttr) - - Add a new attribute node to the element, replacing an existing attribute if - necessary if the :attr:`name` attribute matches. If a replacement occurs, the - old attribute node will be returned. If *newAttr* is already in use, - :exc:`InuseAttributeErr` will be raised. - - -.. method:: Element.setAttributeNodeNS(newAttr) - - Add a new attribute node to the element, replacing an existing attribute if - necessary if the :attr:`namespaceURI` and :attr:`localName` attributes match. - If a replacement occurs, the old attribute node will be returned. If *newAttr* - is already in use, :exc:`InuseAttributeErr` will be raised. - - -.. method:: Element.setAttributeNS(namespaceURI, qname, value) - - Set an attribute value from a string, given a *namespaceURI* and a *qname*. - Note that a qname is the whole attribute name. This is different than above. - - -.. _dom-attr-objects: - -Attr Objects -^^^^^^^^^^^^ - -:class:`Attr` inherits from :class:`Node`, so inherits all its attributes. - - -.. attribute:: Attr.name - - The attribute name. - In a namespace-using document it may include a colon. - - -.. attribute:: Attr.localName - - The part of the name following the colon if there is one, else the - entire name. - This is a read-only attribute. - - -.. attribute:: Attr.prefix - - The part of the name preceding the colon if there is one, else the - empty string. - - -.. attribute:: Attr.value - - The text value of the attribute. This is a synonym for the - :attr:`nodeValue` attribute. - - -.. _dom-attributelist-objects: - -NamedNodeMap Objects -^^^^^^^^^^^^^^^^^^^^ - -:class:`NamedNodeMap` does *not* inherit from :class:`Node`. - - -.. attribute:: NamedNodeMap.length - - The length of the attribute list. - - -.. method:: NamedNodeMap.item(index) - - Return an attribute with a particular index. The order you get the attributes - in is arbitrary but will be consistent for the life of a DOM. Each item is an - attribute node. Get its value with the :attr:`value` attribute. - -There are also experimental methods that give this class more mapping behavior. -You can use them or you can use the standardized :meth:`getAttribute\*` family -of methods on the :class:`Element` objects. - - -.. _dom-comment-objects: - -Comment Objects -^^^^^^^^^^^^^^^ - -:class:`Comment` represents a comment in the XML document. It is a subclass of -:class:`Node`, but cannot have child nodes. - - -.. attribute:: Comment.data - - The content of the comment as a string. The attribute contains all characters - between the leading ````, but does not - include them. - - -.. _dom-text-objects: - -Text and CDATASection Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The :class:`Text` interface represents text in the XML document. If the parser -and DOM implementation support the DOM's XML extension, portions of the text -enclosed in CDATA marked sections are stored in :class:`CDATASection` objects. -These two interfaces are identical, but provide different values for the -:attr:`nodeType` attribute. - -These interfaces extend the :class:`Node` interface. They cannot have child -nodes. - - -.. attribute:: Text.data - - The content of the text node as a string. - -.. note:: - - The use of a :class:`CDATASection` node does not indicate that the node - represents a complete CDATA marked section, only that the content of the node - was part of a CDATA section. A single CDATA section may be represented by more - than one node in the document tree. There is no way to determine whether two - adjacent :class:`CDATASection` nodes represent different CDATA marked sections. - - -.. _dom-pi-objects: - -ProcessingInstruction Objects -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Represents a processing instruction in the XML document; this inherits from the -:class:`Node` interface and cannot have child nodes. - - -.. attribute:: ProcessingInstruction.target - - The content of the processing instruction up to the first whitespace character. - This is a read-only attribute. - - -.. attribute:: ProcessingInstruction.data - - The content of the processing instruction following the first whitespace - character. - - -.. _dom-exceptions: - -Exceptions -^^^^^^^^^^ - -The DOM Level 2 recommendation defines a single exception, :exc:`DOMException`, -and a number of constants that allow applications to determine what sort of -error occurred. :exc:`DOMException` instances carry a :attr:`code` attribute -that provides the appropriate value for the specific exception. - -The Python DOM interface provides the constants, but also expands the set of -exceptions so that a specific exception exists for each of the exception codes -defined by the DOM. The implementations must raise the appropriate specific -exception, each of which carries the appropriate value for the :attr:`code` -attribute. - - -.. exception:: DOMException - - Base exception class used for all specific DOM exceptions. This exception class - cannot be directly instantiated. - - -.. exception:: DomstringSizeErr - - Raised when a specified range of text does not fit into a string. This is not - known to be used in the Python DOM implementations, but may be received from DOM - implementations not written in Python. - - -.. exception:: HierarchyRequestErr - - Raised when an attempt is made to insert a node where the node type is not - allowed. - - -.. exception:: IndexSizeErr - - Raised when an index or size parameter to a method is negative or exceeds the - allowed values. - - -.. exception:: InuseAttributeErr - - Raised when an attempt is made to insert an :class:`Attr` node that is already - present elsewhere in the document. - - -.. exception:: InvalidAccessErr - - Raised if a parameter or an operation is not supported on the underlying object. - - -.. exception:: InvalidCharacterErr - - This exception is raised when a string parameter contains a character that is - not permitted in the context it's being used in by the XML 1.0 recommendation. - For example, attempting to create an :class:`Element` node with a space in the - element type name will cause this error to be raised. - - -.. exception:: InvalidModificationErr - - Raised when an attempt is made to modify the type of a node. - - -.. exception:: InvalidStateErr - - Raised when an attempt is made to use an object that is not defined or is no - longer usable. - - -.. exception:: NamespaceErr - - If an attempt is made to change any object in a way that is not permitted with - regard to the `Namespaces in XML `_ - recommendation, this exception is raised. - - -.. exception:: NotFoundErr - - Exception when a node does not exist in the referenced context. For example, - :meth:`NamedNodeMap.removeNamedItem` will raise this if the node passed in does - not exist in the map. - - -.. exception:: NotSupportedErr - - Raised when the implementation does not support the requested type of object or - operation. - - -.. exception:: NoDataAllowedErr - - This is raised if data is specified for a node which does not support data. - - .. XXX a better explanation is needed! - - -.. exception:: NoModificationAllowedErr - - Raised on attempts to modify an object where modifications are not allowed (such - as for read-only nodes). - - -.. exception:: SyntaxErr - - Raised when an invalid or illegal string is specified. - - .. XXX how is this different from InvalidCharacterErr? - - -.. exception:: WrongDocumentErr - - Raised when a node is inserted in a different document than it currently belongs - to, and the implementation does not support migrating the node from one document - to the other. - -The exception codes defined in the DOM recommendation map to the exceptions -described above according to this table: - -+--------------------------------------+---------------------------------+ -| Constant | Exception | -+======================================+=================================+ -| :const:`DOMSTRING_SIZE_ERR` | :exc:`DomstringSizeErr` | -+--------------------------------------+---------------------------------+ -| :const:`HIERARCHY_REQUEST_ERR` | :exc:`HierarchyRequestErr` | -+--------------------------------------+---------------------------------+ -| :const:`INDEX_SIZE_ERR` | :exc:`IndexSizeErr` | -+--------------------------------------+---------------------------------+ -| :const:`INUSE_ATTRIBUTE_ERR` | :exc:`InuseAttributeErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_ACCESS_ERR` | :exc:`InvalidAccessErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_CHARACTER_ERR` | :exc:`InvalidCharacterErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_MODIFICATION_ERR` | :exc:`InvalidModificationErr` | -+--------------------------------------+---------------------------------+ -| :const:`INVALID_STATE_ERR` | :exc:`InvalidStateErr` | -+--------------------------------------+---------------------------------+ -| :const:`NAMESPACE_ERR` | :exc:`NamespaceErr` | -+--------------------------------------+---------------------------------+ -| :const:`NOT_FOUND_ERR` | :exc:`NotFoundErr` | -+--------------------------------------+---------------------------------+ -| :const:`NOT_SUPPORTED_ERR` | :exc:`NotSupportedErr` | -+--------------------------------------+---------------------------------+ -| :const:`NO_DATA_ALLOWED_ERR` | :exc:`NoDataAllowedErr` | -+--------------------------------------+---------------------------------+ -| :const:`NO_MODIFICATION_ALLOWED_ERR` | :exc:`NoModificationAllowedErr` | -+--------------------------------------+---------------------------------+ -| :const:`SYNTAX_ERR` | :exc:`SyntaxErr` | -+--------------------------------------+---------------------------------+ -| :const:`WRONG_DOCUMENT_ERR` | :exc:`WrongDocumentErr` | -+--------------------------------------+---------------------------------+ - - -.. _dom-conformance: - -Conformance ------------ - -This section describes the conformance requirements and relationships between -the Python DOM API, the W3C DOM recommendations, and the OMG IDL mapping for -Python. - - -.. _dom-type-mapping: - -Type Mapping -^^^^^^^^^^^^ - -The IDL types used in the DOM specification are mapped to Python types -according to the following table. - -+------------------+-------------------------------------------+ -| IDL Type | Python Type | -+==================+===========================================+ -| ``boolean`` | ``bool`` or ``int`` | -+------------------+-------------------------------------------+ -| ``int`` | ``int`` | -+------------------+-------------------------------------------+ -| ``long int`` | ``int`` | -+------------------+-------------------------------------------+ -| ``unsigned int`` | ``int`` | -+------------------+-------------------------------------------+ -| ``DOMString`` | ``str`` or ``bytes`` | -+------------------+-------------------------------------------+ -| ``null`` | ``None`` | -+------------------+-------------------------------------------+ - -.. _dom-accessor-methods: - -Accessor Methods -^^^^^^^^^^^^^^^^ - -The mapping from OMG IDL to Python defines accessor functions for IDL -``attribute`` declarations in much the way the Java mapping does. -Mapping the IDL declarations :: - - readonly attribute string someValue; - attribute string anotherValue; - -yields three accessor functions: a "get" method for :attr:`someValue` -(:meth:`_get_someValue`), and "get" and "set" methods for :attr:`anotherValue` -(:meth:`_get_anotherValue` and :meth:`_set_anotherValue`). The mapping, in -particular, does not require that the IDL attributes are accessible as normal -Python attributes: ``object.someValue`` is *not* required to work, and may -raise an :exc:`AttributeError`. - -The Python DOM API, however, *does* require that normal attribute access work. -This means that the typical surrogates generated by Python IDL compilers are not -likely to work, and wrapper objects may be needed on the client if the DOM -objects are accessed via CORBA. While this does require some additional -consideration for CORBA DOM clients, the implementers with experience using DOM -over CORBA from Python do not consider this a problem. Attributes that are -declared ``readonly`` may not restrict write access in all DOM -implementations. - -In the Python DOM API, accessor functions are not required. If provided, they -should take the form defined by the Python IDL mapping, but these methods are -considered unnecessary since the attributes are accessible directly from Python. -"Set" accessors should never be provided for ``readonly`` attributes. - -The IDL definitions do not fully embody the requirements of the W3C DOM API, -such as the notion of certain objects, such as the return value of -:meth:`getElementsByTagName`, being "live". The Python DOM API does not require -implementations to enforce such requirements. - diff --git a/Doc/library/xml.etree.elementtree.rst b/Doc/library/xml.etree.elementtree.rst deleted file mode 100644 index 31135a7e613c97..00000000000000 --- a/Doc/library/xml.etree.elementtree.rst +++ /dev/null @@ -1,1509 +0,0 @@ -:mod:`xml.etree.ElementTree` --- The ElementTree XML API -======================================================== - -.. module:: xml.etree.ElementTree - :synopsis: Implementation of the ElementTree API. - -.. moduleauthor:: Fredrik Lundh - -**Source code:** :source:`Lib/xml/etree/ElementTree.py` - --------------- - -The :mod:`xml.etree.ElementTree` module implements a simple and efficient API -for parsing and creating XML data. - -.. versionchanged:: 3.3 - This module will use a fast implementation whenever available. - -.. deprecated:: 3.3 - The :mod:`!xml.etree.cElementTree` module is deprecated. - - -.. warning:: - - The :mod:`xml.etree.ElementTree` module is not secure against - maliciously constructed data. If you need to parse untrusted or - unauthenticated data see :ref:`xml-vulnerabilities`. - -Tutorial --------- - -This is a short tutorial for using :mod:`xml.etree.ElementTree` (``ET`` in -short). The goal is to demonstrate some of the building blocks and basic -concepts of the module. - -XML tree and elements -^^^^^^^^^^^^^^^^^^^^^ - -XML is an inherently hierarchical data format, and the most natural way to -represent it is with a tree. ``ET`` has two classes for this purpose - -:class:`ElementTree` represents the whole XML document as a tree, and -:class:`Element` represents a single node in this tree. Interactions with -the whole document (reading and writing to/from files) are usually done -on the :class:`ElementTree` level. Interactions with a single XML element -and its sub-elements are done on the :class:`Element` level. - -.. _elementtree-parsing-xml: - -Parsing XML -^^^^^^^^^^^ - -We'll be using the following XML document as the sample data for this section: - -.. code-block:: xml - - - - - 1 - 2008 - 141100 - - - - - 4 - 2011 - 59900 - - - - 68 - 2011 - 13600 - - - - - -We can import this data by reading from a file:: - - import xml.etree.ElementTree as ET - tree = ET.parse('country_data.xml') - root = tree.getroot() - -Or directly from a string:: - - root = ET.fromstring(country_data_as_string) - -:func:`fromstring` parses XML from a string directly into an :class:`Element`, -which is the root element of the parsed tree. Other parsing functions may -create an :class:`ElementTree`. Check the documentation to be sure. - -As an :class:`Element`, ``root`` has a tag and a dictionary of attributes:: - - >>> root.tag - 'data' - >>> root.attrib - {} - -It also has children nodes over which we can iterate:: - - >>> for child in root: - ... print(child.tag, child.attrib) - ... - country {'name': 'Liechtenstein'} - country {'name': 'Singapore'} - country {'name': 'Panama'} - -Children are nested, and we can access specific child nodes by index:: - - >>> root[0][1].text - '2008' - - -.. note:: - - Not all elements of the XML input will end up as elements of the - parsed tree. Currently, this module skips over any XML comments, - processing instructions, and document type declarations in the - input. Nevertheless, trees built using this module's API rather - than parsing from XML text can have comments and processing - instructions in them; they will be included when generating XML - output. A document type declaration may be accessed by passing a - custom :class:`TreeBuilder` instance to the :class:`XMLParser` - constructor. - - -.. _elementtree-pull-parsing: - -Pull API for non-blocking parsing -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Most parsing functions provided by this module require the whole document -to be read at once before returning any result. It is possible to use an -:class:`XMLParser` and feed data into it incrementally, but it is a push API that -calls methods on a callback target, which is too low-level and inconvenient for -most needs. Sometimes what the user really wants is to be able to parse XML -incrementally, without blocking operations, while enjoying the convenience of -fully constructed :class:`Element` objects. - -The most powerful tool for doing this is :class:`XMLPullParser`. It does not -require a blocking read to obtain the XML data, and is instead fed with data -incrementally with :meth:`XMLPullParser.feed` calls. To get the parsed XML -elements, call :meth:`XMLPullParser.read_events`. Here is an example:: - - >>> parser = ET.XMLPullParser(['start', 'end']) - >>> parser.feed('sometext') - >>> list(parser.read_events()) - [('start', )] - >>> parser.feed(' more text') - >>> for event, elem in parser.read_events(): - ... print(event) - ... print(elem.tag, 'text=', elem.text) - ... - end - mytag text= sometext more text - -The obvious use case is applications that operate in a non-blocking fashion -where the XML data is being received from a socket or read incrementally from -some storage device. In such cases, blocking reads are unacceptable. - -Because it's so flexible, :class:`XMLPullParser` can be inconvenient to use for -simpler use-cases. If you don't mind your application blocking on reading XML -data but would still like to have incremental parsing capabilities, take a look -at :func:`iterparse`. It can be useful when you're reading a large XML document -and don't want to hold it wholly in memory. - -Finding interesting elements -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -:class:`Element` has some useful methods that help iterate recursively over all -the sub-tree below it (its children, their children, and so on). For example, -:meth:`Element.iter`:: - - >>> for neighbor in root.iter('neighbor'): - ... print(neighbor.attrib) - ... - {'name': 'Austria', 'direction': 'E'} - {'name': 'Switzerland', 'direction': 'W'} - {'name': 'Malaysia', 'direction': 'N'} - {'name': 'Costa Rica', 'direction': 'W'} - {'name': 'Colombia', 'direction': 'E'} - -:meth:`Element.findall` finds only elements with a tag which are direct -children of the current element. :meth:`Element.find` finds the *first* child -with a particular tag, and :attr:`Element.text` accesses the element's text -content. :meth:`Element.get` accesses the element's attributes:: - - >>> for country in root.findall('country'): - ... rank = country.find('rank').text - ... name = country.get('name') - ... print(name, rank) - ... - Liechtenstein 1 - Singapore 4 - Panama 68 - -More sophisticated specification of which elements to look for is possible by -using :ref:`XPath `. - -Modifying an XML File -^^^^^^^^^^^^^^^^^^^^^ - -:class:`ElementTree` provides a simple way to build XML documents and write them to files. -The :meth:`ElementTree.write` method serves this purpose. - -Once created, an :class:`Element` object may be manipulated by directly changing -its fields (such as :attr:`Element.text`), adding and modifying attributes -(:meth:`Element.set` method), as well as adding new children (for example -with :meth:`Element.append`). - -Let's say we want to add one to each country's rank, and add an ``updated`` -attribute to the rank element:: - - >>> for rank in root.iter('rank'): - ... new_rank = int(rank.text) + 1 - ... rank.text = str(new_rank) - ... rank.set('updated', 'yes') - ... - >>> tree.write('output.xml') - -Our XML now looks like this: - -.. code-block:: xml - - - - - 2 - 2008 - 141100 - - - - - 5 - 2011 - 59900 - - - - 69 - 2011 - 13600 - - - - - -We can remove elements using :meth:`Element.remove`. Let's say we want to -remove all countries with a rank higher than 50:: - - >>> for country in root.findall('country'): - ... # using root.findall() to avoid removal during traversal - ... rank = int(country.find('rank').text) - ... if rank > 50: - ... root.remove(country) - ... - >>> tree.write('output.xml') - -Note that concurrent modification while iterating can lead to problems, -just like when iterating and modifying Python lists or dicts. -Therefore, the example first collects all matching elements with -``root.findall()``, and only then iterates over the list of matches. - -Our XML now looks like this: - -.. code-block:: xml - - - - - 2 - 2008 - 141100 - - - - - 5 - 2011 - 59900 - - - - -Building XML documents -^^^^^^^^^^^^^^^^^^^^^^ - -The :func:`SubElement` function also provides a convenient way to create new -sub-elements for a given element:: - - >>> a = ET.Element('a') - >>> b = ET.SubElement(a, 'b') - >>> c = ET.SubElement(a, 'c') - >>> d = ET.SubElement(c, 'd') - >>> ET.dump(a) - - -Parsing XML with Namespaces -^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -If the XML input has `namespaces -`__, tags and attributes -with prefixes in the form ``prefix:sometag`` get expanded to -``{uri}sometag`` where the *prefix* is replaced by the full *URI*. -Also, if there is a `default namespace -`__, -that full URI gets prepended to all of the non-prefixed tags. - -Here is an XML example that incorporates two namespaces, one with the -prefix "fictional" and the other serving as the default namespace: - -.. code-block:: xml - - - - - John Cleese - Lancelot - Archie Leach - - - Eric Idle - Sir Robin - Gunther - Commander Clement - - - -One way to search and explore this XML example is to manually add the -URI to every tag or attribute in the xpath of a -:meth:`~Element.find` or :meth:`~Element.findall`:: - - root = fromstring(xml_text) - for actor in root.findall('{http://people.example.com}actor'): - name = actor.find('{http://people.example.com}name') - print(name.text) - for char in actor.findall('{http://characters.example.com}character'): - print(' |-->', char.text) - -A better way to search the namespaced XML example is to create a -dictionary with your own prefixes and use those in the search functions:: - - ns = {'real_person': 'http://people.example.com', - 'role': 'http://characters.example.com'} - - for actor in root.findall('real_person:actor', ns): - name = actor.find('real_person:name', ns) - print(name.text) - for char in actor.findall('role:character', ns): - print(' |-->', char.text) - -These two approaches both output:: - - John Cleese - |--> Lancelot - |--> Archie Leach - Eric Idle - |--> Sir Robin - |--> Gunther - |--> Commander Clement - - -.. _elementtree-xpath: - -XPath support -------------- - -This module provides limited support for -`XPath expressions `_ for locating elements in a -tree. The goal is to support a small subset of the abbreviated syntax; a full -XPath engine is outside the scope of the module. - -Example -^^^^^^^ - -Here's an example that demonstrates some of the XPath capabilities of the -module. We'll be using the ``countrydata`` XML document from the -:ref:`Parsing XML ` section:: - - import xml.etree.ElementTree as ET - - root = ET.fromstring(countrydata) - - # Top-level elements - root.findall(".") - - # All 'neighbor' grand-children of 'country' children of the top-level - # elements - root.findall("./country/neighbor") - - # Nodes with name='Singapore' that have a 'year' child - root.findall(".//year/..[@name='Singapore']") - - # 'year' nodes that are children of nodes with name='Singapore' - root.findall(".//*[@name='Singapore']/year") - - # All 'neighbor' nodes that are the second child of their parent - root.findall(".//neighbor[2]") - -For XML with namespaces, use the usual qualified ``{namespace}tag`` notation:: - - # All dublin-core "title" tags in the document - root.findall(".//{http://purl.org/dc/elements/1.1/}title") - - -Supported XPath syntax -^^^^^^^^^^^^^^^^^^^^^^ - -.. tabularcolumns:: |l|L| - -+-----------------------+------------------------------------------------------+ -| Syntax | Meaning | -+=======================+======================================================+ -| ``tag`` | Selects all child elements with the given tag. | -| | For example, ``spam`` selects all child elements | -| | named ``spam``, and ``spam/egg`` selects all | -| | grandchildren named ``egg`` in all children named | -| | ``spam``. ``{namespace}*`` selects all tags in the | -| | given namespace, ``{*}spam`` selects tags named | -| | ``spam`` in any (or no) namespace, and ``{}*`` | -| | only selects tags that are not in a namespace. | -| | | -| | .. versionchanged:: 3.8 | -| | Support for star-wildcards was added. | -+-----------------------+------------------------------------------------------+ -| ``*`` | Selects all child elements, including comments and | -| | processing instructions. For example, ``*/egg`` | -| | selects all grandchildren named ``egg``. | -+-----------------------+------------------------------------------------------+ -| ``.`` | Selects the current node. This is mostly useful | -| | at the beginning of the path, to indicate that it's | -| | a relative path. | -+-----------------------+------------------------------------------------------+ -| ``//`` | Selects all subelements, on all levels beneath the | -| | current element. For example, ``.//egg`` selects | -| | all ``egg`` elements in the entire tree. | -+-----------------------+------------------------------------------------------+ -| ``..`` | Selects the parent element. Returns ``None`` if the | -| | path attempts to reach the ancestors of the start | -| | element (the element ``find`` was called on). | -+-----------------------+------------------------------------------------------+ -| ``[@attrib]`` | Selects all elements that have the given attribute. | -+-----------------------+------------------------------------------------------+ -| ``[@attrib='value']`` | Selects all elements for which the given attribute | -| | has the given value. The value cannot contain | -| | quotes. | -+-----------------------+------------------------------------------------------+ -| ``[@attrib!='value']``| Selects all elements for which the given attribute | -| | does not have the given value. The value cannot | -| | contain quotes. | -| | | -| | .. versionadded:: 3.10 | -+-----------------------+------------------------------------------------------+ -| ``[tag]`` | Selects all elements that have a child named | -| | ``tag``. Only immediate children are supported. | -+-----------------------+------------------------------------------------------+ -| ``[.='text']`` | Selects all elements whose complete text content, | -| | including descendants, equals the given ``text``. | -| | | -| | .. versionadded:: 3.7 | -+-----------------------+------------------------------------------------------+ -| ``[.!='text']`` | Selects all elements whose complete text content, | -| | including descendants, does not equal the given | -| | ``text``. | -| | | -| | .. versionadded:: 3.10 | -+-----------------------+------------------------------------------------------+ -| ``[tag='text']`` | Selects all elements that have a child named | -| | ``tag`` whose complete text content, including | -| | descendants, equals the given ``text``. | -+-----------------------+------------------------------------------------------+ -| ``[tag!='text']`` | Selects all elements that have a child named | -| | ``tag`` whose complete text content, including | -| | descendants, does not equal the given ``text``. | -| | | -| | .. versionadded:: 3.10 | -+-----------------------+------------------------------------------------------+ -| ``[position]`` | Selects all elements that are located at the given | -| | position. The position can be either an integer | -| | (1 is the first position), the expression ``last()`` | -| | (for the last position), or a position relative to | -| | the last position (e.g. ``last()-1``). | -+-----------------------+------------------------------------------------------+ - -Predicates (expressions within square brackets) must be preceded by a tag -name, an asterisk, or another predicate. ``position`` predicates must be -preceded by a tag name. - -Reference ---------- - -.. _elementtree-functions: - -Functions -^^^^^^^^^ - -.. function:: canonicalize(xml_data=None, *, out=None, from_file=None, **options) - - `C14N 2.0 `_ transformation function. - - Canonicalization is a way to normalise XML output in a way that allows - byte-by-byte comparisons and digital signatures. It reduced the freedom - that XML serializers have and instead generates a more constrained XML - representation. The main restrictions regard the placement of namespace - declarations, the ordering of attributes, and ignorable whitespace. - - This function takes an XML data string (*xml_data*) or a file path or - file-like object (*from_file*) as input, converts it to the canonical - form, and writes it out using the *out* file(-like) object, if provided, - or returns it as a text string if not. The output file receives text, - not bytes. It should therefore be opened in text mode with ``utf-8`` - encoding. - - Typical uses:: - - xml_data = "..." - print(canonicalize(xml_data)) - - with open("c14n_output.xml", mode='w', encoding='utf-8') as out_file: - canonicalize(xml_data, out=out_file) - - with open("c14n_output.xml", mode='w', encoding='utf-8') as out_file: - canonicalize(from_file="inputfile.xml", out=out_file) - - The configuration *options* are as follows: - - - *with_comments*: set to true to include comments (default: false) - - *strip_text*: set to true to strip whitespace before and after text content - (default: false) - - *rewrite_prefixes*: set to true to replace namespace prefixes by "n{number}" - (default: false) - - *qname_aware_tags*: a set of qname aware tag names in which prefixes - should be replaced in text content (default: empty) - - *qname_aware_attrs*: a set of qname aware attribute names in which prefixes - should be replaced in text content (default: empty) - - *exclude_attrs*: a set of attribute names that should not be serialised - - *exclude_tags*: a set of tag names that should not be serialised - - In the option list above, "a set" refers to any collection or iterable of - strings, no ordering is expected. - - .. versionadded:: 3.8 - - -.. function:: Comment(text=None) - - Comment element factory. This factory function creates a special element - that will be serialized as an XML comment by the standard serializer. The - comment string can be either a bytestring or a Unicode string. *text* is a - string containing the comment string. Returns an element instance - representing a comment. - - Note that :class:`XMLParser` skips over comments in the input - instead of creating comment objects for them. An :class:`ElementTree` will - only contain comment nodes if they have been inserted into to - the tree using one of the :class:`Element` methods. - -.. function:: dump(elem) - - Writes an element tree or element structure to sys.stdout. This function - should be used for debugging only. - - The exact output format is implementation dependent. In this version, it's - written as an ordinary XML file. - - *elem* is an element tree or an individual element. - - .. versionchanged:: 3.8 - The :func:`dump` function now preserves the attribute order specified - by the user. - - -.. function:: fromstring(text, parser=None) - - Parses an XML section from a string constant. Same as :func:`XML`. *text* - is a string containing XML data. *parser* is an optional parser instance. - If not given, the standard :class:`XMLParser` parser is used. - Returns an :class:`Element` instance. - - -.. function:: fromstringlist(sequence, parser=None) - - Parses an XML document from a sequence of string fragments. *sequence* is a - list or other sequence containing XML data fragments. *parser* is an - optional parser instance. If not given, the standard :class:`XMLParser` - parser is used. Returns an :class:`Element` instance. - - .. versionadded:: 3.2 - - -.. function:: indent(tree, space=" ", level=0) - - Appends whitespace to the subtree to indent the tree visually. - This can be used to generate pretty-printed XML output. - *tree* can be an Element or ElementTree. *space* is the whitespace - string that will be inserted for each indentation level, two space - characters by default. For indenting partial subtrees inside of an - already indented tree, pass the initial indentation level as *level*. - - .. versionadded:: 3.9 - - -.. function:: iselement(element) - - Check if an object appears to be a valid element object. *element* is an - element instance. Return ``True`` if this is an element object. - - -.. function:: iterparse(source, events=None, parser=None) - - Parses an XML section into an element tree incrementally, and reports what's - going on to the user. *source* is a filename or :term:`file object` - containing XML data. *events* is a sequence of events to report back. The - supported events are the strings ``"start"``, ``"end"``, ``"comment"``, - ``"pi"``, ``"start-ns"`` and ``"end-ns"`` - (the "ns" events are used to get detailed namespace - information). If *events* is omitted, only ``"end"`` events are reported. - *parser* is an optional parser instance. If not given, the standard - :class:`XMLParser` parser is used. *parser* must be a subclass of - :class:`XMLParser` and can only use the default :class:`TreeBuilder` as a - target. Returns an :term:`iterator` providing ``(event, elem)`` pairs; - it has a ``root`` attribute that references the root element of the - resulting XML tree once *source* is fully read. - - Note that while :func:`iterparse` builds the tree incrementally, it issues - blocking reads on *source* (or the file it names). As such, it's unsuitable - for applications where blocking reads can't be made. For fully non-blocking - parsing, see :class:`XMLPullParser`. - - .. note:: - - :func:`iterparse` only guarantees that it has seen the ">" character of a - starting tag when it emits a "start" event, so the attributes are defined, - but the contents of the text and tail attributes are undefined at that - point. The same applies to the element children; they may or may not be - present. - - If you need a fully populated element, look for "end" events instead. - - .. deprecated:: 3.4 - The *parser* argument. - - .. versionchanged:: 3.8 - The ``comment`` and ``pi`` events were added. - - -.. function:: parse(source, parser=None) - - Parses an XML section into an element tree. *source* is a filename or file - object containing XML data. *parser* is an optional parser instance. If - not given, the standard :class:`XMLParser` parser is used. Returns an - :class:`ElementTree` instance. - - -.. function:: ProcessingInstruction(target, text=None) - - PI element factory. This factory function creates a special element that - will be serialized as an XML processing instruction. *target* is a string - containing the PI target. *text* is a string containing the PI contents, if - given. Returns an element instance, representing a processing instruction. - - Note that :class:`XMLParser` skips over processing instructions - in the input instead of creating comment objects for them. An - :class:`ElementTree` will only contain processing instruction nodes if - they have been inserted into to the tree using one of the - :class:`Element` methods. - -.. function:: register_namespace(prefix, uri) - - Registers a namespace prefix. The registry is global, and any existing - mapping for either the given prefix or the namespace URI will be removed. - *prefix* is a namespace prefix. *uri* is a namespace uri. Tags and - attributes in this namespace will be serialized with the given prefix, if at - all possible. - - .. versionadded:: 3.2 - - -.. function:: SubElement(parent, tag, attrib={}, **extra) - - Subelement factory. This function creates an element instance, and appends - it to an existing element. - - The element name, attribute names, and attribute values can be either - bytestrings or Unicode strings. *parent* is the parent element. *tag* is - the subelement name. *attrib* is an optional dictionary, containing element - attributes. *extra* contains additional attributes, given as keyword - arguments. Returns an element instance. - - -.. function:: tostring(element, encoding="us-ascii", method="xml", *, \ - xml_declaration=None, default_namespace=None, \ - short_empty_elements=True) - - Generates a string representation of an XML element, including all - subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is - the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to - generate a Unicode string (otherwise, a bytestring is generated). *method* - is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). - *xml_declaration*, *default_namespace* and *short_empty_elements* has the same - meaning as in :meth:`ElementTree.write`. Returns an (optionally) encoded string - containing the XML data. - - .. versionadded:: 3.4 - The *short_empty_elements* parameter. - - .. versionadded:: 3.8 - The *xml_declaration* and *default_namespace* parameters. - - .. versionchanged:: 3.8 - The :func:`tostring` function now preserves the attribute order - specified by the user. - - -.. function:: tostringlist(element, encoding="us-ascii", method="xml", *, \ - xml_declaration=None, default_namespace=None, \ - short_empty_elements=True) - - Generates a string representation of an XML element, including all - subelements. *element* is an :class:`Element` instance. *encoding* [1]_ is - the output encoding (default is US-ASCII). Use ``encoding="unicode"`` to - generate a Unicode string (otherwise, a bytestring is generated). *method* - is either ``"xml"``, ``"html"`` or ``"text"`` (default is ``"xml"``). - *xml_declaration*, *default_namespace* and *short_empty_elements* has the same - meaning as in :meth:`ElementTree.write`. Returns a list of (optionally) encoded - strings containing the XML data. It does not guarantee any specific sequence, - except that ``b"".join(tostringlist(element)) == tostring(element)``. - - .. versionadded:: 3.2 - - .. versionadded:: 3.4 - The *short_empty_elements* parameter. - - .. versionadded:: 3.8 - The *xml_declaration* and *default_namespace* parameters. - - .. versionchanged:: 3.8 - The :func:`tostringlist` function now preserves the attribute order - specified by the user. - - -.. function:: XML(text, parser=None) - - Parses an XML section from a string constant. This function can be used to - embed "XML literals" in Python code. *text* is a string containing XML - data. *parser* is an optional parser instance. If not given, the standard - :class:`XMLParser` parser is used. Returns an :class:`Element` instance. - - -.. function:: XMLID(text, parser=None) - - Parses an XML section from a string constant, and also returns a dictionary - which maps from element id:s to elements. *text* is a string containing XML - data. *parser* is an optional parser instance. If not given, the standard - :class:`XMLParser` parser is used. Returns a tuple containing an - :class:`Element` instance and a dictionary. - - -.. _elementtree-xinclude: - -XInclude support ----------------- - -This module provides limited support for -`XInclude directives `_, via the :mod:`xml.etree.ElementInclude` helper module. This module can be used to insert subtrees and text strings into element trees, based on information in the tree. - -Example -^^^^^^^ - -Here's an example that demonstrates use of the XInclude module. To include an XML document in the current document, use the ``{http://www.w3.org/2001/XInclude}include`` element and set the **parse** attribute to ``"xml"``, and use the **href** attribute to specify the document to include. - -.. code-block:: xml - - - - - - -By default, the **href** attribute is treated as a file name. You can use custom loaders to override this behaviour. Also note that the standard helper does not support XPointer syntax. - -To process this file, load it as usual, and pass the root element to the :mod:`xml.etree.ElementTree` module: - -.. code-block:: python - - from xml.etree import ElementTree, ElementInclude - - tree = ElementTree.parse("document.xml") - root = tree.getroot() - - ElementInclude.include(root) - -The ElementInclude module replaces the ``{http://www.w3.org/2001/XInclude}include`` element with the root element from the **source.xml** document. The result might look something like this: - -.. code-block:: xml - - - This is a paragraph. - - -If the **parse** attribute is omitted, it defaults to "xml". The href attribute is required. - -To include a text document, use the ``{http://www.w3.org/2001/XInclude}include`` element, and set the **parse** attribute to "text": - -.. code-block:: xml - - - - Copyright (c) . - - -The result might look something like: - -.. code-block:: xml - - - Copyright (c) 2003. - - -Reference ---------- - -.. _elementinclude-functions: - -Functions -^^^^^^^^^ - -.. module:: xml.etree.ElementInclude - -.. function:: xml.etree.ElementInclude.default_loader( href, parse, encoding=None) - :module: - - Default loader. This default loader reads an included resource from disk. *href* is a URL. - *parse* is for parse mode either "xml" or "text". *encoding* - is an optional text encoding. If not given, encoding is ``utf-8``. Returns the - expanded resource. If the parse mode is ``"xml"``, this is an ElementTree - instance. If the parse mode is "text", this is a Unicode string. If the - loader fails, it can return None or raise an exception. - - -.. function:: xml.etree.ElementInclude.include( elem, loader=None, base_url=None, \ - max_depth=6) - :module: - - This function expands XInclude directives. *elem* is the root element. *loader* is - an optional resource loader. If omitted, it defaults to :func:`default_loader`. - If given, it should be a callable that implements the same interface as - :func:`default_loader`. *base_url* is base URL of the original file, to resolve - relative include file references. *max_depth* is the maximum number of recursive - inclusions. Limited to reduce the risk of malicious content explosion. Pass a - negative value to disable the limitation. - - Returns the expanded resource. If the parse mode is - ``"xml"``, this is an ElementTree instance. If the parse mode is "text", - this is a Unicode string. If the loader fails, it can return None or - raise an exception. - - .. versionadded:: 3.9 - The *base_url* and *max_depth* parameters. - - -.. _elementtree-element-objects: - -Element Objects -^^^^^^^^^^^^^^^ - -.. module:: xml.etree.ElementTree - :noindex: - -.. class:: Element(tag, attrib={}, **extra) - - Element class. This class defines the Element interface, and provides a - reference implementation of this interface. - - The element name, attribute names, and attribute values can be either - bytestrings or Unicode strings. *tag* is the element name. *attrib* is - an optional dictionary, containing element attributes. *extra* contains - additional attributes, given as keyword arguments. - - - .. attribute:: tag - - A string identifying what kind of data this element represents (the - element type, in other words). - - - .. attribute:: text - tail - - These attributes can be used to hold additional data associated with - the element. Their values are usually strings but may be any - application-specific object. If the element is created from - an XML file, the *text* attribute holds either the text between - the element's start tag and its first child or end tag, or ``None``, and - the *tail* attribute holds either the text between the element's - end tag and the next tag, or ``None``. For the XML data - - .. code-block:: xml - - 1234 - - the *a* element has ``None`` for both *text* and *tail* attributes, - the *b* element has *text* ``"1"`` and *tail* ``"4"``, - the *c* element has *text* ``"2"`` and *tail* ``None``, - and the *d* element has *text* ``None`` and *tail* ``"3"``. - - To collect the inner text of an element, see :meth:`itertext`, for - example ``"".join(element.itertext())``. - - Applications may store arbitrary objects in these attributes. - - - .. attribute:: attrib - - A dictionary containing the element's attributes. Note that while the - *attrib* value is always a real mutable Python dictionary, an ElementTree - implementation may choose to use another internal representation, and - create the dictionary only if someone asks for it. To take advantage of - such implementations, use the dictionary methods below whenever possible. - - The following dictionary-like methods work on the element attributes. - - - .. method:: clear() - - Resets an element. This function removes all subelements, clears all - attributes, and sets the text and tail attributes to ``None``. - - - .. method:: get(key, default=None) - - Gets the element attribute named *key*. - - Returns the attribute value, or *default* if the attribute was not found. - - - .. method:: items() - - Returns the element attributes as a sequence of (name, value) pairs. The - attributes are returned in an arbitrary order. - - - .. method:: keys() - - Returns the elements attribute names as a list. The names are returned - in an arbitrary order. - - - .. method:: set(key, value) - - Set the attribute *key* on the element to *value*. - - The following methods work on the element's children (subelements). - - - .. method:: append(subelement) - - Adds the element *subelement* to the end of this element's internal list - of subelements. Raises :exc:`TypeError` if *subelement* is not an - :class:`Element`. - - - .. method:: extend(subelements) - - Appends *subelements* from a sequence object with zero or more elements. - Raises :exc:`TypeError` if a subelement is not an :class:`Element`. - - .. versionadded:: 3.2 - - - .. method:: find(match, namespaces=None) - - Finds the first subelement matching *match*. *match* may be a tag name - or a :ref:`path `. Returns an element instance - or ``None``. *namespaces* is an optional mapping from namespace prefix - to full name. Pass ``''`` as prefix to move all unprefixed tag names - in the expression into the given namespace. - - - .. method:: findall(match, namespaces=None) - - Finds all matching subelements, by tag name or - :ref:`path `. Returns a list containing all matching - elements in document order. *namespaces* is an optional mapping from - namespace prefix to full name. Pass ``''`` as prefix to move all - unprefixed tag names in the expression into the given namespace. - - - .. method:: findtext(match, default=None, namespaces=None) - - Finds text for the first subelement matching *match*. *match* may be - a tag name or a :ref:`path `. Returns the text content - of the first matching element, or *default* if no element was found. - Note that if the matching element has no text content an empty string - is returned. *namespaces* is an optional mapping from namespace prefix - to full name. Pass ``''`` as prefix to move all unprefixed tag names - in the expression into the given namespace. - - - .. method:: insert(index, subelement) - - Inserts *subelement* at the given position in this element. Raises - :exc:`TypeError` if *subelement* is not an :class:`Element`. - - - .. method:: iter(tag=None) - - Creates a tree :term:`iterator` with the current element as the root. - The iterator iterates over this element and all elements below it, in - document (depth first) order. If *tag* is not ``None`` or ``'*'``, only - elements whose tag equals *tag* are returned from the iterator. If the - tree structure is modified during iteration, the result is undefined. - - .. versionadded:: 3.2 - - - .. method:: iterfind(match, namespaces=None) - - Finds all matching subelements, by tag name or - :ref:`path `. Returns an iterable yielding all - matching elements in document order. *namespaces* is an optional mapping - from namespace prefix to full name. - - - .. versionadded:: 3.2 - - - .. method:: itertext() - - Creates a text iterator. The iterator loops over this element and all - subelements, in document order, and returns all inner text. - - .. versionadded:: 3.2 - - - .. method:: makeelement(tag, attrib) - - Creates a new element object of the same type as this element. Do not - call this method, use the :func:`SubElement` factory function instead. - - - .. method:: remove(subelement) - - Removes *subelement* from the element. Unlike the find\* methods this - method compares elements based on the instance identity, not on tag value - or contents. - - :class:`Element` objects also support the following sequence type methods - for working with subelements: :meth:`~object.__delitem__`, - :meth:`~object.__getitem__`, :meth:`~object.__setitem__`, - :meth:`~object.__len__`. - - Caution: Elements with no subelements will test as ``False``. This behavior - will change in future versions. Use specific ``len(elem)`` or ``elem is - None`` test instead. :: - - element = root.find('foo') - - if not element: # careful! - print("element not found, or element has no subelements") - - if element is None: - print("element not found") - - Prior to Python 3.8, the serialisation order of the XML attributes of - elements was artificially made predictable by sorting the attributes by - their name. Based on the now guaranteed ordering of dicts, this arbitrary - reordering was removed in Python 3.8 to preserve the order in which - attributes were originally parsed or created by user code. - - In general, user code should try not to depend on a specific ordering of - attributes, given that the `XML Information Set - `_ explicitly excludes the attribute - order from conveying information. Code should be prepared to deal with - any ordering on input. In cases where deterministic XML output is required, - e.g. for cryptographic signing or test data sets, canonical serialisation - is available with the :func:`canonicalize` function. - - In cases where canonical output is not applicable but a specific attribute - order is still desirable on output, code should aim for creating the - attributes directly in the desired order, to avoid perceptual mismatches - for readers of the code. In cases where this is difficult to achieve, a - recipe like the following can be applied prior to serialisation to enforce - an order independently from the Element creation:: - - def reorder_attributes(root): - for el in root.iter(): - attrib = el.attrib - if len(attrib) > 1: - # adjust attribute order, e.g. by sorting - attribs = sorted(attrib.items()) - attrib.clear() - attrib.update(attribs) - - -.. _elementtree-elementtree-objects: - -ElementTree Objects -^^^^^^^^^^^^^^^^^^^ - - -.. class:: ElementTree(element=None, file=None) - - ElementTree wrapper class. This class represents an entire element - hierarchy, and adds some extra support for serialization to and from - standard XML. - - *element* is the root element. The tree is initialized with the contents - of the XML *file* if given. - - - .. method:: _setroot(element) - - Replaces the root element for this tree. This discards the current - contents of the tree, and replaces it with the given element. Use with - care. *element* is an element instance. - - - .. method:: find(match, namespaces=None) - - Same as :meth:`Element.find`, starting at the root of the tree. - - - .. method:: findall(match, namespaces=None) - - Same as :meth:`Element.findall`, starting at the root of the tree. - - - .. method:: findtext(match, default=None, namespaces=None) - - Same as :meth:`Element.findtext`, starting at the root of the tree. - - - .. method:: getroot() - - Returns the root element for this tree. - - - .. method:: iter(tag=None) - - Creates and returns a tree iterator for the root element. The iterator - loops over all elements in this tree, in section order. *tag* is the tag - to look for (default is to return all elements). - - - .. method:: iterfind(match, namespaces=None) - - Same as :meth:`Element.iterfind`, starting at the root of the tree. - - .. versionadded:: 3.2 - - - .. method:: parse(source, parser=None) - - Loads an external XML section into this element tree. *source* is a file - name or :term:`file object`. *parser* is an optional parser instance. - If not given, the standard :class:`XMLParser` parser is used. Returns the - section root element. - - - .. method:: write(file, encoding="us-ascii", xml_declaration=None, \ - default_namespace=None, method="xml", *, \ - short_empty_elements=True) - - Writes the element tree to a file, as XML. *file* is a file name, or a - :term:`file object` opened for writing. *encoding* [1]_ is the output - encoding (default is US-ASCII). - *xml_declaration* controls if an XML declaration should be added to the - file. Use ``False`` for never, ``True`` for always, ``None`` - for only if not US-ASCII or UTF-8 or Unicode (default is ``None``). - *default_namespace* sets the default XML namespace (for "xmlns"). - *method* is either ``"xml"``, ``"html"`` or ``"text"`` (default is - ``"xml"``). - The keyword-only *short_empty_elements* parameter controls the formatting - of elements that contain no content. If ``True`` (the default), they are - emitted as a single self-closed tag, otherwise they are emitted as a pair - of start/end tags. - - The output is either a string (:class:`str`) or binary (:class:`bytes`). - This is controlled by the *encoding* argument. If *encoding* is - ``"unicode"``, the output is a string; otherwise, it's binary. Note that - this may conflict with the type of *file* if it's an open - :term:`file object`; make sure you do not try to write a string to a - binary stream and vice versa. - - .. versionadded:: 3.4 - The *short_empty_elements* parameter. - - .. versionchanged:: 3.8 - The :meth:`write` method now preserves the attribute order specified - by the user. - - -This is the XML file that is going to be manipulated:: - - - - Example page - - -

    Moved to example.org - or example.com.

    - - - -Example of changing the attribute "target" of every link in first paragraph:: - - >>> from xml.etree.ElementTree import ElementTree - >>> tree = ElementTree() - >>> tree.parse("index.xhtml") - - >>> p = tree.find("body/p") # Finds first occurrence of tag p in body - >>> p - - >>> links = list(p.iter("a")) # Returns list of all links - >>> links - [, ] - >>> for i in links: # Iterates through all found links - ... i.attrib["target"] = "blank" - >>> tree.write("output.xhtml") - -.. _elementtree-qname-objects: - -QName Objects -^^^^^^^^^^^^^ - - -.. class:: QName(text_or_uri, tag=None) - - QName wrapper. This can be used to wrap a QName attribute value, in order - to get proper namespace handling on output. *text_or_uri* is a string - containing the QName value, in the form {uri}local, or, if the tag argument - is given, the URI part of a QName. If *tag* is given, the first argument is - interpreted as a URI, and this argument is interpreted as a local name. - :class:`QName` instances are opaque. - - - -.. _elementtree-treebuilder-objects: - -TreeBuilder Objects -^^^^^^^^^^^^^^^^^^^ - - -.. class:: TreeBuilder(element_factory=None, *, comment_factory=None, \ - pi_factory=None, insert_comments=False, insert_pis=False) - - Generic element structure builder. This builder converts a sequence of - start, data, end, comment and pi method calls to a well-formed element - structure. You can use this class to build an element structure using - a custom XML parser, or a parser for some other XML-like format. - - *element_factory*, when given, must be a callable accepting two positional - arguments: a tag and a dict of attributes. It is expected to return a new - element instance. - - The *comment_factory* and *pi_factory* functions, when given, should behave - like the :func:`Comment` and :func:`ProcessingInstruction` functions to - create comments and processing instructions. When not given, the default - factories will be used. When *insert_comments* and/or *insert_pis* is true, - comments/pis will be inserted into the tree if they appear within the root - element (but not outside of it). - - .. method:: close() - - Flushes the builder buffers, and returns the toplevel document - element. Returns an :class:`Element` instance. - - - .. method:: data(data) - - Adds text to the current element. *data* is a string. This should be - either a bytestring, or a Unicode string. - - - .. method:: end(tag) - - Closes the current element. *tag* is the element name. Returns the - closed element. - - - .. method:: start(tag, attrs) - - Opens a new element. *tag* is the element name. *attrs* is a dictionary - containing element attributes. Returns the opened element. - - - .. method:: comment(text) - - Creates a comment with the given *text*. If ``insert_comments`` is true, - this will also add it to the tree. - - .. versionadded:: 3.8 - - - .. method:: pi(target, text) - - Creates a comment with the given *target* name and *text*. If - ``insert_pis`` is true, this will also add it to the tree. - - .. versionadded:: 3.8 - - - In addition, a custom :class:`TreeBuilder` object can provide the - following methods: - - .. method:: doctype(name, pubid, system) - - Handles a doctype declaration. *name* is the doctype name. *pubid* is - the public identifier. *system* is the system identifier. This method - does not exist on the default :class:`TreeBuilder` class. - - .. versionadded:: 3.2 - - .. method:: start_ns(prefix, uri) - - Is called whenever the parser encounters a new namespace declaration, - before the ``start()`` callback for the opening element that defines it. - *prefix* is ``''`` for the default namespace and the declared - namespace prefix name otherwise. *uri* is the namespace URI. - - .. versionadded:: 3.8 - - .. method:: end_ns(prefix) - - Is called after the ``end()`` callback of an element that declared - a namespace prefix mapping, with the name of the *prefix* that went - out of scope. - - .. versionadded:: 3.8 - - -.. class:: C14NWriterTarget(write, *, \ - with_comments=False, strip_text=False, rewrite_prefixes=False, \ - qname_aware_tags=None, qname_aware_attrs=None, \ - exclude_attrs=None, exclude_tags=None) - - A `C14N 2.0 `_ writer. Arguments are the - same as for the :func:`canonicalize` function. This class does not build a - tree but translates the callback events directly into a serialised form - using the *write* function. - - .. versionadded:: 3.8 - - -.. _elementtree-xmlparser-objects: - -XMLParser Objects -^^^^^^^^^^^^^^^^^ - - -.. class:: XMLParser(*, target=None, encoding=None) - - This class is the low-level building block of the module. It uses - :mod:`xml.parsers.expat` for efficient, event-based parsing of XML. It can - be fed XML data incrementally with the :meth:`feed` method, and parsing - events are translated to a push API - by invoking callbacks on the *target* - object. If *target* is omitted, the standard :class:`TreeBuilder` is used. - If *encoding* [1]_ is given, the value overrides the - encoding specified in the XML file. - - .. versionchanged:: 3.8 - Parameters are now :ref:`keyword-only `. - The *html* argument no longer supported. - - - .. method:: close() - - Finishes feeding data to the parser. Returns the result of calling the - ``close()`` method of the *target* passed during construction; by default, - this is the toplevel document element. - - - .. method:: feed(data) - - Feeds data to the parser. *data* is encoded data. - - :meth:`XMLParser.feed` calls *target*\'s ``start(tag, attrs_dict)`` method - for each opening tag, its ``end(tag)`` method for each closing tag, and data - is processed by method ``data(data)``. For further supported callback - methods, see the :class:`TreeBuilder` class. :meth:`XMLParser.close` calls - *target*\'s method ``close()``. :class:`XMLParser` can be used not only for - building a tree structure. This is an example of counting the maximum depth - of an XML file:: - - >>> from xml.etree.ElementTree import XMLParser - >>> class MaxDepth: # The target object of the parser - ... maxDepth = 0 - ... depth = 0 - ... def start(self, tag, attrib): # Called for each opening tag. - ... self.depth += 1 - ... if self.depth > self.maxDepth: - ... self.maxDepth = self.depth - ... def end(self, tag): # Called for each closing tag. - ... self.depth -= 1 - ... def data(self, data): - ... pass # We do not need to do anything with data. - ... def close(self): # Called when all data has been parsed. - ... return self.maxDepth - ... - >>> target = MaxDepth() - >>> parser = XMLParser(target=target) - >>> exampleXml = """ - ... - ... - ... - ... - ... - ... - ... - ... - ... - ... """ - >>> parser.feed(exampleXml) - >>> parser.close() - 4 - - -.. _elementtree-xmlpullparser-objects: - -XMLPullParser Objects -^^^^^^^^^^^^^^^^^^^^^ - -.. class:: XMLPullParser(events=None) - - A pull parser suitable for non-blocking applications. Its input-side API is - similar to that of :class:`XMLParser`, but instead of pushing calls to a - callback target, :class:`XMLPullParser` collects an internal list of parsing - events and lets the user read from it. *events* is a sequence of events to - report back. The supported events are the strings ``"start"``, ``"end"``, - ``"comment"``, ``"pi"``, ``"start-ns"`` and ``"end-ns"`` (the "ns" events - are used to get detailed namespace information). If *events* is omitted, - only ``"end"`` events are reported. - - .. method:: feed(data) - - Feed the given bytes data to the parser. - - .. method:: close() - - Signal the parser that the data stream is terminated. Unlike - :meth:`XMLParser.close`, this method always returns :const:`None`. - Any events not yet retrieved when the parser is closed can still be - read with :meth:`read_events`. - - .. method:: read_events() - - Return an iterator over the events which have been encountered in the - data fed to the - parser. The iterator yields ``(event, elem)`` pairs, where *event* is a - string representing the type of event (e.g. ``"end"``) and *elem* is the - encountered :class:`Element` object, or other context value as follows. - - * ``start``, ``end``: the current Element. - * ``comment``, ``pi``: the current comment / processing instruction - * ``start-ns``: a tuple ``(prefix, uri)`` naming the declared namespace - mapping. - * ``end-ns``: :const:`None` (this may change in a future version) - - Events provided in a previous call to :meth:`read_events` will not be - yielded again. Events are consumed from the internal queue only when - they are retrieved from the iterator, so multiple readers iterating in - parallel over iterators obtained from :meth:`read_events` will have - unpredictable results. - - .. note:: - - :class:`XMLPullParser` only guarantees that it has seen the ">" - character of a starting tag when it emits a "start" event, so the - attributes are defined, but the contents of the text and tail attributes - are undefined at that point. The same applies to the element children; - they may or may not be present. - - If you need a fully populated element, look for "end" events instead. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.8 - The ``comment`` and ``pi`` events were added. - - -Exceptions -^^^^^^^^^^ - -.. class:: ParseError - - XML parse error, raised by the various parsing methods in this module when - parsing fails. The string representation of an instance of this exception - will contain a user-friendly error message. In addition, it will have - the following attributes available: - - .. attribute:: code - - A numeric error code from the expat parser. See the documentation of - :mod:`xml.parsers.expat` for the list of error codes and their meanings. - - .. attribute:: position - - A tuple of *line*, *column* numbers, specifying where the error occurred. - -.. rubric:: Footnotes - -.. [1] The encoding string included in XML output should conform to the - appropriate standards. For example, "UTF-8" is valid, but "UTF8" is - not. See https://www.w3.org/TR/2006/REC-xml11-20060816/#NT-EncodingDecl - and https://www.iana.org/assignments/character-sets/character-sets.xhtml. diff --git a/Doc/library/xml.rst b/Doc/library/xml.rst deleted file mode 100644 index 1e49b6568dfc28..00000000000000 --- a/Doc/library/xml.rst +++ /dev/null @@ -1,135 +0,0 @@ -.. _xml: - -XML Processing Modules -====================== - -.. module:: xml - :synopsis: Package containing XML processing modules - -.. sectionauthor:: Christian Heimes -.. sectionauthor:: Georg Brandl - -**Source code:** :source:`Lib/xml/` - --------------- - -Python's interfaces for processing XML are grouped in the ``xml`` package. - -.. warning:: - - The XML modules are not secure against erroneous or maliciously - constructed data. If you need to parse untrusted or - unauthenticated data see the :ref:`xml-vulnerabilities` and - :ref:`defusedxml-package` sections. - -It is important to note that modules in the :mod:`xml` package require that -there be at least one SAX-compliant XML parser available. The Expat parser is -included with Python, so the :mod:`xml.parsers.expat` module will always be -available. - -The documentation for the :mod:`xml.dom` and :mod:`xml.sax` packages are the -definition of the Python bindings for the DOM and SAX interfaces. - -The XML handling submodules are: - -* :mod:`xml.etree.ElementTree`: the ElementTree API, a simple and lightweight - XML processor - -.. - -* :mod:`xml.dom`: the DOM API definition -* :mod:`xml.dom.minidom`: a minimal DOM implementation -* :mod:`xml.dom.pulldom`: support for building partial DOM trees - -.. - -* :mod:`xml.sax`: SAX2 base classes and convenience functions -* :mod:`xml.parsers.expat`: the Expat parser binding - - -.. _xml-vulnerabilities: - -XML vulnerabilities -------------------- - -The XML processing modules are not secure against maliciously constructed data. -An attacker can abuse XML features to carry out denial of service attacks, -access local files, generate network connections to other machines, or -circumvent firewalls. - -The following table gives an overview of the known attacks and whether -the various modules are vulnerable to them. - -========================= ================== ================== ================== ================== ================== -kind sax etree minidom pulldom xmlrpc -========================= ================== ================== ================== ================== ================== -billion laughs **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) -quadratic blowup **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) **Vulnerable** (1) -external entity expansion Safe (5) Safe (2) Safe (3) Safe (5) Safe (4) -`DTD`_ retrieval Safe (5) Safe Safe Safe (5) Safe -decompression bomb Safe Safe Safe Safe **Vulnerable** -========================= ================== ================== ================== ================== ================== - -1. Expat 2.4.1 and newer is not vulnerable to the "billion laughs" and - "quadratic blowup" vulnerabilities. Items still listed as vulnerable due to - potential reliance on system-provided libraries. Check - :const:`pyexpat.EXPAT_VERSION`. -2. :mod:`xml.etree.ElementTree` doesn't expand external entities and raises a - :exc:`~xml.etree.ElementTree.ParseError` when an entity occurs. -3. :mod:`xml.dom.minidom` doesn't expand external entities and simply returns - the unexpanded entity verbatim. -4. :mod:`xmlrpc.client` doesn't expand external entities and omits them. -5. Since Python 3.7.1, external general entities are no longer processed by - default. - - -billion laughs / exponential entity expansion - The `Billion Laughs`_ attack -- also known as exponential entity expansion -- - uses multiple levels of nested entities. Each entity refers to another entity - several times, and the final entity definition contains a small string. - The exponential expansion results in several gigabytes of text and - consumes lots of memory and CPU time. - -quadratic blowup entity expansion - A quadratic blowup attack is similar to a `Billion Laughs`_ attack; it abuses - entity expansion, too. Instead of nested entities it repeats one large entity - with a couple of thousand chars over and over again. The attack isn't as - efficient as the exponential case but it avoids triggering parser countermeasures - that forbid deeply nested entities. - -external entity expansion - Entity declarations can contain more than just text for replacement. They can - also point to external resources or local files. The XML - parser accesses the resource and embeds the content into the XML document. - -`DTD`_ retrieval - Some XML libraries like Python's :mod:`xml.dom.pulldom` retrieve document type - definitions from remote or local locations. The feature has similar - implications as the external entity expansion issue. - -decompression bomb - Decompression bombs (aka `ZIP bomb`_) apply to all XML libraries - that can parse compressed XML streams such as gzipped HTTP streams or - LZMA-compressed - files. For an attacker it can reduce the amount of transmitted data by three - magnitudes or more. - -The documentation for `defusedxml`_ on PyPI has further information about -all known attack vectors with examples and references. - -.. _defusedxml-package: - -The :mod:`!defusedxml` Package ------------------------------- - -`defusedxml`_ is a pure Python package with modified subclasses of all stdlib -XML parsers that prevent any potentially malicious operation. Use of this -package is recommended for any server code that parses untrusted XML data. The -package also ships with example exploits and extended documentation on more -XML exploits such as XPath injection. - - -.. _defusedxml: https://pypi.org/project/defusedxml/ -.. _Billion Laughs: https://en.wikipedia.org/wiki/Billion_laughs -.. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb -.. _DTD: https://en.wikipedia.org/wiki/Document_type_definition diff --git a/Doc/library/xml.sax.handler.rst b/Doc/library/xml.sax.handler.rst deleted file mode 100644 index e2f28e3244cb09..00000000000000 --- a/Doc/library/xml.sax.handler.rst +++ /dev/null @@ -1,463 +0,0 @@ -:mod:`xml.sax.handler` --- Base classes for SAX handlers -======================================================== - -.. module:: xml.sax.handler - :synopsis: Base classes for SAX event handlers. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/handler.py` - --------------- - -The SAX API defines five kinds of handlers: content handlers, DTD handlers, -error handlers, entity resolvers and lexical handlers. Applications normally -only need to implement those interfaces whose events they are interested in; -they can implement the interfaces in a single object or in multiple objects. -Handler implementations should inherit from the base classes provided in the -module :mod:`xml.sax.handler`, so that all methods get default implementations. - - -.. class:: ContentHandler - - This is the main callback interface in SAX, and the one most important to - applications. The order of events in this interface mirrors the order of the - information in the document. - - -.. class:: DTDHandler - - Handle DTD events. - - This interface specifies only those DTD events required for basic parsing - (unparsed entities and attributes). - - -.. class:: EntityResolver - - Basic interface for resolving entities. If you create an object implementing - this interface, then register the object with your Parser, the parser will call - the method in your object to resolve all external entities. - - -.. class:: ErrorHandler - - Interface used by the parser to present error and warning messages to the - application. The methods of this object control whether errors are immediately - converted to exceptions or are handled in some other way. - - -.. class:: LexicalHandler - - Interface used by the parser to represent low frequency events which may not - be of interest to many applications. - -In addition to these classes, :mod:`xml.sax.handler` provides symbolic constants -for the feature and property names. - - -.. data:: feature_namespaces - - | value: ``"http://xml.org/sax/features/namespaces"`` - | true: Perform Namespace processing. - | false: Optionally do not perform Namespace processing (implies - namespace-prefixes; default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_namespace_prefixes - - | value: ``"http://xml.org/sax/features/namespace-prefixes"`` - | true: Report the original prefixed names and attributes used for Namespace - declarations. - | false: Do not report attributes used for Namespace declarations, and - optionally do not report original prefixed names (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_string_interning - - | value: ``"http://xml.org/sax/features/string-interning"`` - | true: All element names, prefixes, attribute names, Namespace URIs, and - local names are interned using the built-in intern function. - | false: Names are not necessarily interned, although they may be (default). - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_validation - - | value: ``"http://xml.org/sax/features/validation"`` - | true: Report all validation errors (implies external-general-entities and - external-parameter-entities). - | false: Do not report validation errors. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_ges - - | value: ``"http://xml.org/sax/features/external-general-entities"`` - | true: Include all external general (text) entities. - | false: Do not include external general entities. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: feature_external_pes - - | value: ``"http://xml.org/sax/features/external-parameter-entities"`` - | true: Include all external parameter entities, including the external DTD - subset. - | false: Do not include any external parameter entities, even the external - DTD subset. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: all_features - - List of all features. - - -.. data:: property_lexical_handler - - | value: ``"http://xml.org/sax/properties/lexical-handler"`` - | data type: xml.sax.handler.LexicalHandler (not supported in Python 2) - | description: An optional extension handler for lexical events like - comments. - | access: read/write - - -.. data:: property_declaration_handler - - | value: ``"http://xml.org/sax/properties/declaration-handler"`` - | data type: xml.sax.sax2lib.DeclHandler (not supported in Python 2) - | description: An optional extension handler for DTD-related events other - than notations and unparsed entities. - | access: read/write - - -.. data:: property_dom_node - - | value: ``"http://xml.org/sax/properties/dom-node"`` - | data type: org.w3c.dom.Node (not supported in Python 2) - | description: When parsing, the current DOM node being visited if this is - a DOM iterator; when not parsing, the root DOM node for iteration. - | access: (parsing) read-only; (not parsing) read/write - - -.. data:: property_xml_string - - | value: ``"http://xml.org/sax/properties/xml-string"`` - | data type: Bytes - | description: The literal string of characters that was the source for the - current event. - | access: read-only - - -.. data:: all_properties - - List of all known property names. - - -.. _content-handler-objects: - -ContentHandler Objects ----------------------- - -Users are expected to subclass :class:`ContentHandler` to support their -application. The following methods are called by the parser on the appropriate -events in the input document: - - -.. method:: ContentHandler.setDocumentLocator(locator) - - Called by the parser to give the application a locator for locating the origin - of document events. - - SAX parsers are strongly encouraged (though not absolutely required) to supply a - locator: if it does so, it must supply the locator to the application by - invoking this method before invoking any of the other methods in the - DocumentHandler interface. - - The locator allows the application to determine the end position of any - document-related event, even if the parser is not reporting an error. Typically, - the application will use this information for reporting its own errors (such as - character content that does not match an application's business rules). The - information returned by the locator is probably not sufficient for use with a - search engine. - - Note that the locator will return correct information only during the invocation - of the events in this interface. The application should not attempt to use it at - any other time. - - -.. method:: ContentHandler.startDocument() - - Receive notification of the beginning of a document. - - The SAX parser will invoke this method only once, before any other methods in - this interface or in DTDHandler (except for :meth:`setDocumentLocator`). - - -.. method:: ContentHandler.endDocument() - - Receive notification of the end of a document. - - The SAX parser will invoke this method only once, and it will be the last method - invoked during the parse. The parser shall not invoke this method until it has - either abandoned parsing (because of an unrecoverable error) or reached the end - of input. - - -.. method:: ContentHandler.startPrefixMapping(prefix, uri) - - Begin the scope of a prefix-URI Namespace mapping. - - The information from this event is not necessary for normal Namespace - processing: the SAX XML reader will automatically replace prefixes for element - and attribute names when the ``feature_namespaces`` feature is enabled (the - default). - - There are cases, however, when applications need to use prefixes in character - data or in attribute values, where they cannot safely be expanded automatically; - the :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events supply the - information to the application to expand prefixes in those contexts itself, if - necessary. - - .. XXX This is not really the default, is it? MvL - - Note that :meth:`startPrefixMapping` and :meth:`endPrefixMapping` events are not - guaranteed to be properly nested relative to each-other: all - :meth:`startPrefixMapping` events will occur before the corresponding - :meth:`startElement` event, and all :meth:`endPrefixMapping` events will occur - after the corresponding :meth:`endElement` event, but their order is not - guaranteed. - - -.. method:: ContentHandler.endPrefixMapping(prefix) - - End the scope of a prefix-URI mapping. - - See :meth:`startPrefixMapping` for details. This event will always occur after - the corresponding :meth:`endElement` event, but the order of - :meth:`endPrefixMapping` events is not otherwise guaranteed. - - -.. method:: ContentHandler.startElement(name, attrs) - - Signals the start of an element in non-namespace mode. - - The *name* parameter contains the raw XML 1.0 name of the element type as a - string and the *attrs* parameter holds an object of the - :class:`~xml.sax.xmlreader.Attributes` - interface (see :ref:`attributes-objects`) containing the attributes of - the element. The object passed as *attrs* may be re-used by the parser; holding - on to a reference to it is not a reliable way to keep a copy of the attributes. - To keep a copy of the attributes, use the :meth:`copy` method of the *attrs* - object. - - -.. method:: ContentHandler.endElement(name) - - Signals the end of an element in non-namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElement` event. - - -.. method:: ContentHandler.startElementNS(name, qname, attrs) - - Signals the start of an element in namespace mode. - - The *name* parameter contains the name of the element type as a ``(uri, - localname)`` tuple, the *qname* parameter contains the raw XML 1.0 name used in - the source document, and the *attrs* parameter holds an instance of the - :class:`~xml.sax.xmlreader.AttributesNS` interface (see - :ref:`attributes-ns-objects`) - containing the attributes of the element. If no namespace is associated with - the element, the *uri* component of *name* will be ``None``. The object passed - as *attrs* may be re-used by the parser; holding on to a reference to it is not - a reliable way to keep a copy of the attributes. To keep a copy of the - attributes, use the :meth:`copy` method of the *attrs* object. - - Parsers may set the *qname* parameter to ``None``, unless the - ``feature_namespace_prefixes`` feature is activated. - - -.. method:: ContentHandler.endElementNS(name, qname) - - Signals the end of an element in namespace mode. - - The *name* parameter contains the name of the element type, just as with the - :meth:`startElementNS` method, likewise the *qname* parameter. - - -.. method:: ContentHandler.characters(content) - - Receive notification of character data. - - The Parser will call this method to report each chunk of character data. SAX - parsers may return all contiguous character data in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity so that the Locator provides useful - information. - - *content* may be a string or bytes instance; the ``expat`` reader module - always produces strings. - - .. note:: - - The earlier SAX 1 interface provided by the Python XML Special Interest Group - used a more Java-like interface for this method. Since most parsers used from - Python did not take advantage of the older interface, the simpler signature was - chosen to replace it. To convert old code to the new interface, use *content* - instead of slicing content with the old *offset* and *length* parameters. - - -.. method:: ContentHandler.ignorableWhitespace(whitespace) - - Receive notification of ignorable whitespace in element content. - - Validating Parsers must use this method to report each chunk of ignorable - whitespace (see the W3C XML 1.0 recommendation, section 2.10): non-validating - parsers may also use this method if they are capable of parsing and using - content models. - - SAX parsers may return all contiguous whitespace in a single chunk, or they may - split it into several chunks; however, all of the characters in any single event - must come from the same external entity, so that the Locator provides useful - information. - - -.. method:: ContentHandler.processingInstruction(target, data) - - Receive notification of a processing instruction. - - The Parser will invoke this method once for each processing instruction found: - note that processing instructions may occur before or after the main document - element. - - A SAX parser should never report an XML declaration (XML 1.0, section 2.8) or a - text declaration (XML 1.0, section 4.3.1) using this method. - - -.. method:: ContentHandler.skippedEntity(name) - - Receive notification of a skipped entity. - - The Parser will invoke this method once for each entity skipped. Non-validating - processors may skip entities if they have not seen the declarations (because, - for example, the entity was declared in an external DTD subset). All processors - may skip external entities, depending on the values of the - ``feature_external_ges`` and the ``feature_external_pes`` properties. - - -.. _dtd-handler-objects: - -DTDHandler Objects ------------------- - -:class:`DTDHandler` instances provide the following methods: - - -.. method:: DTDHandler.notationDecl(name, publicId, systemId) - - Handle a notation declaration event. - - -.. method:: DTDHandler.unparsedEntityDecl(name, publicId, systemId, ndata) - - Handle an unparsed entity declaration event. - - -.. _entity-resolver-objects: - -EntityResolver Objects ----------------------- - - -.. method:: EntityResolver.resolveEntity(publicId, systemId) - - Resolve the system identifier of an entity and return either the system - identifier to read from as a string, or an InputSource to read from. The default - implementation returns *systemId*. - - -.. _sax-error-handler: - -ErrorHandler Objects --------------------- - -Objects with this interface are used to receive error and warning information -from the :class:`~xml.sax.xmlreader.XMLReader`. If you create an object that -implements this interface, then register the object with your -:class:`~xml.sax.xmlreader.XMLReader`, the parser -will call the methods in your object to report all warnings and errors. There -are three levels of errors available: warnings, (possibly) recoverable errors, -and unrecoverable errors. All methods take a :exc:`~xml.sax.SAXParseException` as the -only parameter. Errors and warnings may be converted to an exception by raising -the passed-in exception object. - - -.. method:: ErrorHandler.error(exception) - - Called when the parser encounters a recoverable error. If this method does not - raise an exception, parsing may continue, but further document information - should not be expected by the application. Allowing the parser to continue may - allow additional errors to be discovered in the input document. - - -.. method:: ErrorHandler.fatalError(exception) - - Called when the parser encounters an error it cannot recover from; parsing is - expected to terminate when this method returns. - - -.. method:: ErrorHandler.warning(exception) - - Called when the parser presents minor warning information to the application. - Parsing is expected to continue when this method returns, and document - information will continue to be passed to the application. Raising an exception - in this method will cause parsing to end. - - -.. _lexical-handler-objects: - -LexicalHandler Objects ----------------------- -Optional SAX2 handler for lexical events. - -This handler is used to obtain lexical information about an XML -document. Lexical information includes information describing the -document encoding used and XML comments embedded in the document, as -well as section boundaries for the DTD and for any CDATA sections. -The lexical handlers are used in the same manner as content handlers. - -Set the LexicalHandler of an XMLReader by using the setProperty method -with the property identifier -``'http://xml.org/sax/properties/lexical-handler'``. - - -.. method:: LexicalHandler.comment(content) - - Reports a comment anywhere in the document (including the DTD and - outside the document element). - -.. method:: LexicalHandler.startDTD(name, public_id, system_id) - - Reports the start of the DTD declarations if the document has an - associated DTD. - -.. method:: LexicalHandler.endDTD() - - Reports the end of DTD declaration. - -.. method:: LexicalHandler.startCDATA() - - Reports the start of a CDATA marked section. - - The contents of the CDATA marked section will be reported through - the characters handler. - -.. method:: LexicalHandler.endCDATA() - - Reports the end of a CDATA marked section. diff --git a/Doc/library/xml.sax.reader.rst b/Doc/library/xml.sax.reader.rst deleted file mode 100644 index 113e9e93fb04ff..00000000000000 --- a/Doc/library/xml.sax.reader.rst +++ /dev/null @@ -1,397 +0,0 @@ -:mod:`xml.sax.xmlreader` --- Interface for XML parsers -====================================================== - -.. module:: xml.sax.xmlreader - :synopsis: Interface which SAX-compliant XML parsers must implement. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/xmlreader.py` - --------------- - -SAX parsers implement the :class:`XMLReader` interface. They are implemented in -a Python module, which must provide a function :func:`create_parser`. This -function is invoked by :func:`xml.sax.make_parser` with no arguments to create -a new parser object. - - -.. class:: XMLReader() - - Base class which can be inherited by SAX parsers. - - -.. class:: IncrementalParser() - - In some cases, it is desirable not to parse an input source at once, but to feed - chunks of the document as they get available. Note that the reader will normally - not read the entire file, but read it in chunks as well; still :meth:`parse` - won't return until the entire document is processed. So these interfaces should - be used if the blocking behaviour of :meth:`parse` is not desirable. - - When the parser is instantiated it is ready to begin accepting data from the - feed method immediately. After parsing has been finished with a call to close - the reset method must be called to make the parser ready to accept new data, - either from feed or using the parse method. - - Note that these methods must *not* be called during parsing, that is, after - parse has been called and before it returns. - - By default, the class also implements the parse method of the XMLReader - interface using the feed, close and reset methods of the IncrementalParser - interface as a convenience to SAX 2.0 driver writers. - - -.. class:: Locator() - - Interface for associating a SAX event with a document location. A locator object - will return valid results only during calls to DocumentHandler methods; at any - other time, the results are unpredictable. If information is not available, - methods may return ``None``. - - -.. class:: InputSource(system_id=None) - - Encapsulation of the information needed by the :class:`XMLReader` to read - entities. - - This class may include information about the public identifier, system - identifier, byte stream (possibly with character encoding information) and/or - the character stream of an entity. - - Applications will create objects of this class for use in the - :meth:`XMLReader.parse` method and for returning from - EntityResolver.resolveEntity. - - An :class:`InputSource` belongs to the application, the :class:`XMLReader` is - not allowed to modify :class:`InputSource` objects passed to it from the - application, although it may make copies and modify those. - - -.. class:: AttributesImpl(attrs) - - This is an implementation of the :class:`Attributes` interface (see section - :ref:`attributes-objects`). This is a dictionary-like object which - represents the element attributes in a :meth:`startElement` call. In addition - to the most useful dictionary operations, it supports a number of other - methods as described by the interface. Objects of this class should be - instantiated by readers; *attrs* must be a dictionary-like object containing - a mapping from attribute names to attribute values. - - -.. class:: AttributesNSImpl(attrs, qnames) - - Namespace-aware variant of :class:`AttributesImpl`, which will be passed to - :meth:`startElementNS`. It is derived from :class:`AttributesImpl`, but - understands attribute names as two-tuples of *namespaceURI* and - *localname*. In addition, it provides a number of methods expecting qualified - names as they appear in the original document. This class implements the - :class:`AttributesNS` interface (see section :ref:`attributes-ns-objects`). - - -.. _xmlreader-objects: - -XMLReader Objects ------------------ - -The :class:`XMLReader` interface supports the following methods: - - -.. method:: XMLReader.parse(source) - - Process an input source, producing SAX events. The *source* object can be a - system identifier (a string identifying the input source -- typically a file - name or a URL), a :class:`pathlib.Path` or :term:`path-like ` - object, or an :class:`InputSource` object. When - :meth:`parse` returns, the input is completely processed, and the parser object - can be discarded or reset. - - .. versionchanged:: 3.5 - Added support of character streams. - - .. versionchanged:: 3.8 - Added support of path-like objects. - - -.. method:: XMLReader.getContentHandler() - - Return the current :class:`~xml.sax.handler.ContentHandler`. - - -.. method:: XMLReader.setContentHandler(handler) - - Set the current :class:`~xml.sax.handler.ContentHandler`. If no - :class:`~xml.sax.handler.ContentHandler` is set, content events will be - discarded. - - -.. method:: XMLReader.getDTDHandler() - - Return the current :class:`~xml.sax.handler.DTDHandler`. - - -.. method:: XMLReader.setDTDHandler(handler) - - Set the current :class:`~xml.sax.handler.DTDHandler`. If no - :class:`~xml.sax.handler.DTDHandler` is set, DTD - events will be discarded. - - -.. method:: XMLReader.getEntityResolver() - - Return the current :class:`~xml.sax.handler.EntityResolver`. - - -.. method:: XMLReader.setEntityResolver(handler) - - Set the current :class:`~xml.sax.handler.EntityResolver`. If no - :class:`~xml.sax.handler.EntityResolver` is set, - attempts to resolve an external entity will result in opening the system - identifier for the entity, and fail if it is not available. - - -.. method:: XMLReader.getErrorHandler() - - Return the current :class:`~xml.sax.handler.ErrorHandler`. - - -.. method:: XMLReader.setErrorHandler(handler) - - Set the current error handler. If no :class:`~xml.sax.handler.ErrorHandler` - is set, errors will be raised as exceptions, and warnings will be printed. - - -.. method:: XMLReader.setLocale(locale) - - Allow an application to set the locale for errors and warnings. - - SAX parsers are not required to provide localization for errors and warnings; if - they cannot support the requested locale, however, they must raise a SAX - exception. Applications may request a locale change in the middle of a parse. - - -.. method:: XMLReader.getFeature(featurename) - - Return the current setting for feature *featurename*. If the feature is not - recognized, :exc:`SAXNotRecognizedException` is raised. The well-known - featurenames are listed in the module :mod:`xml.sax.handler`. - - -.. method:: XMLReader.setFeature(featurename, value) - - Set the *featurename* to *value*. If the feature is not recognized, - :exc:`SAXNotRecognizedException` is raised. If the feature or its setting is not - supported by the parser, *SAXNotSupportedException* is raised. - - -.. method:: XMLReader.getProperty(propertyname) - - Return the current setting for property *propertyname*. If the property is not - recognized, a :exc:`SAXNotRecognizedException` is raised. The well-known - propertynames are listed in the module :mod:`xml.sax.handler`. - - -.. method:: XMLReader.setProperty(propertyname, value) - - Set the *propertyname* to *value*. If the property is not recognized, - :exc:`SAXNotRecognizedException` is raised. If the property or its setting is - not supported by the parser, *SAXNotSupportedException* is raised. - - -.. _incremental-parser-objects: - -IncrementalParser Objects -------------------------- - -Instances of :class:`IncrementalParser` offer the following additional methods: - - -.. method:: IncrementalParser.feed(data) - - Process a chunk of *data*. - - -.. method:: IncrementalParser.close() - - Assume the end of the document. That will check well-formedness conditions that - can be checked only at the end, invoke handlers, and may clean up resources - allocated during parsing. - - -.. method:: IncrementalParser.reset() - - This method is called after close has been called to reset the parser so that it - is ready to parse new documents. The results of calling parse or feed after - close without calling reset are undefined. - - -.. _locator-objects: - -Locator Objects ---------------- - -Instances of :class:`Locator` provide these methods: - - -.. method:: Locator.getColumnNumber() - - Return the column number where the current event begins. - - -.. method:: Locator.getLineNumber() - - Return the line number where the current event begins. - - -.. method:: Locator.getPublicId() - - Return the public identifier for the current event. - - -.. method:: Locator.getSystemId() - - Return the system identifier for the current event. - - -.. _input-source-objects: - -InputSource Objects -------------------- - - -.. method:: InputSource.setPublicId(id) - - Sets the public identifier of this :class:`InputSource`. - - -.. method:: InputSource.getPublicId() - - Returns the public identifier of this :class:`InputSource`. - - -.. method:: InputSource.setSystemId(id) - - Sets the system identifier of this :class:`InputSource`. - - -.. method:: InputSource.getSystemId() - - Returns the system identifier of this :class:`InputSource`. - - -.. method:: InputSource.setEncoding(encoding) - - Sets the character encoding of this :class:`InputSource`. - - The encoding must be a string acceptable for an XML encoding declaration (see - section 4.3.3 of the XML recommendation). - - The encoding attribute of the :class:`InputSource` is ignored if the - :class:`InputSource` also contains a character stream. - - -.. method:: InputSource.getEncoding() - - Get the character encoding of this InputSource. - - -.. method:: InputSource.setByteStream(bytefile) - - Set the byte stream (a :term:`binary file`) for this input source. - - The SAX parser will ignore this if there is also a character stream specified, - but it will use a byte stream in preference to opening a URI connection itself. - - If the application knows the character encoding of the byte stream, it should - set it with the setEncoding method. - - -.. method:: InputSource.getByteStream() - - Get the byte stream for this input source. - - The getEncoding method will return the character encoding for this byte stream, - or ``None`` if unknown. - - -.. method:: InputSource.setCharacterStream(charfile) - - Set the character stream (a :term:`text file`) for this input source. - - If there is a character stream specified, the SAX parser will ignore any byte - stream and will not attempt to open a URI connection to the system identifier. - - -.. method:: InputSource.getCharacterStream() - - Get the character stream for this input source. - - -.. _attributes-objects: - -The :class:`Attributes` Interface ---------------------------------- - -:class:`Attributes` objects implement a portion of the :term:`mapping protocol -`, including the methods :meth:`~collections.abc.Mapping.copy`, -:meth:`~collections.abc.Mapping.get`, :meth:`~object.__contains__`, -:meth:`~collections.abc.Mapping.items`, :meth:`~collections.abc.Mapping.keys`, -and :meth:`~collections.abc.Mapping.values`. The following methods -are also provided: - - -.. method:: Attributes.getLength() - - Return the number of attributes. - - -.. method:: Attributes.getNames() - - Return the names of the attributes. - - -.. method:: Attributes.getType(name) - - Returns the type of the attribute *name*, which is normally ``'CDATA'``. - - -.. method:: Attributes.getValue(name) - - Return the value of attribute *name*. - -.. getValueByQName, getNameByQName, getQNameByName, getQNames available -.. here already, but documented only for derived class. - - -.. _attributes-ns-objects: - -The :class:`AttributesNS` Interface ------------------------------------ - -This interface is a subtype of the :class:`Attributes` interface (see section -:ref:`attributes-objects`). All methods supported by that interface are also -available on :class:`AttributesNS` objects. - -The following methods are also available: - - -.. method:: AttributesNS.getValueByQName(name) - - Return the value for a qualified name. - - -.. method:: AttributesNS.getNameByQName(name) - - Return the ``(namespace, localname)`` pair for a qualified *name*. - - -.. method:: AttributesNS.getQNameByName(name) - - Return the qualified name for a ``(namespace, localname)`` pair. - - -.. method:: AttributesNS.getQNames() - - Return the qualified names of all attributes. - diff --git a/Doc/library/xml.sax.rst b/Doc/library/xml.sax.rst deleted file mode 100644 index 6d351dfb4d7072..00000000000000 --- a/Doc/library/xml.sax.rst +++ /dev/null @@ -1,176 +0,0 @@ -:mod:`xml.sax` --- Support for SAX2 parsers -=========================================== - -.. module:: xml.sax - :synopsis: Package containing SAX2 base classes and convenience functions. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Fred L. Drake, Jr. -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/__init__.py` - --------------- - -The :mod:`xml.sax` package provides a number of modules which implement the -Simple API for XML (SAX) interface for Python. The package itself provides the -SAX exceptions and the convenience functions which will be most used by users of -the SAX API. - - -.. warning:: - - The :mod:`xml.sax` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - -.. versionchanged:: 3.7.1 - - The SAX parser no longer processes general external entities by default - to increase security. Before, the parser created network connections - to fetch remote files or loaded local files from the file - system for DTD and entities. The feature can be enabled again with method - :meth:`~xml.sax.xmlreader.XMLReader.setFeature` on the parser object - and argument :data:`~xml.sax.handler.feature_external_ges`. - -The convenience functions are: - - -.. function:: make_parser(parser_list=[]) - - Create and return a SAX :class:`~xml.sax.xmlreader.XMLReader` object. The - first parser found will - be used. If *parser_list* is provided, it must be an iterable of strings which - name modules that have a function named :func:`create_parser`. Modules listed - in *parser_list* will be used before modules in the default list of parsers. - - .. versionchanged:: 3.8 - The *parser_list* argument can be any iterable, not just a list. - - -.. function:: parse(filename_or_stream, handler, error_handler=handler.ErrorHandler()) - - Create a SAX parser and use it to parse a document. The document, passed in as - *filename_or_stream*, can be a filename or a file object. The *handler* - parameter needs to be a SAX :class:`~handler.ContentHandler` instance. If - *error_handler* is given, it must be a SAX :class:`~handler.ErrorHandler` - instance; if - omitted, :exc:`SAXParseException` will be raised on all errors. There is no - return value; all work must be done by the *handler* passed in. - - -.. function:: parseString(string, handler, error_handler=handler.ErrorHandler()) - - Similar to :func:`parse`, but parses from a buffer *string* received as a - parameter. *string* must be a :class:`str` instance or a - :term:`bytes-like object`. - - .. versionchanged:: 3.5 - Added support of :class:`str` instances. - -A typical SAX application uses three kinds of objects: readers, handlers and -input sources. "Reader" in this context is another term for parser, i.e. some -piece of code that reads the bytes or characters from the input source, and -produces a sequence of events. The events then get distributed to the handler -objects, i.e. the reader invokes a method on the handler. A SAX application -must therefore obtain a reader object, create or open the input sources, create -the handlers, and connect these objects all together. As the final step of -preparation, the reader is called to parse the input. During parsing, methods on -the handler objects are called based on structural and syntactic events from the -input data. - -For these objects, only the interfaces are relevant; they are normally not -instantiated by the application itself. Since Python does not have an explicit -notion of interface, they are formally introduced as classes, but applications -may use implementations which do not inherit from the provided classes. The -:class:`~xml.sax.xmlreader.InputSource`, :class:`~xml.sax.xmlreader.Locator`, -:class:`~xml.sax.xmlreader.Attributes`, :class:`~xml.sax.xmlreader.AttributesNS`, -and :class:`~xml.sax.xmlreader.XMLReader` interfaces are defined in the -module :mod:`xml.sax.xmlreader`. The handler interfaces are defined in -:mod:`xml.sax.handler`. For convenience, -:class:`~xml.sax.xmlreader.InputSource` (which is often -instantiated directly) and the handler classes are also available from -:mod:`xml.sax`. These interfaces are described below. - -In addition to these classes, :mod:`xml.sax` provides the following exception -classes. - - -.. exception:: SAXException(msg, exception=None) - - Encapsulate an XML error or warning. This class can contain basic error or - warning information from either the XML parser or the application: it can be - subclassed to provide additional functionality or to add localization. Note - that although the handlers defined in the - :class:`~xml.sax.handler.ErrorHandler` interface - receive instances of this exception, it is not required to actually raise the - exception --- it is also useful as a container for information. - - When instantiated, *msg* should be a human-readable description of the error. - The optional *exception* parameter, if given, should be ``None`` or an exception - that was caught by the parsing code and is being passed along as information. - - This is the base class for the other SAX exception classes. - - -.. exception:: SAXParseException(msg, exception, locator) - - Subclass of :exc:`SAXException` raised on parse errors. Instances of this - class are passed to the methods of the SAX - :class:`~xml.sax.handler.ErrorHandler` interface to provide information - about the parse error. This class supports the SAX - :class:`~xml.sax.xmlreader.Locator` interface as well as the - :class:`SAXException` interface. - - -.. exception:: SAXNotRecognizedException(msg, exception=None) - - Subclass of :exc:`SAXException` raised when a SAX - :class:`~xml.sax.xmlreader.XMLReader` is - confronted with an unrecognized feature or property. SAX applications and - extensions may use this class for similar purposes. - - -.. exception:: SAXNotSupportedException(msg, exception=None) - - Subclass of :exc:`SAXException` raised when a SAX - :class:`~xml.sax.xmlreader.XMLReader` is asked to - enable a feature that is not supported, or to set a property to a value that the - implementation does not support. SAX applications and extensions may use this - class for similar purposes. - - -.. seealso:: - - `SAX: The Simple API for XML `_ - This site is the focal point for the definition of the SAX API. It provides a - Java implementation and online documentation. Links to implementations and - historical information are also available. - - Module :mod:`xml.sax.handler` - Definitions of the interfaces for application-provided objects. - - Module :mod:`xml.sax.saxutils` - Convenience functions for use in SAX applications. - - Module :mod:`xml.sax.xmlreader` - Definitions of the interfaces for parser-provided objects. - - -.. _sax-exception-objects: - -SAXException Objects --------------------- - -The :class:`SAXException` exception class supports the following methods: - - -.. method:: SAXException.getMessage() - - Return a human-readable message describing the error condition. - - -.. method:: SAXException.getException() - - Return an encapsulated exception object, or ``None``. - diff --git a/Doc/library/xml.sax.utils.rst b/Doc/library/xml.sax.utils.rst deleted file mode 100644 index 8c1be43d1f2776..00000000000000 --- a/Doc/library/xml.sax.utils.rst +++ /dev/null @@ -1,91 +0,0 @@ -:mod:`xml.sax.saxutils` --- SAX Utilities -========================================= - -.. module:: xml.sax.saxutils - :synopsis: Convenience functions and classes for use with SAX. - -.. moduleauthor:: Lars Marius Garshol -.. sectionauthor:: Martin v. Löwis - -**Source code:** :source:`Lib/xml/sax/saxutils.py` - --------------- - -The module :mod:`xml.sax.saxutils` contains a number of classes and functions -that are commonly useful when creating SAX applications, either in direct use, -or as base classes. - - -.. function:: escape(data, entities={}) - - Escape ``'&'``, ``'<'``, and ``'>'`` in a string of data. - - You can escape other strings of data by passing a dictionary as the optional - *entities* parameter. The keys and values must all be strings; each key will be - replaced with its corresponding value. The characters ``'&'``, ``'<'`` and - ``'>'`` are always escaped, even if *entities* is provided. - - -.. function:: unescape(data, entities={}) - - Unescape ``'&'``, ``'<'``, and ``'>'`` in a string of data. - - You can unescape other strings of data by passing a dictionary as the optional - *entities* parameter. The keys and values must all be strings; each key will be - replaced with its corresponding value. ``'&'``, ``'<'``, and ``'>'`` - are always unescaped, even if *entities* is provided. - - -.. function:: quoteattr(data, entities={}) - - Similar to :func:`escape`, but also prepares *data* to be used as an - attribute value. The return value is a quoted version of *data* with any - additional required replacements. :func:`quoteattr` will select a quote - character based on the content of *data*, attempting to avoid encoding any - quote characters in the string. If both single- and double-quote characters - are already in *data*, the double-quote characters will be encoded and *data* - will be wrapped in double-quotes. The resulting string can be used directly - as an attribute value:: - - >>> print("" % quoteattr("ab ' cd \" ef")) - - - This function is useful when generating attribute values for HTML or any SGML - using the reference concrete syntax. - - -.. class:: XMLGenerator(out=None, encoding='iso-8859-1', short_empty_elements=False) - - This class implements the :class:`~xml.sax.handler.ContentHandler` interface - by writing SAX - events back into an XML document. In other words, using an :class:`XMLGenerator` - as the content handler will reproduce the original document being parsed. *out* - should be a file-like object which will default to *sys.stdout*. *encoding* is - the encoding of the output stream which defaults to ``'iso-8859-1'``. - *short_empty_elements* controls the formatting of elements that contain no - content: if ``False`` (the default) they are emitted as a pair of start/end - tags, if set to ``True`` they are emitted as a single self-closed tag. - - .. versionadded:: 3.2 - The *short_empty_elements* parameter. - - -.. class:: XMLFilterBase(base) - - This class is designed to sit between an - :class:`~xml.sax.xmlreader.XMLReader` and the client - application's event handlers. By default, it does nothing but pass requests up - to the reader and events on to the handlers unmodified, but subclasses can - override specific methods to modify the event stream or the configuration - requests as they pass through. - - -.. function:: prepare_input_source(source, base='') - - This function takes an input source and an optional base URL and returns a - fully resolved :class:`~xml.sax.xmlreader.InputSource` object ready for - reading. The input source can be given as a string, a file-like object, or - an :class:`~xml.sax.xmlreader.InputSource` object; parsers will use this - function to implement the polymorphic *source* argument to their - :meth:`~xml.sax.xmlreader.XMLReader.parse` method. - diff --git a/Doc/library/xmlrpc.client.rst b/Doc/library/xmlrpc.client.rst deleted file mode 100644 index 146c4fd768233b..00000000000000 --- a/Doc/library/xmlrpc.client.rst +++ /dev/null @@ -1,608 +0,0 @@ -:mod:`xmlrpc.client` --- XML-RPC client access -============================================== - -.. module:: xmlrpc.client - :synopsis: XML-RPC client access. - -.. moduleauthor:: Fredrik Lundh -.. sectionauthor:: Eric S. Raymond - -**Source code:** :source:`Lib/xmlrpc/client.py` - -.. XXX Not everything is documented yet. It might be good to describe - Marshaller, Unmarshaller, getparser and Transport. - --------------- - -XML-RPC is a Remote Procedure Call method that uses XML passed via HTTP(S) as a -transport. With it, a client can call methods with parameters on a remote -server (the server is named by a URI) and get back structured data. This module -supports writing XML-RPC client code; it handles all the details of translating -between conformable Python objects and XML on the wire. - - -.. warning:: - - The :mod:`xmlrpc.client` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - -.. versionchanged:: 3.5 - - For HTTPS URIs, :mod:`xmlrpc.client` now performs all the necessary - certificate and hostname checks by default. - -.. include:: ../includes/wasm-notavail.rst - -.. class:: ServerProxy(uri, transport=None, encoding=None, verbose=False, \ - allow_none=False, use_datetime=False, \ - use_builtin_types=False, *, headers=(), context=None) - - A :class:`ServerProxy` instance is an object that manages communication with a - remote XML-RPC server. The required first argument is a URI (Uniform Resource - Indicator), and will normally be the URL of the server. The optional second - argument is a transport factory instance; by default it is an internal - :class:`SafeTransport` instance for https: URLs and an internal HTTP - :class:`Transport` instance otherwise. The optional third argument is an - encoding, by default UTF-8. The optional fourth argument is a debugging flag. - - The following parameters govern the use of the returned proxy instance. - If *allow_none* is true, the Python constant ``None`` will be translated into - XML; the default behaviour is for ``None`` to raise a :exc:`TypeError`. This is - a commonly used extension to the XML-RPC specification, but isn't supported by - all clients and servers; see `http://ontosys.com/xml-rpc/extensions.php - `_ - for a description. - The *use_builtin_types* flag can be used to cause date/time values - to be presented as :class:`datetime.datetime` objects and binary data to be - presented as :class:`bytes` objects; this flag is false by default. - :class:`datetime.datetime`, :class:`bytes` and :class:`bytearray` objects - may be passed to calls. - The *headers* parameter is an optional sequence of HTTP headers to send with - each request, expressed as a sequence of 2-tuples representing the header - name and value. (e.g. ``[('Header-Name', 'value')]``). - The obsolete *use_datetime* flag is similar to *use_builtin_types* but it - applies only to date/time values. - -.. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - -.. versionchanged:: 3.8 - The *headers* parameter was added. - - Both the HTTP and HTTPS transports support the URL syntax extension for HTTP - Basic Authentication: ``http://user:pass@host:port/path``. The ``user:pass`` - portion will be base64-encoded as an HTTP 'Authorization' header, and sent to - the remote server as part of the connection process when invoking an XML-RPC - method. You only need to use this if the remote server requires a Basic - Authentication user and password. If an HTTPS URL is provided, *context* may - be :class:`ssl.SSLContext` and configures the SSL settings of the underlying - HTTPS connection. - - The returned instance is a proxy object with methods that can be used to invoke - corresponding RPC calls on the remote server. If the remote server supports the - introspection API, the proxy can also be used to query the remote server for the - methods it supports (service discovery) and fetch other server-associated - metadata. - - Types that are conformable (e.g. that can be marshalled through XML), - include the following (and except where noted, they are unmarshalled - as the same Python type): - - .. tabularcolumns:: |l|L| - - +----------------------+-------------------------------------------------------+ - | XML-RPC type | Python type | - +======================+=======================================================+ - | ``boolean`` | :class:`bool` | - +----------------------+-------------------------------------------------------+ - | ``int``, ``i1``, | :class:`int` in range from -2147483648 to 2147483647. | - | ``i2``, ``i4``, | Values get the ```` tag. | - | ``i8`` or | | - | ``biginteger`` | | - +----------------------+-------------------------------------------------------+ - | ``double`` or | :class:`float`. Values get the ```` tag. | - | ``float`` | | - +----------------------+-------------------------------------------------------+ - | ``string`` | :class:`str` | - +----------------------+-------------------------------------------------------+ - | ``array`` | :class:`list` or :class:`tuple` containing | - | | conformable elements. Arrays are returned as | - | | :class:`lists `. | - +----------------------+-------------------------------------------------------+ - | ``struct`` | :class:`dict`. Keys must be strings, values may be | - | | any conformable type. Objects of user-defined | - | | classes can be passed in; only their | - | | :attr:`~object.__dict__` attribute is transmitted. | - +----------------------+-------------------------------------------------------+ - | ``dateTime.iso8601`` | :class:`DateTime` or :class:`datetime.datetime`. | - | | Returned type depends on values of | - | | *use_builtin_types* and *use_datetime* flags. | - +----------------------+-------------------------------------------------------+ - | ``base64`` | :class:`Binary`, :class:`bytes` or | - | | :class:`bytearray`. Returned type depends on the | - | | value of the *use_builtin_types* flag. | - +----------------------+-------------------------------------------------------+ - | ``nil`` | The ``None`` constant. Passing is allowed only if | - | | *allow_none* is true. | - +----------------------+-------------------------------------------------------+ - | ``bigdecimal`` | :class:`decimal.Decimal`. Returned type only. | - +----------------------+-------------------------------------------------------+ - - This is the full set of data types supported by XML-RPC. Method calls may also - raise a special :exc:`Fault` instance, used to signal XML-RPC server errors, or - :exc:`ProtocolError` used to signal an error in the HTTP/HTTPS transport layer. - Both :exc:`Fault` and :exc:`ProtocolError` derive from a base class called - :exc:`Error`. Note that the xmlrpc client module currently does not marshal - instances of subclasses of built-in types. - - When passing strings, characters special to XML such as ``<``, ``>``, and ``&`` - will be automatically escaped. However, it's the caller's responsibility to - ensure that the string is free of characters that aren't allowed in XML, such as - the control characters with ASCII values between 0 and 31 (except, of course, - tab, newline and carriage return); failing to do this will result in an XML-RPC - request that isn't well-formed XML. If you have to pass arbitrary bytes - via XML-RPC, use :class:`bytes` or :class:`bytearray` classes or the - :class:`Binary` wrapper class described below. - - :class:`Server` is retained as an alias for :class:`ServerProxy` for backwards - compatibility. New code should use :class:`ServerProxy`. - - .. versionchanged:: 3.5 - Added the *context* argument. - - .. versionchanged:: 3.6 - Added support of type tags with prefixes (e.g. ``ex:nil``). - Added support of unmarshalling additional types used by Apache XML-RPC - implementation for numerics: ``i1``, ``i2``, ``i8``, ``biginteger``, - ``float`` and ``bigdecimal``. - See https://ws.apache.org/xmlrpc/types.html for a description. - - -.. seealso:: - - `XML-RPC HOWTO `_ - A good description of XML-RPC operation and client software in several languages. - Contains pretty much everything an XML-RPC client developer needs to know. - - `XML-RPC Introspection `_ - Describes the XML-RPC protocol extension for introspection. - - `XML-RPC Specification `_ - The official specification. - -.. _serverproxy-objects: - -ServerProxy Objects -------------------- - -A :class:`ServerProxy` instance has a method corresponding to each remote -procedure call accepted by the XML-RPC server. Calling the method performs an -RPC, dispatched by both name and argument signature (e.g. the same method name -can be overloaded with multiple argument signatures). The RPC finishes by -returning a value, which may be either returned data in a conformant type or a -:class:`Fault` or :class:`ProtocolError` object indicating an error. - -Servers that support the XML introspection API support some common methods -grouped under the reserved :attr:`~ServerProxy.system` attribute: - - -.. method:: ServerProxy.system.listMethods() - - This method returns a list of strings, one for each (non-system) method - supported by the XML-RPC server. - - -.. method:: ServerProxy.system.methodSignature(name) - - This method takes one parameter, the name of a method implemented by the XML-RPC - server. It returns an array of possible signatures for this method. A signature - is an array of types. The first of these types is the return type of the method, - the rest are parameters. - - Because multiple signatures (ie. overloading) is permitted, this method returns - a list of signatures rather than a singleton. - - Signatures themselves are restricted to the top level parameters expected by a - method. For instance if a method expects one array of structs as a parameter, - and it returns a string, its signature is simply "string, array". If it expects - three integers and returns a string, its signature is "string, int, int, int". - - If no signature is defined for the method, a non-array value is returned. In - Python this means that the type of the returned value will be something other - than list. - - -.. method:: ServerProxy.system.methodHelp(name) - - This method takes one parameter, the name of a method implemented by the XML-RPC - server. It returns a documentation string describing the use of that method. If - no such string is available, an empty string is returned. The documentation - string may contain HTML markup. - -.. versionchanged:: 3.5 - - Instances of :class:`ServerProxy` support the :term:`context manager` protocol - for closing the underlying transport. - - -A working example follows. The server code:: - - from xmlrpc.server import SimpleXMLRPCServer - - def is_even(n): - return n % 2 == 0 - - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_function(is_even, "is_even") - server.serve_forever() - -The client code for the preceding server:: - - import xmlrpc.client - - with xmlrpc.client.ServerProxy("http://localhost:8000/") as proxy: - print("3 is even: %s" % str(proxy.is_even(3))) - print("100 is even: %s" % str(proxy.is_even(100))) - -.. _datetime-objects: - -DateTime Objects ----------------- - -.. class:: DateTime - - This class may be initialized with seconds since the epoch, a time - tuple, an ISO 8601 time/date string, or a :class:`datetime.datetime` - instance. It has the following methods, supported mainly for internal - use by the marshalling/unmarshalling code: - - - .. method:: decode(string) - - Accept a string as the instance's new time value. - - - .. method:: encode(out) - - Write the XML-RPC encoding of this :class:`DateTime` item to the *out* stream - object. - - It also supports certain of Python's built-in operators through rich comparison - and :meth:`__repr__` methods. - -A working example follows. The server code:: - - import datetime - from xmlrpc.server import SimpleXMLRPCServer - import xmlrpc.client - - def today(): - today = datetime.datetime.today() - return xmlrpc.client.DateTime(today) - - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_function(today, "today") - server.serve_forever() - -The client code for the preceding server:: - - import xmlrpc.client - import datetime - - proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") - - today = proxy.today() - # convert the ISO8601 string to a datetime object - converted = datetime.datetime.strptime(today.value, "%Y%m%dT%H:%M:%S") - print("Today: %s" % converted.strftime("%d.%m.%Y, %H:%M")) - -.. _binary-objects: - -Binary Objects --------------- - -.. class:: Binary - - This class may be initialized from bytes data (which may include NULs). The - primary access to the content of a :class:`Binary` object is provided by an - attribute: - - - .. attribute:: data - - The binary data encapsulated by the :class:`Binary` instance. The data is - provided as a :class:`bytes` object. - - :class:`Binary` objects have the following methods, supported mainly for - internal use by the marshalling/unmarshalling code: - - - .. method:: decode(bytes) - - Accept a base64 :class:`bytes` object and decode it as the instance's new data. - - - .. method:: encode(out) - - Write the XML-RPC base 64 encoding of this binary item to the *out* stream object. - - The encoded data will have newlines every 76 characters as per - :rfc:`RFC 2045 section 6.8 <2045#section-6.8>`, - which was the de facto standard base64 specification when the - XML-RPC spec was written. - - It also supports certain of Python's built-in operators through :meth:`__eq__` - and :meth:`__ne__` methods. - -Example usage of the binary objects. We're going to transfer an image over -XMLRPC:: - - from xmlrpc.server import SimpleXMLRPCServer - import xmlrpc.client - - def python_logo(): - with open("python_logo.jpg", "rb") as handle: - return xmlrpc.client.Binary(handle.read()) - - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_function(python_logo, 'python_logo') - - server.serve_forever() - -The client gets the image and saves it to a file:: - - import xmlrpc.client - - proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") - with open("fetched_python_logo.jpg", "wb") as handle: - handle.write(proxy.python_logo().data) - -.. _fault-objects: - -Fault Objects -------------- - -.. class:: Fault - - A :class:`Fault` object encapsulates the content of an XML-RPC fault tag. Fault - objects have the following attributes: - - - .. attribute:: faultCode - - An int indicating the fault type. - - - .. attribute:: faultString - - A string containing a diagnostic message associated with the fault. - -In the following example we're going to intentionally cause a :exc:`Fault` by -returning a complex type object. The server code:: - - from xmlrpc.server import SimpleXMLRPCServer - - # A marshalling error is going to occur because we're returning a - # complex number - def add(x, y): - return x+y+0j - - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_function(add, 'add') - - server.serve_forever() - -The client code for the preceding server:: - - import xmlrpc.client - - proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") - try: - proxy.add(2, 5) - except xmlrpc.client.Fault as err: - print("A fault occurred") - print("Fault code: %d" % err.faultCode) - print("Fault string: %s" % err.faultString) - - - -.. _protocol-error-objects: - -ProtocolError Objects ---------------------- - -.. class:: ProtocolError - - A :class:`ProtocolError` object describes a protocol error in the underlying - transport layer (such as a 404 'not found' error if the server named by the URI - does not exist). It has the following attributes: - - - .. attribute:: url - - The URI or URL that triggered the error. - - - .. attribute:: errcode - - The error code. - - - .. attribute:: errmsg - - The error message or diagnostic string. - - - .. attribute:: headers - - A dict containing the headers of the HTTP/HTTPS request that triggered the - error. - -In the following example we're going to intentionally cause a :exc:`ProtocolError` -by providing an invalid URI:: - - import xmlrpc.client - - # create a ServerProxy with a URI that doesn't respond to XMLRPC requests - proxy = xmlrpc.client.ServerProxy("http://google.com/") - - try: - proxy.some_method() - except xmlrpc.client.ProtocolError as err: - print("A protocol error occurred") - print("URL: %s" % err.url) - print("HTTP/HTTPS headers: %s" % err.headers) - print("Error code: %d" % err.errcode) - print("Error message: %s" % err.errmsg) - -MultiCall Objects ------------------ - -The :class:`MultiCall` object provides a way to encapsulate multiple calls to a -remote server into a single request [#]_. - - -.. class:: MultiCall(server) - - Create an object used to boxcar method calls. *server* is the eventual target of - the call. Calls can be made to the result object, but they will immediately - return ``None``, and only store the call name and parameters in the - :class:`MultiCall` object. Calling the object itself causes all stored calls to - be transmitted as a single ``system.multicall`` request. The result of this call - is a :term:`generator`; iterating over this generator yields the individual - results. - -A usage example of this class follows. The server code:: - - from xmlrpc.server import SimpleXMLRPCServer - - def add(x, y): - return x + y - - def subtract(x, y): - return x - y - - def multiply(x, y): - return x * y - - def divide(x, y): - return x // y - - # A simple server with simple arithmetic functions - server = SimpleXMLRPCServer(("localhost", 8000)) - print("Listening on port 8000...") - server.register_multicall_functions() - server.register_function(add, 'add') - server.register_function(subtract, 'subtract') - server.register_function(multiply, 'multiply') - server.register_function(divide, 'divide') - server.serve_forever() - -The client code for the preceding server:: - - import xmlrpc.client - - proxy = xmlrpc.client.ServerProxy("http://localhost:8000/") - multicall = xmlrpc.client.MultiCall(proxy) - multicall.add(7, 3) - multicall.subtract(7, 3) - multicall.multiply(7, 3) - multicall.divide(7, 3) - result = multicall() - - print("7+3=%d, 7-3=%d, 7*3=%d, 7//3=%d" % tuple(result)) - - -Convenience Functions ---------------------- - -.. function:: dumps(params, methodname=None, methodresponse=None, encoding=None, allow_none=False) - - Convert *params* into an XML-RPC request. or into a response if *methodresponse* - is true. *params* can be either a tuple of arguments or an instance of the - :exc:`Fault` exception class. If *methodresponse* is true, only a single value - can be returned, meaning that *params* must be of length 1. *encoding*, if - supplied, is the encoding to use in the generated XML; the default is UTF-8. - Python's :const:`None` value cannot be used in standard XML-RPC; to allow using - it via an extension, provide a true value for *allow_none*. - - -.. function:: loads(data, use_datetime=False, use_builtin_types=False) - - Convert an XML-RPC request or response into Python objects, a ``(params, - methodname)``. *params* is a tuple of argument; *methodname* is a string, or - ``None`` if no method name is present in the packet. If the XML-RPC packet - represents a fault condition, this function will raise a :exc:`Fault` exception. - The *use_builtin_types* flag can be used to cause date/time values to be - presented as :class:`datetime.datetime` objects and binary data to be - presented as :class:`bytes` objects; this flag is false by default. - - The obsolete *use_datetime* flag is similar to *use_builtin_types* but it - applies only to date/time values. - - .. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - - -.. _xmlrpc-client-example: - -Example of Client Usage ------------------------ - -:: - - # simple test program (from the XML-RPC specification) - from xmlrpc.client import ServerProxy, Error - - # server = ServerProxy("http://localhost:8000") # local server - with ServerProxy("http://betty.userland.com") as proxy: - - print(proxy) - - try: - print(proxy.examples.getStateName(41)) - except Error as v: - print("ERROR", v) - -To access an XML-RPC server through a HTTP proxy, you need to define a custom -transport. The following example shows how:: - - import http.client - import xmlrpc.client - - class ProxiedTransport(xmlrpc.client.Transport): - - def set_proxy(self, host, port=None, headers=None): - self.proxy = host, port - self.proxy_headers = headers - - def make_connection(self, host): - connection = http.client.HTTPConnection(*self.proxy) - connection.set_tunnel(host, headers=self.proxy_headers) - self._connection = host, connection - return connection - - transport = ProxiedTransport() - transport.set_proxy('proxy-server', 8080) - server = xmlrpc.client.ServerProxy('http://betty.userland.com', transport=transport) - print(server.examples.getStateName(41)) - - -Example of Client and Server Usage ----------------------------------- - -See :ref:`simplexmlrpcserver-example`. - - -.. rubric:: Footnotes - -.. [#] This approach has been first presented in `a discussion on xmlrpc.com - `_. -.. the link now points to webarchive since the one at -.. http://www.xmlrpc.com/discuss/msgReader%241208 is broken (and webadmin -.. doesn't reply) diff --git a/Doc/library/xmlrpc.rst b/Doc/library/xmlrpc.rst deleted file mode 100644 index 5f0a2cf68d01f9..00000000000000 --- a/Doc/library/xmlrpc.rst +++ /dev/null @@ -1,12 +0,0 @@ -:mod:`!xmlrpc` --- XMLRPC server and client modules -=================================================== - -XML-RPC is a Remote Procedure Call method that uses XML passed via HTTP as a -transport. With it, a client can call methods with parameters on a remote -server (the server is named by a URI) and get back structured data. - -``xmlrpc`` is a package that collects server and client modules implementing -XML-RPC. The modules are: - -* :mod:`xmlrpc.client` -* :mod:`xmlrpc.server` diff --git a/Doc/library/xmlrpc.server.rst b/Doc/library/xmlrpc.server.rst deleted file mode 100644 index 016369d2b89d2c..00000000000000 --- a/Doc/library/xmlrpc.server.rst +++ /dev/null @@ -1,446 +0,0 @@ -:mod:`xmlrpc.server` --- Basic XML-RPC servers -============================================== - -.. module:: xmlrpc.server - :synopsis: Basic XML-RPC server implementations. - -.. moduleauthor:: Brian Quinlan -.. sectionauthor:: Fred L. Drake, Jr. - -**Source code:** :source:`Lib/xmlrpc/server.py` - --------------- - -The :mod:`xmlrpc.server` module provides a basic server framework for XML-RPC -servers written in Python. Servers can either be free standing, using -:class:`SimpleXMLRPCServer`, or embedded in a CGI environment, using -:class:`CGIXMLRPCRequestHandler`. - - -.. warning:: - - The :mod:`xmlrpc.server` module is not secure against maliciously - constructed data. If you need to parse untrusted or unauthenticated data see - :ref:`xml-vulnerabilities`. - -.. include:: ../includes/wasm-notavail.rst - -.. class:: SimpleXMLRPCServer(addr, requestHandler=SimpleXMLRPCRequestHandler,\ - logRequests=True, allow_none=False, encoding=None,\ - bind_and_activate=True, use_builtin_types=False) - - Create a new server instance. This class provides methods for registration of - functions that can be called by the XML-RPC protocol. The *requestHandler* - parameter should be a factory for request handler instances; it defaults to - :class:`SimpleXMLRPCRequestHandler`. The *addr* and *requestHandler* parameters - are passed to the :class:`socketserver.TCPServer` constructor. If *logRequests* - is true (the default), requests will be logged; setting this parameter to false - will turn off logging. The *allow_none* and *encoding* parameters are passed - on to :mod:`xmlrpc.client` and control the XML-RPC responses that will be returned - from the server. The *bind_and_activate* parameter controls whether - :meth:`server_bind` and :meth:`server_activate` are called immediately by the - constructor; it defaults to true. Setting it to false allows code to manipulate - the *allow_reuse_address* class variable before the address is bound. - The *use_builtin_types* parameter is passed to the - :func:`~xmlrpc.client.loads` function and controls which types are processed - when date/times values or binary data are received; it defaults to false. - - .. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - - -.. class:: CGIXMLRPCRequestHandler(allow_none=False, encoding=None,\ - use_builtin_types=False) - - Create a new instance to handle XML-RPC requests in a CGI environment. The - *allow_none* and *encoding* parameters are passed on to :mod:`xmlrpc.client` - and control the XML-RPC responses that will be returned from the server. - The *use_builtin_types* parameter is passed to the - :func:`~xmlrpc.client.loads` function and controls which types are processed - when date/times values or binary data are received; it defaults to false. - - .. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - - -.. class:: SimpleXMLRPCRequestHandler() - - Create a new request handler instance. This request handler supports ``POST`` - requests and modifies logging so that the *logRequests* parameter to the - :class:`SimpleXMLRPCServer` constructor parameter is honored. - - -.. _simple-xmlrpc-servers: - -SimpleXMLRPCServer Objects --------------------------- - -The :class:`SimpleXMLRPCServer` class is based on -:class:`socketserver.TCPServer` and provides a means of creating simple, stand -alone XML-RPC servers. - - -.. method:: SimpleXMLRPCServer.register_function(function=None, name=None) - - Register a function that can respond to XML-RPC requests. If *name* is given, - it will be the method name associated with *function*, otherwise - ``function.__name__`` will be used. *name* is a string, and may contain - characters not legal in Python identifiers, including the period character. - - This method can also be used as a decorator. When used as a decorator, - *name* can only be given as a keyword argument to register *function* under - *name*. If no *name* is given, ``function.__name__`` will be used. - - .. versionchanged:: 3.7 - :meth:`register_function` can be used as a decorator. - - -.. method:: SimpleXMLRPCServer.register_instance(instance, allow_dotted_names=False) - - Register an object which is used to expose method names which have not been - registered using :meth:`register_function`. If *instance* contains a - :meth:`_dispatch` method, it is called with the requested method name and the - parameters from the request. Its API is ``def _dispatch(self, method, params)`` - (note that *params* does not represent a variable argument list). If it calls - an underlying function to perform its task, that function is called as - ``func(*params)``, expanding the parameter list. The return value from - :meth:`_dispatch` is returned to the client as the result. If *instance* does - not have a :meth:`_dispatch` method, it is searched for an attribute matching - the name of the requested method. - - If the optional *allow_dotted_names* argument is true and the instance does not - have a :meth:`_dispatch` method, then if the requested method name contains - periods, each component of the method name is searched for individually, with - the effect that a simple hierarchical search is performed. The value found from - this search is then called with the parameters from the request, and the return - value is passed back to the client. - - .. warning:: - - Enabling the *allow_dotted_names* option allows intruders to access your - module's global variables and may allow intruders to execute arbitrary code on - your machine. Only use this option on a secure, closed network. - - -.. method:: SimpleXMLRPCServer.register_introspection_functions() - - Registers the XML-RPC introspection functions ``system.listMethods``, - ``system.methodHelp`` and ``system.methodSignature``. - - -.. method:: SimpleXMLRPCServer.register_multicall_functions() - - Registers the XML-RPC multicall function system.multicall. - - -.. attribute:: SimpleXMLRPCRequestHandler.rpc_paths - - An attribute value that must be a tuple listing valid path portions of the URL - for receiving XML-RPC requests. Requests posted to other paths will result in a - 404 "no such page" HTTP error. If this tuple is empty, all paths will be - considered valid. The default value is ``('/', '/RPC2')``. - - -.. _simplexmlrpcserver-example: - -SimpleXMLRPCServer Example -^^^^^^^^^^^^^^^^^^^^^^^^^^ -Server code:: - - from xmlrpc.server import SimpleXMLRPCServer - from xmlrpc.server import SimpleXMLRPCRequestHandler - - # Restrict to a particular path. - class RequestHandler(SimpleXMLRPCRequestHandler): - rpc_paths = ('/RPC2',) - - # Create server - with SimpleXMLRPCServer(('localhost', 8000), - requestHandler=RequestHandler) as server: - server.register_introspection_functions() - - # Register pow() function; this will use the value of - # pow.__name__ as the name, which is just 'pow'. - server.register_function(pow) - - # Register a function under a different name - def adder_function(x, y): - return x + y - server.register_function(adder_function, 'add') - - # Register an instance; all the methods of the instance are - # published as XML-RPC methods (in this case, just 'mul'). - class MyFuncs: - def mul(self, x, y): - return x * y - - server.register_instance(MyFuncs()) - - # Run the server's main loop - server.serve_forever() - -The following client code will call the methods made available by the preceding -server:: - - import xmlrpc.client - - s = xmlrpc.client.ServerProxy('http://localhost:8000') - print(s.pow(2,3)) # Returns 2**3 = 8 - print(s.add(2,3)) # Returns 5 - print(s.mul(5,2)) # Returns 5*2 = 10 - - # Print list of available methods - print(s.system.listMethods()) - -:meth:`register_function` can also be used as a decorator. The previous server -example can register functions in a decorator way:: - - from xmlrpc.server import SimpleXMLRPCServer - from xmlrpc.server import SimpleXMLRPCRequestHandler - - class RequestHandler(SimpleXMLRPCRequestHandler): - rpc_paths = ('/RPC2',) - - with SimpleXMLRPCServer(('localhost', 8000), - requestHandler=RequestHandler) as server: - server.register_introspection_functions() - - # Register pow() function; this will use the value of - # pow.__name__ as the name, which is just 'pow'. - server.register_function(pow) - - # Register a function under a different name, using - # register_function as a decorator. *name* can only be given - # as a keyword argument. - @server.register_function(name='add') - def adder_function(x, y): - return x + y - - # Register a function under function.__name__. - @server.register_function - def mul(x, y): - return x * y - - server.serve_forever() - -The following example included in the :file:`Lib/xmlrpc/server.py` module shows -a server allowing dotted names and registering a multicall function. - -.. warning:: - - Enabling the *allow_dotted_names* option allows intruders to access your - module's global variables and may allow intruders to execute arbitrary code on - your machine. Only use this example only within a secure, closed network. - -:: - - import datetime - - class ExampleService: - def getData(self): - return '42' - - class currentTime: - @staticmethod - def getCurrentTime(): - return datetime.datetime.now() - - with SimpleXMLRPCServer(("localhost", 8000)) as server: - server.register_function(pow) - server.register_function(lambda x,y: x+y, 'add') - server.register_instance(ExampleService(), allow_dotted_names=True) - server.register_multicall_functions() - print('Serving XML-RPC on localhost port 8000') - try: - server.serve_forever() - except KeyboardInterrupt: - print("\nKeyboard interrupt received, exiting.") - sys.exit(0) - -This ExampleService demo can be invoked from the command line:: - - python -m xmlrpc.server - - -The client that interacts with the above server is included in -``Lib/xmlrpc/client.py``:: - - server = ServerProxy("http://localhost:8000") - - try: - print(server.currentTime.getCurrentTime()) - except Error as v: - print("ERROR", v) - - multi = MultiCall(server) - multi.getData() - multi.pow(2,9) - multi.add(1,2) - try: - for response in multi(): - print(response) - except Error as v: - print("ERROR", v) - -This client which interacts with the demo XMLRPC server can be invoked as:: - - python -m xmlrpc.client - - -CGIXMLRPCRequestHandler ------------------------ - -The :class:`CGIXMLRPCRequestHandler` class can be used to handle XML-RPC -requests sent to Python CGI scripts. - - -.. method:: CGIXMLRPCRequestHandler.register_function(function=None, name=None) - - Register a function that can respond to XML-RPC requests. If *name* is given, - it will be the method name associated with *function*, otherwise - ``function.__name__`` will be used. *name* is a string, and may contain - characters not legal in Python identifiers, including the period character. - - This method can also be used as a decorator. When used as a decorator, - *name* can only be given as a keyword argument to register *function* under - *name*. If no *name* is given, ``function.__name__`` will be used. - - .. versionchanged:: 3.7 - :meth:`register_function` can be used as a decorator. - - -.. method:: CGIXMLRPCRequestHandler.register_instance(instance) - - Register an object which is used to expose method names which have not been - registered using :meth:`register_function`. If instance contains a - :meth:`_dispatch` method, it is called with the requested method name and the - parameters from the request; the return value is returned to the client as the - result. If instance does not have a :meth:`_dispatch` method, it is searched - for an attribute matching the name of the requested method; if the requested - method name contains periods, each component of the method name is searched for - individually, with the effect that a simple hierarchical search is performed. - The value found from this search is then called with the parameters from the - request, and the return value is passed back to the client. - - -.. method:: CGIXMLRPCRequestHandler.register_introspection_functions() - - Register the XML-RPC introspection functions ``system.listMethods``, - ``system.methodHelp`` and ``system.methodSignature``. - - -.. method:: CGIXMLRPCRequestHandler.register_multicall_functions() - - Register the XML-RPC multicall function ``system.multicall``. - - -.. method:: CGIXMLRPCRequestHandler.handle_request(request_text=None) - - Handle an XML-RPC request. If *request_text* is given, it should be the POST - data provided by the HTTP server, otherwise the contents of stdin will be used. - -Example:: - - class MyFuncs: - def mul(self, x, y): - return x * y - - - handler = CGIXMLRPCRequestHandler() - handler.register_function(pow) - handler.register_function(lambda x,y: x+y, 'add') - handler.register_introspection_functions() - handler.register_instance(MyFuncs()) - handler.handle_request() - - -Documenting XMLRPC server -------------------------- - -These classes extend the above classes to serve HTML documentation in response -to HTTP GET requests. Servers can either be free standing, using -:class:`DocXMLRPCServer`, or embedded in a CGI environment, using -:class:`DocCGIXMLRPCRequestHandler`. - - -.. class:: DocXMLRPCServer(addr, requestHandler=DocXMLRPCRequestHandler,\ - logRequests=True, allow_none=False, encoding=None,\ - bind_and_activate=True, use_builtin_types=True) - - Create a new server instance. All parameters have the same meaning as for - :class:`SimpleXMLRPCServer`; *requestHandler* defaults to - :class:`DocXMLRPCRequestHandler`. - - .. versionchanged:: 3.3 - The *use_builtin_types* flag was added. - - -.. class:: DocCGIXMLRPCRequestHandler() - - Create a new instance to handle XML-RPC requests in a CGI environment. - - -.. class:: DocXMLRPCRequestHandler() - - Create a new request handler instance. This request handler supports XML-RPC - POST requests, documentation GET requests, and modifies logging so that the - *logRequests* parameter to the :class:`DocXMLRPCServer` constructor parameter is - honored. - - -.. _doc-xmlrpc-servers: - -DocXMLRPCServer Objects ------------------------ - -The :class:`DocXMLRPCServer` class is derived from :class:`SimpleXMLRPCServer` -and provides a means of creating self-documenting, stand alone XML-RPC -servers. HTTP POST requests are handled as XML-RPC method calls. HTTP GET -requests are handled by generating pydoc-style HTML documentation. This allows a -server to provide its own web-based documentation. - - -.. method:: DocXMLRPCServer.set_server_title(server_title) - - Set the title used in the generated HTML documentation. This title will be used - inside the HTML "title" element. - - -.. method:: DocXMLRPCServer.set_server_name(server_name) - - Set the name used in the generated HTML documentation. This name will appear at - the top of the generated documentation inside a "h1" element. - - -.. method:: DocXMLRPCServer.set_server_documentation(server_documentation) - - Set the description used in the generated HTML documentation. This description - will appear as a paragraph, below the server name, in the documentation. - - -DocCGIXMLRPCRequestHandler --------------------------- - -The :class:`DocCGIXMLRPCRequestHandler` class is derived from -:class:`CGIXMLRPCRequestHandler` and provides a means of creating -self-documenting, XML-RPC CGI scripts. HTTP POST requests are handled as XML-RPC -method calls. HTTP GET requests are handled by generating pydoc-style HTML -documentation. This allows a server to provide its own web-based documentation. - - -.. method:: DocCGIXMLRPCRequestHandler.set_server_title(server_title) - - Set the title used in the generated HTML documentation. This title will be used - inside the HTML "title" element. - - -.. method:: DocCGIXMLRPCRequestHandler.set_server_name(server_name) - - Set the name used in the generated HTML documentation. This name will appear at - the top of the generated documentation inside a "h1" element. - - -.. method:: DocCGIXMLRPCRequestHandler.set_server_documentation(server_documentation) - - Set the description used in the generated HTML documentation. This description - will appear as a paragraph, below the server name, in the documentation. diff --git a/Doc/library/zipapp.rst b/Doc/library/zipapp.rst deleted file mode 100644 index c98d8a6036bf83..00000000000000 --- a/Doc/library/zipapp.rst +++ /dev/null @@ -1,444 +0,0 @@ -:mod:`zipapp` --- Manage executable Python zip archives -======================================================= - -.. module:: zipapp - :synopsis: Manage executable Python zip archives - -.. versionadded:: 3.5 - -**Source code:** :source:`Lib/zipapp.py` - -.. index:: - single: Executable Zip Files - --------------- - -This module provides tools to manage the creation of zip files containing -Python code, which can be :ref:`executed directly by the Python interpreter -`. The module provides both a -:ref:`zipapp-command-line-interface` and a :ref:`zipapp-python-api`. - - -Basic Example -------------- - -The following example shows how the :ref:`zipapp-command-line-interface` -can be used to create an executable archive from a directory containing -Python code. When run, the archive will execute the ``main`` function from -the module ``myapp`` in the archive. - -.. code-block:: shell-session - - $ python -m zipapp myapp -m "myapp:main" - $ python myapp.pyz - - - -.. _zipapp-command-line-interface: - -Command-Line Interface ----------------------- - -When called as a program from the command line, the following form is used: - -.. code-block:: shell-session - - $ python -m zipapp source [options] - -If *source* is a directory, this will create an archive from the contents of -*source*. If *source* is a file, it should be an archive, and it will be -copied to the target archive (or the contents of its shebang line will be -displayed if the --info option is specified). - -The following options are understood: - -.. program:: zipapp - -.. option:: -o , --output= - - Write the output to a file named *output*. If this option is not specified, - the output filename will be the same as the input *source*, with the - extension ``.pyz`` added. If an explicit filename is given, it is used as - is (so a ``.pyz`` extension should be included if required). - - An output filename must be specified if the *source* is an archive (and in - that case, *output* must not be the same as *source*). - -.. option:: -p , --python= - - Add a ``#!`` line to the archive specifying *interpreter* as the command - to run. Also, on POSIX, make the archive executable. The default is to - write no ``#!`` line, and not make the file executable. - -.. option:: -m , --main= - - Write a ``__main__.py`` file to the archive that executes *mainfn*. The - *mainfn* argument should have the form "pkg.mod:fn", where "pkg.mod" is a - package/module in the archive, and "fn" is a callable in the given module. - The ``__main__.py`` file will execute that callable. - - :option:`--main` cannot be specified when copying an archive. - -.. option:: -c, --compress - - Compress files with the deflate method, reducing the size of the output - file. By default, files are stored uncompressed in the archive. - - :option:`--compress` has no effect when copying an archive. - - .. versionadded:: 3.7 - -.. option:: --info - - Display the interpreter embedded in the archive, for diagnostic purposes. In - this case, any other options are ignored and SOURCE must be an archive, not a - directory. - -.. option:: -h, --help - - Print a short usage message and exit. - - -.. _zipapp-python-api: - -Python API ----------- - -The module defines two convenience functions: - - -.. function:: create_archive(source, target=None, interpreter=None, main=None, filter=None, compressed=False) - - Create an application archive from *source*. The source can be any - of the following: - - * The name of a directory, or a :term:`path-like object` referring - to a directory, in which case a new application archive will be - created from the content of that directory. - * The name of an existing application archive file, or a :term:`path-like object` - referring to such a file, in which case the file is copied to - the target (modifying it to reflect the value given for the *interpreter* - argument). The file name should include the ``.pyz`` extension, if required. - * A file object open for reading in bytes mode. The content of the - file should be an application archive, and the file object is - assumed to be positioned at the start of the archive. - - The *target* argument determines where the resulting archive will be - written: - - * If it is the name of a file, or a :term:`path-like object`, - the archive will be written to that file. - * If it is an open file object, the archive will be written to that - file object, which must be open for writing in bytes mode. - * If the target is omitted (or ``None``), the source must be a directory - and the target will be a file with the same name as the source, with - a ``.pyz`` extension added. - - The *interpreter* argument specifies the name of the Python - interpreter with which the archive will be executed. It is written as - a "shebang" line at the start of the archive. On POSIX, this will be - interpreted by the OS, and on Windows it will be handled by the Python - launcher. Omitting the *interpreter* results in no shebang line being - written. If an interpreter is specified, and the target is a - filename, the executable bit of the target file will be set. - - The *main* argument specifies the name of a callable which will be - used as the main program for the archive. It can only be specified if - the source is a directory, and the source does not already contain a - ``__main__.py`` file. The *main* argument should take the form - "pkg.module:callable" and the archive will be run by importing - "pkg.module" and executing the given callable with no arguments. It - is an error to omit *main* if the source is a directory and does not - contain a ``__main__.py`` file, as otherwise the resulting archive - would not be executable. - - The optional *filter* argument specifies a callback function that - is passed a Path object representing the path to the file being added - (relative to the source directory). It should return ``True`` if the - file is to be added. - - The optional *compressed* argument determines whether files are - compressed. If set to ``True``, files in the archive are compressed - with the deflate method; otherwise, files are stored uncompressed. - This argument has no effect when copying an existing archive. - - If a file object is specified for *source* or *target*, it is the - caller's responsibility to close it after calling create_archive. - - When copying an existing archive, file objects supplied only need - ``read`` and ``readline``, or ``write`` methods. When creating an - archive from a directory, if the target is a file object it will be - passed to the ``zipfile.ZipFile`` class, and must supply the methods - needed by that class. - - .. versionadded:: 3.7 - Added the *filter* and *compressed* arguments. - -.. function:: get_interpreter(archive) - - Return the interpreter specified in the ``#!`` line at the start of the - archive. If there is no ``#!`` line, return :const:`None`. - The *archive* argument can be a filename or a file-like object open - for reading in bytes mode. It is assumed to be at the start of the archive. - - -.. _zipapp-examples: - -Examples --------- - -Pack up a directory into an archive, and run it. - -.. code-block:: shell-session - - $ python -m zipapp myapp - $ python myapp.pyz - - -The same can be done using the :func:`create_archive` function:: - - >>> import zipapp - >>> zipapp.create_archive('myapp', 'myapp.pyz') - -To make the application directly executable on POSIX, specify an interpreter -to use. - -.. code-block:: shell-session - - $ python -m zipapp myapp -p "/usr/bin/env python" - $ ./myapp.pyz - - -To replace the shebang line on an existing archive, create a modified archive -using the :func:`create_archive` function:: - - >>> import zipapp - >>> zipapp.create_archive('old_archive.pyz', 'new_archive.pyz', '/usr/bin/python3') - -To update the file in place, do the replacement in memory using a :class:`~io.BytesIO` -object, and then overwrite the source afterwards. Note that there is a risk -when overwriting a file in place that an error will result in the loss of -the original file. This code does not protect against such errors, but -production code should do so. Also, this method will only work if the archive -fits in memory:: - - >>> import zipapp - >>> import io - >>> temp = io.BytesIO() - >>> zipapp.create_archive('myapp.pyz', temp, '/usr/bin/python2') - >>> with open('myapp.pyz', 'wb') as f: - >>> f.write(temp.getvalue()) - - -.. _zipapp-specifying-the-interpreter: - -Specifying the Interpreter --------------------------- - -Note that if you specify an interpreter and then distribute your application -archive, you need to ensure that the interpreter used is portable. The Python -launcher for Windows supports most common forms of POSIX ``#!`` line, but there -are other issues to consider: - -* If you use "/usr/bin/env python" (or other forms of the "python" command, - such as "/usr/bin/python"), you need to consider that your users may have - either Python 2 or Python 3 as their default, and write your code to work - under both versions. -* If you use an explicit version, for example "/usr/bin/env python3" your - application will not work for users who do not have that version. (This - may be what you want if you have not made your code Python 2 compatible). -* There is no way to say "python X.Y or later", so be careful of using an - exact version like "/usr/bin/env python3.4" as you will need to change your - shebang line for users of Python 3.5, for example. - -Typically, you should use an "/usr/bin/env python2" or "/usr/bin/env python3", -depending on whether your code is written for Python 2 or 3. - - -Creating Standalone Applications with zipapp --------------------------------------------- - -Using the :mod:`zipapp` module, it is possible to create self-contained Python -programs, which can be distributed to end users who only need to have a -suitable version of Python installed on their system. The key to doing this -is to bundle all of the application's dependencies into the archive, along -with the application code. - -The steps to create a standalone archive are as follows: - -1. Create your application in a directory as normal, so you have a ``myapp`` - directory containing a ``__main__.py`` file, and any supporting application - code. - -2. Install all of your application's dependencies into the ``myapp`` directory, - using pip: - - .. code-block:: shell-session - - $ python -m pip install -r requirements.txt --target myapp - - (this assumes you have your project requirements in a ``requirements.txt`` - file - if not, you can just list the dependencies manually on the pip command - line). - -3. Package the application using: - - .. code-block:: shell-session - - $ python -m zipapp -p "interpreter" myapp - -This will produce a standalone executable, which can be run on any machine with -the appropriate interpreter available. See :ref:`zipapp-specifying-the-interpreter` -for details. It can be shipped to users as a single file. - -On Unix, the ``myapp.pyz`` file is executable as it stands. You can rename the -file to remove the ``.pyz`` extension if you prefer a "plain" command name. On -Windows, the ``myapp.pyz[w]`` file is executable by virtue of the fact that -the Python interpreter registers the ``.pyz`` and ``.pyzw`` file extensions -when installed. - - -Making a Windows executable -~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -On Windows, registration of the ``.pyz`` extension is optional, and -furthermore, there are certain places that don't recognise registered -extensions "transparently" (the simplest example is that -``subprocess.run(['myapp'])`` won't find your application - you need to -explicitly specify the extension). - -On Windows, therefore, it is often preferable to create an executable from the -zipapp. This is relatively easy, although it does require a C compiler. The -basic approach relies on the fact that zipfiles can have arbitrary data -prepended, and Windows exe files can have arbitrary data appended. So by -creating a suitable launcher and tacking the ``.pyz`` file onto the end of it, -you end up with a single-file executable that runs your application. - -A suitable launcher can be as simple as the following:: - - #define Py_LIMITED_API 1 - #include "Python.h" - - #define WIN32_LEAN_AND_MEAN - #include - - #ifdef WINDOWS - int WINAPI wWinMain( - HINSTANCE hInstance, /* handle to current instance */ - HINSTANCE hPrevInstance, /* handle to previous instance */ - LPWSTR lpCmdLine, /* pointer to command line */ - int nCmdShow /* show state of window */ - ) - #else - int wmain() - #endif - { - wchar_t **myargv = _alloca((__argc + 1) * sizeof(wchar_t*)); - myargv[0] = __wargv[0]; - memcpy(myargv + 1, __wargv, __argc * sizeof(wchar_t *)); - return Py_Main(__argc+1, myargv); - } - -If you define the ``WINDOWS`` preprocessor symbol, this will generate a -GUI executable, and without it, a console executable. - -To compile the executable, you can either just use the standard MSVC -command line tools, or you can take advantage of the fact that distutils -knows how to compile Python source:: - - >>> from distutils.ccompiler import new_compiler - >>> import distutils.sysconfig - >>> import sys - >>> import os - >>> from pathlib import Path - - >>> def compile(src): - >>> src = Path(src) - >>> cc = new_compiler() - >>> exe = src.stem - >>> cc.add_include_dir(distutils.sysconfig.get_python_inc()) - >>> cc.add_library_dir(os.path.join(sys.base_exec_prefix, 'libs')) - >>> # First the CLI executable - >>> objs = cc.compile([str(src)]) - >>> cc.link_executable(objs, exe) - >>> # Now the GUI executable - >>> cc.define_macro('WINDOWS') - >>> objs = cc.compile([str(src)]) - >>> cc.link_executable(objs, exe + 'w') - - >>> if __name__ == "__main__": - >>> compile("zastub.c") - -The resulting launcher uses the "Limited ABI", so it will run unchanged with -any version of Python 3.x. All it needs is for Python (``python3.dll``) to be -on the user's ``PATH``. - -For a fully standalone distribution, you can distribute the launcher with your -application appended, bundled with the Python "embedded" distribution. This -will run on any PC with the appropriate architecture (32 bit or 64 bit). - - -Caveats -~~~~~~~ - -There are some limitations to the process of bundling your application into -a single file. In most, if not all, cases they can be addressed without -needing major changes to your application. - -1. If your application depends on a package that includes a C extension, that - package cannot be run from a zip file (this is an OS limitation, as executable - code must be present in the filesystem for the OS loader to load it). In this - case, you can exclude that dependency from the zipfile, and either require - your users to have it installed, or ship it alongside your zipfile and add code - to your ``__main__.py`` to include the directory containing the unzipped - module in ``sys.path``. In this case, you will need to make sure to ship - appropriate binaries for your target architecture(s) (and potentially pick the - correct version to add to ``sys.path`` at runtime, based on the user's machine). - -2. If you are shipping a Windows executable as described above, you either need to - ensure that your users have ``python3.dll`` on their PATH (which is not the - default behaviour of the installer) or you should bundle your application with - the embedded distribution. - -3. The suggested launcher above uses the Python embedding API. This means that in - your application, ``sys.executable`` will be your application, and *not* a - conventional Python interpreter. Your code and its dependencies need to be - prepared for this possibility. For example, if your application uses the - :mod:`multiprocessing` module, it will need to call - :func:`multiprocessing.set_executable` to let the module know where to find the - standard Python interpreter. - - -The Python Zip Application Archive Format ------------------------------------------ - -Python has been able to execute zip files which contain a ``__main__.py`` file -since version 2.6. In order to be executed by Python, an application archive -simply has to be a standard zip file containing a ``__main__.py`` file which -will be run as the entry point for the application. As usual for any Python -script, the parent of the script (in this case the zip file) will be placed on -:data:`sys.path` and thus further modules can be imported from the zip file. - -The zip file format allows arbitrary data to be prepended to a zip file. The -zip application format uses this ability to prepend a standard POSIX "shebang" -line to the file (``#!/path/to/interpreter``). - -Formally, the Python zip application format is therefore: - -1. An optional shebang line, containing the characters ``b'#!'`` followed by an - interpreter name, and then a newline (``b'\n'``) character. The interpreter - name can be anything acceptable to the OS "shebang" processing, or the Python - launcher on Windows. The interpreter should be encoded in UTF-8 on Windows, - and in :func:`sys.getfilesystemencoding()` on POSIX. -2. Standard zipfile data, as generated by the :mod:`zipfile` module. The - zipfile content *must* include a file called ``__main__.py`` (which must be - in the "root" of the zipfile - i.e., it cannot be in a subdirectory). The - zipfile data can be compressed or uncompressed. - -If an application archive has a shebang line, it may have the executable bit set -on POSIX systems, to allow it to be executed directly. - -There is no requirement that the tools in this module are used to create -application archives - the module is a convenience, but archives in the above -format created by any means are acceptable to Python. - diff --git a/Doc/library/zipfile.rst b/Doc/library/zipfile.rst deleted file mode 100644 index ef84115a4b3c13..00000000000000 --- a/Doc/library/zipfile.rst +++ /dev/null @@ -1,978 +0,0 @@ -:mod:`zipfile` --- Work with ZIP archives -========================================= - -.. module:: zipfile - :synopsis: Read and write ZIP-format archive files. - -.. moduleauthor:: James C. Ahlstrom -.. sectionauthor:: James C. Ahlstrom - -**Source code:** :source:`Lib/zipfile.py` - --------------- - -The ZIP file format is a common archive and compression standard. This module -provides tools to create, read, write, append, and list a ZIP file. Any -advanced use of this module will require an understanding of the format, as -defined in `PKZIP Application Note`_. - -This module does not currently handle multi-disk ZIP files. -It can handle ZIP files that use the ZIP64 extensions -(that is ZIP files that are more than 4 GiB in size). It supports -decryption of encrypted files in ZIP archives, but it currently cannot -create an encrypted file. Decryption is extremely slow as it is -implemented in native Python rather than C. - -The module defines the following items: - -.. exception:: BadZipFile - - The error raised for bad ZIP files. - - .. versionadded:: 3.2 - - -.. exception:: BadZipfile - - Alias of :exc:`BadZipFile`, for compatibility with older Python versions. - - .. deprecated:: 3.2 - - -.. exception:: LargeZipFile - - The error raised when a ZIP file would require ZIP64 functionality but that has - not been enabled. - - -.. class:: ZipFile - :noindex: - - The class for reading and writing ZIP files. See section - :ref:`zipfile-objects` for constructor details. - - -.. class:: Path - :noindex: - - Class that implements a subset of the interface provided by - :class:`pathlib.Path`, including the full - :class:`importlib.resources.abc.Traversable` interface. - - .. versionadded:: 3.8 - - -.. class:: PyZipFile - :noindex: - - Class for creating ZIP archives containing Python libraries. - - -.. class:: ZipInfo(filename='NoName', date_time=(1980,1,1,0,0,0)) - - Class used to represent information about a member of an archive. Instances - of this class are returned by the :meth:`.getinfo` and :meth:`.infolist` - methods of :class:`ZipFile` objects. Most users of the :mod:`zipfile` module - will not need to create these, but only use those created by this - module. *filename* should be the full name of the archive member, and - *date_time* should be a tuple containing six fields which describe the time - of the last modification to the file; the fields are described in section - :ref:`zipinfo-objects`. - -.. function:: is_zipfile(filename) - - Returns ``True`` if *filename* is a valid ZIP file based on its magic number, - otherwise returns ``False``. *filename* may be a file or file-like object too. - - .. versionchanged:: 3.1 - Support for file and file-like objects. - - -.. data:: ZIP_STORED - - The numeric constant for an uncompressed archive member. - - -.. data:: ZIP_DEFLATED - - The numeric constant for the usual ZIP compression method. This requires the - :mod:`zlib` module. - - -.. data:: ZIP_BZIP2 - - The numeric constant for the BZIP2 compression method. This requires the - :mod:`bz2` module. - - .. versionadded:: 3.3 - -.. data:: ZIP_LZMA - - The numeric constant for the LZMA compression method. This requires the - :mod:`lzma` module. - - .. versionadded:: 3.3 - - .. note:: - - The ZIP file format specification has included support for bzip2 compression - since 2001, and for LZMA compression since 2006. However, some tools - (including older Python releases) do not support these compression - methods, and may either refuse to process the ZIP file altogether, - or fail to extract individual files. - - -.. seealso:: - - `PKZIP Application Note`_ - Documentation on the ZIP file format by Phil Katz, the creator of the format and - algorithms used. - - `Info-ZIP Home Page `_ - Information about the Info-ZIP project's ZIP archive programs and development - libraries. - - -.. _zipfile-objects: - -ZipFile Objects ---------------- - - -.. class:: ZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ - compresslevel=None, *, strict_timestamps=True, \ - metadata_encoding=None) - - Open a ZIP file, where *file* can be a path to a file (a string), a - file-like object or a :term:`path-like object`. - - The *mode* parameter should be ``'r'`` to read an existing - file, ``'w'`` to truncate and write a new file, ``'a'`` to append to an - existing file, or ``'x'`` to exclusively create and write a new file. - If *mode* is ``'x'`` and *file* refers to an existing file, - a :exc:`FileExistsError` will be raised. - If *mode* is ``'a'`` and *file* refers to an existing ZIP - file, then additional files are added to it. If *file* does not refer to a - ZIP file, then a new ZIP archive is appended to the file. This is meant for - adding a ZIP archive to another file (such as :file:`python.exe`). If - *mode* is ``'a'`` and the file does not exist at all, it is created. - If *mode* is ``'r'`` or ``'a'``, the file should be seekable. - - *compression* is the ZIP compression method to use when writing the archive, - and should be :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, - :const:`ZIP_BZIP2` or :const:`ZIP_LZMA`; unrecognized - values will cause :exc:`NotImplementedError` to be raised. If - :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or :const:`ZIP_LZMA` is specified - but the corresponding module (:mod:`zlib`, :mod:`bz2` or :mod:`lzma`) is not - available, :exc:`RuntimeError` is raised. The default is :const:`ZIP_STORED`. - - If *allowZip64* is ``True`` (the default) zipfile will create ZIP files that - use the ZIP64 extensions when the zipfile is larger than 4 GiB. If it is - ``false`` :mod:`zipfile` will raise an exception when the ZIP file would - require ZIP64 extensions. - - The *compresslevel* parameter controls the compression level to use when - writing files to the archive. - When using :const:`ZIP_STORED` or :const:`ZIP_LZMA` it has no effect. - When using :const:`ZIP_DEFLATED` integers ``0`` through ``9`` are accepted - (see :class:`zlib ` for more information). - When using :const:`ZIP_BZIP2` integers ``1`` through ``9`` are accepted - (see :class:`bz2 ` for more information). - - The *strict_timestamps* argument, when set to ``False``, allows to - zip files older than 1980-01-01 at the cost of setting the - timestamp to 1980-01-01. - Similar behavior occurs with files newer than 2107-12-31, - the timestamp is also set to the limit. - - When mode is ``'r'``, *metadata_encoding* may be set to the name of a codec, - which will be used to decode metadata such as the names of members and ZIP - comments. - - If the file is created with mode ``'w'``, ``'x'`` or ``'a'`` and then - :meth:`closed ` without adding any files to the archive, the appropriate - ZIP structures for an empty archive will be written to the file. - - ZipFile is also a context manager and therefore supports the - :keyword:`with` statement. In the example, *myzip* is closed after the - :keyword:`!with` statement's suite is finished---even if an exception occurs:: - - with ZipFile('spam.zip', 'w') as myzip: - myzip.write('eggs.txt') - - .. note:: - - *metadata_encoding* is an instance-wide setting for the ZipFile. - It is not currently possible to set this on a per-member basis. - - This attribute is a workaround for legacy implementations which produce - archives with names in the current locale encoding or code page (mostly - on Windows). According to the .ZIP standard, the encoding of metadata - may be specified to be either IBM code page (default) or UTF-8 by a flag - in the archive header. - That flag takes precedence over *metadata_encoding*, which is - a Python-specific extension. - - .. versionadded:: 3.2 - Added the ability to use :class:`ZipFile` as a context manager. - - .. versionchanged:: 3.3 - Added support for :mod:`bzip2 ` and :mod:`lzma` compression. - - .. versionchanged:: 3.4 - ZIP64 extensions are enabled by default. - - .. versionchanged:: 3.5 - Added support for writing to unseekable streams. - Added support for the ``'x'`` mode. - - .. versionchanged:: 3.6 - Previously, a plain :exc:`RuntimeError` was raised for unrecognized - compression values. - - .. versionchanged:: 3.6.2 - The *file* parameter accepts a :term:`path-like object`. - - .. versionchanged:: 3.7 - Add the *compresslevel* parameter. - - .. versionadded:: 3.8 - The *strict_timestamps* keyword-only argument - - .. versionchanged:: 3.11 - Added support for specifying member name encoding for reading - metadata in the zipfile's directory and file headers. - - -.. method:: ZipFile.close() - - Close the archive file. You must call :meth:`close` before exiting your program - or essential records will not be written. - - -.. method:: ZipFile.getinfo(name) - - Return a :class:`ZipInfo` object with information about the archive member - *name*. Calling :meth:`getinfo` for a name not currently contained in the - archive will raise a :exc:`KeyError`. - - -.. method:: ZipFile.infolist() - - Return a list containing a :class:`ZipInfo` object for each member of the - archive. The objects are in the same order as their entries in the actual ZIP - file on disk if an existing archive was opened. - - -.. method:: ZipFile.namelist() - - Return a list of archive members by name. - - -.. method:: ZipFile.open(name, mode='r', pwd=None, *, force_zip64=False) - - Access a member of the archive as a binary file-like object. *name* - can be either the name of a file within the archive or a :class:`ZipInfo` - object. The *mode* parameter, if included, must be ``'r'`` (the default) - or ``'w'``. *pwd* is the password used to decrypt encrypted ZIP files as a - :class:`bytes` object. - - :meth:`~ZipFile.open` is also a context manager and therefore supports the - :keyword:`with` statement:: - - with ZipFile('spam.zip') as myzip: - with myzip.open('eggs.txt') as myfile: - print(myfile.read()) - - With *mode* ``'r'`` the file-like object - (``ZipExtFile``) is read-only and provides the following methods: - :meth:`~io.BufferedIOBase.read`, :meth:`~io.IOBase.readline`, - :meth:`~io.IOBase.readlines`, :meth:`~io.IOBase.seek`, - :meth:`~io.IOBase.tell`, :meth:`~container.__iter__`, :meth:`~iterator.__next__`. - These objects can operate independently of the ZipFile. - - With ``mode='w'``, a writable file handle is returned, which supports the - :meth:`~io.BufferedIOBase.write` method. While a writable file handle is open, - attempting to read or write other files in the ZIP file will raise a - :exc:`ValueError`. - - When writing a file, if the file size is not known in advance but may exceed - 2 GiB, pass ``force_zip64=True`` to ensure that the header format is - capable of supporting large files. If the file size is known in advance, - construct a :class:`ZipInfo` object with :attr:`~ZipInfo.file_size` set, and - use that as the *name* parameter. - - .. note:: - - The :meth:`.open`, :meth:`read` and :meth:`extract` methods can take a filename - or a :class:`ZipInfo` object. You will appreciate this when trying to read a - ZIP file that contains members with duplicate names. - - .. versionchanged:: 3.6 - Removed support of ``mode='U'``. Use :class:`io.TextIOWrapper` for reading - compressed text files in :term:`universal newlines` mode. - - .. versionchanged:: 3.6 - :meth:`ZipFile.open` can now be used to write files into the archive with the - ``mode='w'`` option. - - .. versionchanged:: 3.6 - Calling :meth:`.open` on a closed ZipFile will raise a :exc:`ValueError`. - Previously, a :exc:`RuntimeError` was raised. - - -.. method:: ZipFile.extract(member, path=None, pwd=None) - - Extract a member from the archive to the current working directory; *member* - must be its full name or a :class:`ZipInfo` object. Its file information is - extracted as accurately as possible. *path* specifies a different directory - to extract to. *member* can be a filename or a :class:`ZipInfo` object. - *pwd* is the password used for encrypted files as a :class:`bytes` object. - - Returns the normalized path created (a directory or new file). - - .. note:: - - If a member filename is an absolute path, a drive/UNC sharepoint and - leading (back)slashes will be stripped, e.g.: ``///foo/bar`` becomes - ``foo/bar`` on Unix, and ``C:\foo\bar`` becomes ``foo\bar`` on Windows. - And all ``".."`` components in a member filename will be removed, e.g.: - ``../../foo../../ba..r`` becomes ``foo../ba..r``. On Windows illegal - characters (``:``, ``<``, ``>``, ``|``, ``"``, ``?``, and ``*``) - replaced by underscore (``_``). - - .. versionchanged:: 3.6 - Calling :meth:`extract` on a closed ZipFile will raise a - :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. - - .. versionchanged:: 3.6.2 - The *path* parameter accepts a :term:`path-like object`. - - -.. method:: ZipFile.extractall(path=None, members=None, pwd=None) - - Extract all members from the archive to the current working directory. *path* - specifies a different directory to extract to. *members* is optional and must - be a subset of the list returned by :meth:`namelist`. *pwd* is the password - used for encrypted files as a :class:`bytes` object. - - .. warning:: - - Never extract archives from untrusted sources without prior inspection. - It is possible that files are created outside of *path*, e.g. members - that have absolute filenames starting with ``"/"`` or filenames with two - dots ``".."``. This module attempts to prevent that. - See :meth:`extract` note. - - .. versionchanged:: 3.6 - Calling :meth:`extractall` on a closed ZipFile will raise a - :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. - - .. versionchanged:: 3.6.2 - The *path* parameter accepts a :term:`path-like object`. - - -.. method:: ZipFile.printdir() - - Print a table of contents for the archive to ``sys.stdout``. - - -.. method:: ZipFile.setpassword(pwd) - - Set *pwd* (a :class:`bytes` object) as default password to extract encrypted files. - - -.. method:: ZipFile.read(name, pwd=None) - - Return the bytes of the file *name* in the archive. *name* is the name of the - file in the archive, or a :class:`ZipInfo` object. The archive must be open for - read or append. *pwd* is the password used for encrypted files as a :class:`bytes` - object and, if specified, overrides the default password set with :meth:`setpassword`. - Calling :meth:`read` on a ZipFile that uses a compression method other than - :const:`ZIP_STORED`, :const:`ZIP_DEFLATED`, :const:`ZIP_BZIP2` or - :const:`ZIP_LZMA` will raise a :exc:`NotImplementedError`. An error will also - be raised if the corresponding compression module is not available. - - .. versionchanged:: 3.6 - Calling :meth:`read` on a closed ZipFile will raise a :exc:`ValueError`. - Previously, a :exc:`RuntimeError` was raised. - - -.. method:: ZipFile.testzip() - - Read all the files in the archive and check their CRC's and file headers. - Return the name of the first bad file, or else return ``None``. - - .. versionchanged:: 3.6 - Calling :meth:`testzip` on a closed ZipFile will raise a - :exc:`ValueError`. Previously, a :exc:`RuntimeError` was raised. - - -.. method:: ZipFile.write(filename, arcname=None, compress_type=None, \ - compresslevel=None) - - Write the file named *filename* to the archive, giving it the archive name - *arcname* (by default, this will be the same as *filename*, but without a drive - letter and with leading path separators removed). If given, *compress_type* - overrides the value given for the *compression* parameter to the constructor for - the new entry. Similarly, *compresslevel* will override the constructor if - given. - The archive must be open with mode ``'w'``, ``'x'`` or ``'a'``. - - .. note:: - - The ZIP file standard historically did not specify a metadata encoding, - but strongly recommended CP437 (the original IBM PC encoding) for - interoperability. Recent versions allow use of UTF-8 (only). In this - module, UTF-8 will automatically be used to write the member names if - they contain any non-ASCII characters. It is not possible to write - member names in any encoding other than ASCII or UTF-8. - - .. note:: - - Archive names should be relative to the archive root, that is, they should not - start with a path separator. - - .. note:: - - If ``arcname`` (or ``filename``, if ``arcname`` is not given) contains a null - byte, the name of the file in the archive will be truncated at the null byte. - - .. note:: - - A leading slash in the filename may lead to the archive being impossible to - open in some zip programs on Windows systems. - - .. versionchanged:: 3.6 - Calling :meth:`write` on a ZipFile created with mode ``'r'`` or - a closed ZipFile will raise a :exc:`ValueError`. Previously, - a :exc:`RuntimeError` was raised. - - -.. method:: ZipFile.writestr(zinfo_or_arcname, data, compress_type=None, \ - compresslevel=None) - - Write a file into the archive. The contents is *data*, which may be either - a :class:`str` or a :class:`bytes` instance; if it is a :class:`str`, - it is encoded as UTF-8 first. *zinfo_or_arcname* is either the file - name it will be given in the archive, or a :class:`ZipInfo` instance. If it's - an instance, at least the filename, date, and time must be given. If it's a - name, the date and time is set to the current date and time. - The archive must be opened with mode ``'w'``, ``'x'`` or ``'a'``. - - If given, *compress_type* overrides the value given for the *compression* - parameter to the constructor for the new entry, or in the *zinfo_or_arcname* - (if that is a :class:`ZipInfo` instance). Similarly, *compresslevel* will - override the constructor if given. - - .. note:: - - When passing a :class:`ZipInfo` instance as the *zinfo_or_arcname* parameter, - the compression method used will be that specified in the *compress_type* - member of the given :class:`ZipInfo` instance. By default, the - :class:`ZipInfo` constructor sets this member to :const:`ZIP_STORED`. - - .. versionchanged:: 3.2 - The *compress_type* argument. - - .. versionchanged:: 3.6 - Calling :meth:`writestr` on a ZipFile created with mode ``'r'`` or - a closed ZipFile will raise a :exc:`ValueError`. Previously, - a :exc:`RuntimeError` was raised. - -.. method:: ZipFile.mkdir(zinfo_or_directory, mode=511) - - Create a directory inside the archive. If *zinfo_or_directory* is a string, - a directory is created inside the archive with the mode that is specified in - the *mode* argument. If, however, *zinfo_or_directory* is - a :class:`ZipInfo` instance then the *mode* argument is ignored. - - The archive must be opened with mode ``'w'``, ``'x'`` or ``'a'``. - - .. versionadded:: 3.11 - - -The following data attributes are also available: - -.. attribute:: ZipFile.filename - - Name of the ZIP file. - -.. attribute:: ZipFile.debug - - The level of debug output to use. This may be set from ``0`` (the default, no - output) to ``3`` (the most output). Debugging information is written to - ``sys.stdout``. - -.. attribute:: ZipFile.comment - - The comment associated with the ZIP file as a :class:`bytes` object. - If assigning a comment to a - :class:`ZipFile` instance created with mode ``'w'``, ``'x'`` or ``'a'``, - it should be no longer than 65535 bytes. Comments longer than this will be - truncated. - - -.. _path-objects: - -Path Objects ------------- - -.. class:: Path(root, at='') - - Construct a Path object from a ``root`` zipfile (which may be a - :class:`ZipFile` instance or ``file`` suitable for passing to - the :class:`ZipFile` constructor). - - ``at`` specifies the location of this Path within the zipfile, - e.g. 'dir/file.txt', 'dir/', or ''. Defaults to the empty string, - indicating the root. - -Path objects expose the following features of :mod:`pathlib.Path` -objects: - -Path objects are traversable using the ``/`` operator or ``joinpath``. - -.. attribute:: Path.name - - The final path component. - -.. method:: Path.open(mode='r', *, pwd, **) - - Invoke :meth:`ZipFile.open` on the current path. - Allows opening for read or write, text or binary - through supported modes: 'r', 'w', 'rb', 'wb'. - Positional and keyword arguments are passed through to - :class:`io.TextIOWrapper` when opened as text and - ignored otherwise. - ``pwd`` is the ``pwd`` parameter to - :meth:`ZipFile.open`. - - .. versionchanged:: 3.9 - Added support for text and binary modes for open. Default - mode is now text. - - .. versionchanged:: 3.11.2 - The ``encoding`` parameter can be supplied as a positional argument - without causing a :exc:`TypeError`. As it could in 3.9. Code needing to - be compatible with unpatched 3.10 and 3.11 versions must pass all - :class:`io.TextIOWrapper` arguments, ``encoding`` included, as keywords. - -.. method:: Path.iterdir() - - Enumerate the children of the current directory. - -.. method:: Path.is_dir() - - Return ``True`` if the current context references a directory. - -.. method:: Path.is_file() - - Return ``True`` if the current context references a file. - -.. method:: Path.exists() - - Return ``True`` if the current context references a file or - directory in the zip file. - -.. data:: Path.suffix - - The file extension of the final component. - - .. versionadded:: 3.11 - Added :data:`Path.suffix` property. - -.. data:: Path.stem - - The final path component, without its suffix. - - .. versionadded:: 3.11 - Added :data:`Path.stem` property. - -.. data:: Path.suffixes - - A list of the path’s file extensions. - - .. versionadded:: 3.11 - Added :data:`Path.suffixes` property. - -.. method:: Path.read_text(*, **) - - Read the current file as unicode text. Positional and - keyword arguments are passed through to - :class:`io.TextIOWrapper` (except ``buffer``, which is - implied by the context). - - .. versionchanged:: 3.11.2 - The ``encoding`` parameter can be supplied as a positional argument - without causing a :exc:`TypeError`. As it could in 3.9. Code needing to - be compatible with unpatched 3.10 and 3.11 versions must pass all - :class:`io.TextIOWrapper` arguments, ``encoding`` included, as keywords. - -.. method:: Path.read_bytes() - - Read the current file as bytes. - -.. method:: Path.joinpath(*other) - - Return a new Path object with each of the *other* arguments - joined. The following are equivalent:: - - >>> Path(...).joinpath('child').joinpath('grandchild') - >>> Path(...).joinpath('child', 'grandchild') - >>> Path(...) / 'child' / 'grandchild' - - .. versionchanged:: 3.10 - Prior to 3.10, ``joinpath`` was undocumented and accepted - exactly one parameter. - -The `zipp `_ project provides backports -of the latest path object functionality to older Pythons. Use -``zipp.Path`` in place of ``zipfile.Path`` for early access to -changes. - -.. _pyzipfile-objects: - -PyZipFile Objects ------------------ - -The :class:`PyZipFile` constructor takes the same parameters as the -:class:`ZipFile` constructor, and one additional parameter, *optimize*. - -.. class:: PyZipFile(file, mode='r', compression=ZIP_STORED, allowZip64=True, \ - optimize=-1) - - .. versionadded:: 3.2 - The *optimize* parameter. - - .. versionchanged:: 3.4 - ZIP64 extensions are enabled by default. - - Instances have one method in addition to those of :class:`ZipFile` objects: - - .. method:: PyZipFile.writepy(pathname, basename='', filterfunc=None) - - Search for files :file:`\*.py` and add the corresponding file to the - archive. - - If the *optimize* parameter to :class:`PyZipFile` was not given or ``-1``, - the corresponding file is a :file:`\*.pyc` file, compiling if necessary. - - If the *optimize* parameter to :class:`PyZipFile` was ``0``, ``1`` or - ``2``, only files with that optimization level (see :func:`compile`) are - added to the archive, compiling if necessary. - - If *pathname* is a file, the filename must end with :file:`.py`, and - just the (corresponding :file:`\*.pyc`) file is added at the top level - (no path information). If *pathname* is a file that does not end with - :file:`.py`, a :exc:`RuntimeError` will be raised. If it is a directory, - and the directory is not a package directory, then all the files - :file:`\*.pyc` are added at the top level. If the directory is a - package directory, then all :file:`\*.pyc` are added under the package - name as a file path, and if any subdirectories are package directories, - all of these are added recursively in sorted order. - - *basename* is intended for internal use only. - - *filterfunc*, if given, must be a function taking a single string - argument. It will be passed each path (including each individual full - file path) before it is added to the archive. If *filterfunc* returns a - false value, the path will not be added, and if it is a directory its - contents will be ignored. For example, if our test files are all either - in ``test`` directories or start with the string ``test_``, we can use a - *filterfunc* to exclude them:: - - >>> zf = PyZipFile('myprog.zip') - >>> def notests(s): - ... fn = os.path.basename(s) - ... return (not (fn == 'test' or fn.startswith('test_'))) - >>> zf.writepy('myprog', filterfunc=notests) - - The :meth:`writepy` method makes archives with file names like - this:: - - string.pyc # Top level name - test/__init__.pyc # Package directory - test/testall.pyc # Module test.testall - test/bogus/__init__.pyc # Subpackage directory - test/bogus/myfile.pyc # Submodule test.bogus.myfile - - .. versionadded:: 3.4 - The *filterfunc* parameter. - - .. versionchanged:: 3.6.2 - The *pathname* parameter accepts a :term:`path-like object`. - - .. versionchanged:: 3.7 - Recursion sorts directory entries. - - -.. _zipinfo-objects: - -ZipInfo Objects ---------------- - -Instances of the :class:`ZipInfo` class are returned by the :meth:`.getinfo` and -:meth:`.infolist` methods of :class:`ZipFile` objects. Each object stores -information about a single member of the ZIP archive. - -There is one classmethod to make a :class:`ZipInfo` instance for a filesystem -file: - -.. classmethod:: ZipInfo.from_file(filename, arcname=None, *, \ - strict_timestamps=True) - - Construct a :class:`ZipInfo` instance for a file on the filesystem, in - preparation for adding it to a zip file. - - *filename* should be the path to a file or directory on the filesystem. - - If *arcname* is specified, it is used as the name within the archive. - If *arcname* is not specified, the name will be the same as *filename*, but - with any drive letter and leading path separators removed. - - The *strict_timestamps* argument, when set to ``False``, allows to - zip files older than 1980-01-01 at the cost of setting the - timestamp to 1980-01-01. - Similar behavior occurs with files newer than 2107-12-31, - the timestamp is also set to the limit. - - .. versionadded:: 3.6 - - .. versionchanged:: 3.6.2 - The *filename* parameter accepts a :term:`path-like object`. - - .. versionadded:: 3.8 - The *strict_timestamps* keyword-only argument - - -Instances have the following methods and attributes: - -.. method:: ZipInfo.is_dir() - - Return ``True`` if this archive member is a directory. - - This uses the entry's name: directories should always end with ``/``. - - .. versionadded:: 3.6 - - -.. attribute:: ZipInfo.filename - - Name of the file in the archive. - - -.. attribute:: ZipInfo.date_time - - The time and date of the last modification to the archive member. This is a - tuple of six values: - - +-------+--------------------------+ - | Index | Value | - +=======+==========================+ - | ``0`` | Year (>= 1980) | - +-------+--------------------------+ - | ``1`` | Month (one-based) | - +-------+--------------------------+ - | ``2`` | Day of month (one-based) | - +-------+--------------------------+ - | ``3`` | Hours (zero-based) | - +-------+--------------------------+ - | ``4`` | Minutes (zero-based) | - +-------+--------------------------+ - | ``5`` | Seconds (zero-based) | - +-------+--------------------------+ - - .. note:: - - The ZIP file format does not support timestamps before 1980. - - -.. attribute:: ZipInfo.compress_type - - Type of compression for the archive member. - - -.. attribute:: ZipInfo.comment - - Comment for the individual archive member as a :class:`bytes` object. - - -.. attribute:: ZipInfo.extra - - Expansion field data. The `PKZIP Application Note`_ contains - some comments on the internal structure of the data contained in this - :class:`bytes` object. - - -.. attribute:: ZipInfo.create_system - - System which created ZIP archive. - - -.. attribute:: ZipInfo.create_version - - PKZIP version which created ZIP archive. - - -.. attribute:: ZipInfo.extract_version - - PKZIP version needed to extract archive. - - -.. attribute:: ZipInfo.reserved - - Must be zero. - - -.. attribute:: ZipInfo.flag_bits - - ZIP flag bits. - - -.. attribute:: ZipInfo.volume - - Volume number of file header. - - -.. attribute:: ZipInfo.internal_attr - - Internal attributes. - - -.. attribute:: ZipInfo.external_attr - - External file attributes. - - -.. attribute:: ZipInfo.header_offset - - Byte offset to the file header. - - -.. attribute:: ZipInfo.CRC - - CRC-32 of the uncompressed file. - - -.. attribute:: ZipInfo.compress_size - - Size of the compressed data. - - -.. attribute:: ZipInfo.file_size - - Size of the uncompressed file. - - -.. _zipfile-commandline: -.. program:: zipfile - -Command-Line Interface ----------------------- - -The :mod:`zipfile` module provides a simple command-line interface to interact -with ZIP archives. - -If you want to create a new ZIP archive, specify its name after the :option:`-c` -option and then list the filename(s) that should be included: - -.. code-block:: shell-session - - $ python -m zipfile -c monty.zip spam.txt eggs.txt - -Passing a directory is also acceptable: - -.. code-block:: shell-session - - $ python -m zipfile -c monty.zip life-of-brian_1979/ - -If you want to extract a ZIP archive into the specified directory, use -the :option:`-e` option: - -.. code-block:: shell-session - - $ python -m zipfile -e monty.zip target-dir/ - -For a list of the files in a ZIP archive, use the :option:`-l` option: - -.. code-block:: shell-session - - $ python -m zipfile -l monty.zip - - -Command-line options -~~~~~~~~~~~~~~~~~~~~ - -.. option:: -l - --list - - List files in a zipfile. - -.. option:: -c ... - --create ... - - Create zipfile from source files. - -.. option:: -e - --extract - - Extract zipfile into target directory. - -.. option:: -t - --test - - Test whether the zipfile is valid or not. - -.. option:: --metadata-encoding - - Specify encoding of member names for :option:`-l`, :option:`-e` and - :option:`-t`. - - .. versionadded:: 3.11 - - -Decompression pitfalls ----------------------- - -The extraction in zipfile module might fail due to some pitfalls listed below. - -From file itself -~~~~~~~~~~~~~~~~ - -Decompression may fail due to incorrect password / CRC checksum / ZIP format or -unsupported compression method / decryption. - -File System limitations -~~~~~~~~~~~~~~~~~~~~~~~ - -Exceeding limitations on different file systems can cause decompression failed. -Such as allowable characters in the directory entries, length of the file name, -length of the pathname, size of a single file, and number of files, etc. - -.. _zipfile-resources-limitations: - -Resources limitations -~~~~~~~~~~~~~~~~~~~~~ - -The lack of memory or disk volume would lead to decompression -failed. For example, decompression bombs (aka `ZIP bomb`_) -apply to zipfile library that can cause disk volume exhaustion. - -Interruption -~~~~~~~~~~~~ - -Interruption during the decompression, such as pressing control-C or killing the -decompression process may result in incomplete decompression of the archive. - -Default behaviors of extraction -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Not knowing the default extraction behaviors -can cause unexpected decompression results. -For example, when extracting the same archive twice, -it overwrites files without asking. - - -.. _ZIP bomb: https://en.wikipedia.org/wiki/Zip_bomb -.. _PKZIP Application Note: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT diff --git a/Doc/library/zipimport.rst b/Doc/library/zipimport.rst deleted file mode 100644 index 71647445ed37ef..00000000000000 --- a/Doc/library/zipimport.rst +++ /dev/null @@ -1,218 +0,0 @@ -:mod:`zipimport` --- Import modules from Zip archives -===================================================== - -.. module:: zipimport - :synopsis: Support for importing Python modules from ZIP archives. - -.. moduleauthor:: Just van Rossum - -**Source code:** :source:`Lib/zipimport.py` - --------------- - -This module adds the ability to import Python modules (:file:`\*.py`, -:file:`\*.pyc`) and packages from ZIP-format archives. It is usually not -needed to use the :mod:`zipimport` module explicitly; it is automatically used -by the built-in :keyword:`import` mechanism for :data:`sys.path` items that are paths -to ZIP archives. - -Typically, :data:`sys.path` is a list of directory names as strings. This module -also allows an item of :data:`sys.path` to be a string naming a ZIP file archive. -The ZIP archive can contain a subdirectory structure to support package imports, -and a path within the archive can be specified to only import from a -subdirectory. For example, the path :file:`example.zip/lib/` would only -import from the :file:`lib/` subdirectory within the archive. - -Any files may be present in the ZIP archive, but importers are only invoked for -:file:`.py` and :file:`.pyc` files. ZIP import of dynamic modules -(:file:`.pyd`, :file:`.so`) is disallowed. Note that if an archive only contains -:file:`.py` files, Python will not attempt to modify the archive by adding the -corresponding :file:`.pyc` file, meaning that if a ZIP archive -doesn't contain :file:`.pyc` files, importing may be rather slow. - -.. versionchanged:: 3.8 - Previously, ZIP archives with an archive comment were not supported. - -.. seealso:: - - `PKZIP Application Note `_ - Documentation on the ZIP file format by Phil Katz, the creator of the format and - algorithms used. - - :pep:`273` - Import Modules from Zip Archives - Written by James C. Ahlstrom, who also provided an implementation. Python 2.3 - follows the specification in :pep:`273`, but uses an implementation written by Just - van Rossum that uses the import hooks described in :pep:`302`. - - :mod:`importlib` - The implementation of the import machinery - Package providing the relevant protocols for all importers to - implement. - - -This module defines an exception: - -.. exception:: ZipImportError - - Exception raised by zipimporter objects. It's a subclass of :exc:`ImportError`, - so it can be caught as :exc:`ImportError`, too. - - -.. _zipimporter-objects: - -zipimporter Objects -------------------- - -:class:`zipimporter` is the class for importing ZIP files. - -.. class:: zipimporter(archivepath) - - Create a new zipimporter instance. *archivepath* must be a path to a ZIP - file, or to a specific path within a ZIP file. For example, an *archivepath* - of :file:`foo/bar.zip/lib` will look for modules in the :file:`lib` directory - inside the ZIP file :file:`foo/bar.zip` (provided that it exists). - - :exc:`ZipImportError` is raised if *archivepath* doesn't point to a valid ZIP - archive. - - .. method:: create_module(spec) - - Implementation of :meth:`importlib.abc.Loader.create_module` that returns - :const:`None` to explicitly request the default semantics. - - .. versionadded:: 3.10 - - - .. method:: exec_module(module) - - Implementation of :meth:`importlib.abc.Loader.exec_module`. - - .. versionadded:: 3.10 - - - .. method:: find_loader(fullname, path=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_loader`. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_module(fullname, path=None) - - Search for a module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. It returns the zipimporter instance itself - if the module was found, or :const:`None` if it wasn't. The optional - *path* argument is ignored---it's there for compatibility with the - importer protocol. - - .. deprecated:: 3.10 - - Use :meth:`find_spec` instead. - - - .. method:: find_spec(fullname, target=None) - - An implementation of :meth:`importlib.abc.PathEntryFinder.find_spec`. - - .. versionadded:: 3.10 - - - .. method:: get_code(fullname) - - Return the code object for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be imported. - - - .. method:: get_data(pathname) - - Return the data associated with *pathname*. Raise :exc:`OSError` if the - file wasn't found. - - .. versionchanged:: 3.3 - :exc:`IOError` used to be raised, it is now an alias of :exc:`OSError`. - - - .. method:: get_filename(fullname) - - Return the value ``__file__`` would be set to if the specified module - was imported. Raise :exc:`ZipImportError` if the module couldn't be - imported. - - .. versionadded:: 3.1 - - - .. method:: get_source(fullname) - - Return the source code for the specified module. Raise - :exc:`ZipImportError` if the module couldn't be found, return - :const:`None` if the archive does contain the module, but has no source - for it. - - - .. method:: is_package(fullname) - - Return ``True`` if the module specified by *fullname* is a package. Raise - :exc:`ZipImportError` if the module couldn't be found. - - - .. method:: load_module(fullname) - - Load the module specified by *fullname*. *fullname* must be the fully - qualified (dotted) module name. Returns the imported module on success, - raises :exc:`ZipImportError` on failure. - - .. deprecated:: 3.10 - - Use :meth:`exec_module` instead. - - - .. method:: invalidate_caches() - - Clear out the internal cache of information about files found within - the ZIP archive. - - .. versionadded:: 3.10 - - - .. attribute:: archive - - The file name of the importer's associated ZIP file, without a possible - subpath. - - - .. attribute:: prefix - - The subpath within the ZIP file where modules are searched. This is the - empty string for zipimporter objects which point to the root of the ZIP - file. - - The :attr:`archive` and :attr:`prefix` attributes, when combined with a - slash, equal the original *archivepath* argument given to the - :class:`zipimporter` constructor. - - -.. _zipimport-examples: - -Examples --------- - -Here is an example that imports a module from a ZIP archive - note that the -:mod:`zipimport` module is not explicitly used. - -.. code-block:: shell-session - - $ unzip -l example.zip - Archive: example.zip - Length Date Time Name - -------- ---- ---- ---- - 8467 11-26-02 22:30 jwzthreading.py - -------- ------- - 8467 1 file - $ ./python - Python 2.3 (#1, Aug 1 2003, 19:54:32) - >>> import sys - >>> sys.path.insert(0, 'example.zip') # Add .zip file to front of path - >>> import jwzthreading - >>> jwzthreading.__file__ - 'example.zip/jwzthreading.py' diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst deleted file mode 100644 index ac179722dee2be..00000000000000 --- a/Doc/library/zlib.rst +++ /dev/null @@ -1,343 +0,0 @@ -:mod:`zlib` --- Compression compatible with :program:`gzip` -=========================================================== - -.. module:: zlib - :synopsis: Low-level interface to compression and decompression routines - compatible with gzip. - --------------- - -For applications that require data compression, the functions in this module -allow compression and decompression, using the zlib library. The zlib library -has its own home page at https://www.zlib.net. There are known -incompatibilities between the Python module and versions of the zlib library -earlier than 1.1.3; 1.1.3 has a `security vulnerability `_, so we recommend using -1.1.4 or later. - -zlib's functions have many options and often need to be used in a particular -order. This documentation doesn't attempt to cover all of the permutations; -consult the zlib manual at http://www.zlib.net/manual.html for authoritative -information. - -For reading and writing ``.gz`` files see the :mod:`gzip` module. - -The available exception and functions in this module are: - - -.. exception:: error - - Exception raised on compression and decompression errors. - - -.. function:: adler32(data[, value]) - - Computes an Adler-32 checksum of *data*. (An Adler-32 checksum is almost as - reliable as a CRC32 but can be computed much more quickly.) The result - is an unsigned 32-bit integer. If *value* is present, it is used as - the starting value of the checksum; otherwise, a default value of 1 - is used. Passing in *value* allows computing a running checksum over the - concatenation of several inputs. The algorithm is not cryptographically - strong, and should not be used for authentication or digital signatures. Since - the algorithm is designed for use as a checksum algorithm, it is not suitable - for use as a general hash algorithm. - - .. versionchanged:: 3.0 - The result is always unsigned. - -.. function:: compress(data, /, level=-1, wbits=MAX_WBITS) - - Compresses the bytes in *data*, returning a bytes object containing compressed data. - *level* is an integer from ``0`` to ``9`` or ``-1`` controlling the level of compression; - ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, ``9`` (Z_BEST_COMPRESSION) - is slowest and produces the most. ``0`` (Z_NO_COMPRESSION) is no compression. - The default value is ``-1`` (Z_DEFAULT_COMPRESSION). Z_DEFAULT_COMPRESSION represents a default - compromise between speed and compression (currently equivalent to level 6). - - .. _compress-wbits: - - The *wbits* argument controls the size of the history buffer (or the - "window size") used when compressing data, and whether a header and - trailer is included in the output. It can take several ranges of values, - defaulting to ``15`` (MAX_WBITS): - - * +9 to +15: The base-two logarithm of the window size, which - therefore ranges between 512 and 32768. Larger values produce - better compression at the expense of greater memory usage. The - resulting output will include a zlib-specific header and trailer. - - * −9 to −15: Uses the absolute value of *wbits* as the - window size logarithm, while producing a raw output stream with no - header or trailing checksum. - - * +25 to +31 = 16 + (9 to 15): Uses the low 4 bits of the value as the - window size logarithm, while including a basic :program:`gzip` header - and trailing checksum in the output. - - Raises the :exc:`error` exception if any error occurs. - - .. versionchanged:: 3.6 - *level* can now be used as a keyword parameter. - - .. versionchanged:: 3.11 - The *wbits* parameter is now available to set window bits and - compression type. - -.. function:: compressobj(level=-1, method=DEFLATED, wbits=MAX_WBITS, memLevel=DEF_MEM_LEVEL, strategy=Z_DEFAULT_STRATEGY[, zdict]) - - Returns a compression object, to be used for compressing data streams that won't - fit into memory at once. - - *level* is the compression level -- an integer from ``0`` to ``9`` or ``-1``. - A value of ``1`` (Z_BEST_SPEED) is fastest and produces the least compression, - while a value of ``9`` (Z_BEST_COMPRESSION) is slowest and produces the most. - ``0`` (Z_NO_COMPRESSION) is no compression. The default value is ``-1`` (Z_DEFAULT_COMPRESSION). - Z_DEFAULT_COMPRESSION represents a default compromise between speed and compression - (currently equivalent to level 6). - - *method* is the compression algorithm. Currently, the only supported value is - :const:`DEFLATED`. - - The *wbits* parameter controls the size of the history buffer (or the - "window size"), and what header and trailer format will be used. It has - the same meaning as `described for compress() <#compress-wbits>`__. - - The *memLevel* argument controls the amount of memory used for the - internal compression state. Valid values range from ``1`` to ``9``. - Higher values use more memory, but are faster and produce smaller output. - - *strategy* is used to tune the compression algorithm. Possible values are - :const:`Z_DEFAULT_STRATEGY`, :const:`Z_FILTERED`, :const:`Z_HUFFMAN_ONLY`, - :const:`Z_RLE` (zlib 1.2.0.1) and :const:`Z_FIXED` (zlib 1.2.2.2). - - *zdict* is a predefined compression dictionary. This is a sequence of bytes - (such as a :class:`bytes` object) containing subsequences that are expected - to occur frequently in the data that is to be compressed. Those subsequences - that are expected to be most common should come at the end of the dictionary. - - .. versionchanged:: 3.3 - Added the *zdict* parameter and keyword argument support. - - -.. function:: crc32(data[, value]) - - .. index:: - single: Cyclic Redundancy Check - single: checksum; Cyclic Redundancy Check - - Computes a CRC (Cyclic Redundancy Check) checksum of *data*. The - result is an unsigned 32-bit integer. If *value* is present, it is used - as the starting value of the checksum; otherwise, a default value of 0 - is used. Passing in *value* allows computing a running checksum over the - concatenation of several inputs. The algorithm is not cryptographically - strong, and should not be used for authentication or digital signatures. Since - the algorithm is designed for use as a checksum algorithm, it is not suitable - for use as a general hash algorithm. - - .. versionchanged:: 3.0 - The result is always unsigned. - -.. function:: decompress(data, /, wbits=MAX_WBITS, bufsize=DEF_BUF_SIZE) - - Decompresses the bytes in *data*, returning a bytes object containing the - uncompressed data. The *wbits* parameter depends on - the format of *data*, and is discussed further below. - If *bufsize* is given, it is used as the initial size of the output - buffer. Raises the :exc:`error` exception if any error occurs. - - .. _decompress-wbits: - - The *wbits* parameter controls the size of the history buffer - (or "window size"), and what header and trailer format is expected. - It is similar to the parameter for :func:`compressobj`, but accepts - more ranges of values: - - * +8 to +15: The base-two logarithm of the window size. The input - must include a zlib header and trailer. - - * 0: Automatically determine the window size from the zlib header. - Only supported since zlib 1.2.3.5. - - * −8 to −15: Uses the absolute value of *wbits* as the window size - logarithm. The input must be a raw stream with no header or trailer. - - * +24 to +31 = 16 + (8 to 15): Uses the low 4 bits of the value as - the window size logarithm. The input must include a gzip header and - trailer. - - * +40 to +47 = 32 + (8 to 15): Uses the low 4 bits of the value as - the window size logarithm, and automatically accepts either - the zlib or gzip format. - - When decompressing a stream, the window size must not be smaller - than the size originally used to compress the stream; using a too-small - value may result in an :exc:`error` exception. The default *wbits* value - corresponds to the largest window size and requires a zlib header and - trailer to be included. - - *bufsize* is the initial size of the buffer used to hold decompressed data. If - more space is required, the buffer size will be increased as needed, so you - don't have to get this value exactly right; tuning it will only save a few calls - to :c:func:`malloc`. - - .. versionchanged:: 3.6 - *wbits* and *bufsize* can be used as keyword arguments. - -.. function:: decompressobj(wbits=MAX_WBITS[, zdict]) - - Returns a decompression object, to be used for decompressing data streams that - won't fit into memory at once. - - The *wbits* parameter controls the size of the history buffer (or the - "window size"), and what header and trailer format is expected. It has - the same meaning as `described for decompress() <#decompress-wbits>`__. - - The *zdict* parameter specifies a predefined compression dictionary. If - provided, this must be the same dictionary as was used by the compressor that - produced the data that is to be decompressed. - - .. note:: - - If *zdict* is a mutable object (such as a :class:`bytearray`), you must not - modify its contents between the call to :func:`decompressobj` and the first - call to the decompressor's ``decompress()`` method. - - .. versionchanged:: 3.3 - Added the *zdict* parameter. - - -Compression objects support the following methods: - - -.. method:: Compress.compress(data) - - Compress *data*, returning a bytes object containing compressed data for at least - part of the data in *data*. This data should be concatenated to the output - produced by any preceding calls to the :meth:`compress` method. Some input may - be kept in internal buffers for later processing. - - -.. method:: Compress.flush([mode]) - - All pending input is processed, and a bytes object containing the remaining compressed - output is returned. *mode* can be selected from the constants - :const:`Z_NO_FLUSH`, :const:`Z_PARTIAL_FLUSH`, :const:`Z_SYNC_FLUSH`, - :const:`Z_FULL_FLUSH`, :const:`Z_BLOCK` (zlib 1.2.3.4), or :const:`Z_FINISH`, - defaulting to :const:`Z_FINISH`. Except :const:`Z_FINISH`, all constants - allow compressing further bytestrings of data, while :const:`Z_FINISH` finishes the - compressed stream and prevents compressing any more data. After calling :meth:`flush` - with *mode* set to :const:`Z_FINISH`, the :meth:`compress` method cannot be called again; - the only realistic action is to delete the object. - - -.. method:: Compress.copy() - - Returns a copy of the compression object. This can be used to efficiently - compress a set of data that share a common initial prefix. - - -.. versionchanged:: 3.8 - Added :func:`copy.copy` and :func:`copy.deepcopy` support to compression - objects. - - -Decompression objects support the following methods and attributes: - - -.. attribute:: Decompress.unused_data - - A bytes object which contains any bytes past the end of the compressed data. That is, - this remains ``b""`` until the last byte that contains compression data is - available. If the whole bytestring turned out to contain compressed data, this is - ``b""``, an empty bytes object. - - -.. attribute:: Decompress.unconsumed_tail - - A bytes object that contains any data that was not consumed by the last - :meth:`decompress` call because it exceeded the limit for the uncompressed data - buffer. This data has not yet been seen by the zlib machinery, so you must feed - it (possibly with further data concatenated to it) back to a subsequent - :meth:`decompress` method call in order to get correct output. - - -.. attribute:: Decompress.eof - - A boolean indicating whether the end of the compressed data stream has been - reached. - - This makes it possible to distinguish between a properly formed compressed - stream, and an incomplete or truncated one. - - .. versionadded:: 3.3 - - -.. method:: Decompress.decompress(data, max_length=0) - - Decompress *data*, returning a bytes object containing the uncompressed data - corresponding to at least part of the data in *string*. This data should be - concatenated to the output produced by any preceding calls to the - :meth:`decompress` method. Some of the input data may be preserved in internal - buffers for later processing. - - If the optional parameter *max_length* is non-zero then the return value will be - no longer than *max_length*. This may mean that not all of the compressed input - can be processed; and unconsumed data will be stored in the attribute - :attr:`unconsumed_tail`. This bytestring must be passed to a subsequent call to - :meth:`decompress` if decompression is to continue. If *max_length* is zero - then the whole input is decompressed, and :attr:`unconsumed_tail` is empty. - - .. versionchanged:: 3.6 - *max_length* can be used as a keyword argument. - - -.. method:: Decompress.flush([length]) - - All pending input is processed, and a bytes object containing the remaining - uncompressed output is returned. After calling :meth:`flush`, the - :meth:`decompress` method cannot be called again; the only realistic action is - to delete the object. - - The optional parameter *length* sets the initial size of the output buffer. - - -.. method:: Decompress.copy() - - Returns a copy of the decompression object. This can be used to save the state - of the decompressor midway through the data stream in order to speed up random - seeks into the stream at a future point. - - -.. versionchanged:: 3.8 - Added :func:`copy.copy` and :func:`copy.deepcopy` support to decompression - objects. - - -Information about the version of the zlib library in use is available through -the following constants: - - -.. data:: ZLIB_VERSION - - The version string of the zlib library that was used for building the module. - This may be different from the zlib library actually used at runtime, which - is available as :const:`ZLIB_RUNTIME_VERSION`. - - -.. data:: ZLIB_RUNTIME_VERSION - - The version string of the zlib library actually loaded by the interpreter. - - .. versionadded:: 3.3 - - -.. seealso:: - - Module :mod:`gzip` - Reading and writing :program:`gzip`\ -format files. - - http://www.zlib.net - The zlib library home page. - - http://www.zlib.net/manual.html - The zlib manual explains the semantics and usage of the library's many - functions. diff --git a/Doc/library/zoneinfo.rst b/Doc/library/zoneinfo.rst deleted file mode 100644 index d2e5619e7e47c2..00000000000000 --- a/Doc/library/zoneinfo.rst +++ /dev/null @@ -1,417 +0,0 @@ -:mod:`zoneinfo` --- IANA time zone support -========================================== - -.. module:: zoneinfo - :synopsis: IANA time zone support - -.. versionadded:: 3.9 - -.. moduleauthor:: Paul Ganssle -.. sectionauthor:: Paul Ganssle - -**Source code:** :source:`Lib/zoneinfo` - --------------- - -The :mod:`zoneinfo` module provides a concrete time zone implementation to -support the IANA time zone database as originally specified in :pep:`615`. By -default, :mod:`zoneinfo` uses the system's time zone data if available; if no -system time zone data is available, the library will fall back to using the -first-party `tzdata`_ package available on PyPI. - -.. seealso:: - - Module: :mod:`datetime` - Provides the :class:`~datetime.time` and :class:`~datetime.datetime` - types with which the :class:`ZoneInfo` class is designed to be used. - - Package `tzdata`_ - First-party package maintained by the CPython core developers to supply - time zone data via PyPI. - -.. include:: ../includes/wasm-notavail.rst - -Using ``ZoneInfo`` ------------------- - -:class:`ZoneInfo` is a concrete implementation of the :class:`datetime.tzinfo` -abstract base class, and is intended to be attached to ``tzinfo``, either via -the constructor, the :meth:`datetime.replace ` -method or :meth:`datetime.astimezone `:: - - >>> from zoneinfo import ZoneInfo - >>> from datetime import datetime, timedelta - - >>> dt = datetime(2020, 10, 31, 12, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) - 2020-10-31 12:00:00-07:00 - - >>> dt.tzname() - 'PDT' - -Datetimes constructed in this way are compatible with datetime arithmetic and -handle daylight saving time transitions with no further intervention:: - - >>> dt_add = dt + timedelta(days=1) - - >>> print(dt_add) - 2020-11-01 12:00:00-08:00 - - >>> dt_add.tzname() - 'PST' - -These time zones also support the :attr:`~datetime.datetime.fold` attribute -introduced in :pep:`495`. During offset transitions which induce ambiguous -times (such as a daylight saving time to standard time transition), the offset -from *before* the transition is used when ``fold=0``, and the offset *after* -the transition is used when ``fold=1``, for example:: - - >>> dt = datetime(2020, 11, 1, 1, tzinfo=ZoneInfo("America/Los_Angeles")) - >>> print(dt) - 2020-11-01 01:00:00-07:00 - - >>> print(dt.replace(fold=1)) - 2020-11-01 01:00:00-08:00 - -When converting from another time zone, the fold will be set to the correct -value:: - - >>> from datetime import timezone - >>> LOS_ANGELES = ZoneInfo("America/Los_Angeles") - >>> dt_utc = datetime(2020, 11, 1, 8, tzinfo=timezone.utc) - - >>> # Before the PDT -> PST transition - >>> print(dt_utc.astimezone(LOS_ANGELES)) - 2020-11-01 01:00:00-07:00 - - >>> # After the PDT -> PST transition - >>> print((dt_utc + timedelta(hours=1)).astimezone(LOS_ANGELES)) - 2020-11-01 01:00:00-08:00 - -Data sources ------------- - -The ``zoneinfo`` module does not directly provide time zone data, and instead -pulls time zone information from the system time zone database or the -first-party PyPI package `tzdata`_, if available. Some systems, including -notably Windows systems, do not have an IANA database available, and so for -projects targeting cross-platform compatibility that require time zone data, it -is recommended to declare a dependency on tzdata. If neither system data nor -tzdata are available, all calls to :class:`ZoneInfo` will raise -:exc:`ZoneInfoNotFoundError`. - -.. _zoneinfo_data_configuration: - -Configuring the data sources -**************************** - -When ``ZoneInfo(key)`` is called, the constructor first searches the -directories specified in :data:`TZPATH` for a file matching ``key``, and on -failure looks for a match in the tzdata package. This behavior can be -configured in three ways: - -1. The default :data:`TZPATH` when not otherwise specified can be configured at - :ref:`compile time `. -2. :data:`TZPATH` can be configured using :ref:`an environment variable - `. -3. At :ref:`runtime `, the search path can be - manipulated using the :func:`reset_tzpath` function. - -.. _zoneinfo_data_compile_time_config: - -Compile-time configuration -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The default :data:`TZPATH` includes several common deployment locations for the -time zone database (except on Windows, where there are no "well-known" -locations for time zone data). On POSIX systems, downstream distributors and -those building Python from source who know where their system -time zone data is deployed may change the default time zone path by specifying -the compile-time option ``TZPATH`` (or, more likely, the :option:`configure -flag --with-tzpath <--with-tzpath>`), which should be a string delimited by -:data:`os.pathsep`. - -On all platforms, the configured value is available as the ``TZPATH`` key in -:func:`sysconfig.get_config_var`. - -.. _zoneinfo_data_environment_var: - -Environment configuration -^^^^^^^^^^^^^^^^^^^^^^^^^ - -When initializing :data:`TZPATH` (either at import time or whenever -:func:`reset_tzpath` is called with no arguments), the ``zoneinfo`` module will -use the environment variable ``PYTHONTZPATH``, if it exists, to set the search -path. - -.. envvar:: PYTHONTZPATH - - This is an :data:`os.pathsep`-separated string containing the time zone - search path to use. It must consist of only absolute rather than relative - paths. Relative components specified in ``PYTHONTZPATH`` will not be used, - but otherwise the behavior when a relative path is specified is - implementation-defined; CPython will raise :exc:`InvalidTZPathWarning`, but - other implementations are free to silently ignore the erroneous component - or raise an exception. - -To set the system to ignore the system data and use the tzdata package -instead, set ``PYTHONTZPATH=""``. - -.. _zoneinfo_data_runtime_config: - -Runtime configuration -^^^^^^^^^^^^^^^^^^^^^ - -The TZ search path can also be configured at runtime using the -:func:`reset_tzpath` function. This is generally not an advisable operation, -though it is reasonable to use it in test functions that require the use of a -specific time zone path (or require disabling access to the system time zones). - - -The ``ZoneInfo`` class ----------------------- - -.. class:: ZoneInfo(key) - - A concrete :class:`datetime.tzinfo` subclass that represents an IANA time - zone specified by the string ``key``. Calls to the primary constructor will - always return objects that compare identically; put another way, barring - cache invalidation via :meth:`ZoneInfo.clear_cache`, for all values of - ``key``, the following assertion will always be true: - - .. code-block:: python - - a = ZoneInfo(key) - b = ZoneInfo(key) - assert a is b - - ``key`` must be in the form of a relative, normalized POSIX path, with no - up-level references. The constructor will raise :exc:`ValueError` if a - non-conforming key is passed. - - If no file matching ``key`` is found, the constructor will raise - :exc:`ZoneInfoNotFoundError`. - - -The ``ZoneInfo`` class has two alternate constructors: - -.. classmethod:: ZoneInfo.from_file(fobj, /, key=None) - - Constructs a ``ZoneInfo`` object from a file-like object returning bytes - (e.g. a file opened in binary mode or an :class:`io.BytesIO` object). - Unlike the primary constructor, this always constructs a new object. - - The ``key`` parameter sets the name of the zone for the purposes of - :py:meth:`~object.__str__` and :py:meth:`~object.__repr__`. - - Objects created via this constructor cannot be pickled (see `pickling`_). - -.. classmethod:: ZoneInfo.no_cache(key) - - An alternate constructor that bypasses the constructor's cache. It is - identical to the primary constructor, but returns a new object on each - call. This is most likely to be useful for testing or demonstration - purposes, but it can also be used to create a system with a different cache - invalidation strategy. - - Objects created via this constructor will also bypass the cache of a - deserializing process when unpickled. - - .. TODO: Add "See `cache_behavior`_" reference when that section is ready. - - .. caution:: - - Using this constructor may change the semantics of your datetimes in - surprising ways, only use it if you know that you need to. - -The following class methods are also available: - -.. classmethod:: ZoneInfo.clear_cache(*, only_keys=None) - - A method for invalidating the cache on the ``ZoneInfo`` class. If no - arguments are passed, all caches are invalidated and the next call to - the primary constructor for each key will return a new instance. - - If an iterable of key names is passed to the ``only_keys`` parameter, only - the specified keys will be removed from the cache. Keys passed to - ``only_keys`` but not found in the cache are ignored. - - .. TODO: Add "See `cache_behavior`_" reference when that section is ready. - - .. warning:: - - Invoking this function may change the semantics of datetimes using - ``ZoneInfo`` in surprising ways; this modifies process-wide global state - and thus may have wide-ranging effects. Only use it if you know that you - need to. - -The class has one attribute: - -.. attribute:: ZoneInfo.key - - This is a read-only :term:`attribute` that returns the value of ``key`` - passed to the constructor, which should be a lookup key in the IANA time - zone database (e.g. ``America/New_York``, ``Europe/Paris`` or - ``Asia/Tokyo``). - - For zones constructed from file without specifying a ``key`` parameter, - this will be set to ``None``. - - .. note:: - - Although it is a somewhat common practice to expose these to end users, - these values are designed to be primary keys for representing the - relevant zones and not necessarily user-facing elements. Projects like - CLDR (the Unicode Common Locale Data Repository) can be used to get - more user-friendly strings from these keys. - -String representations -********************** - -The string representation returned when calling :py:class:`str` on a -:class:`ZoneInfo` object defaults to using the :attr:`ZoneInfo.key` attribute (see -the note on usage in the attribute documentation):: - - >>> zone = ZoneInfo("Pacific/Kwajalein") - >>> str(zone) - 'Pacific/Kwajalein' - - >>> dt = datetime(2020, 4, 1, 3, 15, tzinfo=zone) - >>> f"{dt.isoformat()} [{dt.tzinfo}]" - '2020-04-01T03:15:00+12:00 [Pacific/Kwajalein]' - -For objects constructed from a file without specifying a ``key`` parameter, -``str`` falls back to calling :func:`repr`. ``ZoneInfo``'s ``repr`` is -implementation-defined and not necessarily stable between versions, but it is -guaranteed not to be a valid ``ZoneInfo`` key. - -.. _pickling: - -Pickle serialization -******************** - -Rather than serializing all transition data, ``ZoneInfo`` objects are -serialized by key, and ``ZoneInfo`` objects constructed from files (even those -with a value for ``key`` specified) cannot be pickled. - -The behavior of a ``ZoneInfo`` file depends on how it was constructed: - -1. ``ZoneInfo(key)``: When constructed with the primary constructor, a - ``ZoneInfo`` object is serialized by key, and when deserialized, the - deserializing process uses the primary and thus it is expected that these - are expected to be the same object as other references to the same time - zone. For example, if ``europe_berlin_pkl`` is a string containing a pickle - constructed from ``ZoneInfo("Europe/Berlin")``, one would expect the - following behavior: - - .. code-block:: pycon - - >>> a = ZoneInfo("Europe/Berlin") - >>> b = pickle.loads(europe_berlin_pkl) - >>> a is b - True - -2. ``ZoneInfo.no_cache(key)``: When constructed from the cache-bypassing - constructor, the ``ZoneInfo`` object is also serialized by key, but when - deserialized, the deserializing process uses the cache bypassing - constructor. If ``europe_berlin_pkl_nc`` is a string containing a pickle - constructed from ``ZoneInfo.no_cache("Europe/Berlin")``, one would expect - the following behavior: - - .. code-block:: pycon - - >>> a = ZoneInfo("Europe/Berlin") - >>> b = pickle.loads(europe_berlin_pkl_nc) - >>> a is b - False - -3. ``ZoneInfo.from_file(fobj, /, key=None)``: When constructed from a file, the - ``ZoneInfo`` object raises an exception on pickling. If an end user wants to - pickle a ``ZoneInfo`` constructed from a file, it is recommended that they - use a wrapper type or a custom serialization function: either serializing by - key or storing the contents of the file object and serializing that. - -This method of serialization requires that the time zone data for the required -key be available on both the serializing and deserializing side, similar to the -way that references to classes and functions are expected to exist in both the -serializing and deserializing environments. It also means that no guarantees -are made about the consistency of results when unpickling a ``ZoneInfo`` -pickled in an environment with a different version of the time zone data. - -Functions ---------- - -.. function:: available_timezones() - - Get a set containing all the valid keys for IANA time zones available - anywhere on the time zone path. This is recalculated on every call to the - function. - - This function only includes canonical zone names and does not include - "special" zones such as those under the ``posix/`` and ``right/`` - directories, or the ``posixrules`` zone. - - .. caution:: - - This function may open a large number of files, as the best way to - determine if a file on the time zone path is a valid time zone is to - read the "magic string" at the beginning. - - .. note:: - - These values are not designed to be exposed to end-users; for user - facing elements, applications should use something like CLDR (the - Unicode Common Locale Data Repository) to get more user-friendly - strings. See also the cautionary note on :attr:`ZoneInfo.key`. - -.. function:: reset_tzpath(to=None) - - Sets or resets the time zone search path (:data:`TZPATH`) for the module. - When called with no arguments, :data:`TZPATH` is set to the default value. - - Calling ``reset_tzpath`` will not invalidate the :class:`ZoneInfo` cache, - and so calls to the primary ``ZoneInfo`` constructor will only use the new - ``TZPATH`` in the case of a cache miss. - - The ``to`` parameter must be a :term:`sequence` of strings or - :class:`os.PathLike` and not a string, all of which must be absolute paths. - :exc:`ValueError` will be raised if something other than an absolute path - is passed. - -Globals -------- - -.. data:: TZPATH - - A read-only sequence representing the time zone search path -- when - constructing a ``ZoneInfo`` from a key, the key is joined to each entry in - the ``TZPATH``, and the first file found is used. - - ``TZPATH`` may contain only absolute paths, never relative paths, - regardless of how it is configured. - - The object that ``zoneinfo.TZPATH`` points to may change in response to a - call to :func:`reset_tzpath`, so it is recommended to use - ``zoneinfo.TZPATH`` rather than importing ``TZPATH`` from ``zoneinfo`` or - assigning a long-lived variable to ``zoneinfo.TZPATH``. - - For more information on configuring the time zone search path, see - :ref:`zoneinfo_data_configuration`. - -Exceptions and warnings ------------------------ - -.. exception:: ZoneInfoNotFoundError - - Raised when construction of a :class:`ZoneInfo` object fails because the - specified key could not be found on the system. This is a subclass of - :exc:`KeyError`. - -.. exception:: InvalidTZPathWarning - - Raised when :envvar:`PYTHONTZPATH` contains an invalid component that will - be filtered out, such as a relative path. - -.. Links and references: - -.. _tzdata: https://pypi.org/project/tzdata/ diff --git a/Doc/license.rst b/Doc/license.rst deleted file mode 100644 index 6644df65170a0c..00000000000000 --- a/Doc/license.rst +++ /dev/null @@ -1,1066 +0,0 @@ -.. highlight:: none - -.. _history-and-license: - -******************* -History and License -******************* - - -History of the software -======================= - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see https://www.cwi.nl/) in the Netherlands as a -successor of a language called ABC. Guido remains Python's principal author, -although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for National -Research Initiatives (CNRI, see https://www.cnri.reston.va.us/) in Reston, -Virginia where he released several versions of the software. - -In May 2000, Guido and the Python core development team moved to BeOpen.com to -form the BeOpen PythonLabs team. In October of the same year, the PythonLabs -team moved to Digital Creations (now Zope Corporation; see -https://www.zope.org/). In 2001, the Python Software Foundation (PSF, see -https://www.python.org/psf/) was formed, a non-profit organization created -specifically to own Python-related Intellectual Property. Zope Corporation is a -sponsoring member of the PSF. - -All Python releases are Open Source (see https://opensource.org/ for the Open -Source Definition). Historically, most, but not all, Python releases have also -been GPL-compatible; the table below summarizes the various releases. - -+----------------+--------------+------------+------------+-----------------+ -| Release | Derived from | Year | Owner | GPL compatible? | -+================+==============+============+============+=================+ -| 0.9.0 thru 1.2 | n/a | 1991-1995 | CWI | yes | -+----------------+--------------+------------+------------+-----------------+ -| 1.3 thru 1.5.2 | 1.2 | 1995-1999 | CNRI | yes | -+----------------+--------------+------------+------------+-----------------+ -| 1.6 | 1.5.2 | 2000 | CNRI | no | -+----------------+--------------+------------+------------+-----------------+ -| 2.0 | 1.6 | 2000 | BeOpen.com | no | -+----------------+--------------+------------+------------+-----------------+ -| 1.6.1 | 1.6 | 2001 | CNRI | no | -+----------------+--------------+------------+------------+-----------------+ -| 2.1 | 2.0+1.6.1 | 2001 | PSF | no | -+----------------+--------------+------------+------------+-----------------+ -| 2.0.1 | 2.0+1.6.1 | 2001 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.1.1 | 2.1+2.0.1 | 2001 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.1.2 | 2.1.1 | 2002 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.1.3 | 2.1.2 | 2002 | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ -| 2.2 and above | 2.1.1 | 2001-now | PSF | yes | -+----------------+--------------+------------+------------+-----------------+ - -.. note:: - - GPL-compatible doesn't mean that we're distributing Python under the GPL. All - Python licenses, unlike the GPL, let you distribute a modified version without - making your changes open source. The GPL-compatible licenses make it possible to - combine Python with other software that is released under the GPL; the others - don't. - -Thanks to the many outside volunteers who have worked under Guido's direction to -make these releases possible. - - -Terms and conditions for accessing or otherwise using Python -============================================================ - -Python software and documentation are licensed under the -:ref:`PSF License Agreement `. - -Starting with Python 3.8.6, examples, recipes, and other code in -the documentation are dual licensed under the PSF License Agreement -and the :ref:`Zero-Clause BSD license `. - -Some software incorporated into Python is under different licenses. -The licenses are listed with code falling under that license. -See :ref:`OtherLicenses` for an incomplete list of these licenses. - - -.. _PSF-license: - -PSF LICENSE AGREEMENT FOR PYTHON |release| ------------------------------------------- - -.. parsed-literal:: - - 1. This LICENSE AGREEMENT is between the Python Software Foundation ("PSF"), and - the Individual or Organization ("Licensee") accessing and otherwise using Python - |release| software in source or binary form and its associated documentation. - - 2. Subject to the terms and conditions of this License Agreement, PSF hereby - grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, - analyze, test, perform and/or display publicly, prepare derivative works, - distribute, and otherwise use Python |release| alone or in any derivative - version, provided, however, that PSF's License Agreement and PSF's notice of - copyright, i.e., "Copyright © 2001-2023 Python Software Foundation; All Rights - Reserved" are retained in Python |release| alone or in any derivative version - prepared by Licensee. - - 3. In the event Licensee prepares a derivative work that is based on or - incorporates Python |release| or any part thereof, and wants to make the - derivative work available to others as provided herein, then Licensee hereby - agrees to include in any such work a brief summary of the changes made to Python - |release|. - - 4. PSF is making Python |release| available to Licensee on an "AS IS" basis. - PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF - EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND DISCLAIMS ANY REPRESENTATION OR - WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE - USE OF PYTHON |release| WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - - 5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON |release| - FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF - MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON |release|, OR ANY DERIVATIVE - THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - - 6. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. - - 7. Nothing in this License Agreement shall be deemed to create any relationship - of agency, partnership, or joint venture between PSF and Licensee. This License - Agreement does not grant permission to use PSF trademarks or trade name in a - trademark sense to endorse or promote products or services of Licensee, or any - third party. - - 8. By copying, installing or otherwise using Python |release|, Licensee agrees - to be bound by the terms and conditions of this License Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -.. parsed-literal:: - - 1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an office at - 160 Saratoga Avenue, Santa Clara, CA 95051, and the Individual or Organization - ("Licensee") accessing and otherwise using this software in source or binary - form and its associated documentation ("the Software"). - - 2. Subject to the terms and conditions of this BeOpen Python License Agreement, - BeOpen hereby grants Licensee a non-exclusive, royalty-free, world-wide license - to reproduce, analyze, test, perform and/or display publicly, prepare derivative - works, distribute, and otherwise use the Software alone or in any derivative - version, provided, however, that the BeOpen Python License is retained in the - Software, alone or in any derivative version prepared by Licensee. - - 3. BeOpen is making the Software available to Licensee on an "AS IS" basis. - BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF - EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND DISCLAIMS ANY REPRESENTATION OR - WARRANTY OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE - USE OF THE SOFTWARE WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - - 4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE SOFTWARE FOR - ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF USING, - MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY DERIVATIVE THEREOF, EVEN IF - ADVISED OF THE POSSIBILITY THEREOF. - - 5. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. - - 6. This License Agreement shall be governed by and interpreted in all respects - by the law of the State of California, excluding conflict of law provisions. - Nothing in this License Agreement shall be deemed to create any relationship of - agency, partnership, or joint venture between BeOpen and Licensee. This License - Agreement does not grant permission to use BeOpen trademarks or trade names in a - trademark sense to endorse or promote products or services of Licensee, or any - third party. As an exception, the "BeOpen Python" logos available at - http://www.pythonlabs.com/logos.html may be used according to the permissions - granted on that web page. - - 7. By copying, installing or otherwise using the software, Licensee agrees to be - bound by the terms and conditions of this License Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -.. parsed-literal:: - - 1. This LICENSE AGREEMENT is between the Corporation for National Research - Initiatives, having an office at 1895 Preston White Drive, Reston, VA 20191 - ("CNRI"), and the Individual or Organization ("Licensee") accessing and - otherwise using Python 1.6.1 software in source or binary form and its - associated documentation. - - 2. Subject to the terms and conditions of this License Agreement, CNRI hereby - grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, - analyze, test, perform and/or display publicly, prepare derivative works, - distribute, and otherwise use Python 1.6.1 alone or in any derivative version, - provided, however, that CNRI's License Agreement and CNRI's notice of copyright, - i.e., "Copyright © 1995-2001 Corporation for National Research Initiatives; All - Rights Reserved" are retained in Python 1.6.1 alone or in any derivative version - prepared by Licensee. Alternately, in lieu of CNRI's License Agreement, - Licensee may substitute the following text (omitting the quotes): "Python 1.6.1 - is made available subject to the terms and conditions in CNRI's License - Agreement. This Agreement together with Python 1.6.1 may be located on the - internet using the following unique, persistent identifier (known as a handle): - 1895.22/1013. This Agreement may also be obtained from a proxy server on the - internet using the following URL: http://hdl.handle.net/1895.22/1013." - - 3. In the event Licensee prepares a derivative work that is based on or - incorporates Python 1.6.1 or any part thereof, and wants to make the derivative - work available to others as provided herein, then Licensee hereby agrees to - include in any such work a brief summary of the changes made to Python 1.6.1. - - 4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" basis. CNRI - MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR IMPLIED. BY WAY OF EXAMPLE, - BUT NOT LIMITATION, CNRI MAKES NO AND DISCLAIMS ANY REPRESENTATION OR WARRANTY - OF MERCHANTABILITY OR FITNESS FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF - PYTHON 1.6.1 WILL NOT INFRINGE ANY THIRD PARTY RIGHTS. - - 5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON 1.6.1 FOR - ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS A RESULT OF - MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, OR ANY DERIVATIVE - THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - - 6. This License Agreement will automatically terminate upon a material breach of - its terms and conditions. - - 7. This License Agreement shall be governed by the federal intellectual property - law of the United States, including without limitation the federal copyright - law, and, to the extent such U.S. federal law does not apply, by the law of the - Commonwealth of Virginia, excluding Virginia's conflict of law provisions. - Notwithstanding the foregoing, with regard to derivative works based on Python - 1.6.1 that incorporate non-separable material that was previously distributed - under the GNU General Public License (GPL), the law of the Commonwealth of - Virginia shall govern this License Agreement only as to issues arising under or - with respect to Paragraphs 4, 5, and 7 of this License Agreement. Nothing in - this License Agreement shall be deemed to create any relationship of agency, - partnership, or joint venture between CNRI and Licensee. This License Agreement - does not grant permission to use CNRI trademarks or trade name in a trademark - sense to endorse or promote products or services of Licensee, or any third - party. - - 8. By clicking on the "ACCEPT" button where indicated, or by copying, installing - or otherwise using Python 1.6.1, Licensee agrees to be bound by the terms and - conditions of this License Agreement. - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -.. parsed-literal:: - - Copyright © 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, The - Netherlands. All rights reserved. - - Permission to use, copy, modify, and distribute this software and its - documentation for any purpose and without fee is hereby granted, provided that - the above copyright notice appear in all copies and that both that copyright - notice and this permission notice appear in supporting documentation, and that - the name of Stichting Mathematisch Centrum or CWI not be used in advertising or - publicity pertaining to distribution of the software without specific, written - prior permission. - - STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS - SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO - EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE FOR ANY SPECIAL, INDIRECT - OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, - DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS - SOFTWARE. - - -.. _BSD0: - -ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON |release| DOCUMENTATION ----------------------------------------------------------------------- - -.. parsed-literal:: - - Permission to use, copy, modify, and/or distribute this software for any - purpose with or without fee is hereby granted. - - THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH - REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, - INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM - LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR - OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - PERFORMANCE OF THIS SOFTWARE. - - -.. _OtherLicenses: - -Licenses and Acknowledgements for Incorporated Software -======================================================= - -This section is an incomplete, but growing list of licenses and acknowledgements -for third-party software incorporated in the Python distribution. - - -Mersenne Twister ----------------- - -The :mod:`_random` module includes code based on a download from -http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/MT2002/emt19937ar.html. The following are -the verbatim comments from the original code:: - - A C-program for MT19937, with initialization improved 2002/1/26. - Coded by Takuji Nishimura and Makoto Matsumoto. - - Before using, initialize the state by using init_genrand(seed) - or init_by_array(init_key, key_length). - - Copyright (C) 1997 - 2002, Makoto Matsumoto and Takuji Nishimura, - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. The names of its contributors may not be used to endorse or promote - products derived from this software without specific prior written - permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR - CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, - EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, - PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - - Any feedback is very welcome. - http://www.math.sci.hiroshima-u.ac.jp/~m-mat/MT/emt.html - email: m-mat @ math.sci.hiroshima-u.ac.jp (remove space) - - -Sockets -------- - -The :mod:`socket` module uses the functions, :c:func:`!getaddrinfo`, and -:c:func:`!getnameinfo`, which are coded in separate source files from the WIDE -Project, https://www.wide.ad.jp/. :: - - Copyright (C) 1995, 1996, 1997, and 1998 WIDE Project. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - 3. Neither the name of the project nor the names of its contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - - -Asynchronous socket services ----------------------------- - -The :mod:`asynchat` and :mod:`asyncore` modules contain the following notice:: - - Copyright 1996 by Sam Rushing - - All Rights Reserved - - Permission to use, copy, modify, and distribute this software and - its documentation for any purpose and without fee is hereby - granted, provided that the above copyright notice appear in all - copies and that both that copyright notice and this permission - notice appear in supporting documentation, and that the name of Sam - Rushing not be used in advertising or publicity pertaining to - distribution of the software without specific, written prior - permission. - - SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, - INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN - NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR - CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS - OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, - NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN - CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - -Cookie management ------------------ - -The :mod:`http.cookies` module contains the following notice:: - - Copyright 2000 by Timothy O'Malley - - All Rights Reserved - - Permission to use, copy, modify, and distribute this software - and its documentation for any purpose and without fee is hereby - granted, provided that the above copyright notice appear in all - copies and that both that copyright notice and this permission - notice appear in supporting documentation, and that the name of - Timothy O'Malley not be used in advertising or publicity - pertaining to distribution of the software without specific, written - prior permission. - - Timothy O'Malley DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS - SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY - AND FITNESS, IN NO EVENT SHALL Timothy O'Malley BE LIABLE FOR - ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, - WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - PERFORMANCE OF THIS SOFTWARE. - - -Execution tracing ------------------ - -The :mod:`trace` module contains the following notice:: - - portions copyright 2001, Autonomous Zones Industries, Inc., all rights... - err... reserved and offered to the public under the terms of the - Python 2.2 license. - Author: Zooko O'Whielacronx - http://zooko.com/ - mailto:zooko@zooko.com - - Copyright 2000, Mojam Media, Inc., all rights reserved. - Author: Skip Montanaro - - Copyright 1999, Bioreason, Inc., all rights reserved. - Author: Andrew Dalke - - Copyright 1995-1997, Automatrix, Inc., all rights reserved. - Author: Skip Montanaro - - Copyright 1991-1995, Stichting Mathematisch Centrum, all rights reserved. - - - Permission to use, copy, modify, and distribute this Python software and - its associated documentation for any purpose without fee is hereby - granted, provided that the above copyright notice appears in all copies, - and that both that copyright notice and this permission notice appear in - supporting documentation, and that the name of neither Automatrix, - Bioreason or Mojam Media be used in advertising or publicity pertaining to - distribution of the software without specific, written prior permission. - - -UUencode and UUdecode functions -------------------------------- - -The :mod:`uu` module contains the following notice:: - - Copyright 1994 by Lance Ellinghouse - Cathedral City, California Republic, United States of America. - All Rights Reserved - Permission to use, copy, modify, and distribute this software and its - documentation for any purpose and without fee is hereby granted, - provided that the above copyright notice appear in all copies and that - both that copyright notice and this permission notice appear in - supporting documentation, and that the name of Lance Ellinghouse - not be used in advertising or publicity pertaining to distribution - of the software without specific, written prior permission. - LANCE ELLINGHOUSE DISCLAIMS ALL WARRANTIES WITH REGARD TO - THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND - FITNESS, IN NO EVENT SHALL LANCE ELLINGHOUSE CENTRUM BE LIABLE - FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES - WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN - ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT - OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - - Modified by Jack Jansen, CWI, July 1995: - - Use binascii module to do the actual line-by-line conversion - between ascii and binary. This results in a 1000-fold speedup. The C - version is still 5 times faster, though. - - Arguments more compliant with Python standard - - -XML Remote Procedure Calls --------------------------- - -The :mod:`xmlrpc.client` module contains the following notice:: - - The XML-RPC client interface is - - Copyright (c) 1999-2002 by Secret Labs AB - Copyright (c) 1999-2002 by Fredrik Lundh - - By obtaining, using, and/or copying this software and/or its - associated documentation, you agree that you have read, understood, - and will comply with the following terms and conditions: - - Permission to use, copy, modify, and distribute this software and - its associated documentation for any purpose and without fee is - hereby granted, provided that the above copyright notice appears in - all copies, and that both that copyright notice and this permission - notice appear in supporting documentation, and that the name of - Secret Labs AB or the author not be used in advertising or publicity - pertaining to distribution of the software without specific, written - prior permission. - - SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD - TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- - ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR - BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY - DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, - WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS - ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE - OF THIS SOFTWARE. - - -test_epoll ----------- - -The :mod:`!test.test_epoll` module contains the following notice:: - - Copyright (c) 2001-2006 Twisted Matrix Laboratories. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be - included in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - -Select kqueue -------------- - -The :mod:`select` module contains the following notice for the kqueue -interface:: - - Copyright (c) 2000 Doug White, 2006 James Knight, 2007 Christian Heimes - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - - -SipHash24 ---------- - -The file :file:`Python/pyhash.c` contains Marek Majkowski' implementation of -Dan Bernstein's SipHash24 algorithm. It contains the following note:: - - - Copyright (c) 2013 Marek Majkowski - - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in - all copies or substantial portions of the Software. - - - Original location: - https://github.com/majek/csiphash/ - - Solution inspired by code from: - Samuel Neves (supercop/crypto_auth/siphash24/little) - djb (supercop/crypto_auth/siphash24/little2) - Jean-Philippe Aumasson (https://131002.net/siphash/siphash24.c) - - -strtod and dtoa ---------------- - -The file :file:`Python/dtoa.c`, which supplies C functions dtoa and -strtod for conversion of C doubles to and from strings, is derived -from the file of the same name by David M. Gay, currently available -from https://web.archive.org/web/20220517033456/http://www.netlib.org/fp/dtoa.c. -The original file, as retrieved on March 16, 2009, contains the following -copyright and licensing notice:: - - /**************************************************************** - * - * The author of this software is David M. Gay. - * - * Copyright (c) 1991, 2000, 2001 by Lucent Technologies. - * - * Permission to use, copy, modify, and distribute this software for any - * purpose without fee is hereby granted, provided that this entire notice - * is included in all copies of any software which is or includes a copy - * or modification of this software and in all copies of the supporting - * documentation for such software. - * - * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR IMPLIED - * WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR LUCENT MAKES ANY - * REPRESENTATION OR WARRANTY OF ANY KIND CONCERNING THE MERCHANTABILITY - * OF THIS SOFTWARE OR ITS FITNESS FOR ANY PARTICULAR PURPOSE. - * - ***************************************************************/ - - -OpenSSL -------- - -The modules :mod:`hashlib`, :mod:`posix`, :mod:`ssl`, :mod:`crypt` use -the OpenSSL library for added performance if made available by the -operating system. Additionally, the Windows and macOS installers for -Python may include a copy of the OpenSSL libraries, so we include a copy -of the OpenSSL license here. For the OpenSSL 3.0 release, -and later releases derived from that, the Apache License v2 applies:: - - - Apache License - Version 2.0, January 2004 - https://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - -expat ------ - -The :mod:`pyexpat ` extension is built using an included copy of the expat -sources unless the build is configured ``--with-system-expat``:: - - Copyright (c) 1998, 1999, 2000 Thai Open Source Software Center Ltd - and Clark Cooper - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - "Software"), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. - IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY - CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, - TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE - SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - -libffi ------- - -The :mod:`_ctypes` extension is built using an included copy of the libffi -sources unless the build is configured ``--with-system-libffi``:: - - Copyright (c) 1996-2008 Red Hat, Inc and others. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of this software and associated documentation files (the - ``Software''), to deal in the Software without restriction, including - without limitation the rights to use, copy, modify, merge, publish, - distribute, sublicense, and/or sell copies of the Software, and to - permit persons to whom the Software is furnished to do so, subject to - the following conditions: - - The above copyright notice and this permission notice shall be included - in all copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT - HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, - WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER - DEALINGS IN THE SOFTWARE. - - -zlib ----- - -The :mod:`zlib` extension is built using an included copy of the zlib -sources if the zlib version found on the system is too old to be -used for the build:: - - Copyright (C) 1995-2011 Jean-loup Gailly and Mark Adler - - This software is provided 'as-is', without any express or implied - warranty. In no event will the authors be held liable for any damages - arising from the use of this software. - - Permission is granted to anyone to use this software for any purpose, - including commercial applications, and to alter it and redistribute it - freely, subject to the following restrictions: - - 1. The origin of this software must not be misrepresented; you must not - claim that you wrote the original software. If you use this software - in a product, an acknowledgment in the product documentation would be - appreciated but is not required. - - 2. Altered source versions must be plainly marked as such, and must not be - misrepresented as being the original software. - - 3. This notice may not be removed or altered from any source distribution. - - Jean-loup Gailly Mark Adler - jloup@gzip.org madler@alumni.caltech.edu - - -cfuhash -------- - -The implementation of the hash table used by the :mod:`tracemalloc` is based -on the cfuhash project:: - - Copyright (c) 2005 Don Owens - All rights reserved. - - This code is released under the BSD license: - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - - * Neither the name of the author nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS - FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE - COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, - INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES - (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR - SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED - OF THE POSSIBILITY OF SUCH DAMAGE. - - -libmpdec --------- - -The :mod:`_decimal` module is built using an included copy of the libmpdec -library unless the build is configured ``--with-system-libmpdec``:: - - Copyright (c) 2008-2020 Stefan Krah. All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS "AS IS" AND - ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - SUCH DAMAGE. - - -W3C C14N test suite -------------------- - -The C14N 2.0 test suite in the :mod:`test` package -(``Lib/test/xmltestdata/c14n-20/``) was retrieved from the W3C website at -https://www.w3.org/TR/xml-c14n2-testcases/ and is distributed under the -3-clause BSD license:: - - Copyright (c) 2013 W3C(R) (MIT, ERCIM, Keio, Beihang), - All Rights Reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions - are met: - - * Redistributions of works must retain the original copyright notice, - this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the original copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - * Neither the name of the W3C nor the names of its contributors may be - used to endorse or promote products derived from this work without - specific prior written permission. - - THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS - "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT - LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR - A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT - OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT - LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, - DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY - THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT - (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE - OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -Audioop -------- - -The audioop module uses the code base in g771.c file of the SoX project:: - - Programming the AdLib/Sound Blaster - FM Music Chips - Version 2.0 (24 Feb 1992) - Copyright (c) 1991, 1992 by Jeffrey S. Lee - jlee@smylex.uucp - Warranty and Copyright Policy - This document is provided on an "as-is" basis, and its author makes - no warranty or representation, express or implied, with respect to - its quality performance or fitness for a particular purpose. In no - event will the author of this document be liable for direct, indirect, - special, incidental, or consequential damages arising out of the use - or inability to use the information contained within. Use of this - document is at your own risk. - This file may be used and copied freely so long as the applicable - copyright notices are retained, and no modifications are made to the - text of the document. No money shall be charged for its distribution - beyond reasonable shipping, handling and duplication costs, nor shall - proprietary changes be made to this document so that it cannot be - distributed freely. This document may not be included in published - material or commercial packages without the written consent of its - author. diff --git a/Doc/make.bat b/Doc/make.bat deleted file mode 100644 index 4f0b3c11f4facb..00000000000000 --- a/Doc/make.bat +++ /dev/null @@ -1,194 +0,0 @@ -@echo off -setlocal - -pushd %~dp0 - -set this=%~n0 - -call ..\PCbuild\find_python.bat %PYTHON% - -if not defined PYTHON set PYTHON=py - -if not defined SPHINXBUILD ( - %PYTHON% -c "import sphinx" > nul 2> nul - if errorlevel 1 ( - echo Installing sphinx with %PYTHON% - %PYTHON% -m pip install -r requirements.txt - if errorlevel 1 exit /B - ) - set SPHINXBUILD=%PYTHON% -c "import sphinx.cmd.build, sys; sys.exit(sphinx.cmd.build.main())" -) - -%PYTHON% -c "import python_docs_theme" > nul 2> nul -if errorlevel 1 ( - echo Installing python-docs-theme with %PYTHON% - %PYTHON% -m pip install python-docs-theme - if errorlevel 1 exit /B -) - -if not defined BLURB ( - %PYTHON% -c "import blurb" > nul 2> nul - if errorlevel 1 ( - echo Installing blurb with %PYTHON% - rem Should have been installed with Sphinx earlier - %PYTHON% -m pip install blurb - if errorlevel 1 exit /B - ) - set BLURB=%PYTHON% -m blurb -) - -if not defined SPHINXLINT ( - %PYTHON% -c "import sphinxlint" > nul 2> nul - if errorlevel 1 ( - echo Installing sphinx-lint with %PYTHON% - rem Should have been installed with Sphinx earlier - %PYTHON% -m pip install sphinx-lint - if errorlevel 1 exit /B - ) - set SPHINXLINT=%PYTHON% -m sphinxlint -) - -if "%1" NEQ "htmlhelp" goto :skiphhcsearch -if exist "%HTMLHELP%" goto :skiphhcsearch - -rem Search for HHC in likely places -set HTMLHELP= -where hhc /q && set "HTMLHELP=hhc" && goto :skiphhcsearch -where /R ..\externals hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles(x86)%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" where /R "%ProgramFiles%" hhc > "%TEMP%\hhc.loc" 2> nul && set /P HTMLHELP= < "%TEMP%\hhc.loc" & del "%TEMP%\hhc.loc" -if not exist "%HTMLHELP%" ( - echo. - echo.The HTML Help Workshop was not found. Set the HTMLHELP variable - echo.to the path to hhc.exe or download and install it from - echo.http://msdn.microsoft.com/en-us/library/ms669985 - exit /B 1 -) -:skiphhcsearch - -if not defined DISTVERSION for /f "usebackq" %%v in (`%PYTHON% tools/extensions/patchlevel.py`) do set DISTVERSION=%%v - -if not defined BUILDDIR set BUILDDIR=build - -rem Targets that don't require sphinx-build -if "%1" EQU "" goto help -if "%1" EQU "help" goto help -if "%1" EQU "check" goto check -if "%1" EQU "serve" goto serve -if "%1" == "clean" ( - rmdir /q /s "%BUILDDIR%" - goto end -) - -%SPHINXBUILD% >nul 2> nul -if errorlevel 9009 ( - echo. - echo.The 'sphinx-build' command was not found. Make sure you have Sphinx - echo.installed, then set the SPHINXBUILD environment variable to point - echo.to the full path of the 'sphinx-build' executable. Alternatively you - echo.may add the Sphinx directory to PATH. - echo. - echo.If you don't have Sphinx installed, grab it from - echo.http://sphinx-doc.org/ - popd - exit /B 1 -) - -rem Targets that do require sphinx-build and have their own label -if "%1" EQU "htmlview" goto htmlview - -rem Everything else -goto build - -:help -echo.usage: %this% BUILDER [filename ...] -echo. -echo.Call %this% with the desired Sphinx builder as the first argument, e.g. -echo.``%this% html`` or ``%this% doctest``. Interesting targets that are -echo.always available include: -echo. -echo. Provided by Sphinx: -echo. html, htmlhelp, latex, text -echo. suspicious, linkcheck, changes, doctest -echo. Provided by this script: -echo. clean, check, htmlview -echo. -echo.All arguments past the first one are passed through to sphinx-build as -echo.filenames to build or are ignored. See README.rst in this directory or -echo.the documentation for your version of Sphinx for more exhaustive lists -echo.of available targets and descriptions of each. -echo. -echo.This script assumes that the SPHINXBUILD environment variable contains -echo.a legitimate command for calling sphinx-build, or that sphinx-build is -echo.on your PATH if SPHINXBUILD is not set. Options for sphinx-build can -echo.be passed by setting the SPHINXOPTS environment variable. -goto end - -:build -if not exist "%BUILDDIR%" mkdir "%BUILDDIR%" - -rem PY_MISC_NEWS_DIR is also used by our Sphinx extension in tools/extensions/pyspecific.py -if not defined PY_MISC_NEWS_DIR set PY_MISC_NEWS_DIR=%BUILDDIR%\%1 -if not exist "%PY_MISC_NEWS_DIR%" mkdir "%PY_MISC_NEWS_DIR%" -if exist ..\Misc\NEWS ( - echo.Copying Misc\NEWS to %PY_MISC_NEWS_DIR%\NEWS - copy ..\Misc\NEWS "%PY_MISC_NEWS_DIR%\NEWS" > nul -) else if exist ..\Misc\NEWS.D ( - if defined BLURB ( - echo.Merging Misc/NEWS with %BLURB% - %BLURB% merge -f "%PY_MISC_NEWS_DIR%\NEWS" - ) else ( - echo.No Misc/NEWS file and Blurb is not available. - exit /B 1 - ) -) - -if defined PAPER ( - set SPHINXOPTS=-D latex_elements.papersize=%PAPER% %SPHINXOPTS% -) -if "%1" EQU "htmlhelp" ( - set SPHINXOPTS=-D html_theme_options.body_max_width=none %SPHINXOPTS% -) -cmd /S /C "%SPHINXBUILD% %SPHINXOPTS% -b%1 -dbuild\doctrees . "%BUILDDIR%\%1" %2 %3 %4 %5 %6 %7 %8 %9" - -if "%1" EQU "htmlhelp" ( - "%HTMLHELP%" "%BUILDDIR%\htmlhelp\python%DISTVERSION:.=%.hhp" - rem hhc.exe seems to always exit with code 1, reset to 0 for less than 2 - if not errorlevel 2 cmd /C exit /b 0 -) - -echo. -if errorlevel 1 ( - echo.Build failed (exit code %ERRORLEVEL%^), check for error messages - echo.above. Any output will be found in %BUILDDIR%\%1 -) else ( - echo.Build succeeded. All output should be in %BUILDDIR%\%1 -) -goto end - -:htmlview -if NOT "%2" EQU "" ( - echo.Can't specify filenames to build with htmlview target, ignoring. -) -cmd /C %this% html - -if EXIST "%BUILDDIR%\html\index.html" ( - echo.Opening "%BUILDDIR%\html\index.html" in the default web browser... - start "" "%BUILDDIR%\html\index.html" -) - -goto end - -:check -rem Check the docs and NEWS files with sphinx-lint. -rem Ignore the tools dir and check that the default role is not used. -cmd /S /C "%SPHINXLINT% -i tools --enable default-role" -cmd /S /C "%SPHINXLINT% --enable default-role ..\Misc\NEWS.d\next\ " -goto end - -:serve -echo.The serve target was removed, use htmlview instead (see bpo-36329) -goto end - -:end -popd diff --git a/Doc/reference/compound_stmts.rst b/Doc/reference/compound_stmts.rst deleted file mode 100644 index fd3091105d2273..00000000000000 --- a/Doc/reference/compound_stmts.rst +++ /dev/null @@ -1,1638 +0,0 @@ -.. _compound: - -******************* -Compound statements -******************* - -.. index:: pair: compound; statement - -Compound statements contain (groups of) other statements; they affect or control -the execution of those other statements in some way. In general, compound -statements span multiple lines, although in simple incarnations a whole compound -statement may be contained in one line. - -The :keyword:`if`, :keyword:`while` and :keyword:`for` statements implement -traditional control flow constructs. :keyword:`try` specifies exception -handlers and/or cleanup code for a group of statements, while the -:keyword:`with` statement allows the execution of initialization and -finalization code around a block of code. Function and class definitions are -also syntactically compound statements. - -.. index:: - single: clause - single: suite - single: ; (semicolon) - -A compound statement consists of one or more 'clauses.' A clause consists of a -header and a 'suite.' The clause headers of a particular compound statement are -all at the same indentation level. Each clause header begins with a uniquely -identifying keyword and ends with a colon. A suite is a group of statements -controlled by a clause. A suite can be one or more semicolon-separated simple -statements on the same line as the header, following the header's colon, or it -can be one or more indented statements on subsequent lines. Only the latter -form of a suite can contain nested compound statements; the following is illegal, -mostly because it wouldn't be clear to which :keyword:`if` clause a following -:keyword:`else` clause would belong:: - - if test1: if test2: print(x) - -Also note that the semicolon binds tighter than the colon in this context, so -that in the following example, either all or none of the :func:`print` calls are -executed:: - - if x < y < z: print(x); print(y); print(z) - -Summarizing: - - -.. productionlist:: python-grammar - compound_stmt: `if_stmt` - : | `while_stmt` - : | `for_stmt` - : | `try_stmt` - : | `with_stmt` - : | `match_stmt` - : | `funcdef` - : | `classdef` - : | `async_with_stmt` - : | `async_for_stmt` - : | `async_funcdef` - suite: `stmt_list` NEWLINE | NEWLINE INDENT `statement`+ DEDENT - statement: `stmt_list` NEWLINE | `compound_stmt` - stmt_list: `simple_stmt` (";" `simple_stmt`)* [";"] - -.. index:: - single: NEWLINE token - single: DEDENT token - pair: dangling; else - -Note that statements always end in a ``NEWLINE`` possibly followed by a -``DEDENT``. Also note that optional continuation clauses always begin with a -keyword that cannot start a statement, thus there are no ambiguities (the -'dangling :keyword:`else`' problem is solved in Python by requiring nested -:keyword:`if` statements to be indented). - -The formatting of the grammar rules in the following sections places each clause -on a separate line for clarity. - - -.. _if: -.. _elif: -.. _else: - -The :keyword:`!if` statement -============================ - -.. index:: - ! pair: statement; if - pair: keyword; elif - pair: keyword; else - single: : (colon); compound statement - -The :keyword:`if` statement is used for conditional execution: - -.. productionlist:: python-grammar - if_stmt: "if" `assignment_expression` ":" `suite` - : ("elif" `assignment_expression` ":" `suite`)* - : ["else" ":" `suite`] - -It selects exactly one of the suites by evaluating the expressions one by one -until one is found to be true (see section :ref:`booleans` for the definition of -true and false); then that suite is executed (and no other part of the -:keyword:`if` statement is executed or evaluated). If all expressions are -false, the suite of the :keyword:`else` clause, if present, is executed. - - -.. _while: - -The :keyword:`!while` statement -=============================== - -.. index:: - ! pair: statement; while - pair: keyword; else - pair: loop; statement - single: : (colon); compound statement - -The :keyword:`while` statement is used for repeated execution as long as an -expression is true: - -.. productionlist:: python-grammar - while_stmt: "while" `assignment_expression` ":" `suite` - : ["else" ":" `suite`] - -This repeatedly tests the expression and, if it is true, executes the first -suite; if the expression is false (which may be the first time it is tested) the -suite of the :keyword:`!else` clause, if present, is executed and the loop -terminates. - -.. index:: - pair: statement; break - pair: statement; continue - -A :keyword:`break` statement executed in the first suite terminates the loop -without executing the :keyword:`!else` clause's suite. A :keyword:`continue` -statement executed in the first suite skips the rest of the suite and goes back -to testing the expression. - - -.. _for: - -The :keyword:`!for` statement -============================= - -.. index:: - ! pair: statement; for - pair: keyword; in - pair: keyword; else - pair: target; list - pair: loop; statement - pair: object; sequence - single: : (colon); compound statement - -The :keyword:`for` statement is used to iterate over the elements of a sequence -(such as a string, tuple or list) or other iterable object: - -.. productionlist:: python-grammar - for_stmt: "for" `target_list` "in" `starred_list` ":" `suite` - : ["else" ":" `suite`] - -The ``starred_list`` expression is evaluated once; it should yield an -:term:`iterable` object. An :term:`iterator` is created for that iterable. -The first item provided -by the iterator is then assigned to the target list using the standard -rules for assignments (see :ref:`assignment`), and the suite is executed. This -repeats for each item provided by the iterator. When the iterator is exhausted, -the suite in the :keyword:`!else` clause, -if present, is executed, and the loop terminates. - -.. index:: - pair: statement; break - pair: statement; continue - -A :keyword:`break` statement executed in the first suite terminates the loop -without executing the :keyword:`!else` clause's suite. A :keyword:`continue` -statement executed in the first suite skips the rest of the suite and continues -with the next item, or with the :keyword:`!else` clause if there is no next -item. - -The for-loop makes assignments to the variables in the target list. -This overwrites all previous assignments to those variables including -those made in the suite of the for-loop:: - - for i in range(10): - print(i) - i = 5 # this will not affect the for-loop - # because i will be overwritten with the next - # index in the range - - -.. index:: - pair: built-in function; range - -Names in the target list are not deleted when the loop is finished, but if the -sequence is empty, they will not have been assigned to at all by the loop. Hint: -the built-in type :func:`range` represents immutable arithmetic sequences of integers. -For instance, iterating ``range(3)`` successively yields 0, 1, and then 2. - -.. versionchanged:: 3.11 - Starred elements are now allowed in the expression list. - - -.. _try: - -The :keyword:`!try` statement -============================= - -.. index:: - ! pair: statement; try - pair: keyword; except - pair: keyword; finally - pair: keyword; else - pair: keyword; as - single: : (colon); compound statement - -The :keyword:`!try` statement specifies exception handlers and/or cleanup code -for a group of statements: - -.. productionlist:: python-grammar - try_stmt: `try1_stmt` | `try2_stmt` | `try3_stmt` - try1_stmt: "try" ":" `suite` - : ("except" [`expression` ["as" `identifier`]] ":" `suite`)+ - : ["else" ":" `suite`] - : ["finally" ":" `suite`] - try2_stmt: "try" ":" `suite` - : ("except" "*" `expression` ["as" `identifier`] ":" `suite`)+ - : ["else" ":" `suite`] - : ["finally" ":" `suite`] - try3_stmt: "try" ":" `suite` - : "finally" ":" `suite` - -Additional information on exceptions can be found in section :ref:`exceptions`, -and information on using the :keyword:`raise` statement to generate exceptions -may be found in section :ref:`raise`. - - -.. _except: - -:keyword:`!except` clause -------------------------- - -The :keyword:`!except` clause(s) specify one or more exception handlers. When no -exception occurs in the :keyword:`try` clause, no exception handler is executed. -When an exception occurs in the :keyword:`!try` suite, a search for an exception -handler is started. This search inspects the :keyword:`!except` clauses in turn -until one is found that matches the exception. -An expression-less :keyword:`!except` clause, if present, must be last; -it matches any exception. -For an :keyword:`!except` clause with an expression, -that expression is evaluated, and the clause matches the exception -if the resulting object is "compatible" with the exception. An object is -compatible with an exception if the object is the class or a -:term:`non-virtual base class ` of the exception object, -or a tuple containing an item that is the class or a non-virtual base class -of the exception object. - -If no :keyword:`!except` clause matches the exception, -the search for an exception handler -continues in the surrounding code and on the invocation stack. [#]_ - -If the evaluation of an expression -in the header of an :keyword:`!except` clause raises an exception, -the original search for a handler is canceled and a search starts for -the new exception in the surrounding code and on the call stack (it is treated -as if the entire :keyword:`try` statement raised the exception). - -.. index:: single: as; except clause - -When a matching :keyword:`!except` clause is found, -the exception is assigned to the target -specified after the :keyword:`!as` keyword in that :keyword:`!except` clause, -if present, and the :keyword:`!except` clause's suite is executed. -All :keyword:`!except` clauses must have an executable block. -When the end of this block is reached, execution continues -normally after the entire :keyword:`try` statement. -(This means that if two nested handlers exist for the same exception, -and the exception occurs in the :keyword:`!try` clause of the inner handler, -the outer handler will not handle the exception.) - -When an exception has been assigned using ``as target``, it is cleared at the -end of the :keyword:`!except` clause. This is as if :: - - except E as N: - foo - -was translated to :: - - except E as N: - try: - foo - finally: - del N - -This means the exception must be assigned to a different name to be able to -refer to it after the :keyword:`!except` clause. -Exceptions are cleared because with the -traceback attached to them, they form a reference cycle with the stack frame, -keeping all locals in that frame alive until the next garbage collection occurs. - -.. index:: - pair: module; sys - pair: object; traceback - -Before an :keyword:`!except` clause's suite is executed, -the exception is stored in the :mod:`sys` module, where it can be accessed -from within the body of the :keyword:`!except` clause by calling -:func:`sys.exception`. When leaving an exception handler, the exception -stored in the :mod:`sys` module is reset to its previous value:: - - >>> print(sys.exception()) - None - >>> try: - ... raise TypeError - ... except: - ... print(repr(sys.exception())) - ... try: - ... raise ValueError - ... except: - ... print(repr(sys.exception())) - ... print(repr(sys.exception())) - ... - TypeError() - ValueError() - TypeError() - >>> print(sys.exception()) - None - - -.. index:: - pair: keyword; except_star - -.. _except_star: - -:keyword:`!except*` clause --------------------------- - -The :keyword:`!except*` clause(s) are used for handling -:exc:`ExceptionGroup`\s. The exception type for matching is interpreted as in -the case of :keyword:`except`, but in the case of exception groups we can have -partial matches when the type matches some of the exceptions in the group. -This means that multiple :keyword:`!except*` clauses can execute, -each handling part of the exception group. -Each clause executes at most once and handles an exception group -of all matching exceptions. Each exception in the group is handled by at most -one :keyword:`!except*` clause, the first that matches it. :: - - >>> try: - ... raise ExceptionGroup("eg", - ... [ValueError(1), TypeError(2), OSError(3), OSError(4)]) - ... except* TypeError as e: - ... print(f'caught {type(e)} with nested {e.exceptions}') - ... except* OSError as e: - ... print(f'caught {type(e)} with nested {e.exceptions}') - ... - caught with nested (TypeError(2),) - caught with nested (OSError(3), OSError(4)) - + Exception Group Traceback (most recent call last): - | File "", line 2, in - | ExceptionGroup: eg - +-+---------------- 1 ---------------- - | ValueError: 1 - +------------------------------------ - - -Any remaining exceptions that were not handled by any :keyword:`!except*` -clause are re-raised at the end, combined into an exception group along with -all exceptions that were raised from within :keyword:`!except*` clauses. - -From version 3.11.4, when the entire :exc:`ExceptionGroup` is handled and -only one exception is raised from an :keyword:`!except*` clause, this -exception is no longer wrapped to form a new :exc:`ExceptionGroup`. - -If the raised exception is not an exception group and its type matches -one of the :keyword:`!except*` clauses, it is caught and wrapped by an -exception group with an empty message string. :: - - >>> try: - ... raise BlockingIOError - ... except* BlockingIOError as e: - ... print(repr(e)) - ... - ExceptionGroup('', (BlockingIOError())) - -An :keyword:`!except*` clause must have a matching type, -and this type cannot be a subclass of :exc:`BaseExceptionGroup`. -It is not possible to mix :keyword:`except` and :keyword:`!except*` -in the same :keyword:`try`. -:keyword:`break`, :keyword:`continue` and :keyword:`return` -cannot appear in an :keyword:`!except*` clause. - - -.. index:: - pair: keyword; else - pair: statement; return - pair: statement; break - pair: statement; continue - -.. _except_else: - -:keyword:`!else` clause ------------------------ - -The optional :keyword:`!else` clause is executed if the control flow leaves the -:keyword:`try` suite, no exception was raised, and no :keyword:`return`, -:keyword:`continue`, or :keyword:`break` statement was executed. Exceptions in -the :keyword:`!else` clause are not handled by the preceding :keyword:`except` -clauses. - - -.. index:: pair: keyword; finally - -.. _finally: - -:keyword:`!finally` clause --------------------------- - -If :keyword:`!finally` is present, it specifies a 'cleanup' handler. The -:keyword:`try` clause is executed, including any :keyword:`except` and -:keyword:`else` clauses. If an exception occurs in any of the clauses and is -not handled, the exception is temporarily saved. The :keyword:`!finally` clause -is executed. If there is a saved exception it is re-raised at the end of the -:keyword:`!finally` clause. If the :keyword:`!finally` clause raises another -exception, the saved exception is set as the context of the new exception. -If the :keyword:`!finally` clause executes a :keyword:`return`, :keyword:`break` -or :keyword:`continue` statement, the saved exception is discarded:: - - >>> def f(): - ... try: - ... 1/0 - ... finally: - ... return 42 - ... - >>> f() - 42 - -The exception information is not available to the program during execution of -the :keyword:`!finally` clause. - -.. index:: - pair: statement; return - pair: statement; break - pair: statement; continue - -When a :keyword:`return`, :keyword:`break` or :keyword:`continue` statement is -executed in the :keyword:`try` suite of a :keyword:`!try`...\ :keyword:`!finally` -statement, the :keyword:`!finally` clause is also executed 'on the way out.' - -The return value of a function is determined by the last :keyword:`return` -statement executed. Since the :keyword:`!finally` clause always executes, a -:keyword:`!return` statement executed in the :keyword:`!finally` clause will -always be the last one executed:: - - >>> def foo(): - ... try: - ... return 'try' - ... finally: - ... return 'finally' - ... - >>> foo() - 'finally' - -.. versionchanged:: 3.8 - Prior to Python 3.8, a :keyword:`continue` statement was illegal in the - :keyword:`!finally` clause due to a problem with the implementation. - - -.. _with: -.. _as: - -The :keyword:`!with` statement -============================== - -.. index:: - ! pair: statement; with - pair: keyword; as - single: as; with statement - single: , (comma); with statement - single: : (colon); compound statement - -The :keyword:`with` statement is used to wrap the execution of a block with -methods defined by a context manager (see section :ref:`context-managers`). -This allows common :keyword:`try`...\ :keyword:`except`...\ :keyword:`finally` -usage patterns to be encapsulated for convenient reuse. - -.. productionlist:: python-grammar - with_stmt: "with" ( "(" `with_stmt_contents` ","? ")" | `with_stmt_contents` ) ":" `suite` - with_stmt_contents: `with_item` ("," `with_item`)* - with_item: `expression` ["as" `target`] - -The execution of the :keyword:`with` statement with one "item" proceeds as follows: - -#. The context expression (the expression given in the - :token:`~python-grammar:with_item`) is evaluated to obtain a context manager. - -#. The context manager's :meth:`~object.__enter__` is loaded for later use. - -#. The context manager's :meth:`~object.__exit__` is loaded for later use. - -#. The context manager's :meth:`~object.__enter__` method is invoked. - -#. If a target was included in the :keyword:`with` statement, the return value - from :meth:`~object.__enter__` is assigned to it. - - .. note:: - - The :keyword:`with` statement guarantees that if the :meth:`~object.__enter__` - method returns without an error, then :meth:`~object.__exit__` will always be - called. Thus, if an error occurs during the assignment to the target list, - it will be treated the same as an error occurring within the suite would - be. See step 7 below. - -#. The suite is executed. - -#. The context manager's :meth:`~object.__exit__` method is invoked. If an exception - caused the suite to be exited, its type, value, and traceback are passed as - arguments to :meth:`~object.__exit__`. Otherwise, three :const:`None` arguments are - supplied. - - If the suite was exited due to an exception, and the return value from the - :meth:`~object.__exit__` method was false, the exception is reraised. If the return - value was true, the exception is suppressed, and execution continues with the - statement following the :keyword:`with` statement. - - If the suite was exited for any reason other than an exception, the return - value from :meth:`~object.__exit__` is ignored, and execution proceeds at the normal - location for the kind of exit that was taken. - -The following code:: - - with EXPRESSION as TARGET: - SUITE - -is semantically equivalent to:: - - manager = (EXPRESSION) - enter = type(manager).__enter__ - exit = type(manager).__exit__ - value = enter(manager) - hit_except = False - - try: - TARGET = value - SUITE - except: - hit_except = True - if not exit(manager, *sys.exc_info()): - raise - finally: - if not hit_except: - exit(manager, None, None, None) - -With more than one item, the context managers are processed as if multiple -:keyword:`with` statements were nested:: - - with A() as a, B() as b: - SUITE - -is semantically equivalent to:: - - with A() as a: - with B() as b: - SUITE - -You can also write multi-item context managers in multiple lines if -the items are surrounded by parentheses. For example:: - - with ( - A() as a, - B() as b, - ): - SUITE - -.. versionchanged:: 3.1 - Support for multiple context expressions. - -.. versionchanged:: 3.10 - Support for using grouping parentheses to break the statement in multiple lines. - -.. seealso:: - - :pep:`343` - The "with" statement - The specification, background, and examples for the Python :keyword:`with` - statement. - -.. _match: - -The :keyword:`!match` statement -=============================== - -.. index:: - ! pair: statement; match - ! pair: keyword; case - ! single: pattern matching - pair: keyword; if - pair: keyword; as - pair: match; case - single: as; match statement - single: : (colon); compound statement - -.. versionadded:: 3.10 - -The match statement is used for pattern matching. Syntax: - -.. productionlist:: python-grammar - match_stmt: 'match' `subject_expr` ":" NEWLINE INDENT `case_block`+ DEDENT - subject_expr: `star_named_expression` "," `star_named_expressions`? - : | `named_expression` - case_block: 'case' `patterns` [`guard`] ":" `block` - -.. note:: - This section uses single quotes to denote - :ref:`soft keywords `. - -Pattern matching takes a pattern as input (following ``case``) and a subject -value (following ``match``). The pattern (which may contain subpatterns) is -matched against the subject value. The outcomes are: - -* A match success or failure (also termed a pattern success or failure). - -* Possible binding of matched values to a name. The prerequisites for this are - further discussed below. - -The ``match`` and ``case`` keywords are :ref:`soft keywords `. - -.. seealso:: - - * :pep:`634` -- Structural Pattern Matching: Specification - * :pep:`636` -- Structural Pattern Matching: Tutorial - - -Overview --------- - -Here's an overview of the logical flow of a match statement: - - -#. The subject expression ``subject_expr`` is evaluated and a resulting subject - value obtained. If the subject expression contains a comma, a tuple is - constructed using :ref:`the standard rules `. - -#. Each pattern in a ``case_block`` is attempted to match with the subject value. The - specific rules for success or failure are described below. The match attempt can also - bind some or all of the standalone names within the pattern. The precise - pattern binding rules vary per pattern type and are - specified below. **Name bindings made during a successful pattern match - outlive the executed block and can be used after the match statement**. - - .. note:: - - During failed pattern matches, some subpatterns may succeed. Do not - rely on bindings being made for a failed match. Conversely, do not - rely on variables remaining unchanged after a failed match. The exact - behavior is dependent on implementation and may vary. This is an - intentional decision made to allow different implementations to add - optimizations. - -#. If the pattern succeeds, the corresponding guard (if present) is evaluated. In - this case all name bindings are guaranteed to have happened. - - * If the guard evaluates as true or is missing, the ``block`` inside - ``case_block`` is executed. - - * Otherwise, the next ``case_block`` is attempted as described above. - - * If there are no further case blocks, the match statement is completed. - -.. note:: - - Users should generally never rely on a pattern being evaluated. Depending on - implementation, the interpreter may cache values or use other optimizations - which skip repeated evaluations. - -A sample match statement:: - - >>> flag = False - >>> match (100, 200): - ... case (100, 300): # Mismatch: 200 != 300 - ... print('Case 1') - ... case (100, 200) if flag: # Successful match, but guard fails - ... print('Case 2') - ... case (100, y): # Matches and binds y to 200 - ... print(f'Case 3, y: {y}') - ... case _: # Pattern not attempted - ... print('Case 4, I match anything!') - ... - Case 3, y: 200 - - -In this case, ``if flag`` is a guard. Read more about that in the next section. - -Guards ------- - -.. index:: ! guard - -.. productionlist:: python-grammar - guard: "if" `named_expression` - -A ``guard`` (which is part of the ``case``) must succeed for code inside -the ``case`` block to execute. It takes the form: :keyword:`if` followed by an -expression. - - -The logical flow of a ``case`` block with a ``guard`` follows: - -#. Check that the pattern in the ``case`` block succeeded. If the pattern - failed, the ``guard`` is not evaluated and the next ``case`` block is - checked. - -#. If the pattern succeeded, evaluate the ``guard``. - - * If the ``guard`` condition evaluates as true, the case block is - selected. - - * If the ``guard`` condition evaluates as false, the case block is not - selected. - - * If the ``guard`` raises an exception during evaluation, the exception - bubbles up. - -Guards are allowed to have side effects as they are expressions. Guard -evaluation must proceed from the first to the last case block, one at a time, -skipping case blocks whose pattern(s) don't all succeed. (I.e., -guard evaluation must happen in order.) Guard evaluation must stop once a case -block is selected. - - -.. _irrefutable_case: - -Irrefutable Case Blocks ------------------------ - -.. index:: irrefutable case block, case block - -An irrefutable case block is a match-all case block. A match statement may have -at most one irrefutable case block, and it must be last. - -A case block is considered irrefutable if it has no guard and its pattern is -irrefutable. A pattern is considered irrefutable if we can prove from its -syntax alone that it will always succeed. Only the following patterns are -irrefutable: - -* :ref:`as-patterns` whose left-hand side is irrefutable - -* :ref:`or-patterns` containing at least one irrefutable pattern - -* :ref:`capture-patterns` - -* :ref:`wildcard-patterns` - -* parenthesized irrefutable patterns - - -Patterns --------- - -.. index:: - single: ! patterns - single: AS pattern, OR pattern, capture pattern, wildcard pattern - -.. note:: - This section uses grammar notations beyond standard EBNF: - - * the notation ``SEP.RULE+`` is shorthand for ``RULE (SEP RULE)*`` - - * the notation ``!RULE`` is shorthand for a negative lookahead assertion - - -The top-level syntax for ``patterns`` is: - -.. productionlist:: python-grammar - patterns: `open_sequence_pattern` | `pattern` - pattern: `as_pattern` | `or_pattern` - closed_pattern: | `literal_pattern` - : | `capture_pattern` - : | `wildcard_pattern` - : | `value_pattern` - : | `group_pattern` - : | `sequence_pattern` - : | `mapping_pattern` - : | `class_pattern` - -The descriptions below will include a description "in simple terms" of what a pattern -does for illustration purposes (credits to Raymond Hettinger for a document that -inspired most of the descriptions). Note that these descriptions are purely for -illustration purposes and **may not** reflect -the underlying implementation. Furthermore, they do not cover all valid forms. - - -.. _or-patterns: - -OR Patterns -^^^^^^^^^^^ - -An OR pattern is two or more patterns separated by vertical -bars ``|``. Syntax: - -.. productionlist:: python-grammar - or_pattern: "|".`closed_pattern`+ - -Only the final subpattern may be :ref:`irrefutable `, and each -subpattern must bind the same set of names to avoid ambiguity. - -An OR pattern matches each of its subpatterns in turn to the subject value, -until one succeeds. The OR pattern is then considered successful. Otherwise, -if none of the subpatterns succeed, the OR pattern fails. - -In simple terms, ``P1 | P2 | ...`` will try to match ``P1``, if it fails it will try to -match ``P2``, succeeding immediately if any succeeds, failing otherwise. - -.. _as-patterns: - -AS Patterns -^^^^^^^^^^^ - -An AS pattern matches an OR pattern on the left of the :keyword:`as` -keyword against a subject. Syntax: - -.. productionlist:: python-grammar - as_pattern: `or_pattern` "as" `capture_pattern` - -If the OR pattern fails, the AS pattern fails. Otherwise, the AS pattern binds -the subject to the name on the right of the as keyword and succeeds. -``capture_pattern`` cannot be a ``_``. - -In simple terms ``P as NAME`` will match with ``P``, and on success it will -set ``NAME = ``. - - -.. _literal-patterns: - -Literal Patterns -^^^^^^^^^^^^^^^^ - -A literal pattern corresponds to most -:ref:`literals ` in Python. Syntax: - -.. productionlist:: python-grammar - literal_pattern: `signed_number` - : | `signed_number` "+" NUMBER - : | `signed_number` "-" NUMBER - : | `strings` - : | "None" - : | "True" - : | "False" - : | `signed_number`: NUMBER | "-" NUMBER - -The rule ``strings`` and the token ``NUMBER`` are defined in the -:doc:`standard Python grammar <./grammar>`. Triple-quoted strings are -supported. Raw strings and byte strings are supported. :ref:`f-strings` are -not supported. - -The forms ``signed_number '+' NUMBER`` and ``signed_number '-' NUMBER`` are -for expressing :ref:`complex numbers `; they require a real number -on the left and an imaginary number on the right. E.g. ``3 + 4j``. - -In simple terms, ``LITERAL`` will succeed only if `` == LITERAL``. For -the singletons ``None``, ``True`` and ``False``, the :keyword:`is` operator is used. - -.. _capture-patterns: - -Capture Patterns -^^^^^^^^^^^^^^^^ - -A capture pattern binds the subject value to a name. -Syntax: - -.. productionlist:: python-grammar - capture_pattern: !'_' NAME - -A single underscore ``_`` is not a capture pattern (this is what ``!'_'`` -expresses). It is instead treated as a -:token:`~python-grammar:wildcard_pattern`. - -In a given pattern, a given name can only be bound once. E.g. -``case x, x: ...`` is invalid while ``case [x] | x: ...`` is allowed. - -Capture patterns always succeed. The binding follows scoping rules -established by the assignment expression operator in :pep:`572`; the -name becomes a local variable in the closest containing function scope unless -there's an applicable :keyword:`global` or :keyword:`nonlocal` statement. - -In simple terms ``NAME`` will always succeed and it will set ``NAME = ``. - -.. _wildcard-patterns: - -Wildcard Patterns -^^^^^^^^^^^^^^^^^ - -A wildcard pattern always succeeds (matches anything) -and binds no name. Syntax: - -.. productionlist:: python-grammar - wildcard_pattern: '_' - -``_`` is a :ref:`soft keyword ` within any pattern, -but only within patterns. It is an identifier, as usual, even within -``match`` subject expressions, ``guard``\ s, and ``case`` blocks. - -In simple terms, ``_`` will always succeed. - -.. _value-patterns: - -Value Patterns -^^^^^^^^^^^^^^ - -A value pattern represents a named value in Python. -Syntax: - -.. productionlist:: python-grammar - value_pattern: `attr` - attr: `name_or_attr` "." NAME - name_or_attr: `attr` | NAME - -The dotted name in the pattern is looked up using standard Python -:ref:`name resolution rules `. The pattern succeeds if the -value found compares equal to the subject value (using the ``==`` equality -operator). - -In simple terms ``NAME1.NAME2`` will succeed only if `` == NAME1.NAME2`` - -.. note:: - - If the same value occurs multiple times in the same match statement, the - interpreter may cache the first value found and reuse it rather than repeat - the same lookup. This cache is strictly tied to a given execution of a - given match statement. - -.. _group-patterns: - -Group Patterns -^^^^^^^^^^^^^^ - -A group pattern allows users to add parentheses around patterns to -emphasize the intended grouping. Otherwise, it has no additional syntax. -Syntax: - -.. productionlist:: python-grammar - group_pattern: "(" `pattern` ")" - -In simple terms ``(P)`` has the same effect as ``P``. - -.. _sequence-patterns: - -Sequence Patterns -^^^^^^^^^^^^^^^^^ - -A sequence pattern contains several subpatterns to be matched against sequence elements. -The syntax is similar to the unpacking of a list or tuple. - -.. productionlist:: python-grammar - sequence_pattern: "[" [`maybe_sequence_pattern`] "]" - : | "(" [`open_sequence_pattern`] ")" - open_sequence_pattern: `maybe_star_pattern` "," [`maybe_sequence_pattern`] - maybe_sequence_pattern: ",".`maybe_star_pattern`+ ","? - maybe_star_pattern: `star_pattern` | `pattern` - star_pattern: "*" (`capture_pattern` | `wildcard_pattern`) - -There is no difference if parentheses or square brackets -are used for sequence patterns (i.e. ``(...)`` vs ``[...]`` ). - -.. note:: - A single pattern enclosed in parentheses without a trailing comma - (e.g. ``(3 | 4)``) is a :ref:`group pattern `. - While a single pattern enclosed in square brackets (e.g. ``[3 | 4]``) is - still a sequence pattern. - -At most one star subpattern may be in a sequence pattern. The star subpattern -may occur in any position. If no star subpattern is present, the sequence -pattern is a fixed-length sequence pattern; otherwise it is a variable-length -sequence pattern. - -The following is the logical flow for matching a sequence pattern against a -subject value: - -#. If the subject value is not a sequence [#]_, the sequence pattern - fails. - -#. If the subject value is an instance of ``str``, ``bytes`` or ``bytearray`` - the sequence pattern fails. - -#. The subsequent steps depend on whether the sequence pattern is fixed or - variable-length. - - If the sequence pattern is fixed-length: - - #. If the length of the subject sequence is not equal to the number of - subpatterns, the sequence pattern fails - - #. Subpatterns in the sequence pattern are matched to their corresponding - items in the subject sequence from left to right. Matching stops as soon - as a subpattern fails. If all subpatterns succeed in matching their - corresponding item, the sequence pattern succeeds. - - Otherwise, if the sequence pattern is variable-length: - - #. If the length of the subject sequence is less than the number of non-star - subpatterns, the sequence pattern fails. - - #. The leading non-star subpatterns are matched to their corresponding items - as for fixed-length sequences. - - #. If the previous step succeeds, the star subpattern matches a list formed - of the remaining subject items, excluding the remaining items - corresponding to non-star subpatterns following the star subpattern. - - #. Remaining non-star subpatterns are matched to their corresponding subject - items, as for a fixed-length sequence. - - .. note:: The length of the subject sequence is obtained via - :func:`len` (i.e. via the :meth:`__len__` protocol). This length may be - cached by the interpreter in a similar manner as - :ref:`value patterns `. - - -In simple terms ``[P1, P2, P3,`` ... ``, P]`` matches only if all the following -happens: - -* check ```` is a sequence -* ``len(subject) == `` -* ``P1`` matches ``[0]`` (note that this match can also bind names) -* ``P2`` matches ``[1]`` (note that this match can also bind names) -* ... and so on for the corresponding pattern/element. - -.. _mapping-patterns: - -Mapping Patterns -^^^^^^^^^^^^^^^^ - -A mapping pattern contains one or more key-value patterns. The syntax is -similar to the construction of a dictionary. -Syntax: - -.. productionlist:: python-grammar - mapping_pattern: "{" [`items_pattern`] "}" - items_pattern: ",".`key_value_pattern`+ ","? - key_value_pattern: (`literal_pattern` | `value_pattern`) ":" `pattern` - : | `double_star_pattern` - double_star_pattern: "**" `capture_pattern` - -At most one double star pattern may be in a mapping pattern. The double star -pattern must be the last subpattern in the mapping pattern. - -Duplicate keys in mapping patterns are disallowed. Duplicate literal keys will -raise a :exc:`SyntaxError`. Two keys that otherwise have the same value will -raise a :exc:`ValueError` at runtime. - -The following is the logical flow for matching a mapping pattern against a -subject value: - -#. If the subject value is not a mapping [#]_,the mapping pattern fails. - -#. If every key given in the mapping pattern is present in the subject mapping, - and the pattern for each key matches the corresponding item of the subject - mapping, the mapping pattern succeeds. - -#. If duplicate keys are detected in the mapping pattern, the pattern is - considered invalid. A :exc:`SyntaxError` is raised for duplicate literal - values; or a :exc:`ValueError` for named keys of the same value. - -.. note:: Key-value pairs are matched using the two-argument form of the mapping - subject's ``get()`` method. Matched key-value pairs must already be present - in the mapping, and not created on-the-fly via :meth:`__missing__` or - :meth:`~object.__getitem__`. - -In simple terms ``{KEY1: P1, KEY2: P2, ... }`` matches only if all the following -happens: - -* check ```` is a mapping -* ``KEY1 in `` -* ``P1`` matches ``[KEY1]`` -* ... and so on for the corresponding KEY/pattern pair. - - -.. _class-patterns: - -Class Patterns -^^^^^^^^^^^^^^ - -A class pattern represents a class and its positional and keyword arguments -(if any). Syntax: - -.. productionlist:: python-grammar - class_pattern: `name_or_attr` "(" [`pattern_arguments` ","?] ")" - pattern_arguments: `positional_patterns` ["," `keyword_patterns`] - : | `keyword_patterns` - positional_patterns: ",".`pattern`+ - keyword_patterns: ",".`keyword_pattern`+ - keyword_pattern: NAME "=" `pattern` - -The same keyword should not be repeated in class patterns. - -The following is the logical flow for matching a class pattern against a -subject value: - -#. If ``name_or_attr`` is not an instance of the builtin :class:`type` , raise - :exc:`TypeError`. - -#. If the subject value is not an instance of ``name_or_attr`` (tested via - :func:`isinstance`), the class pattern fails. - -#. If no pattern arguments are present, the pattern succeeds. Otherwise, - the subsequent steps depend on whether keyword or positional argument patterns - are present. - - For a number of built-in types (specified below), a single positional - subpattern is accepted which will match the entire subject; for these types - keyword patterns also work as for other types. - - If only keyword patterns are present, they are processed as follows, - one by one: - - I. The keyword is looked up as an attribute on the subject. - - * If this raises an exception other than :exc:`AttributeError`, the - exception bubbles up. - - * If this raises :exc:`AttributeError`, the class pattern has failed. - - * Else, the subpattern associated with the keyword pattern is matched - against the subject's attribute value. If this fails, the class - pattern fails; if this succeeds, the match proceeds to the next keyword. - - - II. If all keyword patterns succeed, the class pattern succeeds. - - If any positional patterns are present, they are converted to keyword - patterns using the :data:`~object.__match_args__` attribute on the class - ``name_or_attr`` before matching: - - I. The equivalent of ``getattr(cls, "__match_args__", ())`` is called. - - * If this raises an exception, the exception bubbles up. - - * If the returned value is not a tuple, the conversion fails and - :exc:`TypeError` is raised. - - * If there are more positional patterns than ``len(cls.__match_args__)``, - :exc:`TypeError` is raised. - - * Otherwise, positional pattern ``i`` is converted to a keyword pattern - using ``__match_args__[i]`` as the keyword. ``__match_args__[i]`` must - be a string; if not :exc:`TypeError` is raised. - - * If there are duplicate keywords, :exc:`TypeError` is raised. - - .. seealso:: :ref:`class-pattern-matching` - - II. Once all positional patterns have been converted to keyword patterns, - the match proceeds as if there were only keyword patterns. - - For the following built-in types the handling of positional subpatterns is - different: - - * :class:`bool` - * :class:`bytearray` - * :class:`bytes` - * :class:`dict` - * :class:`float` - * :class:`frozenset` - * :class:`int` - * :class:`list` - * :class:`set` - * :class:`str` - * :class:`tuple` - - These classes accept a single positional argument, and the pattern there is matched - against the whole object rather than an attribute. For example ``int(0|1)`` matches - the value ``0``, but not the value ``0.0``. - -In simple terms ``CLS(P1, attr=P2)`` matches only if the following happens: - -* ``isinstance(, CLS)`` -* convert ``P1`` to a keyword pattern using ``CLS.__match_args__`` -* For each keyword argument ``attr=P2``: - - * ``hasattr(, "attr")`` - * ``P2`` matches ``.attr`` - -* ... and so on for the corresponding keyword argument/pattern pair. - -.. seealso:: - - * :pep:`634` -- Structural Pattern Matching: Specification - * :pep:`636` -- Structural Pattern Matching: Tutorial - - -.. index:: - single: parameter; function definition - -.. _function: -.. _def: - -Function definitions -==================== - -.. index:: - pair: statement; def - pair: function; definition - pair: function; name - pair: name; binding - pair: object; user-defined function - pair: object; function - pair: function; name - pair: name; binding - single: () (parentheses); function definition - single: , (comma); parameter list - single: : (colon); compound statement - -A function definition defines a user-defined function object (see section -:ref:`types`): - -.. productionlist:: python-grammar - funcdef: [`decorators`] "def" `funcname` "(" [`parameter_list`] ")" - : ["->" `expression`] ":" `suite` - decorators: `decorator`+ - decorator: "@" `assignment_expression` NEWLINE - parameter_list: `defparameter` ("," `defparameter`)* "," "/" ["," [`parameter_list_no_posonly`]] - : | `parameter_list_no_posonly` - parameter_list_no_posonly: `defparameter` ("," `defparameter`)* ["," [`parameter_list_starargs`]] - : | `parameter_list_starargs` - parameter_list_starargs: "*" [`parameter`] ("," `defparameter`)* ["," ["**" `parameter` [","]]] - : | "**" `parameter` [","] - parameter: `identifier` [":" `expression`] - defparameter: `parameter` ["=" `expression`] - funcname: `identifier` - - -A function definition is an executable statement. Its execution binds the -function name in the current local namespace to a function object (a wrapper -around the executable code for the function). This function object contains a -reference to the current global namespace as the global namespace to be used -when the function is called. - -The function definition does not execute the function body; this gets executed -only when the function is called. [#]_ - -.. index:: - single: @ (at); function definition - -A function definition may be wrapped by one or more :term:`decorator` expressions. -Decorator expressions are evaluated when the function is defined, in the scope -that contains the function definition. The result must be a callable, which is -invoked with the function object as the only argument. The returned value is -bound to the function name instead of the function object. Multiple decorators -are applied in nested fashion. For example, the following code :: - - @f1(arg) - @f2 - def func(): pass - -is roughly equivalent to :: - - def func(): pass - func = f1(arg)(f2(func)) - -except that the original function is not temporarily bound to the name ``func``. - -.. versionchanged:: 3.9 - Functions may be decorated with any valid - :token:`~python-grammar:assignment_expression`. Previously, the grammar was - much more restrictive; see :pep:`614` for details. - -.. index:: - triple: default; parameter; value - single: argument; function definition - single: = (equals); function definition - -When one or more :term:`parameters ` have the form *parameter* ``=`` -*expression*, the function is said to have "default parameter values." For a -parameter with a default value, the corresponding :term:`argument` may be -omitted from a call, in which -case the parameter's default value is substituted. If a parameter has a default -value, all following parameters up until the "``*``" must also have a default -value --- this is a syntactic restriction that is not expressed by the grammar. - -**Default parameter values are evaluated from left to right when the function -definition is executed.** This means that the expression is evaluated once, when -the function is defined, and that the same "pre-computed" value is used for each -call. This is especially important to understand when a default parameter value is a -mutable object, such as a list or a dictionary: if the function modifies the -object (e.g. by appending an item to a list), the default parameter value is in effect -modified. This is generally not what was intended. A way around this is to use -``None`` as the default, and explicitly test for it in the body of the function, -e.g.:: - - def whats_on_the_telly(penguin=None): - if penguin is None: - penguin = [] - penguin.append("property of the zoo") - return penguin - -.. index:: - single: / (slash); function definition - single: * (asterisk); function definition - single: **; function definition - -Function call semantics are described in more detail in section :ref:`calls`. A -function call always assigns values to all parameters mentioned in the parameter -list, either from positional arguments, from keyword arguments, or from default -values. If the form "``*identifier``" is present, it is initialized to a tuple -receiving any excess positional parameters, defaulting to the empty tuple. -If the form "``**identifier``" is present, it is initialized to a new -ordered mapping receiving any excess keyword arguments, defaulting to a -new empty mapping of the same type. Parameters after "``*``" or -"``*identifier``" are keyword-only parameters and may only be passed -by keyword arguments. Parameters before "``/``" are positional-only parameters -and may only be passed by positional arguments. - -.. versionchanged:: 3.8 - The ``/`` function parameter syntax may be used to indicate positional-only - parameters. See :pep:`570` for details. - -.. index:: - pair: function; annotations - single: ->; function annotations - single: : (colon); function annotations - -Parameters may have an :term:`annotation ` of the form "``: expression``" -following the parameter name. Any parameter may have an annotation, even those of the form -``*identifier`` or ``**identifier``. Functions may have "return" annotation of -the form "``-> expression``" after the parameter list. These annotations can be -any valid Python expression. The presence of annotations does not change the -semantics of a function. The annotation values are available as values of -a dictionary keyed by the parameters' names in the :attr:`__annotations__` -attribute of the function object. If the ``annotations`` import from -:mod:`__future__` is used, annotations are preserved as strings at runtime which -enables postponed evaluation. Otherwise, they are evaluated when the function -definition is executed. In this case annotations may be evaluated in -a different order than they appear in the source code. - -.. index:: pair: lambda; expression - -It is also possible to create anonymous functions (functions not bound to a -name), for immediate use in expressions. This uses lambda expressions, described in -section :ref:`lambda`. Note that the lambda expression is merely a shorthand for a -simplified function definition; a function defined in a ":keyword:`def`" -statement can be passed around or assigned to another name just like a function -defined by a lambda expression. The ":keyword:`!def`" form is actually more powerful -since it allows the execution of multiple statements and annotations. - -**Programmer's note:** Functions are first-class objects. A "``def``" statement -executed inside a function definition defines a local function that can be -returned or passed around. Free variables used in the nested function can -access the local variables of the function containing the def. See section -:ref:`naming` for details. - -.. seealso:: - - :pep:`3107` - Function Annotations - The original specification for function annotations. - - :pep:`484` - Type Hints - Definition of a standard meaning for annotations: type hints. - - :pep:`526` - Syntax for Variable Annotations - Ability to type hint variable declarations, including class - variables and instance variables - - :pep:`563` - Postponed Evaluation of Annotations - Support for forward references within annotations by preserving - annotations in a string form at runtime instead of eager evaluation. - - -.. _class: - -Class definitions -================= - -.. index:: - pair: object; class - pair: statement; class - pair: class; definition - pair: class; name - pair: name; binding - pair: execution; frame - single: inheritance - single: docstring - single: () (parentheses); class definition - single: , (comma); expression list - single: : (colon); compound statement - -A class definition defines a class object (see section :ref:`types`): - -.. productionlist:: python-grammar - classdef: [`decorators`] "class" `classname` [`inheritance`] ":" `suite` - inheritance: "(" [`argument_list`] ")" - classname: `identifier` - -A class definition is an executable statement. The inheritance list usually -gives a list of base classes (see :ref:`metaclasses` for more advanced uses), so -each item in the list should evaluate to a class object which allows -subclassing. Classes without an inheritance list inherit, by default, from the -base class :class:`object`; hence, :: - - class Foo: - pass - -is equivalent to :: - - class Foo(object): - pass - -The class's suite is then executed in a new execution frame (see :ref:`naming`), -using a newly created local namespace and the original global namespace. -(Usually, the suite contains mostly function definitions.) When the class's -suite finishes execution, its execution frame is discarded but its local -namespace is saved. [#]_ A class object is then created using the inheritance -list for the base classes and the saved local namespace for the attribute -dictionary. The class name is bound to this class object in the original local -namespace. - -The order in which attributes are defined in the class body is preserved -in the new class's ``__dict__``. Note that this is reliable only right -after the class is created and only for classes that were defined using -the definition syntax. - -Class creation can be customized heavily using :ref:`metaclasses `. - -.. index:: - single: @ (at); class definition - -Classes can also be decorated: just like when decorating functions, :: - - @f1(arg) - @f2 - class Foo: pass - -is roughly equivalent to :: - - class Foo: pass - Foo = f1(arg)(f2(Foo)) - -The evaluation rules for the decorator expressions are the same as for function -decorators. The result is then bound to the class name. - -.. versionchanged:: 3.9 - Classes may be decorated with any valid - :token:`~python-grammar:assignment_expression`. Previously, the grammar was - much more restrictive; see :pep:`614` for details. - -**Programmer's note:** Variables defined in the class definition are class -attributes; they are shared by instances. Instance attributes can be set in a -method with ``self.name = value``. Both class and instance attributes are -accessible through the notation "``self.name``", and an instance attribute hides -a class attribute with the same name when accessed in this way. Class -attributes can be used as defaults for instance attributes, but using mutable -values there can lead to unexpected results. :ref:`Descriptors ` -can be used to create instance variables with different implementation details. - - -.. seealso:: - - :pep:`3115` - Metaclasses in Python 3000 - The proposal that changed the declaration of metaclasses to the current - syntax, and the semantics for how classes with metaclasses are - constructed. - - :pep:`3129` - Class Decorators - The proposal that added class decorators. Function and method decorators - were introduced in :pep:`318`. - - -.. _async: - -Coroutines -========== - -.. versionadded:: 3.5 - -.. index:: pair: statement; async def -.. _`async def`: - -Coroutine function definition ------------------------------ - -.. productionlist:: python-grammar - async_funcdef: [`decorators`] "async" "def" `funcname` "(" [`parameter_list`] ")" - : ["->" `expression`] ":" `suite` - -.. index:: - pair: keyword; async - pair: keyword; await - -Execution of Python coroutines can be suspended and resumed at many points -(see :term:`coroutine`). :keyword:`await` expressions, :keyword:`async for` and -:keyword:`async with` can only be used in the body of a coroutine function. - -Functions defined with ``async def`` syntax are always coroutine functions, -even if they do not contain ``await`` or ``async`` keywords. - -It is a :exc:`SyntaxError` to use a ``yield from`` expression inside the body -of a coroutine function. - -An example of a coroutine function:: - - async def func(param1, param2): - do_stuff() - await some_coroutine() - -.. versionchanged:: 3.7 - ``await`` and ``async`` are now keywords; previously they were only - treated as such inside the body of a coroutine function. - -.. index:: pair: statement; async for -.. _`async for`: - -The :keyword:`!async for` statement ------------------------------------ - -.. productionlist:: python-grammar - async_for_stmt: "async" `for_stmt` - -An :term:`asynchronous iterable` provides an ``__aiter__`` method that directly -returns an :term:`asynchronous iterator`, which can call asynchronous code in -its ``__anext__`` method. - -The ``async for`` statement allows convenient iteration over asynchronous -iterables. - -The following code:: - - async for TARGET in ITER: - SUITE - else: - SUITE2 - -Is semantically equivalent to:: - - iter = (ITER) - iter = type(iter).__aiter__(iter) - running = True - - while running: - try: - TARGET = await type(iter).__anext__(iter) - except StopAsyncIteration: - running = False - else: - SUITE - else: - SUITE2 - -See also :meth:`~object.__aiter__` and :meth:`~object.__anext__` for details. - -It is a :exc:`SyntaxError` to use an ``async for`` statement outside the -body of a coroutine function. - - -.. index:: pair: statement; async with -.. _`async with`: - -The :keyword:`!async with` statement ------------------------------------- - -.. productionlist:: python-grammar - async_with_stmt: "async" `with_stmt` - -An :term:`asynchronous context manager` is a :term:`context manager` that is -able to suspend execution in its *enter* and *exit* methods. - -The following code:: - - async with EXPRESSION as TARGET: - SUITE - -is semantically equivalent to:: - - manager = (EXPRESSION) - aenter = type(manager).__aenter__ - aexit = type(manager).__aexit__ - value = await aenter(manager) - hit_except = False - - try: - TARGET = value - SUITE - except: - hit_except = True - if not await aexit(manager, *sys.exc_info()): - raise - finally: - if not hit_except: - await aexit(manager, None, None, None) - -See also :meth:`~object.__aenter__` and :meth:`~object.__aexit__` for details. - -It is a :exc:`SyntaxError` to use an ``async with`` statement outside the -body of a coroutine function. - -.. seealso:: - - :pep:`492` - Coroutines with async and await syntax - The proposal that made coroutines a proper standalone concept in Python, - and added supporting syntax. - - -.. rubric:: Footnotes - -.. [#] The exception is propagated to the invocation stack unless - there is a :keyword:`finally` clause which happens to raise another - exception. That new exception causes the old one to be lost. - -.. [#] In pattern matching, a sequence is defined as one of the following: - - * a class that inherits from :class:`collections.abc.Sequence` - * a Python class that has been registered as :class:`collections.abc.Sequence` - * a builtin class that has its (CPython) :c:macro:`Py_TPFLAGS_SEQUENCE` bit set - * a class that inherits from any of the above - - The following standard library classes are sequences: - - * :class:`array.array` - * :class:`collections.deque` - * :class:`list` - * :class:`memoryview` - * :class:`range` - * :class:`tuple` - - .. note:: Subject values of type ``str``, ``bytes``, and ``bytearray`` - do not match sequence patterns. - -.. [#] In pattern matching, a mapping is defined as one of the following: - - * a class that inherits from :class:`collections.abc.Mapping` - * a Python class that has been registered as :class:`collections.abc.Mapping` - * a builtin class that has its (CPython) :c:macro:`Py_TPFLAGS_MAPPING` bit set - * a class that inherits from any of the above - - The standard library classes :class:`dict` and :class:`types.MappingProxyType` - are mappings. - -.. [#] A string literal appearing as the first statement in the function body is - transformed into the function's ``__doc__`` attribute and therefore the - function's :term:`docstring`. - -.. [#] A string literal appearing as the first statement in the class body is - transformed into the namespace's ``__doc__`` item and therefore the class's - :term:`docstring`. diff --git a/Doc/reference/datamodel.rst b/Doc/reference/datamodel.rst deleted file mode 100644 index ce92a120609046..00000000000000 --- a/Doc/reference/datamodel.rst +++ /dev/null @@ -1,3236 +0,0 @@ - -.. _datamodel: - -********** -Data model -********** - - -.. _objects: - -Objects, values and types -========================= - -.. index:: - single: object - single: data - -:dfn:`Objects` are Python's abstraction for data. All data in a Python program -is represented by objects or by relations between objects. (In a sense, and in -conformance to Von Neumann's model of a "stored program computer", code is also -represented by objects.) - -.. index:: - pair: built-in function; id - pair: built-in function; type - single: identity of an object - single: value of an object - single: type of an object - single: mutable object - single: immutable object - -.. XXX it *is* now possible in some cases to change an object's - type, under certain controlled conditions - -Every object has an identity, a type and a value. An object's *identity* never -changes once it has been created; you may think of it as the object's address in -memory. The ':keyword:`is`' operator compares the identity of two objects; the -:func:`id` function returns an integer representing its identity. - -.. impl-detail:: - - For CPython, ``id(x)`` is the memory address where ``x`` is stored. - -An object's type determines the operations that the object supports (e.g., "does -it have a length?") and also defines the possible values for objects of that -type. The :func:`type` function returns an object's type (which is an object -itself). Like its identity, an object's :dfn:`type` is also unchangeable. -[#]_ - -The *value* of some objects can change. Objects whose value can -change are said to be *mutable*; objects whose value is unchangeable once they -are created are called *immutable*. (The value of an immutable container object -that contains a reference to a mutable object can change when the latter's value -is changed; however the container is still considered immutable, because the -collection of objects it contains cannot be changed. So, immutability is not -strictly the same as having an unchangeable value, it is more subtle.) An -object's mutability is determined by its type; for instance, numbers, strings -and tuples are immutable, while dictionaries and lists are mutable. - -.. index:: - single: garbage collection - single: reference counting - single: unreachable object - -Objects are never explicitly destroyed; however, when they become unreachable -they may be garbage-collected. An implementation is allowed to postpone garbage -collection or omit it altogether --- it is a matter of implementation quality -how garbage collection is implemented, as long as no objects are collected that -are still reachable. - -.. impl-detail:: - - CPython currently uses a reference-counting scheme with (optional) delayed - detection of cyclically linked garbage, which collects most objects as soon - as they become unreachable, but is not guaranteed to collect garbage - containing circular references. See the documentation of the :mod:`gc` - module for information on controlling the collection of cyclic garbage. - Other implementations act differently and CPython may change. - Do not depend on immediate finalization of objects when they become - unreachable (so you should always close files explicitly). - -Note that the use of the implementation's tracing or debugging facilities may -keep objects alive that would normally be collectable. Also note that catching -an exception with a ':keyword:`try`...\ :keyword:`except`' statement may keep -objects alive. - -Some objects contain references to "external" resources such as open files or -windows. It is understood that these resources are freed when the object is -garbage-collected, but since garbage collection is not guaranteed to happen, -such objects also provide an explicit way to release the external resource, -usually a :meth:`close` method. Programs are strongly recommended to explicitly -close such objects. The ':keyword:`try`...\ :keyword:`finally`' statement -and the ':keyword:`with`' statement provide convenient ways to do this. - -.. index:: single: container - -Some objects contain references to other objects; these are called *containers*. -Examples of containers are tuples, lists and dictionaries. The references are -part of a container's value. In most cases, when we talk about the value of a -container, we imply the values, not the identities of the contained objects; -however, when we talk about the mutability of a container, only the identities -of the immediately contained objects are implied. So, if an immutable container -(like a tuple) contains a reference to a mutable object, its value changes if -that mutable object is changed. - -Types affect almost all aspects of object behavior. Even the importance of -object identity is affected in some sense: for immutable types, operations that -compute new values may actually return a reference to any existing object with -the same type and value, while for mutable objects this is not allowed. E.g., -after ``a = 1; b = 1``, ``a`` and ``b`` may or may not refer to the same object -with the value one, depending on the implementation, but after ``c = []; d = -[]``, ``c`` and ``d`` are guaranteed to refer to two different, unique, newly -created empty lists. (Note that ``c = d = []`` assigns the same object to both -``c`` and ``d``.) - - -.. _types: - -The standard type hierarchy -=========================== - -.. index:: - single: type - pair: data; type - pair: type; hierarchy - pair: extension; module - pair: C; language - -Below is a list of the types that are built into Python. Extension modules -(written in C, Java, or other languages, depending on the implementation) can -define additional types. Future versions of Python may add types to the type -hierarchy (e.g., rational numbers, efficiently stored arrays of integers, etc.), -although such additions will often be provided via the standard library instead. - -.. index:: - single: attribute - pair: special; attribute - triple: generic; special; attribute - -Some of the type descriptions below contain a paragraph listing 'special -attributes.' These are attributes that provide access to the implementation and -are not intended for general use. Their definition may change in the future. - - -None ----- - -.. index:: pair: object; None - -This type has a single value. There is a single object with this value. This -object is accessed through the built-in name ``None``. It is used to signify the -absence of a value in many situations, e.g., it is returned from functions that -don't explicitly return anything. Its truth value is false. - - -NotImplemented --------------- - -.. index:: pair: object; NotImplemented - -This type has a single value. There is a single object with this value. This -object is accessed through the built-in name ``NotImplemented``. Numeric methods -and rich comparison methods should return this value if they do not implement the -operation for the operands provided. (The interpreter will then try the -reflected operation, or some other fallback, depending on the operator.) It -should not be evaluated in a boolean context. - -See -:ref:`implementing-the-arithmetic-operations` -for more details. - -.. versionchanged:: 3.9 - Evaluating ``NotImplemented`` in a boolean context is deprecated. While - it currently evaluates as true, it will emit a :exc:`DeprecationWarning`. - It will raise a :exc:`TypeError` in a future version of Python. - - -Ellipsis --------- -.. index:: - pair: object; Ellipsis - single: ...; ellipsis literal - -This type has a single value. There is a single object with this value. This -object is accessed through the literal ``...`` or the built-in name -``Ellipsis``. Its truth value is true. - - -:class:`numbers.Number` ------------------------ - -.. index:: pair: object; numeric - -These are created by numeric literals and returned as results by arithmetic -operators and arithmetic built-in functions. Numeric objects are immutable; -once created their value never changes. Python numbers are of course strongly -related to mathematical numbers, but subject to the limitations of numerical -representation in computers. - -The string representations of the numeric classes, computed by -:meth:`~object.__repr__` and :meth:`~object.__str__`, have the following -properties: - -* They are valid numeric literals which, when passed to their - class constructor, produce an object having the value of the - original numeric. - -* The representation is in base 10, when possible. - -* Leading zeros, possibly excepting a single zero before a - decimal point, are not shown. - -* Trailing zeros, possibly excepting a single zero after a - decimal point, are not shown. - -* A sign is shown only when the number is negative. - -Python distinguishes between integers, floating point numbers, and complex -numbers: - - -:class:`numbers.Integral` -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: pair: object; integer - -These represent elements from the mathematical set of integers (positive and -negative). - -.. note:: - .. index:: pair: integer; representation - - The rules for integer representation are intended to give the most meaningful - interpretation of shift and mask operations involving negative integers. - -There are two types of integers: - -Integers (:class:`int`) - These represent numbers in an unlimited range, subject to available (virtual) - memory only. For the purpose of shift and mask operations, a binary - representation is assumed, and negative numbers are represented in a variant of - 2's complement which gives the illusion of an infinite string of sign bits - extending to the left. - -Booleans (:class:`bool`) - .. index:: - pair: object; Boolean - single: False - single: True - - These represent the truth values False and True. The two objects representing - the values ``False`` and ``True`` are the only Boolean objects. The Boolean type is a - subtype of the integer type, and Boolean values behave like the values 0 and 1, - respectively, in almost all contexts, the exception being that when converted to - a string, the strings ``"False"`` or ``"True"`` are returned, respectively. - - -:class:`numbers.Real` (:class:`float`) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - pair: object; floating point - pair: floating point; number - pair: C; language - pair: Java; language - -These represent machine-level double precision floating point numbers. You are -at the mercy of the underlying machine architecture (and C or Java -implementation) for the accepted range and handling of overflow. Python does not -support single-precision floating point numbers; the savings in processor and -memory usage that are usually the reason for using these are dwarfed by the -overhead of using objects in Python, so there is no reason to complicate the -language with two kinds of floating point numbers. - - -:class:`numbers.Complex` (:class:`complex`) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - pair: object; complex - pair: complex; number - -These represent complex numbers as a pair of machine-level double precision -floating point numbers. The same caveats apply as for floating point numbers. -The real and imaginary parts of a complex number ``z`` can be retrieved through -the read-only attributes ``z.real`` and ``z.imag``. - - -Sequences ---------- - -.. index:: - pair: built-in function; len - pair: object; sequence - single: index operation - single: item selection - single: subscription - -These represent finite ordered sets indexed by non-negative numbers. The -built-in function :func:`len` returns the number of items of a sequence. When -the length of a sequence is *n*, the index set contains the numbers 0, 1, -..., *n*-1. Item *i* of sequence *a* is selected by ``a[i]``. - -.. index:: single: slicing - -Sequences also support slicing: ``a[i:j]`` selects all items with index *k* such -that *i* ``<=`` *k* ``<`` *j*. When used as an expression, a slice is a -sequence of the same type. This implies that the index set is renumbered so -that it starts at 0. - -Some sequences also support "extended slicing" with a third "step" parameter: -``a[i:j:k]`` selects all items of *a* with index *x* where ``x = i + n*k``, *n* -``>=`` ``0`` and *i* ``<=`` *x* ``<`` *j*. - -Sequences are distinguished according to their mutability: - - -Immutable sequences -^^^^^^^^^^^^^^^^^^^ - -.. index:: - pair: object; immutable sequence - pair: object; immutable - -An object of an immutable sequence type cannot change once it is created. (If -the object contains references to other objects, these other objects may be -mutable and may be changed; however, the collection of objects directly -referenced by an immutable object cannot change.) - -The following types are immutable sequences: - -.. index:: - single: string; immutable sequences - -Strings - .. index:: - pair: built-in function; chr - pair: built-in function; ord - single: character - single: integer - single: Unicode - - A string is a sequence of values that represent Unicode code points. - All the code points in the range ``U+0000 - U+10FFFF`` can be - represented in a string. Python doesn't have a :c:expr:`char` type; - instead, every code point in the string is represented as a string - object with length ``1``. The built-in function :func:`ord` - converts a code point from its string form to an integer in the - range ``0 - 10FFFF``; :func:`chr` converts an integer in the range - ``0 - 10FFFF`` to the corresponding length ``1`` string object. - :meth:`str.encode` can be used to convert a :class:`str` to - :class:`bytes` using the given text encoding, and - :meth:`bytes.decode` can be used to achieve the opposite. - -Tuples - .. index:: - pair: object; tuple - pair: singleton; tuple - pair: empty; tuple - - The items of a tuple are arbitrary Python objects. Tuples of two or - more items are formed by comma-separated lists of expressions. A tuple - of one item (a 'singleton') can be formed by affixing a comma to an - expression (an expression by itself does not create a tuple, since - parentheses must be usable for grouping of expressions). An empty - tuple can be formed by an empty pair of parentheses. - -Bytes - .. index:: bytes, byte - - A bytes object is an immutable array. The items are 8-bit bytes, - represented by integers in the range 0 <= x < 256. Bytes literals - (like ``b'abc'``) and the built-in :func:`bytes()` constructor - can be used to create bytes objects. Also, bytes objects can be - decoded to strings via the :meth:`~bytes.decode` method. - - -Mutable sequences -^^^^^^^^^^^^^^^^^ - -.. index:: - pair: object; mutable sequence - pair: object; mutable - pair: assignment; statement - single: subscription - single: slicing - -Mutable sequences can be changed after they are created. The subscription and -slicing notations can be used as the target of assignment and :keyword:`del` -(delete) statements. - -.. note:: - .. index:: pair: module; array - .. index:: pair: module; collections - - The :mod:`collections` and :mod:`array` module provide - additional examples of mutable sequence types. - -There are currently two intrinsic mutable sequence types: - -Lists - .. index:: pair: object; list - - The items of a list are arbitrary Python objects. Lists are formed by - placing a comma-separated list of expressions in square brackets. (Note - that there are no special cases needed to form lists of length 0 or 1.) - -Byte Arrays - .. index:: bytearray - - A bytearray object is a mutable array. They are created by the built-in - :func:`bytearray` constructor. Aside from being mutable - (and hence unhashable), byte arrays otherwise provide the same interface - and functionality as immutable :class:`bytes` objects. - - -Set types ---------- - -.. index:: - pair: built-in function; len - pair: object; set type - -These represent unordered, finite sets of unique, immutable objects. As such, -they cannot be indexed by any subscript. However, they can be iterated over, and -the built-in function :func:`len` returns the number of items in a set. Common -uses for sets are fast membership testing, removing duplicates from a sequence, -and computing mathematical operations such as intersection, union, difference, -and symmetric difference. - -For set elements, the same immutability rules apply as for dictionary keys. Note -that numeric types obey the normal rules for numeric comparison: if two numbers -compare equal (e.g., ``1`` and ``1.0``), only one of them can be contained in a -set. - -There are currently two intrinsic set types: - - -Sets - .. index:: pair: object; set - - These represent a mutable set. They are created by the built-in :func:`set` - constructor and can be modified afterwards by several methods, such as - :meth:`~set.add`. - - -Frozen sets - .. index:: pair: object; frozenset - - These represent an immutable set. They are created by the built-in - :func:`frozenset` constructor. As a frozenset is immutable and - :term:`hashable`, it can be used again as an element of another set, or as - a dictionary key. - - -Mappings --------- - -.. index:: - pair: built-in function; len - single: subscription - pair: object; mapping - -These represent finite sets of objects indexed by arbitrary index sets. The -subscript notation ``a[k]`` selects the item indexed by ``k`` from the mapping -``a``; this can be used in expressions and as the target of assignments or -:keyword:`del` statements. The built-in function :func:`len` returns the number -of items in a mapping. - -There is currently a single intrinsic mapping type: - - -Dictionaries -^^^^^^^^^^^^ - -.. index:: pair: object; dictionary - -These represent finite sets of objects indexed by nearly arbitrary values. The -only types of values not acceptable as keys are values containing lists or -dictionaries or other mutable types that are compared by value rather than by -object identity, the reason being that the efficient implementation of -dictionaries requires a key's hash value to remain constant. Numeric types used -for keys obey the normal rules for numeric comparison: if two numbers compare -equal (e.g., ``1`` and ``1.0``) then they can be used interchangeably to index -the same dictionary entry. - -Dictionaries preserve insertion order, meaning that keys will be produced -in the same order they were added sequentially over the dictionary. -Replacing an existing key does not change the order, however removing a key -and re-inserting it will add it to the end instead of keeping its old place. - -Dictionaries are mutable; they can be created by the ``{...}`` notation (see -section :ref:`dict`). - -.. index:: - pair: module; dbm.ndbm - pair: module; dbm.gnu - -The extension modules :mod:`dbm.ndbm` and :mod:`dbm.gnu` provide -additional examples of mapping types, as does the :mod:`collections` -module. - -.. versionchanged:: 3.7 - Dictionaries did not preserve insertion order in versions of Python before 3.6. - In CPython 3.6, insertion order was preserved, but it was considered - an implementation detail at that time rather than a language guarantee. - - -Callable types --------------- - -.. index:: - pair: object; callable - pair: function; call - single: invocation - pair: function; argument - -These are the types to which the function call operation (see section -:ref:`calls`) can be applied: - - -User-defined functions -^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - pair: user-defined; function - pair: object; function - pair: object; user-defined function - -A user-defined function object is created by a function definition (see -section :ref:`function`). It should be called with an argument list -containing the same number of items as the function's formal parameter -list. - -Special attributes: - -.. tabularcolumns:: |l|L|l| - -.. index:: - single: __doc__ (function attribute) - single: __name__ (function attribute) - single: __module__ (function attribute) - single: __dict__ (function attribute) - single: __defaults__ (function attribute) - single: __closure__ (function attribute) - single: __code__ (function attribute) - single: __globals__ (function attribute) - single: __annotations__ (function attribute) - single: __kwdefaults__ (function attribute) - pair: global; namespace - -+-------------------------+-------------------------------+-----------+ -| Attribute | Meaning | | -+=========================+===============================+===========+ -| :attr:`__doc__` | The function's documentation | Writable | -| | string, or ``None`` if | | -| | unavailable; not inherited by | | -| | subclasses. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~definition.\ | The function's name. | Writable | -| __name__` | | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~definition.\ | The function's | Writable | -| __qualname__` | :term:`qualified name`. | | -| | | | -| | .. versionadded:: 3.3 | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__module__` | The name of the module the | Writable | -| | function was defined in, or | | -| | ``None`` if unavailable. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__defaults__` | A tuple containing default | Writable | -| | argument values for those | | -| | arguments that have defaults, | | -| | or ``None`` if no arguments | | -| | have a default value. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__code__` | The code object representing | Writable | -| | the compiled function body. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__globals__` | A reference to the dictionary | Read-only | -| | that holds the function's | | -| | global variables --- the | | -| | global namespace of the | | -| | module in which the function | | -| | was defined. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`~object.__dict__`| The namespace supporting | Writable | -| | arbitrary function | | -| | attributes. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__closure__` | ``None`` or a tuple of cells | Read-only | -| | that contain bindings for the | | -| | function's free variables. | | -| | See below for information on | | -| | the ``cell_contents`` | | -| | attribute. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__annotations__` | A dict containing annotations | Writable | -| | of parameters. The keys of | | -| | the dict are the parameter | | -| | names, and ``'return'`` for | | -| | the return annotation, if | | -| | provided. For more | | -| | information on working with | | -| | this attribute, see | | -| | :ref:`annotations-howto`. | | -+-------------------------+-------------------------------+-----------+ -| :attr:`__kwdefaults__` | A dict containing defaults | Writable | -| | for keyword-only parameters. | | -+-------------------------+-------------------------------+-----------+ - -Most of the attributes labelled "Writable" check the type of the assigned value. - -Function objects also support getting and setting arbitrary attributes, which -can be used, for example, to attach metadata to functions. Regular attribute -dot-notation is used to get and set such attributes. *Note that the current -implementation only supports function attributes on user-defined functions. -Function attributes on built-in functions may be supported in the future.* - -A cell object has the attribute ``cell_contents``. This can be used to get -the value of the cell, as well as set the value. - -Additional information about a function's definition can be retrieved from its -code object; see the description of internal types below. The -:data:`cell ` type can be accessed in the :mod:`types` -module. - - -Instance methods -^^^^^^^^^^^^^^^^ - -.. index:: - pair: object; method - pair: object; user-defined method - pair: user-defined; method - -An instance method object combines a class, a class instance and any -callable object (normally a user-defined function). - -.. index:: - single: __func__ (method attribute) - single: __self__ (method attribute) - single: __doc__ (method attribute) - single: __name__ (method attribute) - single: __module__ (method attribute) - -Special read-only attributes: :attr:`__self__` is the class instance object, -:attr:`__func__` is the function object; :attr:`__doc__` is the method's -documentation (same as ``__func__.__doc__``); :attr:`~definition.__name__` is the -method name (same as ``__func__.__name__``); :attr:`__module__` is the -name of the module the method was defined in, or ``None`` if unavailable. - -Methods also support accessing (but not setting) the arbitrary function -attributes on the underlying function object. - -User-defined method objects may be created when getting an attribute of a -class (perhaps via an instance of that class), if that attribute is a -user-defined function object or a class method object. - -When an instance method object is created by retrieving a user-defined -function object from a class via one of its instances, its -:attr:`__self__` attribute is the instance, and the method object is said -to be bound. The new method's :attr:`__func__` attribute is the original -function object. - -When an instance method object is created by retrieving a class method -object from a class or instance, its :attr:`__self__` attribute is the -class itself, and its :attr:`__func__` attribute is the function object -underlying the class method. - -When an instance method object is called, the underlying function -(:attr:`__func__`) is called, inserting the class instance -(:attr:`__self__`) in front of the argument list. For instance, when -:class:`C` is a class which contains a definition for a function -:meth:`f`, and ``x`` is an instance of :class:`C`, calling ``x.f(1)`` is -equivalent to calling ``C.f(x, 1)``. - -When an instance method object is derived from a class method object, the -"class instance" stored in :attr:`__self__` will actually be the class -itself, so that calling either ``x.f(1)`` or ``C.f(1)`` is equivalent to -calling ``f(C,1)`` where ``f`` is the underlying function. - -Note that the transformation from function object to instance method -object happens each time the attribute is retrieved from the instance. In -some cases, a fruitful optimization is to assign the attribute to a local -variable and call that local variable. Also notice that this -transformation only happens for user-defined functions; other callable -objects (and all non-callable objects) are retrieved without -transformation. It is also important to note that user-defined functions -which are attributes of a class instance are not converted to bound -methods; this *only* happens when the function is an attribute of the -class. - - -Generator functions -^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: generator; function - single: generator; iterator - -A function or method which uses the :keyword:`yield` statement (see section -:ref:`yield`) is called a :dfn:`generator function`. Such a function, when -called, always returns an :term:`iterator` object which can be used to -execute the body of the function: calling the iterator's -:meth:`iterator.__next__` method will cause the function to execute until -it provides a value using the :keyword:`!yield` statement. When the -function executes a :keyword:`return` statement or falls off the end, a -:exc:`StopIteration` exception is raised and the iterator will have -reached the end of the set of values to be returned. - - -Coroutine functions -^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: coroutine; function - -A function or method which is defined using :keyword:`async def` is called -a :dfn:`coroutine function`. Such a function, when called, returns a -:term:`coroutine` object. It may contain :keyword:`await` expressions, -as well as :keyword:`async with` and :keyword:`async for` statements. See -also the :ref:`coroutine-objects` section. - - -Asynchronous generator functions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: asynchronous generator; function - single: asynchronous generator; asynchronous iterator - -A function or method which is defined using :keyword:`async def` and -which uses the :keyword:`yield` statement is called a -:dfn:`asynchronous generator function`. Such a function, when called, -returns an :term:`asynchronous iterator` object which can be used in an -:keyword:`async for` statement to execute the body of the function. - -Calling the asynchronous iterator's -:meth:`aiterator.__anext__ ` method -will return an :term:`awaitable` which when awaited -will execute until it provides a value using the :keyword:`yield` -expression. When the function executes an empty :keyword:`return` -statement or falls off the end, a :exc:`StopAsyncIteration` exception -is raised and the asynchronous iterator will have reached the end of -the set of values to be yielded. - - -Built-in functions -^^^^^^^^^^^^^^^^^^ - -.. index:: - pair: object; built-in function - pair: object; function - pair: C; language - -A built-in function object is a wrapper around a C function. Examples of -built-in functions are :func:`len` and :func:`math.sin` (:mod:`math` is a -standard built-in module). The number and type of the arguments are -determined by the C function. Special read-only attributes: -:attr:`__doc__` is the function's documentation string, or ``None`` if -unavailable; :attr:`~definition.__name__` is the function's name; :attr:`__self__` is -set to ``None`` (but see the next item); :attr:`__module__` is the name of -the module the function was defined in or ``None`` if unavailable. - - -Built-in methods -^^^^^^^^^^^^^^^^ - -.. index:: - pair: object; built-in method - pair: object; method - pair: built-in; method - -This is really a different disguise of a built-in function, this time containing -an object passed to the C function as an implicit extra argument. An example of -a built-in method is ``alist.append()``, assuming *alist* is a list object. In -this case, the special read-only attribute :attr:`__self__` is set to the object -denoted by *alist*. - - -Classes -^^^^^^^ - -Classes are callable. These objects normally act as factories for new -instances of themselves, but variations are possible for class types that -override :meth:`~object.__new__`. The arguments of the call are passed to -:meth:`__new__` and, in the typical case, to :meth:`~object.__init__` to -initialize the new instance. - - -Class Instances -^^^^^^^^^^^^^^^ - -Instances of arbitrary classes can be made callable by defining a -:meth:`~object.__call__` method in their class. - - -Modules -------- - -.. index:: - pair: statement; import - pair: object; module - -Modules are a basic organizational unit of Python code, and are created by -the :ref:`import system ` as invoked either by the -:keyword:`import` statement, or by calling -functions such as :func:`importlib.import_module` and built-in -:func:`__import__`. A module object has a namespace implemented by a -dictionary object (this is the dictionary referenced by the ``__globals__`` -attribute of functions defined in the module). Attribute references are -translated to lookups in this dictionary, e.g., ``m.x`` is equivalent to -``m.__dict__["x"]``. A module object does not contain the code object used -to initialize the module (since it isn't needed once the initialization is -done). - -Attribute assignment updates the module's namespace dictionary, e.g., -``m.x = 1`` is equivalent to ``m.__dict__["x"] = 1``. - -.. index:: - single: __name__ (module attribute) - single: __doc__ (module attribute) - single: __file__ (module attribute) - single: __annotations__ (module attribute) - pair: module; namespace - -Predefined (writable) attributes: - - :attr:`__name__` - The module's name. - - :attr:`__doc__` - The module's documentation string, or ``None`` if - unavailable. - - :attr:`__file__` - The pathname of the file from which the - module was loaded, if it was loaded from a file. - The :attr:`__file__` - attribute may be missing for certain types of modules, such as C modules - that are statically linked into the interpreter. For extension modules - loaded dynamically from a shared library, it's the pathname of the shared - library file. - - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` collected during - module body execution. For best practices on working - with :attr:`__annotations__`, please see :ref:`annotations-howto`. - -.. index:: single: __dict__ (module attribute) - -Special read-only attribute: :attr:`~object.__dict__` is the module's -namespace as a dictionary object. - -.. impl-detail:: - - Because of the way CPython clears module dictionaries, the module - dictionary will be cleared when the module falls out of scope even if the - dictionary still has live references. To avoid this, copy the dictionary - or keep the module around while using its dictionary directly. - - -Custom classes --------------- - -Custom class types are typically created by class definitions (see section -:ref:`class`). A class has a namespace implemented by a dictionary object. -Class attribute references are translated to lookups in this dictionary, e.g., -``C.x`` is translated to ``C.__dict__["x"]`` (although there are a number of -hooks which allow for other means of locating attributes). When the attribute -name is not found there, the attribute search continues in the base classes. -This search of the base classes uses the C3 method resolution order which -behaves correctly even in the presence of 'diamond' inheritance structures -where there are multiple inheritance paths leading back to a common ancestor. -Additional details on the C3 MRO used by Python can be found in the -documentation accompanying the 2.3 release at -https://www.python.org/download/releases/2.3/mro/. - -.. XXX: Could we add that MRO doc as an appendix to the language ref? - -.. index:: - pair: object; class - pair: object; class instance - pair: object; instance - pair: class object; call - single: container - pair: object; dictionary - pair: class; attribute - -When a class attribute reference (for class :class:`C`, say) would yield a -class method object, it is transformed into an instance method object whose -:attr:`__self__` attribute is :class:`C`. When it would yield a static -method object, it is transformed into the object wrapped by the static method -object. See section :ref:`descriptors` for another way in which attributes -retrieved from a class may differ from those actually contained in its -:attr:`~object.__dict__`. - -.. index:: triple: class; attribute; assignment - -Class attribute assignments update the class's dictionary, never the dictionary -of a base class. - -.. index:: pair: class object; call - -A class object can be called (see above) to yield a class instance (see below). - - .. index:: - single: __name__ (class attribute) - single: __module__ (class attribute) - single: __dict__ (class attribute) - single: __bases__ (class attribute) - single: __doc__ (class attribute) - single: __annotations__ (class attribute) - -Special attributes: - - :attr:`~definition.__name__` - The class name. - - :attr:`__module__` - The name of the module in which the class was defined. - - :attr:`~object.__dict__` - The dictionary containing the class's namespace. - - :attr:`~class.__bases__` - A tuple containing the base classes, in the order of - their occurrence in the base class list. - - :attr:`__doc__` - The class's documentation string, or ``None`` if undefined. - - :attr:`__annotations__` - A dictionary containing - :term:`variable annotations ` - collected during class body execution. For best practices on - working with :attr:`__annotations__`, please see - :ref:`annotations-howto`. - - -Class instances ---------------- - -.. index:: - pair: object; class instance - pair: object; instance - pair: class; instance - pair: class instance; attribute - -A class instance is created by calling a class object (see above). A class -instance has a namespace implemented as a dictionary which is the first place -in which attribute references are searched. When an attribute is not found -there, and the instance's class has an attribute by that name, the search -continues with the class attributes. If a class attribute is found that is a -user-defined function object, it is transformed into an instance method -object whose :attr:`__self__` attribute is the instance. Static method and -class method objects are also transformed; see above under "Classes". See -section :ref:`descriptors` for another way in which attributes of a class -retrieved via its instances may differ from the objects actually stored in -the class's :attr:`~object.__dict__`. If no class attribute is found, and the -object's class has a :meth:`~object.__getattr__` method, that is called to satisfy -the lookup. - -.. index:: triple: class instance; attribute; assignment - -Attribute assignments and deletions update the instance's dictionary, never a -class's dictionary. If the class has a :meth:`~object.__setattr__` or -:meth:`~object.__delattr__` method, this is called instead of updating the instance -dictionary directly. - -.. index:: - pair: object; numeric - pair: object; sequence - pair: object; mapping - -Class instances can pretend to be numbers, sequences, or mappings if they have -methods with certain special names. See section :ref:`specialnames`. - -.. index:: - single: __dict__ (instance attribute) - single: __class__ (instance attribute) - -Special attributes: :attr:`~object.__dict__` is the attribute dictionary; -:attr:`~instance.__class__` is the instance's class. - - -I/O objects (also known as file objects) ----------------------------------------- - -.. index:: - pair: built-in function; open - pair: module; io - single: popen() (in module os) - single: makefile() (socket method) - single: sys.stdin - single: sys.stdout - single: sys.stderr - single: stdio - single: stdin (in module sys) - single: stdout (in module sys) - single: stderr (in module sys) - -A :term:`file object` represents an open file. Various shortcuts are -available to create file objects: the :func:`open` built-in function, and -also :func:`os.popen`, :func:`os.fdopen`, and the -:meth:`~socket.socket.makefile` method of socket objects (and perhaps by -other functions or methods provided by extension modules). - -The objects ``sys.stdin``, ``sys.stdout`` and ``sys.stderr`` are -initialized to file objects corresponding to the interpreter's standard -input, output and error streams; they are all open in text mode and -therefore follow the interface defined by the :class:`io.TextIOBase` -abstract class. - - -Internal types --------------- - -.. index:: - single: internal type - single: types, internal - -A few types used internally by the interpreter are exposed to the user. Their -definitions may change with future versions of the interpreter, but they are -mentioned here for completeness. - -.. index:: bytecode, object; code, code object - - -Code objects -^^^^^^^^^^^^ - -Code objects represent *byte-compiled* executable Python code, or :term:`bytecode`. -The difference between a code object and a function object is that the function -object contains an explicit reference to the function's globals (the module in -which it was defined), while a code object contains no context; also the default -argument values are stored in the function object, not in the code object -(because they represent values calculated at run-time). Unlike function -objects, code objects are immutable and contain no references (directly or -indirectly) to mutable objects. - -.. index:: - single: co_argcount (code object attribute) - single: co_posonlyargcount (code object attribute) - single: co_kwonlyargcount (code object attribute) - single: co_code (code object attribute) - single: co_consts (code object attribute) - single: co_filename (code object attribute) - single: co_firstlineno (code object attribute) - single: co_flags (code object attribute) - single: co_lnotab (code object attribute) - single: co_name (code object attribute) - single: co_names (code object attribute) - single: co_nlocals (code object attribute) - single: co_stacksize (code object attribute) - single: co_varnames (code object attribute) - single: co_cellvars (code object attribute) - single: co_freevars (code object attribute) - single: co_qualname (code object attribute) - -Special read-only attributes: :attr:`co_name` gives the function name; -:attr:`co_qualname` gives the fully qualified function name; -:attr:`co_argcount` is the total number of positional arguments -(including positional-only arguments and arguments with default values); -:attr:`co_posonlyargcount` is the number of positional-only arguments -(including arguments with default values); :attr:`co_kwonlyargcount` is -the number of keyword-only arguments (including arguments with default -values); :attr:`co_nlocals` is the number of local variables used by the -function (including arguments); :attr:`co_varnames` is a tuple containing -the names of the local variables (starting with the argument names); -:attr:`co_cellvars` is a tuple containing the names of local variables -that are referenced by nested functions; :attr:`co_freevars` is a tuple -containing the names of free variables; :attr:`co_code` is a string -representing the sequence of bytecode instructions; :attr:`co_consts` is -a tuple containing the literals used by the bytecode; :attr:`co_names` is -a tuple containing the names used by the bytecode; :attr:`co_filename` is -the filename from which the code was compiled; :attr:`co_firstlineno` is -the first line number of the function; :attr:`co_lnotab` is a string -encoding the mapping from bytecode offsets to line numbers (for details -see the source code of the interpreter); :attr:`co_stacksize` is the -required stack size; :attr:`co_flags` is an integer encoding a number -of flags for the interpreter. - -.. index:: pair: object; generator - -The following flag bits are defined for :attr:`co_flags`: bit ``0x04`` is set if -the function uses the ``*arguments`` syntax to accept an arbitrary number of -positional arguments; bit ``0x08`` is set if the function uses the -``**keywords`` syntax to accept arbitrary keyword arguments; bit ``0x20`` is set -if the function is a generator. - -Future feature declarations (``from __future__ import division``) also use bits -in :attr:`co_flags` to indicate whether a code object was compiled with a -particular feature enabled: bit ``0x2000`` is set if the function was compiled -with future division enabled; bits ``0x10`` and ``0x1000`` were used in earlier -versions of Python. - -Other bits in :attr:`co_flags` are reserved for internal use. - -.. index:: single: documentation string - -If a code object represents a function, the first item in :attr:`co_consts` is -the documentation string of the function, or ``None`` if undefined. - -.. method:: codeobject.co_positions() - - Returns an iterable over the source code positions of each bytecode - instruction in the code object. - - The iterator returns tuples containing the ``(start_line, end_line, - start_column, end_column)``. The *i-th* tuple corresponds to the - position of the source code that compiled to the *i-th* instruction. - Column information is 0-indexed utf-8 byte offsets on the given source - line. - - This positional information can be missing. A non-exhaustive lists of - cases where this may happen: - - - Running the interpreter with :option:`-X` ``no_debug_ranges``. - - Loading a pyc file compiled while using :option:`-X` ``no_debug_ranges``. - - Position tuples corresponding to artificial instructions. - - Line and column numbers that can't be represented due to - implementation specific limitations. - - When this occurs, some or all of the tuple elements can be - :const:`None`. - - .. versionadded:: 3.11 - - .. note:: - This feature requires storing column positions in code objects which may - result in a small increase of disk usage of compiled Python files or - interpreter memory usage. To avoid storing the extra information and/or - deactivate printing the extra traceback information, the - :option:`-X` ``no_debug_ranges`` command line flag or the :envvar:`PYTHONNODEBUGRANGES` - environment variable can be used. - - -.. _frame-objects: - -Frame objects -^^^^^^^^^^^^^ - -.. index:: pair: object; frame - -Frame objects represent execution frames. They may occur in traceback objects -(see below), and are also passed to registered trace functions. - -.. index:: - single: f_back (frame attribute) - single: f_code (frame attribute) - single: f_globals (frame attribute) - single: f_locals (frame attribute) - single: f_lasti (frame attribute) - single: f_builtins (frame attribute) - -Special read-only attributes: :attr:`f_back` is to the previous stack frame -(towards the caller), or ``None`` if this is the bottom stack frame; -:attr:`f_code` is the code object being executed in this frame; :attr:`f_locals` -is the dictionary used to look up local variables; :attr:`f_globals` is used for -global variables; :attr:`f_builtins` is used for built-in (intrinsic) names; -:attr:`f_lasti` gives the precise instruction (this is an index into the -bytecode string of the code object). - -Accessing ``f_code`` raises an :ref:`auditing event ` -``object.__getattr__`` with arguments ``obj`` and ``"f_code"``. - -.. index:: - single: f_trace (frame attribute) - single: f_trace_lines (frame attribute) - single: f_trace_opcodes (frame attribute) - single: f_lineno (frame attribute) - -Special writable attributes: :attr:`f_trace`, if not ``None``, is a function -called for various events during code execution (this is used by the debugger). -Normally an event is triggered for each new source line - this can be -disabled by setting :attr:`f_trace_lines` to :const:`False`. - -Implementations *may* allow per-opcode events to be requested by setting -:attr:`f_trace_opcodes` to :const:`True`. Note that this may lead to -undefined interpreter behaviour if exceptions raised by the trace -function escape to the function being traced. - -:attr:`f_lineno` is the current line number of the frame --- writing to this -from within a trace function jumps to the given line (only for the bottom-most -frame). A debugger can implement a Jump command (aka Set Next Statement) -by writing to f_lineno. - -Frame objects support one method: - -.. method:: frame.clear() - - This method clears all references to local variables held by the - frame. Also, if the frame belonged to a generator, the generator - is finalized. This helps break reference cycles involving frame - objects (for example when catching an exception and storing its - traceback for later use). - - :exc:`RuntimeError` is raised if the frame is currently executing. - - .. versionadded:: 3.4 - - -.. _traceback-objects: - -Traceback objects -^^^^^^^^^^^^^^^^^ - -.. index:: - pair: object; traceback - pair: stack; trace - pair: exception; handler - pair: execution; stack - single: exc_info (in module sys) - single: last_traceback (in module sys) - single: sys.exc_info - single: sys.exception - single: sys.last_traceback - -Traceback objects represent a stack trace of an exception. A traceback object -is implicitly created when an exception occurs, and may also be explicitly -created by calling :class:`types.TracebackType`. - -For implicitly created tracebacks, when the search for an exception handler -unwinds the execution stack, at each unwound level a traceback object is -inserted in front of the current traceback. When an exception handler is -entered, the stack trace is made available to the program. (See section -:ref:`try`.) It is accessible as the third item of the -tuple returned by ``sys.exc_info()``, and as the ``__traceback__`` attribute -of the caught exception. - -When the program contains no suitable -handler, the stack trace is written (nicely formatted) to the standard error -stream; if the interpreter is interactive, it is also made available to the user -as ``sys.last_traceback``. - -For explicitly created tracebacks, it is up to the creator of the traceback -to determine how the ``tb_next`` attributes should be linked to form a -full stack trace. - -.. index:: - single: tb_frame (traceback attribute) - single: tb_lineno (traceback attribute) - single: tb_lasti (traceback attribute) - pair: statement; try - -Special read-only attributes: -:attr:`tb_frame` points to the execution frame of the current level; -:attr:`tb_lineno` gives the line number where the exception occurred; -:attr:`tb_lasti` indicates the precise instruction. -The line number and last instruction in the traceback may differ from the -line number of its frame object if the exception occurred in a -:keyword:`try` statement with no matching except clause or with a -finally clause. - -Accessing ``tb_frame`` raises an :ref:`auditing event ` -``object.__getattr__`` with arguments ``obj`` and ``"tb_frame"``. - -.. index:: - single: tb_next (traceback attribute) - -Special writable attribute: :attr:`tb_next` is the next level in the stack -trace (towards the frame where the exception occurred), or ``None`` if -there is no next level. - -.. versionchanged:: 3.7 - Traceback objects can now be explicitly instantiated from Python code, - and the ``tb_next`` attribute of existing instances can be updated. - - -Slice objects -^^^^^^^^^^^^^ - -.. index:: pair: built-in function; slice - -Slice objects are used to represent slices for -:meth:`~object.__getitem__` -methods. They are also created by the built-in :func:`slice` function. - -.. index:: - single: start (slice object attribute) - single: stop (slice object attribute) - single: step (slice object attribute) - -Special read-only attributes: :attr:`~slice.start` is the lower bound; -:attr:`~slice.stop` is the upper bound; :attr:`~slice.step` is the step -value; each is ``None`` if omitted. These attributes can have any type. - -Slice objects support one method: - -.. method:: slice.indices(self, length) - - This method takes a single integer argument *length* and computes - information about the slice that the slice object would describe if - applied to a sequence of *length* items. It returns a tuple of three - integers; respectively these are the *start* and *stop* indices and the - *step* or stride length of the slice. Missing or out-of-bounds indices - are handled in a manner consistent with regular slices. - - -Static method objects -^^^^^^^^^^^^^^^^^^^^^ - -Static method objects provide a way of defeating the transformation of function -objects to method objects described above. A static method object is a wrapper -around any other object, usually a user-defined method object. When a static -method object is retrieved from a class or a class instance, the object actually -returned is the wrapped object, which is not subject to any further -transformation. Static method objects are also callable. Static method -objects are created by the built-in :func:`staticmethod` constructor. - - -Class method objects -^^^^^^^^^^^^^^^^^^^^ - -A class method object, like a static method object, is a wrapper around another -object that alters the way in which that object is retrieved from classes and -class instances. The behaviour of class method objects upon such retrieval is -described above, under "User-defined methods". Class method objects are created -by the built-in :func:`classmethod` constructor. - - -.. _specialnames: - -Special method names -==================== - -.. index:: - pair: operator; overloading - single: __getitem__() (mapping object method) - -A class can implement certain operations that are invoked by special syntax -(such as arithmetic operations or subscripting and slicing) by defining methods -with special names. This is Python's approach to :dfn:`operator overloading`, -allowing classes to define their own behavior with respect to language -operators. For instance, if a class defines a method named -:meth:`~object.__getitem__`, -and ``x`` is an instance of this class, then ``x[i]`` is roughly equivalent -to ``type(x).__getitem__(x, i)``. Except where mentioned, attempts to execute an -operation raise an exception when no appropriate method is defined (typically -:exc:`AttributeError` or :exc:`TypeError`). - -Setting a special method to ``None`` indicates that the corresponding -operation is not available. For example, if a class sets -:meth:`~object.__iter__` to ``None``, the class is not iterable, so calling -:func:`iter` on its instances will raise a :exc:`TypeError` (without -falling back to :meth:`~object.__getitem__`). [#]_ - -When implementing a class that emulates any built-in type, it is important that -the emulation only be implemented to the degree that it makes sense for the -object being modelled. For example, some sequences may work well with retrieval -of individual elements, but extracting a slice may not make sense. (One example -of this is the :class:`~xml.dom.NodeList` interface in the W3C's Document -Object Model.) - - -.. _customization: - -Basic customization -------------------- - -.. method:: object.__new__(cls[, ...]) - - .. index:: pair: subclassing; immutable types - - Called to create a new instance of class *cls*. :meth:`__new__` is a static - method (special-cased so you need not declare it as such) that takes the class - of which an instance was requested as its first argument. The remaining - arguments are those passed to the object constructor expression (the call to the - class). The return value of :meth:`__new__` should be the new object instance - (usually an instance of *cls*). - - Typical implementations create a new instance of the class by invoking the - superclass's :meth:`__new__` method using ``super().__new__(cls[, ...])`` - with appropriate arguments and then modifying the newly created instance - as necessary before returning it. - - If :meth:`__new__` is invoked during object construction and it returns an - instance of *cls*, then the new instance’s :meth:`__init__` method - will be invoked like ``__init__(self[, ...])``, where *self* is the new instance - and the remaining arguments are the same as were passed to the object constructor. - - If :meth:`__new__` does not return an instance of *cls*, then the new instance's - :meth:`__init__` method will not be invoked. - - :meth:`__new__` is intended mainly to allow subclasses of immutable types (like - int, str, or tuple) to customize instance creation. It is also commonly - overridden in custom metaclasses in order to customize class creation. - - -.. method:: object.__init__(self[, ...]) - - .. index:: pair: class; constructor - - Called after the instance has been created (by :meth:`__new__`), but before - it is returned to the caller. The arguments are those passed to the - class constructor expression. If a base class has an :meth:`__init__` - method, the derived class's :meth:`__init__` method, if any, must explicitly - call it to ensure proper initialization of the base class part of the - instance; for example: ``super().__init__([args...])``. - - Because :meth:`__new__` and :meth:`__init__` work together in constructing - objects (:meth:`__new__` to create it, and :meth:`__init__` to customize it), - no non-``None`` value may be returned by :meth:`__init__`; doing so will - cause a :exc:`TypeError` to be raised at runtime. - - -.. method:: object.__del__(self) - - .. index:: - single: destructor - single: finalizer - pair: statement; del - - Called when the instance is about to be destroyed. This is also called a - finalizer or (improperly) a destructor. If a base class has a - :meth:`__del__` method, the derived class's :meth:`__del__` method, - if any, must explicitly call it to ensure proper deletion of the base - class part of the instance. - - It is possible (though not recommended!) for the :meth:`__del__` method - to postpone destruction of the instance by creating a new reference to - it. This is called object *resurrection*. It is implementation-dependent - whether :meth:`__del__` is called a second time when a resurrected object - is about to be destroyed; the current :term:`CPython` implementation - only calls it once. - - It is not guaranteed that :meth:`__del__` methods are called for objects - that still exist when the interpreter exits. - - .. note:: - - ``del x`` doesn't directly call ``x.__del__()`` --- the former decrements - the reference count for ``x`` by one, and the latter is only called when - ``x``'s reference count reaches zero. - - .. impl-detail:: - It is possible for a reference cycle to prevent the reference count - of an object from going to zero. In this case, the cycle will be - later detected and deleted by the :term:`cyclic garbage collector - `. A common cause of reference cycles is when - an exception has been caught in a local variable. The frame's - locals then reference the exception, which references its own - traceback, which references the locals of all frames caught in the - traceback. - - .. seealso:: - Documentation for the :mod:`gc` module. - - .. warning:: - - Due to the precarious circumstances under which :meth:`__del__` methods are - invoked, exceptions that occur during their execution are ignored, and a warning - is printed to ``sys.stderr`` instead. In particular: - - * :meth:`__del__` can be invoked when arbitrary code is being executed, - including from any arbitrary thread. If :meth:`__del__` needs to take - a lock or invoke any other blocking resource, it may deadlock as - the resource may already be taken by the code that gets interrupted - to execute :meth:`__del__`. - - * :meth:`__del__` can be executed during interpreter shutdown. As a - consequence, the global variables it needs to access (including other - modules) may already have been deleted or set to ``None``. Python - guarantees that globals whose name begins with a single underscore - are deleted from their module before other globals are deleted; if - no other references to such globals exist, this may help in assuring - that imported modules are still available at the time when the - :meth:`__del__` method is called. - - - .. index:: - single: repr() (built-in function); __repr__() (object method) - -.. method:: object.__repr__(self) - - Called by the :func:`repr` built-in function to compute the "official" string - representation of an object. If at all possible, this should look like a - valid Python expression that could be used to recreate an object with the - same value (given an appropriate environment). If this is not possible, a - string of the form ``<...some useful description...>`` should be returned. - The return value must be a string object. If a class defines :meth:`__repr__` - but not :meth:`__str__`, then :meth:`__repr__` is also used when an - "informal" string representation of instances of that class is required. - - This is typically used for debugging, so it is important that the representation - is information-rich and unambiguous. - - .. index:: - single: string; __str__() (object method) - single: format() (built-in function); __str__() (object method) - single: print() (built-in function); __str__() (object method) - - -.. method:: object.__str__(self) - - Called by :func:`str(object) ` and the built-in functions - :func:`format` and :func:`print` to compute the "informal" or nicely - printable string representation of an object. The return value must be a - :ref:`string ` object. - - This method differs from :meth:`object.__repr__` in that there is no - expectation that :meth:`__str__` return a valid Python expression: a more - convenient or concise representation can be used. - - The default implementation defined by the built-in type :class:`object` - calls :meth:`object.__repr__`. - - .. XXX what about subclasses of string? - - -.. method:: object.__bytes__(self) - - .. index:: pair: built-in function; bytes - - Called by :ref:`bytes ` to compute a byte-string representation - of an object. This should return a :class:`bytes` object. - - .. index:: - single: string; __format__() (object method) - pair: string; conversion - pair: built-in function; print - - -.. method:: object.__format__(self, format_spec) - - Called by the :func:`format` built-in function, - and by extension, evaluation of :ref:`formatted string literals - ` and the :meth:`str.format` method, to produce a "formatted" - string representation of an object. The *format_spec* argument is - a string that contains a description of the formatting options desired. - The interpretation of the *format_spec* argument is up to the type - implementing :meth:`__format__`, however most classes will either - delegate formatting to one of the built-in types, or use a similar - formatting option syntax. - - See :ref:`formatspec` for a description of the standard formatting syntax. - - The return value must be a string object. - - .. versionchanged:: 3.4 - The __format__ method of ``object`` itself raises a :exc:`TypeError` - if passed any non-empty string. - - .. versionchanged:: 3.7 - ``object.__format__(x, '')`` is now equivalent to ``str(x)`` rather - than ``format(str(x), '')``. - - -.. _richcmpfuncs: -.. method:: object.__lt__(self, other) - object.__le__(self, other) - object.__eq__(self, other) - object.__ne__(self, other) - object.__gt__(self, other) - object.__ge__(self, other) - - .. index:: - single: comparisons - - These are the so-called "rich comparison" methods. The correspondence between - operator symbols and method names is as follows: ``xy`` calls ``x.__gt__(y)``, and ``x>=y`` calls - ``x.__ge__(y)``. - - A rich comparison method may return the singleton ``NotImplemented`` if it does - not implement the operation for a given pair of arguments. By convention, - ``False`` and ``True`` are returned for a successful comparison. However, these - methods can return any value, so if the comparison operator is used in a Boolean - context (e.g., in the condition of an ``if`` statement), Python will call - :func:`bool` on the value to determine if the result is true or false. - - By default, ``object`` implements :meth:`__eq__` by using ``is``, returning - ``NotImplemented`` in the case of a false comparison: - ``True if x is y else NotImplemented``. For :meth:`__ne__`, by default it - delegates to :meth:`__eq__` and inverts the result unless it is - ``NotImplemented``. There are no other implied relationships among the - comparison operators or default implementations; for example, the truth of - ``(x.__hash__``. - - If a class that does not override :meth:`__eq__` wishes to suppress hash - support, it should include ``__hash__ = None`` in the class definition. - A class which defines its own :meth:`__hash__` that explicitly raises - a :exc:`TypeError` would be incorrectly identified as hashable by - an ``isinstance(obj, collections.abc.Hashable)`` call. - - - .. note:: - - By default, the :meth:`__hash__` values of str and bytes objects are - "salted" with an unpredictable random value. Although they - remain constant within an individual Python process, they are not - predictable between repeated invocations of Python. - - This is intended to provide protection against a denial-of-service caused - by carefully chosen inputs that exploit the worst case performance of a - dict insertion, O(n\ :sup:`2`) complexity. See - http://ocert.org/advisories/ocert-2011-003.html for details. - - Changing hash values affects the iteration order of sets. - Python has never made guarantees about this ordering - (and it typically varies between 32-bit and 64-bit builds). - - See also :envvar:`PYTHONHASHSEED`. - - .. versionchanged:: 3.3 - Hash randomization is enabled by default. - - -.. method:: object.__bool__(self) - - .. index:: single: __len__() (mapping object method) - - Called to implement truth value testing and the built-in operation - ``bool()``; should return ``False`` or ``True``. When this method is not - defined, :meth:`~object.__len__` is called, if it is defined, and the object is - considered true if its result is nonzero. If a class defines neither - :meth:`!__len__` nor :meth:`!__bool__`, all its instances are considered - true. - - -.. _attribute-access: - -Customizing attribute access ----------------------------- - -The following methods can be defined to customize the meaning of attribute -access (use of, assignment to, or deletion of ``x.name``) for class instances. - -.. XXX explain how descriptors interfere here! - - -.. method:: object.__getattr__(self, name) - - Called when the default attribute access fails with an :exc:`AttributeError` - (either :meth:`__getattribute__` raises an :exc:`AttributeError` because - *name* is not an instance attribute or an attribute in the class tree - for ``self``; or :meth:`__get__` of a *name* property raises - :exc:`AttributeError`). This method should either return the (computed) - attribute value or raise an :exc:`AttributeError` exception. - - Note that if the attribute is found through the normal mechanism, - :meth:`__getattr__` is not called. (This is an intentional asymmetry between - :meth:`__getattr__` and :meth:`__setattr__`.) This is done both for efficiency - reasons and because otherwise :meth:`__getattr__` would have no way to access - other attributes of the instance. Note that at least for instance variables, - you can fake total control by not inserting any values in the instance attribute - dictionary (but instead inserting them in another object). See the - :meth:`__getattribute__` method below for a way to actually get total control - over attribute access. - - -.. method:: object.__getattribute__(self, name) - - Called unconditionally to implement attribute accesses for instances of the - class. If the class also defines :meth:`__getattr__`, the latter will not be - called unless :meth:`__getattribute__` either calls it explicitly or raises an - :exc:`AttributeError`. This method should return the (computed) attribute value - or raise an :exc:`AttributeError` exception. In order to avoid infinite - recursion in this method, its implementation should always call the base class - method with the same name to access any attributes it needs, for example, - ``object.__getattribute__(self, name)``. - - .. note:: - - This method may still be bypassed when looking up special methods as the - result of implicit invocation via language syntax or built-in functions. - See :ref:`special-lookup`. - - .. audit-event:: object.__getattr__ obj,name object.__getattribute__ - - For certain sensitive attribute accesses, raises an - :ref:`auditing event ` ``object.__getattr__`` with arguments - ``obj`` and ``name``. - - -.. method:: object.__setattr__(self, name, value) - - Called when an attribute assignment is attempted. This is called instead of - the normal mechanism (i.e. store the value in the instance dictionary). - *name* is the attribute name, *value* is the value to be assigned to it. - - If :meth:`__setattr__` wants to assign to an instance attribute, it should - call the base class method with the same name, for example, - ``object.__setattr__(self, name, value)``. - - .. audit-event:: object.__setattr__ obj,name,value object.__setattr__ - - For certain sensitive attribute assignments, raises an - :ref:`auditing event ` ``object.__setattr__`` with arguments - ``obj``, ``name``, ``value``. - - -.. method:: object.__delattr__(self, name) - - Like :meth:`__setattr__` but for attribute deletion instead of assignment. This - should only be implemented if ``del obj.name`` is meaningful for the object. - - .. audit-event:: object.__delattr__ obj,name object.__delattr__ - - For certain sensitive attribute deletions, raises an - :ref:`auditing event ` ``object.__delattr__`` with arguments - ``obj`` and ``name``. - - -.. method:: object.__dir__(self) - - Called when :func:`dir` is called on the object. A sequence must be - returned. :func:`dir` converts the returned sequence to a list and sorts it. - - -Customizing module attribute access -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: __getattr__ (module attribute) - single: __dir__ (module attribute) - single: __class__ (module attribute) - -Special names ``__getattr__`` and ``__dir__`` can be also used to customize -access to module attributes. The ``__getattr__`` function at the module level -should accept one argument which is the name of an attribute and return the -computed value or raise an :exc:`AttributeError`. If an attribute is -not found on a module object through the normal lookup, i.e. -:meth:`object.__getattribute__`, then ``__getattr__`` is searched in -the module ``__dict__`` before raising an :exc:`AttributeError`. If found, -it is called with the attribute name and the result is returned. - -The ``__dir__`` function should accept no arguments, and return a sequence of -strings that represents the names accessible on module. If present, this -function overrides the standard :func:`dir` search on a module. - -For a more fine grained customization of the module behavior (setting -attributes, properties, etc.), one can set the ``__class__`` attribute of -a module object to a subclass of :class:`types.ModuleType`. For example:: - - import sys - from types import ModuleType - - class VerboseModule(ModuleType): - def __repr__(self): - return f'Verbose {self.__name__}' - - def __setattr__(self, attr, value): - print(f'Setting {attr}...') - super().__setattr__(attr, value) - - sys.modules[__name__].__class__ = VerboseModule - -.. note:: - Defining module ``__getattr__`` and setting module ``__class__`` only - affect lookups made using the attribute access syntax -- directly accessing - the module globals (whether by code within the module, or via a reference - to the module's globals dictionary) is unaffected. - -.. versionchanged:: 3.5 - ``__class__`` module attribute is now writable. - -.. versionadded:: 3.7 - ``__getattr__`` and ``__dir__`` module attributes. - -.. seealso:: - - :pep:`562` - Module __getattr__ and __dir__ - Describes the ``__getattr__`` and ``__dir__`` functions on modules. - - -.. _descriptors: - -Implementing Descriptors -^^^^^^^^^^^^^^^^^^^^^^^^ - -The following methods only apply when an instance of the class containing the -method (a so-called *descriptor* class) appears in an *owner* class (the -descriptor must be in either the owner's class dictionary or in the class -dictionary for one of its parents). In the examples below, "the attribute" -refers to the attribute whose name is the key of the property in the owner -class' :attr:`~object.__dict__`. - - -.. method:: object.__get__(self, instance, owner=None) - - Called to get the attribute of the owner class (class attribute access) or - of an instance of that class (instance attribute access). The optional - *owner* argument is the owner class, while *instance* is the instance that - the attribute was accessed through, or ``None`` when the attribute is - accessed through the *owner*. - - This method should return the computed attribute value or raise an - :exc:`AttributeError` exception. - - :PEP:`252` specifies that :meth:`__get__` is callable with one or two - arguments. Python's own built-in descriptors support this specification; - however, it is likely that some third-party tools have descriptors - that require both arguments. Python's own :meth:`__getattribute__` - implementation always passes in both arguments whether they are required - or not. - -.. method:: object.__set__(self, instance, value) - - Called to set the attribute on an instance *instance* of the owner class to a - new value, *value*. - - Note, adding :meth:`__set__` or :meth:`__delete__` changes the kind of - descriptor to a "data descriptor". See :ref:`descriptor-invocation` for - more details. - -.. method:: object.__delete__(self, instance) - - Called to delete the attribute on an instance *instance* of the owner class. - - -The attribute :attr:`__objclass__` is interpreted by the :mod:`inspect` module -as specifying the class where this object was defined (setting this -appropriately can assist in runtime introspection of dynamic class attributes). -For callables, it may indicate that an instance of the given type (or a -subclass) is expected or required as the first positional argument (for example, -CPython sets this attribute for unbound methods that are implemented in C). - - -.. _descriptor-invocation: - -Invoking Descriptors -^^^^^^^^^^^^^^^^^^^^ - -In general, a descriptor is an object attribute with "binding behavior", one -whose attribute access has been overridden by methods in the descriptor -protocol: :meth:`~object.__get__`, :meth:`~object.__set__`, and -:meth:`~object.__delete__`. If any of -those methods are defined for an object, it is said to be a descriptor. - -The default behavior for attribute access is to get, set, or delete the -attribute from an object's dictionary. For instance, ``a.x`` has a lookup chain -starting with ``a.__dict__['x']``, then ``type(a).__dict__['x']``, and -continuing through the base classes of ``type(a)`` excluding metaclasses. - -However, if the looked-up value is an object defining one of the descriptor -methods, then Python may override the default behavior and invoke the descriptor -method instead. Where this occurs in the precedence chain depends on which -descriptor methods were defined and how they were called. - -The starting point for descriptor invocation is a binding, ``a.x``. How the -arguments are assembled depends on ``a``: - -Direct Call - The simplest and least common call is when user code directly invokes a - descriptor method: ``x.__get__(a)``. - -Instance Binding - If binding to an object instance, ``a.x`` is transformed into the call: - ``type(a).__dict__['x'].__get__(a, type(a))``. - -Class Binding - If binding to a class, ``A.x`` is transformed into the call: - ``A.__dict__['x'].__get__(None, A)``. - -Super Binding - A dotted lookup such as ``super(A, a).x`` searches - ``a.__class__.__mro__`` for a base class ``B`` following ``A`` and then - returns ``B.__dict__['x'].__get__(a, A)``. If not a descriptor, ``x`` is - returned unchanged. - -.. testcode:: - :hide: - - class Desc: - def __get__(*args): - return args - - class B: - - x = Desc() - - class A(B): - - x = 999 - - def m(self): - 'Demonstrate these two descriptor invocations are equivalent' - result1 = super(A, self).x - result2 = B.__dict__['x'].__get__(self, A) - return result1 == result2 - -.. doctest:: - :hide: - - >>> a = A() - >>> a.__class__.__mro__.index(B) > a.__class__.__mro__.index(A) - True - >>> super(A, a).x == B.__dict__['x'].__get__(a, A) - True - >>> a.m() - True - -For instance bindings, the precedence of descriptor invocation depends on -which descriptor methods are defined. A descriptor can define any combination -of :meth:`~object.__get__`, :meth:`~object.__set__` and -:meth:`~object.__delete__`. If it does not -define :meth:`__get__`, then accessing the attribute will return the descriptor -object itself unless there is a value in the object's instance dictionary. If -the descriptor defines :meth:`__set__` and/or :meth:`__delete__`, it is a data -descriptor; if it defines neither, it is a non-data descriptor. Normally, data -descriptors define both :meth:`__get__` and :meth:`__set__`, while non-data -descriptors have just the :meth:`__get__` method. Data descriptors with -:meth:`__get__` and :meth:`__set__` (and/or :meth:`__delete__`) defined always override a redefinition in an -instance dictionary. In contrast, non-data descriptors can be overridden by -instances. - -Python methods (including those decorated with -:func:`@staticmethod ` and :func:`@classmethod `) are -implemented as non-data descriptors. Accordingly, instances can redefine and -override methods. This allows individual instances to acquire behaviors that -differ from other instances of the same class. - -The :func:`property` function is implemented as a data descriptor. Accordingly, -instances cannot override the behavior of a property. - - -.. _slots: - -__slots__ -^^^^^^^^^ - -*__slots__* allow us to explicitly declare data members (like -properties) and deny the creation of :attr:`~object.__dict__` and *__weakref__* -(unless explicitly declared in *__slots__* or available in a parent.) - -The space saved over using :attr:`~object.__dict__` can be significant. -Attribute lookup speed can be significantly improved as well. - -.. data:: object.__slots__ - - This class variable can be assigned a string, iterable, or sequence of - strings with variable names used by instances. *__slots__* reserves space - for the declared variables and prevents the automatic creation of - :attr:`~object.__dict__` - and *__weakref__* for each instance. - - -.. _datamodel-note-slots: - -Notes on using *__slots__*: - -* When inheriting from a class without *__slots__*, the - :attr:`~object.__dict__` and - *__weakref__* attribute of the instances will always be accessible. - -* Without a :attr:`~object.__dict__` variable, instances cannot be assigned new - variables not - listed in the *__slots__* definition. Attempts to assign to an unlisted - variable name raises :exc:`AttributeError`. If dynamic assignment of new - variables is desired, then add ``'__dict__'`` to the sequence of strings in - the *__slots__* declaration. - -* Without a *__weakref__* variable for each instance, classes defining - *__slots__* do not support :mod:`weak references ` to its instances. - If weak reference - support is needed, then add ``'__weakref__'`` to the sequence of strings in the - *__slots__* declaration. - -* *__slots__* are implemented at the class level by creating :ref:`descriptors ` - for each variable name. As a result, class attributes - cannot be used to set default values for instance variables defined by - *__slots__*; otherwise, the class attribute would overwrite the descriptor - assignment. - -* The action of a *__slots__* declaration is not limited to the class - where it is defined. *__slots__* declared in parents are available in - child classes. However, child subclasses will get a :attr:`~object.__dict__` and - *__weakref__* unless they also define *__slots__* (which should only - contain names of any *additional* slots). - -* If a class defines a slot also defined in a base class, the instance variable - defined by the base class slot is inaccessible (except by retrieving its - descriptor directly from the base class). This renders the meaning of the - program undefined. In the future, a check may be added to prevent this. - -* :exc:`TypeError` will be raised if nonempty *__slots__* are defined for a - class derived from a - :c:member:`"variable-length" built-in type ` such as - :class:`int`, :class:`bytes`, and :class:`tuple`. - -* Any non-string :term:`iterable` may be assigned to *__slots__*. - -* If a :class:`dictionary ` is used to assign *__slots__*, the dictionary - keys will be used as the slot names. The values of the dictionary can be used - to provide per-attribute docstrings that will be recognised by - :func:`inspect.getdoc` and displayed in the output of :func:`help`. - -* :attr:`~instance.__class__` assignment works only if both classes have the - same *__slots__*. - -* :ref:`Multiple inheritance ` with multiple slotted parent - classes can be used, - but only one parent is allowed to have attributes created by slots - (the other bases must have empty slot layouts) - violations raise - :exc:`TypeError`. - -* If an :term:`iterator` is used for *__slots__* then a :term:`descriptor` is - created for each - of the iterator's values. However, the *__slots__* attribute will be an empty - iterator. - -.. _class-customization: - -Customizing class creation --------------------------- - -Whenever a class inherits from another class, :meth:`~object.__init_subclass__` is -called on the parent class. This way, it is possible to write classes which -change the behavior of subclasses. This is closely related to class -decorators, but where class decorators only affect the specific class they're -applied to, ``__init_subclass__`` solely applies to future subclasses of the -class defining the method. - -.. classmethod:: object.__init_subclass__(cls) - - This method is called whenever the containing class is subclassed. - *cls* is then the new subclass. If defined as a normal instance method, - this method is implicitly converted to a class method. - - Keyword arguments which are given to a new class are passed to - the parent's class ``__init_subclass__``. For compatibility with - other classes using ``__init_subclass__``, one should take out the - needed keyword arguments and pass the others over to the base - class, as in:: - - class Philosopher: - def __init_subclass__(cls, /, default_name, **kwargs): - super().__init_subclass__(**kwargs) - cls.default_name = default_name - - class AustralianPhilosopher(Philosopher, default_name="Bruce"): - pass - - The default implementation ``object.__init_subclass__`` does - nothing, but raises an error if it is called with any arguments. - - .. note:: - - The metaclass hint ``metaclass`` is consumed by the rest of the type - machinery, and is never passed to ``__init_subclass__`` implementations. - The actual metaclass (rather than the explicit hint) can be accessed as - ``type(cls)``. - - .. versionadded:: 3.6 - - -When a class is created, :meth:`type.__new__` scans the class variables -and makes callbacks to those with a :meth:`~object.__set_name__` hook. - -.. method:: object.__set_name__(self, owner, name) - - Automatically called at the time the owning class *owner* is - created. The object has been assigned to *name* in that class:: - - class A: - x = C() # Automatically calls: x.__set_name__(A, 'x') - - If the class variable is assigned after the class is created, - :meth:`__set_name__` will not be called automatically. - If needed, :meth:`__set_name__` can be called directly:: - - class A: - pass - - c = C() - A.x = c # The hook is not called - c.__set_name__(A, 'x') # Manually invoke the hook - - See :ref:`class-object-creation` for more details. - - .. versionadded:: 3.6 - - -.. _metaclasses: - -Metaclasses -^^^^^^^^^^^ - -.. index:: - single: metaclass - pair: built-in function; type - single: = (equals); class definition - -By default, classes are constructed using :func:`type`. The class body is -executed in a new namespace and the class name is bound locally to the -result of ``type(name, bases, namespace)``. - -The class creation process can be customized by passing the ``metaclass`` -keyword argument in the class definition line, or by inheriting from an -existing class that included such an argument. In the following example, -both ``MyClass`` and ``MySubclass`` are instances of ``Meta``:: - - class Meta(type): - pass - - class MyClass(metaclass=Meta): - pass - - class MySubclass(MyClass): - pass - -Any other keyword arguments that are specified in the class definition are -passed through to all metaclass operations described below. - -When a class definition is executed, the following steps occur: - -* MRO entries are resolved; -* the appropriate metaclass is determined; -* the class namespace is prepared; -* the class body is executed; -* the class object is created. - - -Resolving MRO entries -^^^^^^^^^^^^^^^^^^^^^ - -.. method:: object.__mro_entries__(self, bases) - - If a base that appears in a class definition is not an instance of - :class:`type`, then an :meth:`!__mro_entries__` method is searched on the base. - If an :meth:`!__mro_entries__` method is found, the base is substituted with the - result of a call to :meth:`!__mro_entries__` when creating the class. - The method is called with the original bases tuple - passed to the *bases* parameter, and must return a tuple - of classes that will be used instead of the base. The returned tuple may be - empty: in these cases, the original base is ignored. - -.. seealso:: - - :func:`types.resolve_bases` - Dynamically resolve bases that are not instances of :class:`type`. - - :pep:`560` - Core support for typing module and generic types. - - -Determining the appropriate metaclass -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -.. index:: - single: metaclass hint - -The appropriate metaclass for a class definition is determined as follows: - -* if no bases and no explicit metaclass are given, then :func:`type` is used; -* if an explicit metaclass is given and it is *not* an instance of - :func:`type`, then it is used directly as the metaclass; -* if an instance of :func:`type` is given as the explicit metaclass, or - bases are defined, then the most derived metaclass is used. - -The most derived metaclass is selected from the explicitly specified -metaclass (if any) and the metaclasses (i.e. ``type(cls)``) of all specified -base classes. The most derived metaclass is one which is a subtype of *all* -of these candidate metaclasses. If none of the candidate metaclasses meets -that criterion, then the class definition will fail with ``TypeError``. - - -.. _prepare: - -Preparing the class namespace -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: __prepare__ (metaclass method) - -Once the appropriate metaclass has been identified, then the class namespace -is prepared. If the metaclass has a ``__prepare__`` attribute, it is called -as ``namespace = metaclass.__prepare__(name, bases, **kwds)`` (where the -additional keyword arguments, if any, come from the class definition). The -``__prepare__`` method should be implemented as a -:func:`classmethod `. The -namespace returned by ``__prepare__`` is passed in to ``__new__``, but when -the final class object is created the namespace is copied into a new ``dict``. - -If the metaclass has no ``__prepare__`` attribute, then the class namespace -is initialised as an empty ordered mapping. - -.. seealso:: - - :pep:`3115` - Metaclasses in Python 3000 - Introduced the ``__prepare__`` namespace hook - - -Executing the class body -^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: class; body - -The class body is executed (approximately) as -``exec(body, globals(), namespace)``. The key difference from a normal -call to :func:`exec` is that lexical scoping allows the class body (including -any methods) to reference names from the current and outer scopes when the -class definition occurs inside a function. - -However, even when the class definition occurs inside the function, methods -defined inside the class still cannot see names defined at the class scope. -Class variables must be accessed through the first parameter of instance or -class methods, or through the implicit lexically scoped ``__class__`` reference -described in the next section. - -.. _class-object-creation: - -Creating the class object -^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. index:: - single: __class__ (method cell) - single: __classcell__ (class namespace entry) - - -Once the class namespace has been populated by executing the class body, -the class object is created by calling -``metaclass(name, bases, namespace, **kwds)`` (the additional keywords -passed here are the same as those passed to ``__prepare__``). - -This class object is the one that will be referenced by the zero-argument -form of :func:`super`. ``__class__`` is an implicit closure reference -created by the compiler if any methods in a class body refer to either -``__class__`` or ``super``. This allows the zero argument form of -:func:`super` to correctly identify the class being defined based on -lexical scoping, while the class or instance that was used to make the -current call is identified based on the first argument passed to the method. - -.. impl-detail:: - - In CPython 3.6 and later, the ``__class__`` cell is passed to the metaclass - as a ``__classcell__`` entry in the class namespace. If present, this must - be propagated up to the ``type.__new__`` call in order for the class to be - initialised correctly. - Failing to do so will result in a :exc:`RuntimeError` in Python 3.8. - -When using the default metaclass :class:`type`, or any metaclass that ultimately -calls ``type.__new__``, the following additional customization steps are -invoked after creating the class object: - -1) The ``type.__new__`` method collects all of the attributes in the class - namespace that define a :meth:`~object.__set_name__` method; -2) Those ``__set_name__`` methods are called with the class - being defined and the assigned name of that particular attribute; -3) The :meth:`~object.__init_subclass__` hook is called on the - immediate parent of the new class in its method resolution order. - -After the class object is created, it is passed to the class decorators -included in the class definition (if any) and the resulting object is bound -in the local namespace as the defined class. - -When a new class is created by ``type.__new__``, the object provided as the -namespace parameter is copied to a new ordered mapping and the original -object is discarded. The new copy is wrapped in a read-only proxy, which -becomes the :attr:`~object.__dict__` attribute of the class object. - -.. seealso:: - - :pep:`3135` - New super - Describes the implicit ``__class__`` closure reference - - -Uses for metaclasses -^^^^^^^^^^^^^^^^^^^^ - -The potential uses for metaclasses are boundless. Some ideas that have been -explored include enum, logging, interface checking, automatic delegation, -automatic property creation, proxies, frameworks, and automatic resource -locking/synchronization. - - -Customizing instance and subclass checks ----------------------------------------- - -The following methods are used to override the default behavior of the -:func:`isinstance` and :func:`issubclass` built-in functions. - -In particular, the metaclass :class:`abc.ABCMeta` implements these methods in -order to allow the addition of Abstract Base Classes (ABCs) as "virtual base -classes" to any class or type (including built-in types), including other -ABCs. - -.. method:: class.__instancecheck__(self, instance) - - Return true if *instance* should be considered a (direct or indirect) - instance of *class*. If defined, called to implement ``isinstance(instance, - class)``. - - -.. method:: class.__subclasscheck__(self, subclass) - - Return true if *subclass* should be considered a (direct or indirect) - subclass of *class*. If defined, called to implement ``issubclass(subclass, - class)``. - - -Note that these methods are looked up on the type (metaclass) of a class. They -cannot be defined as class methods in the actual class. This is consistent with -the lookup of special methods that are called on instances, only in this -case the instance is itself a class. - -.. seealso:: - - :pep:`3119` - Introducing Abstract Base Classes - Includes the specification for customizing :func:`isinstance` and - :func:`issubclass` behavior through :meth:`~class.__instancecheck__` and - :meth:`~class.__subclasscheck__`, with motivation for this functionality - in the context of adding Abstract Base Classes (see the :mod:`abc` - module) to the language. - - -Emulating generic types ------------------------ - -When using :term:`type annotations`, it is often useful to -*parameterize* a :term:`generic type` using Python's square-brackets notation. -For example, the annotation ``list[int]`` might be used to signify a -:class:`list` in which all the elements are of type :class:`int`. - -.. seealso:: - - :pep:`484` - Type Hints - Introducing Python's framework for type annotations - - :ref:`Generic Alias Types` - Documentation for objects representing parameterized generic classes - - :ref:`Generics`, :ref:`user-defined generics` and :class:`typing.Generic` - Documentation on how to implement generic classes that can be - parameterized at runtime and understood by static type-checkers. - -A class can *generally* only be parameterized if it defines the special -class method ``__class_getitem__()``. - -.. classmethod:: object.__class_getitem__(cls, key) - - Return an object representing the specialization of a generic class - by type arguments found in *key*. - - When defined on a class, ``__class_getitem__()`` is automatically a class - method. As such, there is no need for it to be decorated with - :func:`@classmethod` when it is defined. - - -The purpose of *__class_getitem__* -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The purpose of :meth:`~object.__class_getitem__` is to allow runtime -parameterization of standard-library generic classes in order to more easily -apply :term:`type hints` to these classes. - -To implement custom generic classes that can be parameterized at runtime and -understood by static type-checkers, users should either inherit from a standard -library class that already implements :meth:`~object.__class_getitem__`, or -inherit from :class:`typing.Generic`, which has its own implementation of -``__class_getitem__()``. - -Custom implementations of :meth:`~object.__class_getitem__` on classes defined -outside of the standard library may not be understood by third-party -type-checkers such as mypy. Using ``__class_getitem__()`` on any class for -purposes other than type hinting is discouraged. - - -.. _classgetitem-versus-getitem: - - -*__class_getitem__* versus *__getitem__* -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Usually, the :ref:`subscription` of an object using square -brackets will call the :meth:`~object.__getitem__` instance method defined on -the object's class. However, if the object being subscribed is itself a class, -the class method :meth:`~object.__class_getitem__` may be called instead. -``__class_getitem__()`` should return a :ref:`GenericAlias` -object if it is properly defined. - -Presented with the :term:`expression` ``obj[x]``, the Python interpreter -follows something like the following process to decide whether -:meth:`~object.__getitem__` or :meth:`~object.__class_getitem__` should be -called:: - - from inspect import isclass - - def subscribe(obj, x): - """Return the result of the expression 'obj[x]'""" - - class_of_obj = type(obj) - - # If the class of obj defines __getitem__, - # call class_of_obj.__getitem__(obj, x) - if hasattr(class_of_obj, '__getitem__'): - return class_of_obj.__getitem__(obj, x) - - # Else, if obj is a class and defines __class_getitem__, - # call obj.__class_getitem__(x) - elif isclass(obj) and hasattr(obj, '__class_getitem__'): - return obj.__class_getitem__(x) - - # Else, raise an exception - else: - raise TypeError( - f"'{class_of_obj.__name__}' object is not subscriptable" - ) - -In Python, all classes are themselves instances of other classes. The class of -a class is known as that class's :term:`metaclass`, and most classes have the -:class:`type` class as their metaclass. :class:`type` does not define -:meth:`~object.__getitem__`, meaning that expressions such as ``list[int]``, -``dict[str, float]`` and ``tuple[str, bytes]`` all result in -:meth:`~object.__class_getitem__` being called:: - - >>> # list has class "type" as its metaclass, like most classes: - >>> type(list) - - >>> type(dict) == type(list) == type(tuple) == type(str) == type(bytes) - True - >>> # "list[int]" calls "list.__class_getitem__(int)" - >>> list[int] - list[int] - >>> # list.__class_getitem__ returns a GenericAlias object: - >>> type(list[int]) - - -However, if a class has a custom metaclass that defines -:meth:`~object.__getitem__`, subscribing the class may result in different -behaviour. An example of this can be found in the :mod:`enum` module:: - - >>> from enum import Enum - >>> class Menu(Enum): - ... """A breakfast menu""" - ... SPAM = 'spam' - ... BACON = 'bacon' - ... - >>> # Enum classes have a custom metaclass: - >>> type(Menu) - - >>> # EnumMeta defines __getitem__, - >>> # so __class_getitem__ is not called, - >>> # and the result is not a GenericAlias object: - >>> Menu['SPAM'] - - >>> type(Menu['SPAM']) - - - -.. seealso:: - :pep:`560` - Core Support for typing module and generic types - Introducing :meth:`~object.__class_getitem__`, and outlining when a - :ref:`subscription` results in ``__class_getitem__()`` - being called instead of :meth:`~object.__getitem__` - - -.. _callable-types: - -Emulating callable objects --------------------------- - - -.. method:: object.__call__(self[, args...]) - - .. index:: pair: call; instance - - Called when the instance is "called" as a function; if this method is defined, - ``x(arg1, arg2, ...)`` roughly translates to ``type(x).__call__(x, arg1, ...)``. - - -.. _sequence-types: - -Emulating container types -------------------------- - -The following methods can be defined to implement container objects. Containers -usually are :term:`sequences ` (such as :class:`lists ` or -:class:`tuples `) or :term:`mappings ` (like -:class:`dictionaries `), -but can represent other containers as well. The first set of methods is used -either to emulate a sequence or to emulate a mapping; the difference is that for -a sequence, the allowable keys should be the integers *k* for which ``0 <= k < -N`` where *N* is the length of the sequence, or :class:`slice` objects, which define a -range of items. It is also recommended that mappings provide the methods -:meth:`keys`, :meth:`values`, :meth:`items`, :meth:`get`, :meth:`clear`, -:meth:`setdefault`, :meth:`pop`, :meth:`popitem`, :meth:`!copy`, and -:meth:`update` behaving similar to those for Python's standard :class:`dictionary ` -objects. The :mod:`collections.abc` module provides a -:class:`~collections.abc.MutableMapping` -:term:`abstract base class` to help create those methods from a base set of -:meth:`~object.__getitem__`, :meth:`~object.__setitem__`, :meth:`~object.__delitem__`, and :meth:`keys`. -Mutable sequences should provide methods :meth:`append`, :meth:`count`, -:meth:`index`, :meth:`extend`, :meth:`insert`, :meth:`pop`, :meth:`remove`, -:meth:`reverse` and :meth:`sort`, like Python standard :class:`list` -objects. Finally, -sequence types should implement addition (meaning concatenation) and -multiplication (meaning repetition) by defining the methods -:meth:`~object.__add__`, :meth:`~object.__radd__`, :meth:`~object.__iadd__`, -:meth:`~object.__mul__`, :meth:`~object.__rmul__` and :meth:`~object.__imul__` -described below; they should not define other numerical -operators. It is recommended that both mappings and sequences implement the -:meth:`~object.__contains__` method to allow efficient use of the ``in`` -operator; for -mappings, ``in`` should search the mapping's keys; for sequences, it should -search through the values. It is further recommended that both mappings and -sequences implement the :meth:`~object.__iter__` method to allow efficient iteration -through the container; for mappings, :meth:`__iter__` should iterate -through the object's keys; for sequences, it should iterate through the values. - -.. method:: object.__len__(self) - - .. index:: - pair: built-in function; len - single: __bool__() (object method) - - Called to implement the built-in function :func:`len`. Should return the length - of the object, an integer ``>=`` 0. Also, an object that doesn't define a - :meth:`~object.__bool__` method and whose :meth:`!__len__` method returns zero is - considered to be false in a Boolean context. - - .. impl-detail:: - - In CPython, the length is required to be at most :data:`sys.maxsize`. - If the length is larger than :data:`!sys.maxsize` some features (such as - :func:`len`) may raise :exc:`OverflowError`. To prevent raising - :exc:`!OverflowError` by truth value testing, an object must define a - :meth:`~object.__bool__` method. - - -.. method:: object.__length_hint__(self) - - Called to implement :func:`operator.length_hint`. Should return an estimated - length for the object (which may be greater or less than the actual length). - The length must be an integer ``>=`` 0. The return value may also be - :const:`NotImplemented`, which is treated the same as if the - ``__length_hint__`` method didn't exist at all. This method is purely an - optimization and is never required for correctness. - - .. versionadded:: 3.4 - - -.. index:: pair: object; slice - -.. note:: - - Slicing is done exclusively with the following three methods. A call like :: - - a[1:2] = b - - is translated to :: - - a[slice(1, 2, None)] = b - - and so forth. Missing slice items are always filled in with ``None``. - - -.. method:: object.__getitem__(self, key) - - Called to implement evaluation of ``self[key]``. For :term:`sequence` types, - the accepted keys should be integers and slice objects. Note that the - special interpretation of negative indexes (if the class wishes to emulate a - :term:`sequence` type) is up to the :meth:`__getitem__` method. If *key* is - of an inappropriate type, :exc:`TypeError` may be raised; if of a value - outside the set of indexes for the sequence (after any special - interpretation of negative values), :exc:`IndexError` should be raised. For - :term:`mapping` types, if *key* is missing (not in the container), - :exc:`KeyError` should be raised. - - .. note:: - - :keyword:`for` loops expect that an :exc:`IndexError` will be raised for - illegal indexes to allow proper detection of the end of the sequence. - - .. note:: - - When :ref:`subscripting` a *class*, the special - class method :meth:`~object.__class_getitem__` may be called instead of - ``__getitem__()``. See :ref:`classgetitem-versus-getitem` for more - details. - - -.. method:: object.__setitem__(self, key, value) - - Called to implement assignment to ``self[key]``. Same note as for - :meth:`__getitem__`. This should only be implemented for mappings if the - objects support changes to the values for keys, or if new keys can be added, or - for sequences if elements can be replaced. The same exceptions should be raised - for improper *key* values as for the :meth:`__getitem__` method. - - -.. method:: object.__delitem__(self, key) - - Called to implement deletion of ``self[key]``. Same note as for - :meth:`__getitem__`. This should only be implemented for mappings if the - objects support removal of keys, or for sequences if elements can be removed - from the sequence. The same exceptions should be raised for improper *key* - values as for the :meth:`__getitem__` method. - - -.. method:: object.__missing__(self, key) - - Called by :class:`dict`\ .\ :meth:`__getitem__` to implement ``self[key]`` for dict subclasses - when key is not in the dictionary. - - -.. method:: object.__iter__(self) - - This method is called when an :term:`iterator` is required for a container. - This method should return a new iterator object that can iterate over all the - objects in the container. For mappings, it should iterate over the keys of - the container. - - -.. method:: object.__reversed__(self) - - Called (if present) by the :func:`reversed` built-in to implement - reverse iteration. It should return a new iterator object that iterates - over all the objects in the container in reverse order. - - If the :meth:`__reversed__` method is not provided, the :func:`reversed` - built-in will fall back to using the sequence protocol (:meth:`__len__` and - :meth:`__getitem__`). Objects that support the sequence protocol should - only provide :meth:`__reversed__` if they can provide an implementation - that is more efficient than the one provided by :func:`reversed`. - - -The membership test operators (:keyword:`in` and :keyword:`not in`) are normally -implemented as an iteration through a container. However, container objects can -supply the following special method with a more efficient implementation, which -also does not require the object be iterable. - -.. method:: object.__contains__(self, item) - - Called to implement membership test operators. Should return true if *item* - is in *self*, false otherwise. For mapping objects, this should consider the - keys of the mapping rather than the values or the key-item pairs. - - For objects that don't define :meth:`__contains__`, the membership test first - tries iteration via :meth:`__iter__`, then the old sequence iteration - protocol via :meth:`__getitem__`, see :ref:`this section in the language - reference `. - - -.. _numeric-types: - -Emulating numeric types ------------------------ - -The following methods can be defined to emulate numeric objects. Methods -corresponding to operations that are not supported by the particular kind of -number implemented (e.g., bitwise operations for non-integral numbers) should be -left undefined. - - -.. method:: object.__add__(self, other) - object.__sub__(self, other) - object.__mul__(self, other) - object.__matmul__(self, other) - object.__truediv__(self, other) - object.__floordiv__(self, other) - object.__mod__(self, other) - object.__divmod__(self, other) - object.__pow__(self, other[, modulo]) - object.__lshift__(self, other) - object.__rshift__(self, other) - object.__and__(self, other) - object.__xor__(self, other) - object.__or__(self, other) - - .. index:: - pair: built-in function; divmod - pair: built-in function; pow - pair: built-in function; pow - - These methods are called to implement the binary arithmetic operations - (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, - :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``). For instance, to - evaluate the expression ``x + y``, where *x* is an instance of a class that - has an :meth:`__add__` method, ``type(x).__add__(x, y)`` is called. The - :meth:`__divmod__` method should be the equivalent to using - :meth:`__floordiv__` and :meth:`__mod__`; it should not be related to - :meth:`__truediv__`. Note that :meth:`__pow__` should be defined to accept - an optional third argument if the ternary version of the built-in :func:`pow` - function is to be supported. - - If one of those methods does not support the operation with the supplied - arguments, it should return ``NotImplemented``. - - -.. method:: object.__radd__(self, other) - object.__rsub__(self, other) - object.__rmul__(self, other) - object.__rmatmul__(self, other) - object.__rtruediv__(self, other) - object.__rfloordiv__(self, other) - object.__rmod__(self, other) - object.__rdivmod__(self, other) - object.__rpow__(self, other[, modulo]) - object.__rlshift__(self, other) - object.__rrshift__(self, other) - object.__rand__(self, other) - object.__rxor__(self, other) - object.__ror__(self, other) - - .. index:: - pair: built-in function; divmod - pair: built-in function; pow - - These methods are called to implement the binary arithmetic operations - (``+``, ``-``, ``*``, ``@``, ``/``, ``//``, ``%``, :func:`divmod`, - :func:`pow`, ``**``, ``<<``, ``>>``, ``&``, ``^``, ``|``) with reflected - (swapped) operands. These functions are only called if the left operand does - not support the corresponding operation [#]_ and the operands are of different - types. [#]_ For instance, to evaluate the expression ``x - y``, where *y* is - an instance of a class that has an :meth:`__rsub__` method, - ``type(y).__rsub__(y, x)`` is called if ``type(x).__sub__(x, y)`` returns - *NotImplemented*. - - .. index:: pair: built-in function; pow - - Note that ternary :func:`pow` will not try calling :meth:`__rpow__` (the - coercion rules would become too complicated). - - .. note:: - - If the right operand's type is a subclass of the left operand's type and - that subclass provides a different implementation of the reflected method - for the operation, this method will be called before the left operand's - non-reflected method. This behavior allows subclasses to override their - ancestors' operations. - - -.. method:: object.__iadd__(self, other) - object.__isub__(self, other) - object.__imul__(self, other) - object.__imatmul__(self, other) - object.__itruediv__(self, other) - object.__ifloordiv__(self, other) - object.__imod__(self, other) - object.__ipow__(self, other[, modulo]) - object.__ilshift__(self, other) - object.__irshift__(self, other) - object.__iand__(self, other) - object.__ixor__(self, other) - object.__ior__(self, other) - - These methods are called to implement the augmented arithmetic assignments - (``+=``, ``-=``, ``*=``, ``@=``, ``/=``, ``//=``, ``%=``, ``**=``, ``<<=``, - ``>>=``, ``&=``, ``^=``, ``|=``). These methods should attempt to do the - operation in-place (modifying *self*) and return the result (which could be, - but does not have to be, *self*). If a specific method is not defined, the - augmented assignment falls back to the normal methods. For instance, if *x* - is an instance of a class with an :meth:`__iadd__` method, ``x += y`` is - equivalent to ``x = x.__iadd__(y)`` . Otherwise, ``x.__add__(y)`` and - ``y.__radd__(x)`` are considered, as with the evaluation of ``x + y``. In - certain situations, augmented assignment can result in unexpected errors (see - :ref:`faq-augmented-assignment-tuple-error`), but this behavior is in fact - part of the data model. - - -.. method:: object.__neg__(self) - object.__pos__(self) - object.__abs__(self) - object.__invert__(self) - - .. index:: pair: built-in function; abs - - Called to implement the unary arithmetic operations (``-``, ``+``, :func:`abs` - and ``~``). - - -.. method:: object.__complex__(self) - object.__int__(self) - object.__float__(self) - - .. index:: - pair: built-in function; complex - pair: built-in function; int - pair: built-in function; float - - Called to implement the built-in functions :func:`complex`, - :func:`int` and :func:`float`. Should return a value - of the appropriate type. - - -.. method:: object.__index__(self) - - Called to implement :func:`operator.index`, and whenever Python needs to - losslessly convert the numeric object to an integer object (such as in - slicing, or in the built-in :func:`bin`, :func:`hex` and :func:`oct` - functions). Presence of this method indicates that the numeric object is - an integer type. Must return an integer. - - If :meth:`__int__`, :meth:`__float__` and :meth:`__complex__` are not - defined then corresponding built-in functions :func:`int`, :func:`float` - and :func:`complex` fall back to :meth:`__index__`. - - -.. method:: object.__round__(self, [,ndigits]) - object.__trunc__(self) - object.__floor__(self) - object.__ceil__(self) - - .. index:: pair: built-in function; round - - Called to implement the built-in function :func:`round` and :mod:`math` - functions :func:`~math.trunc`, :func:`~math.floor` and :func:`~math.ceil`. - Unless *ndigits* is passed to :meth:`!__round__` all these methods should - return the value of the object truncated to an :class:`~numbers.Integral` - (typically an :class:`int`). - - The built-in function :func:`int` falls back to :meth:`__trunc__` if neither - :meth:`__int__` nor :meth:`__index__` is defined. - - .. versionchanged:: 3.11 - The delegation of :func:`int` to :meth:`__trunc__` is deprecated. - - -.. _context-managers: - -With Statement Context Managers -------------------------------- - -A :dfn:`context manager` is an object that defines the runtime context to be -established when executing a :keyword:`with` statement. The context manager -handles the entry into, and the exit from, the desired runtime context for the -execution of the block of code. Context managers are normally invoked using the -:keyword:`!with` statement (described in section :ref:`with`), but can also be -used by directly invoking their methods. - -.. index:: - pair: statement; with - single: context manager - -Typical uses of context managers include saving and restoring various kinds of -global state, locking and unlocking resources, closing opened files, etc. - -For more information on context managers, see :ref:`typecontextmanager`. - - -.. method:: object.__enter__(self) - - Enter the runtime context related to this object. The :keyword:`with` statement - will bind this method's return value to the target(s) specified in the - :keyword:`!as` clause of the statement, if any. - - -.. method:: object.__exit__(self, exc_type, exc_value, traceback) - - Exit the runtime context related to this object. The parameters describe the - exception that caused the context to be exited. If the context was exited - without an exception, all three arguments will be :const:`None`. - - If an exception is supplied, and the method wishes to suppress the exception - (i.e., prevent it from being propagated), it should return a true value. - Otherwise, the exception will be processed normally upon exit from this method. - - Note that :meth:`~object.__exit__` methods should not reraise the passed-in exception; - this is the caller's responsibility. - - -.. seealso:: - - :pep:`343` - The "with" statement - The specification, background, and examples for the Python :keyword:`with` - statement. - - -.. _class-pattern-matching: - -Customizing positional arguments in class pattern matching ----------------------------------------------------------- - -When using a class name in a pattern, positional arguments in the pattern are not -allowed by default, i.e. ``case MyClass(x, y)`` is typically invalid without special -support in ``MyClass``. To be able to use that kind of pattern, the class needs to -define a *__match_args__* attribute. - -.. data:: object.__match_args__ - - This class variable can be assigned a tuple of strings. When this class is - used in a class pattern with positional arguments, each positional argument will - be converted into a keyword argument, using the corresponding value in - *__match_args__* as the keyword. The absence of this attribute is equivalent to - setting it to ``()``. - -For example, if ``MyClass.__match_args__`` is ``("left", "center", "right")`` that means -that ``case MyClass(x, y)`` is equivalent to ``case MyClass(left=x, center=y)``. Note -that the number of arguments in the pattern must be smaller than or equal to the number -of elements in *__match_args__*; if it is larger, the pattern match attempt will raise -a :exc:`TypeError`. - -.. versionadded:: 3.10 - -.. seealso:: - - :pep:`634` - Structural Pattern Matching - The specification for the Python ``match`` statement. - - -.. _special-lookup: - -Special method lookup ---------------------- - -For custom classes, implicit invocations of special methods are only guaranteed -to work correctly if defined on an object's type, not in the object's instance -dictionary. That behaviour is the reason why the following code raises an -exception:: - - >>> class C: - ... pass - ... - >>> c = C() - >>> c.__len__ = lambda: 5 - >>> len(c) - Traceback (most recent call last): - File "", line 1, in - TypeError: object of type 'C' has no len() - -The rationale behind this behaviour lies with a number of special methods such -as :meth:`~object.__hash__` and :meth:`~object.__repr__` that are implemented -by all objects, -including type objects. If the implicit lookup of these methods used the -conventional lookup process, they would fail when invoked on the type object -itself:: - - >>> 1 .__hash__() == hash(1) - True - >>> int.__hash__() == hash(int) - Traceback (most recent call last): - File "", line 1, in - TypeError: descriptor '__hash__' of 'int' object needs an argument - -Incorrectly attempting to invoke an unbound method of a class in this way is -sometimes referred to as 'metaclass confusion', and is avoided by bypassing -the instance when looking up special methods:: - - >>> type(1).__hash__(1) == hash(1) - True - >>> type(int).__hash__(int) == hash(int) - True - -In addition to bypassing any instance attributes in the interest of -correctness, implicit special method lookup generally also bypasses the -:meth:`~object.__getattribute__` method even of the object's metaclass:: - - >>> class Meta(type): - ... def __getattribute__(*args): - ... print("Metaclass getattribute invoked") - ... return type.__getattribute__(*args) - ... - >>> class C(object, metaclass=Meta): - ... def __len__(self): - ... return 10 - ... def __getattribute__(*args): - ... print("Class getattribute invoked") - ... return object.__getattribute__(*args) - ... - >>> c = C() - >>> c.__len__() # Explicit lookup via instance - Class getattribute invoked - 10 - >>> type(c).__len__(c) # Explicit lookup via type - Metaclass getattribute invoked - 10 - >>> len(c) # Implicit lookup - 10 - -Bypassing the :meth:`~object.__getattribute__` machinery in this fashion -provides significant scope for speed optimisations within the -interpreter, at the cost of some flexibility in the handling of -special methods (the special method *must* be set on the class -object itself in order to be consistently invoked by the interpreter). - - -.. index:: - single: coroutine - -Coroutines -========== - - -Awaitable Objects ------------------ - -An :term:`awaitable` object generally implements an :meth:`~object.__await__` method. -:term:`Coroutine objects ` returned from :keyword:`async def` functions -are awaitable. - -.. note:: - - The :term:`generator iterator` objects returned from generators - decorated with :func:`types.coroutine` - are also awaitable, but they do not implement :meth:`~object.__await__`. - -.. method:: object.__await__(self) - - Must return an :term:`iterator`. Should be used to implement - :term:`awaitable` objects. For instance, :class:`asyncio.Future` implements - this method to be compatible with the :keyword:`await` expression. - - .. note:: - - The language doesn't place any restriction on the type or value of the - objects yielded by the iterator returned by ``__await__``, as this is - specific to the implementation of the asynchronous execution framework - (e.g. :mod:`asyncio`) that will be managing the :term:`awaitable` object. - - -.. versionadded:: 3.5 - -.. seealso:: :pep:`492` for additional information about awaitable objects. - - -.. _coroutine-objects: - -Coroutine Objects ------------------ - -:term:`Coroutine objects ` are :term:`awaitable` objects. -A coroutine's execution can be controlled by calling :meth:`~object.__await__` and -iterating over the result. When the coroutine has finished executing and -returns, the iterator raises :exc:`StopIteration`, and the exception's -:attr:`~StopIteration.value` attribute holds the return value. If the -coroutine raises an exception, it is propagated by the iterator. Coroutines -should not directly raise unhandled :exc:`StopIteration` exceptions. - -Coroutines also have the methods listed below, which are analogous to -those of generators (see :ref:`generator-methods`). However, unlike -generators, coroutines do not directly support iteration. - -.. versionchanged:: 3.5.2 - It is a :exc:`RuntimeError` to await on a coroutine more than once. - - -.. method:: coroutine.send(value) - - Starts or resumes execution of the coroutine. If *value* is ``None``, - this is equivalent to advancing the iterator returned by - :meth:`~object.__await__`. If *value* is not ``None``, this method delegates - to the :meth:`~generator.send` method of the iterator that caused - the coroutine to suspend. The result (return value, - :exc:`StopIteration`, or other exception) is the same as when - iterating over the :meth:`__await__` return value, described above. - -.. method:: coroutine.throw(value) - coroutine.throw(type[, value[, traceback]]) - - Raises the specified exception in the coroutine. This method delegates - to the :meth:`~generator.throw` method of the iterator that caused - the coroutine to suspend, if it has such a method. Otherwise, - the exception is raised at the suspension point. The result - (return value, :exc:`StopIteration`, or other exception) is the same as - when iterating over the :meth:`~object.__await__` return value, described - above. If the exception is not caught in the coroutine, it propagates - back to the caller. - -.. method:: coroutine.close() - - Causes the coroutine to clean itself up and exit. If the coroutine - is suspended, this method first delegates to the :meth:`~generator.close` - method of the iterator that caused the coroutine to suspend, if it - has such a method. Then it raises :exc:`GeneratorExit` at the - suspension point, causing the coroutine to immediately clean itself up. - Finally, the coroutine is marked as having finished executing, even if - it was never started. - - Coroutine objects are automatically closed using the above process when - they are about to be destroyed. - -.. _async-iterators: - -Asynchronous Iterators ----------------------- - -An *asynchronous iterator* can call asynchronous code in -its ``__anext__`` method. - -Asynchronous iterators can be used in an :keyword:`async for` statement. - -.. method:: object.__aiter__(self) - - Must return an *asynchronous iterator* object. - -.. method:: object.__anext__(self) - - Must return an *awaitable* resulting in a next value of the iterator. Should - raise a :exc:`StopAsyncIteration` error when the iteration is over. - -An example of an asynchronous iterable object:: - - class Reader: - async def readline(self): - ... - - def __aiter__(self): - return self - - async def __anext__(self): - val = await self.readline() - if val == b'': - raise StopAsyncIteration - return val - -.. versionadded:: 3.5 - -.. versionchanged:: 3.7 - Prior to Python 3.7, :meth:`~object.__aiter__` could return an *awaitable* - that would resolve to an - :term:`asynchronous iterator `. - - Starting with Python 3.7, :meth:`~object.__aiter__` must return an - asynchronous iterator object. Returning anything else - will result in a :exc:`TypeError` error. - - -.. _async-context-managers: - -Asynchronous Context Managers ------------------------------ - -An *asynchronous context manager* is a *context manager* that is able to -suspend execution in its ``__aenter__`` and ``__aexit__`` methods. - -Asynchronous context managers can be used in an :keyword:`async with` statement. - -.. method:: object.__aenter__(self) - - Semantically similar to :meth:`~object.__enter__`, the only - difference being that it must return an *awaitable*. - -.. method:: object.__aexit__(self, exc_type, exc_value, traceback) - - Semantically similar to :meth:`~object.__exit__`, the only - difference being that it must return an *awaitable*. - -An example of an asynchronous context manager class:: - - class AsyncContextManager: - async def __aenter__(self): - await log('entering context') - - async def __aexit__(self, exc_type, exc, tb): - await log('exiting context') - -.. versionadded:: 3.5 - - -.. rubric:: Footnotes - -.. [#] It *is* possible in some cases to change an object's type, under certain - controlled conditions. It generally isn't a good idea though, since it can - lead to some very strange behaviour if it is handled incorrectly. - -.. [#] The :meth:`~object.__hash__`, :meth:`~object.__iter__`, - :meth:`~object.__reversed__`, and :meth:`~object.__contains__` methods have - special handling for this; others - will still raise a :exc:`TypeError`, but may do so by relying on - the behavior that ``None`` is not callable. - -.. [#] "Does not support" here means that the class has no such method, or - the method returns ``NotImplemented``. Do not set the method to - ``None`` if you want to force fallback to the right operand's reflected - method—that will instead have the opposite effect of explicitly - *blocking* such fallback. - -.. [#] For operands of the same type, it is assumed that if the non-reflected - method -- such as :meth:`~object.__add__` -- fails then the overall - operation is not - supported, which is why the reflected method is not called. diff --git a/Doc/reference/executionmodel.rst b/Doc/reference/executionmodel.rst deleted file mode 100644 index 8917243999d399..00000000000000 --- a/Doc/reference/executionmodel.rst +++ /dev/null @@ -1,281 +0,0 @@ - -.. _execmodel: - -*************** -Execution model -*************** - -.. index:: - single: execution model - pair: code; block - -.. _prog_structure: - -Structure of a program -====================== - -.. index:: block - -A Python program is constructed from code blocks. -A :dfn:`block` is a piece of Python program text that is executed as a unit. -The following are blocks: a module, a function body, and a class definition. -Each command typed interactively is a block. A script file (a file given as -standard input to the interpreter or specified as a command line argument to the -interpreter) is a code block. A script command (a command specified on the -interpreter command line with the :option:`-c` option) is a code block. -A module run as a top level script (as module ``__main__``) from the command -line using a :option:`-m` argument is also a code block. The string -argument passed to the built-in functions :func:`eval` and :func:`exec` is a -code block. - -.. index:: pair: execution; frame - -A code block is executed in an :dfn:`execution frame`. A frame contains some -administrative information (used for debugging) and determines where and how -execution continues after the code block's execution has completed. - -.. _naming: - -Naming and binding -================== - -.. index:: - single: namespace - single: scope - -.. _bind_names: - -Binding of names ----------------- - -.. index:: - single: name - pair: binding; name - -:dfn:`Names` refer to objects. Names are introduced by name binding operations. - -.. index:: single: from; import statement - -The following constructs bind names: - -* formal parameters to functions, -* class definitions, -* function definitions, -* assignment expressions, -* :ref:`targets ` that are identifiers if occurring in - an assignment: - - + :keyword:`for` loop header, - + after :keyword:`!as` in a :keyword:`with` statement, :keyword:`except` - clause, :keyword:`except* ` clause, or in the as-pattern in structural pattern matching, - + in a capture pattern in structural pattern matching - -* :keyword:`import` statements. - -The :keyword:`!import` statement of the form ``from ... import *`` binds all -names defined in the imported module, except those beginning with an underscore. -This form may only be used at the module level. - -A target occurring in a :keyword:`del` statement is also considered bound for -this purpose (though the actual semantics are to unbind the name). - -Each assignment or import statement occurs within a block defined by a class or -function definition or at the module level (the top-level code block). - -.. index:: pair: free; variable - -If a name is bound in a block, it is a local variable of that block, unless -declared as :keyword:`nonlocal` or :keyword:`global`. If a name is bound at -the module level, it is a global variable. (The variables of the module code -block are local and global.) If a variable is used in a code block but not -defined there, it is a :dfn:`free variable`. - -Each occurrence of a name in the program text refers to the :dfn:`binding` of -that name established by the following name resolution rules. - -.. _resolve_names: - -Resolution of names -------------------- - -.. index:: scope - -A :dfn:`scope` defines the visibility of a name within a block. If a local -variable is defined in a block, its scope includes that block. If the -definition occurs in a function block, the scope extends to any blocks contained -within the defining one, unless a contained block introduces a different binding -for the name. - -.. index:: single: environment - -When a name is used in a code block, it is resolved using the nearest enclosing -scope. The set of all such scopes visible to a code block is called the block's -:dfn:`environment`. - -.. index:: - single: NameError (built-in exception) - single: UnboundLocalError - -When a name is not found at all, a :exc:`NameError` exception is raised. -If the current scope is a function scope, and the name refers to a local -variable that has not yet been bound to a value at the point where the name is -used, an :exc:`UnboundLocalError` exception is raised. -:exc:`UnboundLocalError` is a subclass of :exc:`NameError`. - -If a name binding operation occurs anywhere within a code block, all uses of the -name within the block are treated as references to the current block. This can -lead to errors when a name is used within a block before it is bound. This rule -is subtle. Python lacks declarations and allows name binding operations to -occur anywhere within a code block. The local variables of a code block can be -determined by scanning the entire text of the block for name binding operations. -See :ref:`the FAQ entry on UnboundLocalError ` -for examples. - -If the :keyword:`global` statement occurs within a block, all uses of the names -specified in the statement refer to the bindings of those names in the top-level -namespace. Names are resolved in the top-level namespace by searching the -global namespace, i.e. the namespace of the module containing the code block, -and the builtins namespace, the namespace of the module :mod:`builtins`. The -global namespace is searched first. If the names are not found there, the -builtins namespace is searched. The :keyword:`!global` statement must precede -all uses of the listed names. - -The :keyword:`global` statement has the same scope as a name binding operation -in the same block. If the nearest enclosing scope for a free variable contains -a global statement, the free variable is treated as a global. - -.. XXX say more about "nonlocal" semantics here - -The :keyword:`nonlocal` statement causes corresponding names to refer -to previously bound variables in the nearest enclosing function scope. -:exc:`SyntaxError` is raised at compile time if the given name does not -exist in any enclosing function scope. - -.. index:: pair: module; __main__ - -The namespace for a module is automatically created the first time a module is -imported. The main module for a script is always called :mod:`__main__`. - -Class definition blocks and arguments to :func:`exec` and :func:`eval` are -special in the context of name resolution. -A class definition is an executable statement that may use and define names. -These references follow the normal rules for name resolution with an exception -that unbound local variables are looked up in the global namespace. -The namespace of the class definition becomes the attribute dictionary of -the class. The scope of names defined in a class block is limited to the -class block; it does not extend to the code blocks of methods -- this includes -comprehensions and generator expressions since they are implemented using a -function scope. This means that the following will fail:: - - class A: - a = 42 - b = list(a + i for i in range(10)) - -.. _restrict_exec: - -Builtins and restricted execution ---------------------------------- - -.. index:: pair: restricted; execution - -.. impl-detail:: - - Users should not touch ``__builtins__``; it is strictly an implementation - detail. Users wanting to override values in the builtins namespace should - :keyword:`import` the :mod:`builtins` module and modify its - attributes appropriately. - -The builtins namespace associated with the execution of a code block -is actually found by looking up the name ``__builtins__`` in its -global namespace; this should be a dictionary or a module (in the -latter case the module's dictionary is used). By default, when in the -:mod:`__main__` module, ``__builtins__`` is the built-in module -:mod:`builtins`; when in any other module, ``__builtins__`` is an -alias for the dictionary of the :mod:`builtins` module itself. - - -.. _dynamic-features: - -Interaction with dynamic features ---------------------------------- - -Name resolution of free variables occurs at runtime, not at compile time. -This means that the following code will print 42:: - - i = 10 - def f(): - print(i) - i = 42 - f() - -.. XXX from * also invalid with relative imports (at least currently) - -The :func:`eval` and :func:`exec` functions do not have access to the full -environment for resolving names. Names may be resolved in the local and global -namespaces of the caller. Free variables are not resolved in the nearest -enclosing namespace, but in the global namespace. [#]_ The :func:`exec` and -:func:`eval` functions have optional arguments to override the global and local -namespace. If only one namespace is specified, it is used for both. - - -.. _exceptions: - -Exceptions -========== - -.. index:: single: exception - -.. index:: - single: raise an exception - single: handle an exception - single: exception handler - single: errors - single: error handling - -Exceptions are a means of breaking out of the normal flow of control of a code -block in order to handle errors or other exceptional conditions. An exception -is *raised* at the point where the error is detected; it may be *handled* by the -surrounding code block or by any code block that directly or indirectly invoked -the code block where the error occurred. - -The Python interpreter raises an exception when it detects a run-time error -(such as division by zero). A Python program can also explicitly raise an -exception with the :keyword:`raise` statement. Exception handlers are specified -with the :keyword:`try` ... :keyword:`except` statement. The :keyword:`finally` -clause of such a statement can be used to specify cleanup code which does not -handle the exception, but is executed whether an exception occurred or not in -the preceding code. - -.. index:: single: termination model - -Python uses the "termination" model of error handling: an exception handler can -find out what happened and continue execution at an outer level, but it cannot -repair the cause of the error and retry the failing operation (except by -re-entering the offending piece of code from the top). - -.. index:: single: SystemExit (built-in exception) - -When an exception is not handled at all, the interpreter terminates execution of -the program, or returns to its interactive main loop. In either case, it prints -a stack traceback, except when the exception is :exc:`SystemExit`. - -Exceptions are identified by class instances. The :keyword:`except` clause is -selected depending on the class of the instance: it must reference the class of -the instance or a :term:`non-virtual base class ` thereof. -The instance can be received by the handler and can carry additional information -about the exceptional condition. - -.. note:: - - Exception messages are not part of the Python API. Their contents may change - from one version of Python to the next without warning and should not be - relied on by code which will run under multiple versions of the interpreter. - -See also the description of the :keyword:`try` statement in section :ref:`try` -and :keyword:`raise` statement in section :ref:`raise`. - - -.. rubric:: Footnotes - -.. [#] This limitation occurs because the code that is executed by these operations - is not available at the time the module is compiled. diff --git a/Doc/reference/expressions.rst b/Doc/reference/expressions.rst deleted file mode 100644 index 5cba4a6934f05b..00000000000000 --- a/Doc/reference/expressions.rst +++ /dev/null @@ -1,2009 +0,0 @@ - -.. _expressions: - -*********** -Expressions -*********** - -.. index:: expression, BNF - -This chapter explains the meaning of the elements of expressions in Python. - -**Syntax Notes:** In this and the following chapters, extended BNF notation will -be used to describe syntax, not lexical analysis. When (one alternative of) a -syntax rule has the form - -.. productionlist:: python-grammar - name: `othername` - -and no semantics are given, the semantics of this form of ``name`` are the same -as for ``othername``. - - -.. _conversions: - -Arithmetic conversions -====================== - -.. index:: pair: arithmetic; conversion - -When a description of an arithmetic operator below uses the phrase "the numeric -arguments are converted to a common type", this means that the operator -implementation for built-in types works as follows: - -* If either argument is a complex number, the other is converted to complex; - -* otherwise, if either argument is a floating point number, the other is - converted to floating point; - -* otherwise, both must be integers and no conversion is necessary. - -Some additional rules apply for certain operators (e.g., a string as a left -argument to the '%' operator). Extensions must define their own conversion -behavior. - - -.. _atoms: - -Atoms -===== - -.. index:: atom - -Atoms are the most basic elements of expressions. The simplest atoms are -identifiers or literals. Forms enclosed in parentheses, brackets or braces are -also categorized syntactically as atoms. The syntax for atoms is: - -.. productionlist:: python-grammar - atom: `identifier` | `literal` | `enclosure` - enclosure: `parenth_form` | `list_display` | `dict_display` | `set_display` - : | `generator_expression` | `yield_atom` - - -.. _atom-identifiers: - -Identifiers (Names) -------------------- - -.. index:: name, identifier - -An identifier occurring as an atom is a name. See section :ref:`identifiers` -for lexical definition and section :ref:`naming` for documentation of naming and -binding. - -.. index:: pair: exception; NameError - -When the name is bound to an object, evaluation of the atom yields that object. -When a name is not bound, an attempt to evaluate it raises a :exc:`NameError` -exception. - -.. _private-name-mangling: - -.. index:: - pair: name; mangling - pair: private; names - -**Private name mangling:** When an identifier that textually occurs in a class -definition begins with two or more underscore characters and does not end in two -or more underscores, it is considered a :dfn:`private name` of that class. -Private names are transformed to a longer form before code is generated for -them. The transformation inserts the class name, with leading underscores -removed and a single underscore inserted, in front of the name. For example, -the identifier ``__spam`` occurring in a class named ``Ham`` will be transformed -to ``_Ham__spam``. This transformation is independent of the syntactical -context in which the identifier is used. If the transformed name is extremely -long (longer than 255 characters), implementation defined truncation may happen. -If the class name consists only of underscores, no transformation is done. - - -.. _atom-literals: - -Literals --------- - -.. index:: single: literal - -Python supports string and bytes literals and various numeric literals: - -.. productionlist:: python-grammar - literal: `stringliteral` | `bytesliteral` - : | `integer` | `floatnumber` | `imagnumber` - -Evaluation of a literal yields an object of the given type (string, bytes, -integer, floating point number, complex number) with the given value. The value -may be approximated in the case of floating point and imaginary (complex) -literals. See section :ref:`literals` for details. - -.. index:: - triple: immutable; data; type - pair: immutable; object - -All literals correspond to immutable data types, and hence the object's identity -is less important than its value. Multiple evaluations of literals with the -same value (either the same occurrence in the program text or a different -occurrence) may obtain the same object or a different object with the same -value. - - -.. _parenthesized: - -Parenthesized forms -------------------- - -.. index:: - single: parenthesized form - single: () (parentheses); tuple display - -A parenthesized form is an optional expression list enclosed in parentheses: - -.. productionlist:: python-grammar - parenth_form: "(" [`starred_expression`] ")" - -A parenthesized expression list yields whatever that expression list yields: if -the list contains at least one comma, it yields a tuple; otherwise, it yields -the single expression that makes up the expression list. - -.. index:: pair: empty; tuple - -An empty pair of parentheses yields an empty tuple object. Since tuples are -immutable, the same rules as for literals apply (i.e., two occurrences of the empty -tuple may or may not yield the same object). - -.. index:: - single: comma - single: , (comma) - -Note that tuples are not formed by the parentheses, but rather by use of the -comma. The exception is the empty tuple, for which parentheses *are* -required --- allowing unparenthesized "nothing" in expressions would cause -ambiguities and allow common typos to pass uncaught. - - -.. _comprehensions: - -Displays for lists, sets and dictionaries ------------------------------------------ - -.. index:: single: comprehensions - -For constructing a list, a set or a dictionary Python provides special syntax -called "displays", each of them in two flavors: - -* either the container contents are listed explicitly, or - -* they are computed via a set of looping and filtering instructions, called a - :dfn:`comprehension`. - -.. index:: - single: for; in comprehensions - single: if; in comprehensions - single: async for; in comprehensions - -Common syntax elements for comprehensions are: - -.. productionlist:: python-grammar - comprehension: `assignment_expression` `comp_for` - comp_for: ["async"] "for" `target_list` "in" `or_test` [`comp_iter`] - comp_iter: `comp_for` | `comp_if` - comp_if: "if" `or_test` [`comp_iter`] - -The comprehension consists of a single expression followed by at least one -:keyword:`!for` clause and zero or more :keyword:`!for` or :keyword:`!if` clauses. -In this case, the elements of the new container are those that would be produced -by considering each of the :keyword:`!for` or :keyword:`!if` clauses a block, -nesting from left to right, and evaluating the expression to produce an element -each time the innermost block is reached. - -However, aside from the iterable expression in the leftmost :keyword:`!for` clause, -the comprehension is executed in a separate implicitly nested scope. This ensures -that names assigned to in the target list don't "leak" into the enclosing scope. - -The iterable expression in the leftmost :keyword:`!for` clause is evaluated -directly in the enclosing scope and then passed as an argument to the implicitly -nested scope. Subsequent :keyword:`!for` clauses and any filter condition in the -leftmost :keyword:`!for` clause cannot be evaluated in the enclosing scope as -they may depend on the values obtained from the leftmost iterable. For example: -``[x*y for x in range(10) for y in range(x, x+10)]``. - -To ensure the comprehension always results in a container of the appropriate -type, ``yield`` and ``yield from`` expressions are prohibited in the implicitly -nested scope. - -.. index:: - single: await; in comprehensions - -Since Python 3.6, in an :keyword:`async def` function, an :keyword:`!async for` -clause may be used to iterate over a :term:`asynchronous iterator`. -A comprehension in an :keyword:`!async def` function may consist of either a -:keyword:`!for` or :keyword:`!async for` clause following the leading -expression, may contain additional :keyword:`!for` or :keyword:`!async for` -clauses, and may also use :keyword:`await` expressions. -If a comprehension contains either :keyword:`!async for` clauses or -:keyword:`!await` expressions or other asynchronous comprehensions it is called -an :dfn:`asynchronous comprehension`. An asynchronous comprehension may -suspend the execution of the coroutine function in which it appears. -See also :pep:`530`. - -.. versionadded:: 3.6 - Asynchronous comprehensions were introduced. - -.. versionchanged:: 3.8 - ``yield`` and ``yield from`` prohibited in the implicitly nested scope. - -.. versionchanged:: 3.11 - Asynchronous comprehensions are now allowed inside comprehensions in - asynchronous functions. Outer comprehensions implicitly become - asynchronous. - - -.. _lists: - -List displays -------------- - -.. index:: - pair: list; display - pair: list; comprehensions - pair: empty; list - pair: object; list - single: [] (square brackets); list expression - single: , (comma); expression list - -A list display is a possibly empty series of expressions enclosed in square -brackets: - -.. productionlist:: python-grammar - list_display: "[" [`starred_list` | `comprehension`] "]" - -A list display yields a new list object, the contents being specified by either -a list of expressions or a comprehension. When a comma-separated list of -expressions is supplied, its elements are evaluated from left to right and -placed into the list object in that order. When a comprehension is supplied, -the list is constructed from the elements resulting from the comprehension. - - -.. _set: - -Set displays ------------- - -.. index:: - pair: set; display - pair: set; comprehensions - pair: object; set - single: {} (curly brackets); set expression - single: , (comma); expression list - -A set display is denoted by curly braces and distinguishable from dictionary -displays by the lack of colons separating keys and values: - -.. productionlist:: python-grammar - set_display: "{" (`starred_list` | `comprehension`) "}" - -A set display yields a new mutable set object, the contents being specified by -either a sequence of expressions or a comprehension. When a comma-separated -list of expressions is supplied, its elements are evaluated from left to right -and added to the set object. When a comprehension is supplied, the set is -constructed from the elements resulting from the comprehension. - -An empty set cannot be constructed with ``{}``; this literal constructs an empty -dictionary. - - -.. _dict: - -Dictionary displays -------------------- - -.. index:: - pair: dictionary; display - pair: dictionary; comprehensions - key, value, key/value pair - pair: object; dictionary - single: {} (curly brackets); dictionary expression - single: : (colon); in dictionary expressions - single: , (comma); in dictionary displays - -A dictionary display is a possibly empty series of dict items (key/value pairs) -enclosed in curly braces: - -.. productionlist:: python-grammar - dict_display: "{" [`dict_item_list` | `dict_comprehension`] "}" - dict_item_list: `dict_item` ("," `dict_item`)* [","] - dict_item: `expression` ":" `expression` | "**" `or_expr` - dict_comprehension: `expression` ":" `expression` `comp_for` - -A dictionary display yields a new dictionary object. - -If a comma-separated sequence of dict items is given, they are evaluated -from left to right to define the entries of the dictionary: each key object is -used as a key into the dictionary to store the corresponding value. This means -that you can specify the same key multiple times in the dict item list, and the -final dictionary's value for that key will be the last one given. - -.. index:: - unpacking; dictionary - single: **; in dictionary displays - -A double asterisk ``**`` denotes :dfn:`dictionary unpacking`. -Its operand must be a :term:`mapping`. Each mapping item is added -to the new dictionary. Later values replace values already set by -earlier dict items and earlier dictionary unpackings. - -.. versionadded:: 3.5 - Unpacking into dictionary displays, originally proposed by :pep:`448`. - -A dict comprehension, in contrast to list and set comprehensions, needs two -expressions separated with a colon followed by the usual "for" and "if" clauses. -When the comprehension is run, the resulting key and value elements are inserted -in the new dictionary in the order they are produced. - -.. index:: pair: immutable; object - hashable - -Restrictions on the types of the key values are listed earlier in section -:ref:`types`. (To summarize, the key type should be :term:`hashable`, which excludes -all mutable objects.) Clashes between duplicate keys are not detected; the last -value (textually rightmost in the display) stored for a given key value -prevails. - -.. versionchanged:: 3.8 - Prior to Python 3.8, in dict comprehensions, the evaluation order of key - and value was not well-defined. In CPython, the value was evaluated before - the key. Starting with 3.8, the key is evaluated before the value, as - proposed by :pep:`572`. - - -.. _genexpr: - -Generator expressions ---------------------- - -.. index:: - pair: generator; expression - pair: object; generator - single: () (parentheses); generator expression - -A generator expression is a compact generator notation in parentheses: - -.. productionlist:: python-grammar - generator_expression: "(" `expression` `comp_for` ")" - -A generator expression yields a new generator object. Its syntax is the same as -for comprehensions, except that it is enclosed in parentheses instead of -brackets or curly braces. - -Variables used in the generator expression are evaluated lazily when the -:meth:`~generator.__next__` method is called for the generator object (in the same -fashion as normal generators). However, the iterable expression in the -leftmost :keyword:`!for` clause is immediately evaluated, so that an error -produced by it will be emitted at the point where the generator expression -is defined, rather than at the point where the first value is retrieved. -Subsequent :keyword:`!for` clauses and any filter condition in the leftmost -:keyword:`!for` clause cannot be evaluated in the enclosing scope as they may -depend on the values obtained from the leftmost iterable. For example: -``(x*y for x in range(10) for y in range(x, x+10))``. - -The parentheses can be omitted on calls with only one argument. See section -:ref:`calls` for details. - -To avoid interfering with the expected operation of the generator expression -itself, ``yield`` and ``yield from`` expressions are prohibited in the -implicitly defined generator. - -If a generator expression contains either :keyword:`!async for` -clauses or :keyword:`await` expressions it is called an -:dfn:`asynchronous generator expression`. An asynchronous generator -expression returns a new asynchronous generator object, -which is an asynchronous iterator (see :ref:`async-iterators`). - -.. versionadded:: 3.6 - Asynchronous generator expressions were introduced. - -.. versionchanged:: 3.7 - Prior to Python 3.7, asynchronous generator expressions could - only appear in :keyword:`async def` coroutines. Starting - with 3.7, any function can use asynchronous generator expressions. - -.. versionchanged:: 3.8 - ``yield`` and ``yield from`` prohibited in the implicitly nested scope. - - -.. _yieldexpr: - -Yield expressions ------------------ - -.. index:: - pair: keyword; yield - pair: keyword; from - pair: yield; expression - pair: generator; function - -.. productionlist:: python-grammar - yield_atom: "(" `yield_expression` ")" - yield_expression: "yield" [`expression_list` | "from" `expression`] - -The yield expression is used when defining a :term:`generator` function -or an :term:`asynchronous generator` function and -thus can only be used in the body of a function definition. Using a yield -expression in a function's body causes that function to be a generator function, -and using it in an :keyword:`async def` function's body causes that -coroutine function to be an asynchronous generator function. For example:: - - def gen(): # defines a generator function - yield 123 - - async def agen(): # defines an asynchronous generator function - yield 123 - -Due to their side effects on the containing scope, ``yield`` expressions -are not permitted as part of the implicitly defined scopes used to -implement comprehensions and generator expressions. - -.. versionchanged:: 3.8 - Yield expressions prohibited in the implicitly nested scopes used to - implement comprehensions and generator expressions. - -Generator functions are described below, while asynchronous generator -functions are described separately in section -:ref:`asynchronous-generator-functions`. - -When a generator function is called, it returns an iterator known as a -generator. That generator then controls the execution of the generator -function. The execution starts when one of the generator's methods is called. -At that time, the execution proceeds to the first yield expression, where it is -suspended again, returning the value of :token:`~python-grammar:expression_list` -to the generator's caller, -or ``None`` if :token:`~python-grammar:expression_list` is omitted. -By suspended, we mean that all local state is -retained, including the current bindings of local variables, the instruction -pointer, the internal evaluation stack, and the state of any exception handling. -When the execution is resumed by calling one of the generator's methods, the -function can proceed exactly as if the yield expression were just another -external call. The value of the yield expression after resuming depends on the -method which resumed the execution. If :meth:`~generator.__next__` is used -(typically via either a :keyword:`for` or the :func:`next` builtin) then the -result is :const:`None`. Otherwise, if :meth:`~generator.send` is used, then -the result will be the value passed in to that method. - -.. index:: single: coroutine - -All of this makes generator functions quite similar to coroutines; they yield -multiple times, they have more than one entry point and their execution can be -suspended. The only difference is that a generator function cannot control -where the execution should continue after it yields; the control is always -transferred to the generator's caller. - -Yield expressions are allowed anywhere in a :keyword:`try` construct. If the -generator is not resumed before it is -finalized (by reaching a zero reference count or by being garbage collected), -the generator-iterator's :meth:`~generator.close` method will be called, -allowing any pending :keyword:`finally` clauses to execute. - -.. index:: - single: from; yield from expression - -When ``yield from `` is used, the supplied expression must be an -iterable. The values produced by iterating that iterable are passed directly -to the caller of the current generator's methods. Any values passed in with -:meth:`~generator.send` and any exceptions passed in with -:meth:`~generator.throw` are passed to the underlying iterator if it has the -appropriate methods. If this is not the case, then :meth:`~generator.send` -will raise :exc:`AttributeError` or :exc:`TypeError`, while -:meth:`~generator.throw` will just raise the passed in exception immediately. - -When the underlying iterator is complete, the :attr:`~StopIteration.value` -attribute of the raised :exc:`StopIteration` instance becomes the value of -the yield expression. It can be either set explicitly when raising -:exc:`StopIteration`, or automatically when the subiterator is a generator -(by returning a value from the subgenerator). - -.. versionchanged:: 3.3 - Added ``yield from `` to delegate control flow to a subiterator. - -The parentheses may be omitted when the yield expression is the sole expression -on the right hand side of an assignment statement. - -.. seealso:: - - :pep:`255` - Simple Generators - The proposal for adding generators and the :keyword:`yield` statement to Python. - - :pep:`342` - Coroutines via Enhanced Generators - The proposal to enhance the API and syntax of generators, making them - usable as simple coroutines. - - :pep:`380` - Syntax for Delegating to a Subgenerator - The proposal to introduce the :token:`~python-grammar:yield_from` syntax, - making delegation to subgenerators easy. - - :pep:`525` - Asynchronous Generators - The proposal that expanded on :pep:`492` by adding generator capabilities to - coroutine functions. - -.. index:: pair: object; generator -.. _generator-methods: - -Generator-iterator methods -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the methods of a generator iterator. They can -be used to control the execution of a generator function. - -Note that calling any of the generator methods below when the generator -is already executing raises a :exc:`ValueError` exception. - -.. index:: pair: exception; StopIteration - - -.. method:: generator.__next__() - - Starts the execution of a generator function or resumes it at the last - executed yield expression. When a generator function is resumed with a - :meth:`~generator.__next__` method, the current yield expression always - evaluates to :const:`None`. The execution then continues to the next yield - expression, where the generator is suspended again, and the value of the - :token:`~python-grammar:expression_list` is returned to :meth:`__next__`'s - caller. If the generator exits without yielding another value, a - :exc:`StopIteration` exception is raised. - - This method is normally called implicitly, e.g. by a :keyword:`for` loop, or - by the built-in :func:`next` function. - - -.. method:: generator.send(value) - - Resumes the execution and "sends" a value into the generator function. The - *value* argument becomes the result of the current yield expression. The - :meth:`send` method returns the next value yielded by the generator, or - raises :exc:`StopIteration` if the generator exits without yielding another - value. When :meth:`send` is called to start the generator, it must be called - with :const:`None` as the argument, because there is no yield expression that - could receive the value. - - -.. method:: generator.throw(value) - generator.throw(type[, value[, traceback]]) - - Raises an exception at the point where the generator was paused, - and returns the next value yielded by the generator function. If the generator - exits without yielding another value, a :exc:`StopIteration` exception is - raised. If the generator function does not catch the passed-in exception, or - raises a different exception, then that exception propagates to the caller. - - In typical use, this is called with a single exception instance similar to the - way the :keyword:`raise` keyword is used. - - For backwards compatibility, however, the second signature is - supported, following a convention from older versions of Python. - The *type* argument should be an exception class, and *value* - should be an exception instance. If the *value* is not provided, the - *type* constructor is called to get an instance. If *traceback* - is provided, it is set on the exception, otherwise any existing - :attr:`~BaseException.__traceback__` attribute stored in *value* may - be cleared. - -.. index:: pair: exception; GeneratorExit - - -.. method:: generator.close() - - Raises a :exc:`GeneratorExit` at the point where the generator function was - paused. If the generator function then exits gracefully, is already closed, - or raises :exc:`GeneratorExit` (by not catching the exception), close - returns to its caller. If the generator yields a value, a - :exc:`RuntimeError` is raised. If the generator raises any other exception, - it is propagated to the caller. :meth:`close` does nothing if the generator - has already exited due to an exception or normal exit. - -.. index:: single: yield; examples - -Examples -^^^^^^^^ - -Here is a simple example that demonstrates the behavior of generators and -generator functions:: - - >>> def echo(value=None): - ... print("Execution starts when 'next()' is called for the first time.") - ... try: - ... while True: - ... try: - ... value = (yield value) - ... except Exception as e: - ... value = e - ... finally: - ... print("Don't forget to clean up when 'close()' is called.") - ... - >>> generator = echo(1) - >>> print(next(generator)) - Execution starts when 'next()' is called for the first time. - 1 - >>> print(next(generator)) - None - >>> print(generator.send(2)) - 2 - >>> generator.throw(TypeError, "spam") - TypeError('spam',) - >>> generator.close() - Don't forget to clean up when 'close()' is called. - -For examples using ``yield from``, see :ref:`pep-380` in "What's New in -Python." - -.. _asynchronous-generator-functions: - -Asynchronous generator functions -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -The presence of a yield expression in a function or method defined using -:keyword:`async def` further defines the function as an -:term:`asynchronous generator` function. - -When an asynchronous generator function is called, it returns an -asynchronous iterator known as an asynchronous generator object. -That object then controls the execution of the generator function. -An asynchronous generator object is typically used in an -:keyword:`async for` statement in a coroutine function analogously to -how a generator object would be used in a :keyword:`for` statement. - -Calling one of the asynchronous generator's methods returns an :term:`awaitable` -object, and the execution starts when this object is awaited on. At that time, -the execution proceeds to the first yield expression, where it is suspended -again, returning the value of :token:`~python-grammar:expression_list` to the -awaiting coroutine. As with a generator, suspension means that all local state -is retained, including the current bindings of local variables, the instruction -pointer, the internal evaluation stack, and the state of any exception handling. -When the execution is resumed by awaiting on the next object returned by the -asynchronous generator's methods, the function can proceed exactly as if the -yield expression were just another external call. The value of the yield -expression after resuming depends on the method which resumed the execution. If -:meth:`~agen.__anext__` is used then the result is :const:`None`. Otherwise, if -:meth:`~agen.asend` is used, then the result will be the value passed in to that -method. - -If an asynchronous generator happens to exit early by :keyword:`break`, the caller -task being cancelled, or other exceptions, the generator's async cleanup code -will run and possibly raise exceptions or access context variables in an -unexpected context--perhaps after the lifetime of tasks it depends, or -during the event loop shutdown when the async-generator garbage collection hook -is called. -To prevent this, the caller must explicitly close the async generator by calling -:meth:`~agen.aclose` method to finalize the generator and ultimately detach it -from the event loop. - -In an asynchronous generator function, yield expressions are allowed anywhere -in a :keyword:`try` construct. However, if an asynchronous generator is not -resumed before it is finalized (by reaching a zero reference count or by -being garbage collected), then a yield expression within a :keyword:`!try` -construct could result in a failure to execute pending :keyword:`finally` -clauses. In this case, it is the responsibility of the event loop or -scheduler running the asynchronous generator to call the asynchronous -generator-iterator's :meth:`~agen.aclose` method and run the resulting -coroutine object, thus allowing any pending :keyword:`!finally` clauses -to execute. - -To take care of finalization upon event loop termination, an event loop should -define a *finalizer* function which takes an asynchronous generator-iterator and -presumably calls :meth:`~agen.aclose` and executes the coroutine. -This *finalizer* may be registered by calling :func:`sys.set_asyncgen_hooks`. -When first iterated over, an asynchronous generator-iterator will store the -registered *finalizer* to be called upon finalization. For a reference example -of a *finalizer* method see the implementation of -``asyncio.Loop.shutdown_asyncgens`` in :source:`Lib/asyncio/base_events.py`. - -The expression ``yield from `` is a syntax error when used in an -asynchronous generator function. - -.. index:: pair: object; asynchronous-generator -.. _asynchronous-generator-methods: - -Asynchronous generator-iterator methods -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -This subsection describes the methods of an asynchronous generator iterator, -which are used to control the execution of a generator function. - - -.. index:: pair: exception; StopAsyncIteration - -.. coroutinemethod:: agen.__anext__() - - Returns an awaitable which when run starts to execute the asynchronous - generator or resumes it at the last executed yield expression. When an - asynchronous generator function is resumed with an :meth:`~agen.__anext__` - method, the current yield expression always evaluates to :const:`None` in the - returned awaitable, which when run will continue to the next yield - expression. The value of the :token:`~python-grammar:expression_list` of the - yield expression is the value of the :exc:`StopIteration` exception raised by - the completing coroutine. If the asynchronous generator exits without - yielding another value, the awaitable instead raises a - :exc:`StopAsyncIteration` exception, signalling that the asynchronous - iteration has completed. - - This method is normally called implicitly by a :keyword:`async for` loop. - - -.. coroutinemethod:: agen.asend(value) - - Returns an awaitable which when run resumes the execution of the - asynchronous generator. As with the :meth:`~generator.send()` method for a - generator, this "sends" a value into the asynchronous generator function, - and the *value* argument becomes the result of the current yield expression. - The awaitable returned by the :meth:`asend` method will return the next - value yielded by the generator as the value of the raised - :exc:`StopIteration`, or raises :exc:`StopAsyncIteration` if the - asynchronous generator exits without yielding another value. When - :meth:`asend` is called to start the asynchronous - generator, it must be called with :const:`None` as the argument, - because there is no yield expression that could receive the value. - - -.. coroutinemethod:: agen.athrow(value) - agen.athrow(type[, value[, traceback]]) - - Returns an awaitable that raises an exception of type ``type`` at the point - where the asynchronous generator was paused, and returns the next value - yielded by the generator function as the value of the raised - :exc:`StopIteration` exception. If the asynchronous generator exits - without yielding another value, a :exc:`StopAsyncIteration` exception is - raised by the awaitable. - If the generator function does not catch the passed-in exception, or - raises a different exception, then when the awaitable is run that exception - propagates to the caller of the awaitable. - -.. index:: pair: exception; GeneratorExit - - -.. coroutinemethod:: agen.aclose() - - Returns an awaitable that when run will throw a :exc:`GeneratorExit` into - the asynchronous generator function at the point where it was paused. - If the asynchronous generator function then exits gracefully, is already - closed, or raises :exc:`GeneratorExit` (by not catching the exception), - then the returned awaitable will raise a :exc:`StopIteration` exception. - Any further awaitables returned by subsequent calls to the asynchronous - generator will raise a :exc:`StopAsyncIteration` exception. If the - asynchronous generator yields a value, a :exc:`RuntimeError` is raised - by the awaitable. If the asynchronous generator raises any other exception, - it is propagated to the caller of the awaitable. If the asynchronous - generator has already exited due to an exception or normal exit, then - further calls to :meth:`aclose` will return an awaitable that does nothing. - -.. _primaries: - -Primaries -========= - -.. index:: single: primary - -Primaries represent the most tightly bound operations of the language. Their -syntax is: - -.. productionlist:: python-grammar - primary: `atom` | `attributeref` | `subscription` | `slicing` | `call` - - -.. _attribute-references: - -Attribute references --------------------- - -.. index:: - pair: attribute; reference - single: . (dot); attribute reference - -An attribute reference is a primary followed by a period and a name: - -.. productionlist:: python-grammar - attributeref: `primary` "." `identifier` - -.. index:: - pair: exception; AttributeError - pair: object; module - pair: object; list - -The primary must evaluate to an object of a type that supports attribute -references, which most objects do. This object is then asked to produce the -attribute whose name is the identifier. This production can be customized by -overriding the :meth:`__getattr__` method. If this attribute is not available, -the exception :exc:`AttributeError` is raised. Otherwise, the type and value of -the object produced is determined by the object. Multiple evaluations of the -same attribute reference may yield different objects. - - -.. _subscriptions: - -Subscriptions -------------- - -.. index:: - single: subscription - single: [] (square brackets); subscription - -.. index:: - pair: object; sequence - pair: object; mapping - pair: object; string - pair: object; tuple - pair: object; list - pair: object; dictionary - pair: sequence; item - -The subscription of an instance of a :ref:`container class ` -will generally select an element from the container. The subscription of a -:term:`generic class ` will generally return a -:ref:`GenericAlias ` object. - -.. productionlist:: python-grammar - subscription: `primary` "[" `expression_list` "]" - -When an object is subscripted, the interpreter will evaluate the primary and -the expression list. - -The primary must evaluate to an object that supports subscription. An object -may support subscription through defining one or both of -:meth:`~object.__getitem__` and :meth:`~object.__class_getitem__`. When the -primary is subscripted, the evaluated result of the expression list will be -passed to one of these methods. For more details on when ``__class_getitem__`` -is called instead of ``__getitem__``, see :ref:`classgetitem-versus-getitem`. - -If the expression list contains at least one comma, it will evaluate to a -:class:`tuple` containing the items of the expression list. Otherwise, the -expression list will evaluate to the value of the list's sole member. - -For built-in objects, there are two types of objects that support subscription -via :meth:`~object.__getitem__`: - -1. Mappings. If the primary is a :term:`mapping`, the expression list must - evaluate to an object whose value is one of the keys of the mapping, and the - subscription selects the value in the mapping that corresponds to that key. - An example of a builtin mapping class is the :class:`dict` class. -2. Sequences. If the primary is a :term:`sequence`, the expression list must - evaluate to an :class:`int` or a :class:`slice` (as discussed in the - following section). Examples of builtin sequence classes include the - :class:`str`, :class:`list` and :class:`tuple` classes. - -The formal syntax makes no special provision for negative indices in -:term:`sequences `. However, built-in sequences all provide a :meth:`~object.__getitem__` -method that interprets negative indices by adding the length of the sequence -to the index so that, for example, ``x[-1]`` selects the last item of ``x``. The -resulting value must be a nonnegative integer less than the number of items in -the sequence, and the subscription selects the item whose index is that value -(counting from zero). Since the support for negative indices and slicing -occurs in the object's :meth:`~object.__getitem__` method, subclasses overriding -this method will need to explicitly add that support. - -.. index:: - single: character - pair: string; item - -A :class:`string ` is a special kind of sequence whose items are -*characters*. A character is not a separate data type but a -string of exactly one character. - - -.. _slicings: - -Slicings --------- - -.. index:: - single: slicing - single: slice - single: : (colon); slicing - single: , (comma); slicing - -.. index:: - pair: object; sequence - pair: object; string - pair: object; tuple - pair: object; list - -A slicing selects a range of items in a sequence object (e.g., a string, tuple -or list). Slicings may be used as expressions or as targets in assignment or -:keyword:`del` statements. The syntax for a slicing: - -.. productionlist:: python-grammar - slicing: `primary` "[" `slice_list` "]" - slice_list: `slice_item` ("," `slice_item`)* [","] - slice_item: `expression` | `proper_slice` - proper_slice: [`lower_bound`] ":" [`upper_bound`] [ ":" [`stride`] ] - lower_bound: `expression` - upper_bound: `expression` - stride: `expression` - -There is ambiguity in the formal syntax here: anything that looks like an -expression list also looks like a slice list, so any subscription can be -interpreted as a slicing. Rather than further complicating the syntax, this is -disambiguated by defining that in this case the interpretation as a subscription -takes priority over the interpretation as a slicing (this is the case if the -slice list contains no proper slice). - -.. index:: - single: start (slice object attribute) - single: stop (slice object attribute) - single: step (slice object attribute) - -The semantics for a slicing are as follows. The primary is indexed (using the -same :meth:`~object.__getitem__` method as -normal subscription) with a key that is constructed from the slice list, as -follows. If the slice list contains at least one comma, the key is a tuple -containing the conversion of the slice items; otherwise, the conversion of the -lone slice item is the key. The conversion of a slice item that is an -expression is that expression. The conversion of a proper slice is a slice -object (see section :ref:`types`) whose :attr:`~slice.start`, -:attr:`~slice.stop` and :attr:`~slice.step` attributes are the values of the -expressions given as lower bound, upper bound and stride, respectively, -substituting ``None`` for missing expressions. - - -.. index:: - pair: object; callable - single: call - single: argument; call semantics - single: () (parentheses); call - single: , (comma); argument list - single: = (equals); in function calls - -.. _calls: - -Calls ------ - -A call calls a callable object (e.g., a :term:`function`) with a possibly empty -series of :term:`arguments `: - -.. productionlist:: python-grammar - call: `primary` "(" [`argument_list` [","] | `comprehension`] ")" - argument_list: `positional_arguments` ["," `starred_and_keywords`] - : ["," `keywords_arguments`] - : | `starred_and_keywords` ["," `keywords_arguments`] - : | `keywords_arguments` - positional_arguments: positional_item ("," positional_item)* - positional_item: `assignment_expression` | "*" `expression` - starred_and_keywords: ("*" `expression` | `keyword_item`) - : ("," "*" `expression` | "," `keyword_item`)* - keywords_arguments: (`keyword_item` | "**" `expression`) - : ("," `keyword_item` | "," "**" `expression`)* - keyword_item: `identifier` "=" `expression` - -An optional trailing comma may be present after the positional and keyword arguments -but does not affect the semantics. - -.. index:: - single: parameter; call semantics - -The primary must evaluate to a callable object (user-defined functions, built-in -functions, methods of built-in objects, class objects, methods of class -instances, and all objects having a :meth:`__call__` method are callable). All -argument expressions are evaluated before the call is attempted. Please refer -to section :ref:`function` for the syntax of formal :term:`parameter` lists. - -.. XXX update with kwonly args PEP - -If keyword arguments are present, they are first converted to positional -arguments, as follows. First, a list of unfilled slots is created for the -formal parameters. If there are N positional arguments, they are placed in the -first N slots. Next, for each keyword argument, the identifier is used to -determine the corresponding slot (if the identifier is the same as the first -formal parameter name, the first slot is used, and so on). If the slot is -already filled, a :exc:`TypeError` exception is raised. Otherwise, the -argument is placed in the slot, filling it (even if the expression is -``None``, it fills the slot). When all arguments have been processed, the slots -that are still unfilled are filled with the corresponding default value from the -function definition. (Default values are calculated, once, when the function is -defined; thus, a mutable object such as a list or dictionary used as default -value will be shared by all calls that don't specify an argument value for the -corresponding slot; this should usually be avoided.) If there are any unfilled -slots for which no default value is specified, a :exc:`TypeError` exception is -raised. Otherwise, the list of filled slots is used as the argument list for -the call. - -.. impl-detail:: - - An implementation may provide built-in functions whose positional parameters - do not have names, even if they are 'named' for the purpose of documentation, - and which therefore cannot be supplied by keyword. In CPython, this is the - case for functions implemented in C that use :c:func:`PyArg_ParseTuple` to - parse their arguments. - -If there are more positional arguments than there are formal parameter slots, a -:exc:`TypeError` exception is raised, unless a formal parameter using the syntax -``*identifier`` is present; in this case, that formal parameter receives a tuple -containing the excess positional arguments (or an empty tuple if there were no -excess positional arguments). - -If any keyword argument does not correspond to a formal parameter name, a -:exc:`TypeError` exception is raised, unless a formal parameter using the syntax -``**identifier`` is present; in this case, that formal parameter receives a -dictionary containing the excess keyword arguments (using the keywords as keys -and the argument values as corresponding values), or a (new) empty dictionary if -there were no excess keyword arguments. - -.. index:: - single: * (asterisk); in function calls - single: unpacking; in function calls - -If the syntax ``*expression`` appears in the function call, ``expression`` must -evaluate to an :term:`iterable`. Elements from these iterables are -treated as if they were additional positional arguments. For the call -``f(x1, x2, *y, x3, x4)``, if *y* evaluates to a sequence *y1*, ..., *yM*, -this is equivalent to a call with M+4 positional arguments *x1*, *x2*, -*y1*, ..., *yM*, *x3*, *x4*. - -A consequence of this is that although the ``*expression`` syntax may appear -*after* explicit keyword arguments, it is processed *before* the -keyword arguments (and any ``**expression`` arguments -- see below). So:: - - >>> def f(a, b): - ... print(a, b) - ... - >>> f(b=1, *(2,)) - 2 1 - >>> f(a=1, *(2,)) - Traceback (most recent call last): - File "", line 1, in - TypeError: f() got multiple values for keyword argument 'a' - >>> f(1, *(2,)) - 1 2 - -It is unusual for both keyword arguments and the ``*expression`` syntax to be -used in the same call, so in practice this confusion does not often arise. - -.. index:: - single: **; in function calls - -If the syntax ``**expression`` appears in the function call, ``expression`` must -evaluate to a :term:`mapping`, the contents of which are treated as -additional keyword arguments. If a parameter matching a key has already been -given a value (by an explicit keyword argument, or from another unpacking), -a :exc:`TypeError` exception is raised. - -When ``**expression`` is used, each key in this mapping must be -a string. -Each value from the mapping is assigned to the first formal parameter -eligible for keyword assignment whose name is equal to the key. -A key need not be a Python identifier (e.g. ``"max-temp °F"`` is acceptable, -although it will not match any formal parameter that could be declared). -If there is no match to a formal parameter -the key-value pair is collected by the ``**`` parameter, if there is one, -or if there is not, a :exc:`TypeError` exception is raised. - -Formal parameters using the syntax ``*identifier`` or ``**identifier`` cannot be -used as positional argument slots or as keyword argument names. - -.. versionchanged:: 3.5 - Function calls accept any number of ``*`` and ``**`` unpackings, - positional arguments may follow iterable unpackings (``*``), - and keyword arguments may follow dictionary unpackings (``**``). - Originally proposed by :pep:`448`. - -A call always returns some value, possibly ``None``, unless it raises an -exception. How this value is computed depends on the type of the callable -object. - -If it is--- - -a user-defined function: - .. index:: - pair: function; call - triple: user-defined; function; call - pair: object; user-defined function - pair: object; function - - The code block for the function is executed, passing it the argument list. The - first thing the code block will do is bind the formal parameters to the - arguments; this is described in section :ref:`function`. When the code block - executes a :keyword:`return` statement, this specifies the return value of the - function call. - -a built-in function or method: - .. index:: - pair: function; call - pair: built-in function; call - pair: method; call - pair: built-in method; call - pair: object; built-in method - pair: object; built-in function - pair: object; method - pair: object; function - - The result is up to the interpreter; see :ref:`built-in-funcs` for the - descriptions of built-in functions and methods. - -a class object: - .. index:: - pair: object; class - pair: class object; call - - A new instance of that class is returned. - -a class instance method: - .. index:: - pair: object; class instance - pair: object; instance - pair: class instance; call - - The corresponding user-defined function is called, with an argument list that is - one longer than the argument list of the call: the instance becomes the first - argument. - -a class instance: - .. index:: - pair: instance; call - single: __call__() (object method) - - The class must define a :meth:`__call__` method; the effect is then the same as - if that method was called. - - -.. index:: pair: keyword; await -.. _await: - -Await expression -================ - -Suspend the execution of :term:`coroutine` on an :term:`awaitable` object. -Can only be used inside a :term:`coroutine function`. - -.. productionlist:: python-grammar - await_expr: "await" `primary` - -.. versionadded:: 3.5 - - -.. _power: - -The power operator -================== - -.. index:: - pair: power; operation - pair: operator; ** - -The power operator binds more tightly than unary operators on its left; it binds -less tightly than unary operators on its right. The syntax is: - -.. productionlist:: python-grammar - power: (`await_expr` | `primary`) ["**" `u_expr`] - -Thus, in an unparenthesized sequence of power and unary operators, the operators -are evaluated from right to left (this does not constrain the evaluation order -for the operands): ``-1**2`` results in ``-1``. - -The power operator has the same semantics as the built-in :func:`pow` function, -when called with two arguments: it yields its left argument raised to the power -of its right argument. The numeric arguments are first converted to a common -type, and the result is of that type. - -For int operands, the result has the same type as the operands unless the second -argument is negative; in that case, all arguments are converted to float and a -float result is delivered. For example, ``10**2`` returns ``100``, but -``10**-2`` returns ``0.01``. - -Raising ``0.0`` to a negative power results in a :exc:`ZeroDivisionError`. -Raising a negative number to a fractional power results in a :class:`complex` -number. (In earlier versions it raised a :exc:`ValueError`.) - -This operation can be customized using the special :meth:`__pow__` method. - -.. _unary: - -Unary arithmetic and bitwise operations -======================================= - -.. index:: - triple: unary; arithmetic; operation - triple: unary; bitwise; operation - -All unary arithmetic and bitwise operations have the same priority: - -.. productionlist:: python-grammar - u_expr: `power` | "-" `u_expr` | "+" `u_expr` | "~" `u_expr` - -.. index:: - single: negation - single: minus - single: operator; - (minus) - single: - (minus); unary operator - -The unary ``-`` (minus) operator yields the negation of its numeric argument; the -operation can be overridden with the :meth:`__neg__` special method. - -.. index:: - single: plus - single: operator; + (plus) - single: + (plus); unary operator - -The unary ``+`` (plus) operator yields its numeric argument unchanged; the -operation can be overridden with the :meth:`__pos__` special method. - -.. index:: - single: inversion - pair: operator; ~ (tilde) - -The unary ``~`` (invert) operator yields the bitwise inversion of its integer -argument. The bitwise inversion of ``x`` is defined as ``-(x+1)``. It only -applies to integral numbers or to custom objects that override the -:meth:`__invert__` special method. - - - -.. index:: pair: exception; TypeError - -In all three cases, if the argument does not have the proper type, a -:exc:`TypeError` exception is raised. - - -.. _binary: - -Binary arithmetic operations -============================ - -.. index:: triple: binary; arithmetic; operation - -The binary arithmetic operations have the conventional priority levels. Note -that some of these operations also apply to certain non-numeric types. Apart -from the power operator, there are only two levels, one for multiplicative -operators and one for additive operators: - -.. productionlist:: python-grammar - m_expr: `u_expr` | `m_expr` "*" `u_expr` | `m_expr` "@" `m_expr` | - : `m_expr` "//" `u_expr` | `m_expr` "/" `u_expr` | - : `m_expr` "%" `u_expr` - a_expr: `m_expr` | `a_expr` "+" `m_expr` | `a_expr` "-" `m_expr` - -.. index:: - single: multiplication - pair: operator; * (asterisk) - -The ``*`` (multiplication) operator yields the product of its arguments. The -arguments must either both be numbers, or one argument must be an integer and -the other must be a sequence. In the former case, the numbers are converted to a -common type and then multiplied together. In the latter case, sequence -repetition is performed; a negative repetition factor yields an empty sequence. - -This operation can be customized using the special :meth:`__mul__` and -:meth:`__rmul__` methods. - -.. index:: - single: matrix multiplication - pair: operator; @ (at) - -The ``@`` (at) operator is intended to be used for matrix multiplication. No -builtin Python types implement this operator. - -.. versionadded:: 3.5 - -.. index:: - pair: exception; ZeroDivisionError - single: division - pair: operator; / (slash) - pair: operator; // - -The ``/`` (division) and ``//`` (floor division) operators yield the quotient of -their arguments. The numeric arguments are first converted to a common type. -Division of integers yields a float, while floor division of integers results in an -integer; the result is that of mathematical division with the 'floor' function -applied to the result. Division by zero raises the :exc:`ZeroDivisionError` -exception. - -This operation can be customized using the special :meth:`__truediv__` and -:meth:`__floordiv__` methods. - -.. index:: - single: modulo - pair: operator; % (percent) - -The ``%`` (modulo) operator yields the remainder from the division of the first -argument by the second. The numeric arguments are first converted to a common -type. A zero right argument raises the :exc:`ZeroDivisionError` exception. The -arguments may be floating point numbers, e.g., ``3.14%0.7`` equals ``0.34`` -(since ``3.14`` equals ``4*0.7 + 0.34``.) The modulo operator always yields a -result with the same sign as its second operand (or zero); the absolute value of -the result is strictly smaller than the absolute value of the second operand -[#]_. - -The floor division and modulo operators are connected by the following -identity: ``x == (x//y)*y + (x%y)``. Floor division and modulo are also -connected with the built-in function :func:`divmod`: ``divmod(x, y) == (x//y, -x%y)``. [#]_. - -In addition to performing the modulo operation on numbers, the ``%`` operator is -also overloaded by string objects to perform old-style string formatting (also -known as interpolation). The syntax for string formatting is described in the -Python Library Reference, section :ref:`old-string-formatting`. - -The *modulo* operation can be customized using the special :meth:`__mod__` method. - -The floor division operator, the modulo operator, and the :func:`divmod` -function are not defined for complex numbers. Instead, convert to a floating -point number using the :func:`abs` function if appropriate. - -.. index:: - single: addition - single: operator; + (plus) - single: + (plus); binary operator - -The ``+`` (addition) operator yields the sum of its arguments. The arguments -must either both be numbers or both be sequences of the same type. In the -former case, the numbers are converted to a common type and then added together. -In the latter case, the sequences are concatenated. - -This operation can be customized using the special :meth:`__add__` and -:meth:`__radd__` methods. - -.. index:: - single: subtraction - single: operator; - (minus) - single: - (minus); binary operator - -The ``-`` (subtraction) operator yields the difference of its arguments. The -numeric arguments are first converted to a common type. - -This operation can be customized using the special :meth:`__sub__` method. - - -.. _shifting: - -Shifting operations -=================== - -.. index:: - pair: shifting; operation - pair: operator; << - pair: operator; >> - -The shifting operations have lower priority than the arithmetic operations: - -.. productionlist:: python-grammar - shift_expr: `a_expr` | `shift_expr` ("<<" | ">>") `a_expr` - -These operators accept integers as arguments. They shift the first argument to -the left or right by the number of bits given by the second argument. - -This operation can be customized using the special :meth:`__lshift__` and -:meth:`__rshift__` methods. - -.. index:: pair: exception; ValueError - -A right shift by *n* bits is defined as floor division by ``pow(2,n)``. A left -shift by *n* bits is defined as multiplication with ``pow(2,n)``. - - -.. _bitwise: - -Binary bitwise operations -========================= - -.. index:: triple: binary; bitwise; operation - -Each of the three bitwise operations has a different priority level: - -.. productionlist:: python-grammar - and_expr: `shift_expr` | `and_expr` "&" `shift_expr` - xor_expr: `and_expr` | `xor_expr` "^" `and_expr` - or_expr: `xor_expr` | `or_expr` "|" `xor_expr` - -.. index:: - pair: bitwise; and - pair: operator; & (ampersand) - -The ``&`` operator yields the bitwise AND of its arguments, which must be -integers or one of them must be a custom object overriding :meth:`__and__` or -:meth:`__rand__` special methods. - -.. index:: - pair: bitwise; xor - pair: exclusive; or - pair: operator; ^ (caret) - -The ``^`` operator yields the bitwise XOR (exclusive OR) of its arguments, which -must be integers or one of them must be a custom object overriding :meth:`__xor__` or -:meth:`__rxor__` special methods. - -.. index:: - pair: bitwise; or - pair: inclusive; or - pair: operator; | (vertical bar) - -The ``|`` operator yields the bitwise (inclusive) OR of its arguments, which -must be integers or one of them must be a custom object overriding :meth:`__or__` or -:meth:`__ror__` special methods. - - -.. _comparisons: - -Comparisons -=========== - -.. index:: - single: comparison - pair: C; language - pair: operator; < (less) - pair: operator; > (greater) - pair: operator; <= - pair: operator; >= - pair: operator; == - pair: operator; != - -Unlike C, all comparison operations in Python have the same priority, which is -lower than that of any arithmetic, shifting or bitwise operation. Also unlike -C, expressions like ``a < b < c`` have the interpretation that is conventional -in mathematics: - -.. productionlist:: python-grammar - comparison: `or_expr` (`comp_operator` `or_expr`)* - comp_operator: "<" | ">" | "==" | ">=" | "<=" | "!=" - : | "is" ["not"] | ["not"] "in" - -Comparisons yield boolean values: ``True`` or ``False``. Custom -:dfn:`rich comparison methods` may return non-boolean values. In this case -Python will call :func:`bool` on such value in boolean contexts. - -.. index:: pair: chaining; comparisons - -Comparisons can be chained arbitrarily, e.g., ``x < y <= z`` is equivalent to -``x < y and y <= z``, except that ``y`` is evaluated only once (but in both -cases ``z`` is not evaluated at all when ``x < y`` is found to be false). - -Formally, if *a*, *b*, *c*, ..., *y*, *z* are expressions and *op1*, *op2*, ..., -*opN* are comparison operators, then ``a op1 b op2 c ... y opN z`` is equivalent -to ``a op1 b and b op2 c and ... y opN z``, except that each expression is -evaluated at most once. - -Note that ``a op1 b op2 c`` doesn't imply any kind of comparison between *a* and -*c*, so that, e.g., ``x < y > z`` is perfectly legal (though perhaps not -pretty). - -.. _expressions-value-comparisons: - -Value comparisons ------------------ - -The operators ``<``, ``>``, ``==``, ``>=``, ``<=``, and ``!=`` compare the -values of two objects. The objects do not need to have the same type. - -Chapter :ref:`objects` states that objects have a value (in addition to type -and identity). The value of an object is a rather abstract notion in Python: -For example, there is no canonical access method for an object's value. Also, -there is no requirement that the value of an object should be constructed in a -particular way, e.g. comprised of all its data attributes. Comparison operators -implement a particular notion of what the value of an object is. One can think -of them as defining the value of an object indirectly, by means of their -comparison implementation. - -Because all types are (direct or indirect) subtypes of :class:`object`, they -inherit the default comparison behavior from :class:`object`. Types can -customize their comparison behavior by implementing -:dfn:`rich comparison methods` like :meth:`__lt__`, described in -:ref:`customization`. - -The default behavior for equality comparison (``==`` and ``!=``) is based on -the identity of the objects. Hence, equality comparison of instances with the -same identity results in equality, and equality comparison of instances with -different identities results in inequality. A motivation for this default -behavior is the desire that all objects should be reflexive (i.e. ``x is y`` -implies ``x == y``). - -A default order comparison (``<``, ``>``, ``<=``, and ``>=``) is not provided; -an attempt raises :exc:`TypeError`. A motivation for this default behavior is -the lack of a similar invariant as for equality. - -The behavior of the default equality comparison, that instances with different -identities are always unequal, may be in contrast to what types will need that -have a sensible definition of object value and value-based equality. Such -types will need to customize their comparison behavior, and in fact, a number -of built-in types have done that. - -The following list describes the comparison behavior of the most important -built-in types. - -* Numbers of built-in numeric types (:ref:`typesnumeric`) and of the standard - library types :class:`fractions.Fraction` and :class:`decimal.Decimal` can be - compared within and across their types, with the restriction that complex - numbers do not support order comparison. Within the limits of the types - involved, they compare mathematically (algorithmically) correct without loss - of precision. - - The not-a-number values ``float('NaN')`` and ``decimal.Decimal('NaN')`` are - special. Any ordered comparison of a number to a not-a-number value is false. - A counter-intuitive implication is that not-a-number values are not equal to - themselves. For example, if ``x = float('NaN')``, ``3 < x``, ``x < 3`` and - ``x == x`` are all false, while ``x != x`` is true. This behavior is - compliant with IEEE 754. - -* ``None`` and ``NotImplemented`` are singletons. :PEP:`8` advises that - comparisons for singletons should always be done with ``is`` or ``is not``, - never the equality operators. - -* Binary sequences (instances of :class:`bytes` or :class:`bytearray`) can be - compared within and across their types. They compare lexicographically using - the numeric values of their elements. - -* Strings (instances of :class:`str`) compare lexicographically using the - numerical Unicode code points (the result of the built-in function - :func:`ord`) of their characters. [#]_ - - Strings and binary sequences cannot be directly compared. - -* Sequences (instances of :class:`tuple`, :class:`list`, or :class:`range`) can - be compared only within each of their types, with the restriction that ranges - do not support order comparison. Equality comparison across these types - results in inequality, and ordering comparison across these types raises - :exc:`TypeError`. - - Sequences compare lexicographically using comparison of corresponding - elements. The built-in containers typically assume identical objects are - equal to themselves. That lets them bypass equality tests for identical - objects to improve performance and to maintain their internal invariants. - - Lexicographical comparison between built-in collections works as follows: - - - For two collections to compare equal, they must be of the same type, have - the same length, and each pair of corresponding elements must compare - equal (for example, ``[1,2] == (1,2)`` is false because the type is not the - same). - - - Collections that support order comparison are ordered the same as their - first unequal elements (for example, ``[1,2,x] <= [1,2,y]`` has the same - value as ``x <= y``). If a corresponding element does not exist, the - shorter collection is ordered first (for example, ``[1,2] < [1,2,3]`` is - true). - -* Mappings (instances of :class:`dict`) compare equal if and only if they have - equal ``(key, value)`` pairs. Equality comparison of the keys and values - enforces reflexivity. - - Order comparisons (``<``, ``>``, ``<=``, and ``>=``) raise :exc:`TypeError`. - -* Sets (instances of :class:`set` or :class:`frozenset`) can be compared within - and across their types. - - They define order - comparison operators to mean subset and superset tests. Those relations do - not define total orderings (for example, the two sets ``{1,2}`` and ``{2,3}`` - are not equal, nor subsets of one another, nor supersets of one - another). Accordingly, sets are not appropriate arguments for functions - which depend on total ordering (for example, :func:`min`, :func:`max`, and - :func:`sorted` produce undefined results given a list of sets as inputs). - - Comparison of sets enforces reflexivity of its elements. - -* Most other built-in types have no comparison methods implemented, so they - inherit the default comparison behavior. - -User-defined classes that customize their comparison behavior should follow -some consistency rules, if possible: - -* Equality comparison should be reflexive. - In other words, identical objects should compare equal: - - ``x is y`` implies ``x == y`` - -* Comparison should be symmetric. - In other words, the following expressions should have the same result: - - ``x == y`` and ``y == x`` - - ``x != y`` and ``y != x`` - - ``x < y`` and ``y > x`` - - ``x <= y`` and ``y >= x`` - -* Comparison should be transitive. - The following (non-exhaustive) examples illustrate that: - - ``x > y and y > z`` implies ``x > z`` - - ``x < y and y <= z`` implies ``x < z`` - -* Inverse comparison should result in the boolean negation. - In other words, the following expressions should have the same result: - - ``x == y`` and ``not x != y`` - - ``x < y`` and ``not x >= y`` (for total ordering) - - ``x > y`` and ``not x <= y`` (for total ordering) - - The last two expressions apply to totally ordered collections (e.g. to - sequences, but not to sets or mappings). See also the - :func:`~functools.total_ordering` decorator. - -* The :func:`hash` result should be consistent with equality. - Objects that are equal should either have the same hash value, - or be marked as unhashable. - -Python does not enforce these consistency rules. In fact, the not-a-number -values are an example for not following these rules. - - -.. _in: -.. _not in: -.. _membership-test-details: - -Membership test operations --------------------------- - -The operators :keyword:`in` and :keyword:`not in` test for membership. ``x in -s`` evaluates to ``True`` if *x* is a member of *s*, and ``False`` otherwise. -``x not in s`` returns the negation of ``x in s``. All built-in sequences and -set types support this as well as dictionary, for which :keyword:`!in` tests -whether the dictionary has a given key. For container types such as list, tuple, -set, frozenset, dict, or collections.deque, the expression ``x in y`` is equivalent -to ``any(x is e or x == e for e in y)``. - -For the string and bytes types, ``x in y`` is ``True`` if and only if *x* is a -substring of *y*. An equivalent test is ``y.find(x) != -1``. Empty strings are -always considered to be a substring of any other string, so ``"" in "abc"`` will -return ``True``. - -For user-defined classes which define the :meth:`__contains__` method, ``x in -y`` returns ``True`` if ``y.__contains__(x)`` returns a true value, and -``False`` otherwise. - -For user-defined classes which do not define :meth:`__contains__` but do define -:meth:`__iter__`, ``x in y`` is ``True`` if some value ``z``, for which the -expression ``x is z or x == z`` is true, is produced while iterating over ``y``. -If an exception is raised during the iteration, it is as if :keyword:`in` raised -that exception. - -Lastly, the old-style iteration protocol is tried: if a class defines -:meth:`~object.__getitem__`, ``x in y`` is ``True`` if and only if there is a non-negative -integer index *i* such that ``x is y[i] or x == y[i]``, and no lower integer index -raises the :exc:`IndexError` exception. (If any other exception is raised, it is as -if :keyword:`in` raised that exception). - -.. index:: - pair: operator; in - pair: operator; not in - pair: membership; test - pair: object; sequence - -The operator :keyword:`not in` is defined to have the inverse truth value of -:keyword:`in`. - -.. index:: - pair: operator; is - pair: operator; is not - pair: identity; test - - -.. _is: -.. _is not: - -Identity comparisons --------------------- - -The operators :keyword:`is` and :keyword:`is not` test for an object's identity: ``x -is y`` is true if and only if *x* and *y* are the same object. An Object's identity -is determined using the :meth:`id` function. ``x is not y`` yields the inverse -truth value. [#]_ - - -.. _booleans: -.. _and: -.. _or: -.. _not: - -Boolean operations -================== - -.. index:: - pair: Conditional; expression - pair: Boolean; operation - -.. productionlist:: python-grammar - or_test: `and_test` | `or_test` "or" `and_test` - and_test: `not_test` | `and_test` "and" `not_test` - not_test: `comparison` | "not" `not_test` - -In the context of Boolean operations, and also when expressions are used by -control flow statements, the following values are interpreted as false: -``False``, ``None``, numeric zero of all types, and empty strings and containers -(including strings, tuples, lists, dictionaries, sets and frozensets). All -other values are interpreted as true. User-defined objects can customize their -truth value by providing a :meth:`~object.__bool__` method. - -.. index:: pair: operator; not - -The operator :keyword:`not` yields ``True`` if its argument is false, ``False`` -otherwise. - -.. index:: pair: operator; and - -The expression ``x and y`` first evaluates *x*; if *x* is false, its value is -returned; otherwise, *y* is evaluated and the resulting value is returned. - -.. index:: pair: operator; or - -The expression ``x or y`` first evaluates *x*; if *x* is true, its value is -returned; otherwise, *y* is evaluated and the resulting value is returned. - -Note that neither :keyword:`and` nor :keyword:`or` restrict the value and type -they return to ``False`` and ``True``, but rather return the last evaluated -argument. This is sometimes useful, e.g., if ``s`` is a string that should be -replaced by a default value if it is empty, the expression ``s or 'foo'`` yields -the desired value. Because :keyword:`not` has to create a new value, it -returns a boolean value regardless of the type of its argument -(for example, ``not 'foo'`` produces ``False`` rather than ``''``.) - - -.. index:: - single: := (colon equals) - single: assignment expression - single: walrus operator - single: named expression - -Assignment expressions -====================== - -.. productionlist:: python-grammar - assignment_expression: [`identifier` ":="] `expression` - -An assignment expression (sometimes also called a "named expression" or -"walrus") assigns an :token:`~python-grammar:expression` to an -:token:`~python-grammar:identifier`, while also returning the value of the -:token:`~python-grammar:expression`. - -One common use case is when handling matched regular expressions: - -.. code-block:: python - - if matching := pattern.search(data): - do_something(matching) - -Or, when processing a file stream in chunks: - -.. code-block:: python - - while chunk := file.read(9000): - process(chunk) - -Assignment expressions must be surrounded by parentheses when used -as sub-expressions in slicing, conditional, lambda, -keyword-argument, and comprehension-if expressions -and in ``assert`` and ``with`` statements. -In all other places where they can be used, parentheses are not required, -including in ``if`` and ``while`` statements. - -.. versionadded:: 3.8 - See :pep:`572` for more details about assignment expressions. - - -.. _if_expr: - -Conditional expressions -======================= - -.. index:: - pair: conditional; expression - pair: ternary; operator - single: if; conditional expression - single: else; conditional expression - -.. productionlist:: python-grammar - conditional_expression: `or_test` ["if" `or_test` "else" `expression`] - expression: `conditional_expression` | `lambda_expr` - -Conditional expressions (sometimes called a "ternary operator") have the lowest -priority of all Python operations. - -The expression ``x if C else y`` first evaluates the condition, *C* rather than *x*. -If *C* is true, *x* is evaluated and its value is returned; otherwise, *y* is -evaluated and its value is returned. - -See :pep:`308` for more details about conditional expressions. - - -.. _lambdas: -.. _lambda: - -Lambdas -======= - -.. index:: - pair: lambda; expression - pair: lambda; form - pair: anonymous; function - single: : (colon); lambda expression - -.. productionlist:: python-grammar - lambda_expr: "lambda" [`parameter_list`] ":" `expression` - -Lambda expressions (sometimes called lambda forms) are used to create anonymous -functions. The expression ``lambda parameters: expression`` yields a function -object. The unnamed object behaves like a function object defined with: - -.. code-block:: none - - def (parameters): - return expression - -See section :ref:`function` for the syntax of parameter lists. Note that -functions created with lambda expressions cannot contain statements or -annotations. - - -.. _exprlists: - -Expression lists -================ - -.. index:: - pair: expression; list - single: , (comma); expression list - -.. productionlist:: python-grammar - expression_list: `expression` ("," `expression`)* [","] - starred_list: `starred_item` ("," `starred_item`)* [","] - starred_expression: `expression` | (`starred_item` ",")* [`starred_item`] - starred_item: `assignment_expression` | "*" `or_expr` - -.. index:: pair: object; tuple - -Except when part of a list or set display, an expression list -containing at least one comma yields a tuple. The length of -the tuple is the number of expressions in the list. The expressions are -evaluated from left to right. - -.. index:: - pair: iterable; unpacking - single: * (asterisk); in expression lists - -An asterisk ``*`` denotes :dfn:`iterable unpacking`. Its operand must be -an :term:`iterable`. The iterable is expanded into a sequence of items, -which are included in the new tuple, list, or set, at the site of -the unpacking. - -.. versionadded:: 3.5 - Iterable unpacking in expression lists, originally proposed by :pep:`448`. - -.. index:: pair: trailing; comma - -The trailing comma is required only to create a single tuple (a.k.a. a -*singleton*); it is optional in all other cases. A single expression without a -trailing comma doesn't create a tuple, but rather yields the value of that -expression. (To create an empty tuple, use an empty pair of parentheses: -``()``.) - - -.. _evalorder: - -Evaluation order -================ - -.. index:: pair: evaluation; order - -Python evaluates expressions from left to right. Notice that while evaluating -an assignment, the right-hand side is evaluated before the left-hand side. - -In the following lines, expressions will be evaluated in the arithmetic order of -their suffixes:: - - expr1, expr2, expr3, expr4 - (expr1, expr2, expr3, expr4) - {expr1: expr2, expr3: expr4} - expr1 + expr2 * (expr3 - expr4) - expr1(expr2, expr3, *expr4, **expr5) - expr3, expr4 = expr1, expr2 - - -.. _operator-summary: - -Operator precedence -=================== - -.. index:: - pair: operator; precedence - -The following table summarizes the operator precedence in Python, from highest -precedence (most binding) to lowest precedence (least binding). Operators in -the same box have the same precedence. Unless the syntax is explicitly given, -operators are binary. Operators in the same box group left to right (except for -exponentiation and conditional expressions, which group from right to left). - -Note that comparisons, membership tests, and identity tests, all have the same -precedence and have a left-to-right chaining feature as described in the -:ref:`comparisons` section. - - -+-----------------------------------------------+-------------------------------------+ -| Operator | Description | -+===============================================+=====================================+ -| ``(expressions...)``, | Binding or parenthesized | -| | expression, | -| ``[expressions...]``, | list display, | -| ``{key: value...}``, | dictionary display, | -| ``{expressions...}`` | set display | -+-----------------------------------------------+-------------------------------------+ -| ``x[index]``, ``x[index:index]``, | Subscription, slicing, | -| ``x(arguments...)``, ``x.attribute`` | call, attribute reference | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`await x ` | Await expression | -+-----------------------------------------------+-------------------------------------+ -| ``**`` | Exponentiation [#]_ | -+-----------------------------------------------+-------------------------------------+ -| ``+x``, ``-x``, ``~x`` | Positive, negative, bitwise NOT | -+-----------------------------------------------+-------------------------------------+ -| ``*``, ``@``, ``/``, ``//``, ``%`` | Multiplication, matrix | -| | multiplication, division, floor | -| | division, remainder [#]_ | -+-----------------------------------------------+-------------------------------------+ -| ``+``, ``-`` | Addition and subtraction | -+-----------------------------------------------+-------------------------------------+ -| ``<<``, ``>>`` | Shifts | -+-----------------------------------------------+-------------------------------------+ -| ``&`` | Bitwise AND | -+-----------------------------------------------+-------------------------------------+ -| ``^`` | Bitwise XOR | -+-----------------------------------------------+-------------------------------------+ -| ``|`` | Bitwise OR | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`in`, :keyword:`not in`, | Comparisons, including membership | -| :keyword:`is`, :keyword:`is not`, ``<``, | tests and identity tests | -| ``<=``, ``>``, ``>=``, ``!=``, ``==`` | | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`not x ` | Boolean NOT | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`and` | Boolean AND | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`or` | Boolean OR | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`if ` -- :keyword:`!else` | Conditional expression | -+-----------------------------------------------+-------------------------------------+ -| :keyword:`lambda` | Lambda expression | -+-----------------------------------------------+-------------------------------------+ -| ``:=`` | Assignment expression | -+-----------------------------------------------+-------------------------------------+ - - -.. rubric:: Footnotes - -.. [#] While ``abs(x%y) < abs(y)`` is true mathematically, for floats it may not be - true numerically due to roundoff. For example, and assuming a platform on which - a Python float is an IEEE 754 double-precision number, in order that ``-1e-100 % - 1e100`` have the same sign as ``1e100``, the computed result is ``-1e-100 + - 1e100``, which is numerically exactly equal to ``1e100``. The function - :func:`math.fmod` returns a result whose sign matches the sign of the - first argument instead, and so returns ``-1e-100`` in this case. Which approach - is more appropriate depends on the application. - -.. [#] If x is very close to an exact integer multiple of y, it's possible for - ``x//y`` to be one larger than ``(x-x%y)//y`` due to rounding. In such - cases, Python returns the latter result, in order to preserve that - ``divmod(x,y)[0] * y + x % y`` be very close to ``x``. - -.. [#] The Unicode standard distinguishes between :dfn:`code points` - (e.g. U+0041) and :dfn:`abstract characters` (e.g. "LATIN CAPITAL LETTER A"). - While most abstract characters in Unicode are only represented using one - code point, there is a number of abstract characters that can in addition be - represented using a sequence of more than one code point. For example, the - abstract character "LATIN CAPITAL LETTER C WITH CEDILLA" can be represented - as a single :dfn:`precomposed character` at code position U+00C7, or as a - sequence of a :dfn:`base character` at code position U+0043 (LATIN CAPITAL - LETTER C), followed by a :dfn:`combining character` at code position U+0327 - (COMBINING CEDILLA). - - The comparison operators on strings compare at the level of Unicode code - points. This may be counter-intuitive to humans. For example, - ``"\u00C7" == "\u0043\u0327"`` is ``False``, even though both strings - represent the same abstract character "LATIN CAPITAL LETTER C WITH CEDILLA". - - To compare strings at the level of abstract characters (that is, in a way - intuitive to humans), use :func:`unicodedata.normalize`. - -.. [#] Due to automatic garbage-collection, free lists, and the dynamic nature of - descriptors, you may notice seemingly unusual behaviour in certain uses of - the :keyword:`is` operator, like those involving comparisons between instance - methods, or constants. Check their documentation for more info. - -.. [#] The power operator ``**`` binds less tightly than an arithmetic or - bitwise unary operator on its right, that is, ``2**-1`` is ``0.5``. - -.. [#] The ``%`` operator is also used for string formatting; the same - precedence applies. diff --git a/Doc/reference/grammar.rst b/Doc/reference/grammar.rst deleted file mode 100644 index bc1db7b039cd5a..00000000000000 --- a/Doc/reference/grammar.rst +++ /dev/null @@ -1,20 +0,0 @@ -Full Grammar specification -========================== - -This is the full Python grammar, derived directly from the grammar -used to generate the CPython parser (see :source:`Grammar/python.gram`). -The version here omits details related to code generation and -error recovery. - -The notation is a mixture of `EBNF -`_ -and `PEG `_. -In particular, ``&`` followed by a symbol, token or parenthesized -group indicates a positive lookahead (i.e., is required to match but -not consumed), while ``!`` indicates a negative lookahead (i.e., is -required *not* to match). We use the ``|`` separator to mean PEG's -"ordered choice" (written as ``/`` in traditional PEG grammars). See -:pep:`617` for more details on the grammar's syntax. - -.. literalinclude:: ../../Grammar/python.gram - :language: peg diff --git a/Doc/reference/import.rst b/Doc/reference/import.rst deleted file mode 100644 index c44fb9815e9f99..00000000000000 --- a/Doc/reference/import.rst +++ /dev/null @@ -1,1066 +0,0 @@ - -.. _importsystem: - -***************** -The import system -***************** - -.. index:: single: import machinery - -Python code in one :term:`module` gains access to the code in another module -by the process of :term:`importing` it. The :keyword:`import` statement is -the most common way of invoking the import machinery, but it is not the only -way. Functions such as :func:`importlib.import_module` and built-in -:func:`__import__` can also be used to invoke the import machinery. - -The :keyword:`import` statement combines two operations; it searches for the -named module, then it binds the results of that search to a name in the local -scope. The search operation of the :keyword:`!import` statement is defined as -a call to the :func:`__import__` function, with the appropriate arguments. -The return value of :func:`__import__` is used to perform the name -binding operation of the :keyword:`!import` statement. See the -:keyword:`!import` statement for the exact details of that name binding -operation. - -A direct call to :func:`__import__` performs only the module search and, if -found, the module creation operation. While certain side-effects may occur, -such as the importing of parent packages, and the updating of various caches -(including :data:`sys.modules`), only the :keyword:`import` statement performs -a name binding operation. - -When an :keyword:`import` statement is executed, the standard builtin -:func:`__import__` function is called. Other mechanisms for invoking the -import system (such as :func:`importlib.import_module`) may choose to bypass -:func:`__import__` and use their own solutions to implement import semantics. - -When a module is first imported, Python searches for the module and if found, -it creates a module object [#fnmo]_, initializing it. If the named module -cannot be found, a :exc:`ModuleNotFoundError` is raised. Python implements various -strategies to search for the named module when the import machinery is -invoked. These strategies can be modified and extended by using various hooks -described in the sections below. - -.. versionchanged:: 3.3 - The import system has been updated to fully implement the second phase - of :pep:`302`. There is no longer any implicit import machinery - the full - import system is exposed through :data:`sys.meta_path`. In addition, - native namespace package support has been implemented (see :pep:`420`). - - -:mod:`importlib` -================ - -The :mod:`importlib` module provides a rich API for interacting with the -import system. For example :func:`importlib.import_module` provides a -recommended, simpler API than built-in :func:`__import__` for invoking the -import machinery. Refer to the :mod:`importlib` library documentation for -additional detail. - - - -Packages -======== - -.. index:: - single: package - -Python has only one type of module object, and all modules are of this type, -regardless of whether the module is implemented in Python, C, or something -else. To help organize modules and provide a naming hierarchy, Python has a -concept of :term:`packages `. - -You can think of packages as the directories on a file system and modules as -files within directories, but don't take this analogy too literally since -packages and modules need not originate from the file system. For the -purposes of this documentation, we'll use this convenient analogy of -directories and files. Like file system directories, packages are organized -hierarchically, and packages may themselves contain subpackages, as well as -regular modules. - -It's important to keep in mind that all packages are modules, but not all -modules are packages. Or put another way, packages are just a special kind of -module. Specifically, any module that contains a ``__path__`` attribute is -considered a package. - -All modules have a name. Subpackage names are separated from their parent -package name by a dot, akin to Python's standard attribute access syntax. Thus -you might have a package called :mod:`email`, which in turn has a subpackage -called :mod:`email.mime` and a module within that subpackage called -:mod:`email.mime.text`. - - -Regular packages ----------------- - -.. index:: - pair: package; regular - -Python defines two types of packages, :term:`regular packages ` and :term:`namespace packages `. Regular -packages are traditional packages as they existed in Python 3.2 and earlier. -A regular package is typically implemented as a directory containing an -``__init__.py`` file. When a regular package is imported, this -``__init__.py`` file is implicitly executed, and the objects it defines are -bound to names in the package's namespace. The ``__init__.py`` file can -contain the same Python code that any other module can contain, and Python -will add some additional attributes to the module when it is imported. - -For example, the following file system layout defines a top level ``parent`` -package with three subpackages:: - - parent/ - __init__.py - one/ - __init__.py - two/ - __init__.py - three/ - __init__.py - -Importing ``parent.one`` will implicitly execute ``parent/__init__.py`` and -``parent/one/__init__.py``. Subsequent imports of ``parent.two`` or -``parent.three`` will execute ``parent/two/__init__.py`` and -``parent/three/__init__.py`` respectively. - - -Namespace packages ------------------- - -.. index:: - pair: package; namespace - pair: package; portion - -A namespace package is a composite of various :term:`portions `, -where each portion contributes a subpackage to the parent package. Portions -may reside in different locations on the file system. Portions may also be -found in zip files, on the network, or anywhere else that Python searches -during import. Namespace packages may or may not correspond directly to -objects on the file system; they may be virtual modules that have no concrete -representation. - -Namespace packages do not use an ordinary list for their ``__path__`` -attribute. They instead use a custom iterable type which will automatically -perform a new search for package portions on the next import attempt within -that package if the path of their parent package (or :data:`sys.path` for a -top level package) changes. - -With namespace packages, there is no ``parent/__init__.py`` file. In fact, -there may be multiple ``parent`` directories found during import search, where -each one is provided by a different portion. Thus ``parent/one`` may not be -physically located next to ``parent/two``. In this case, Python will create a -namespace package for the top-level ``parent`` package whenever it or one of -its subpackages is imported. - -See also :pep:`420` for the namespace package specification. - - -Searching -========= - -To begin the search, Python needs the :term:`fully qualified ` -name of the module (or package, but for the purposes of this discussion, the -difference is immaterial) being imported. This name may come from various -arguments to the :keyword:`import` statement, or from the parameters to the -:func:`importlib.import_module` or :func:`__import__` functions. - -This name will be used in various phases of the import search, and it may be -the dotted path to a submodule, e.g. ``foo.bar.baz``. In this case, Python -first tries to import ``foo``, then ``foo.bar``, and finally ``foo.bar.baz``. -If any of the intermediate imports fail, a :exc:`ModuleNotFoundError` is raised. - - -The module cache ----------------- - -.. index:: - single: sys.modules - -The first place checked during import search is :data:`sys.modules`. This -mapping serves as a cache of all modules that have been previously imported, -including the intermediate paths. So if ``foo.bar.baz`` was previously -imported, :data:`sys.modules` will contain entries for ``foo``, ``foo.bar``, -and ``foo.bar.baz``. Each key will have as its value the corresponding module -object. - -During import, the module name is looked up in :data:`sys.modules` and if -present, the associated value is the module satisfying the import, and the -process completes. However, if the value is ``None``, then a -:exc:`ModuleNotFoundError` is raised. If the module name is missing, Python will -continue searching for the module. - -:data:`sys.modules` is writable. Deleting a key may not destroy the -associated module (as other modules may hold references to it), -but it will invalidate the cache entry for the named module, causing -Python to search anew for the named module upon its next -import. The key can also be assigned to ``None``, forcing the next import -of the module to result in a :exc:`ModuleNotFoundError`. - -Beware though, as if you keep a reference to the module object, -invalidate its cache entry in :data:`sys.modules`, and then re-import the -named module, the two module objects will *not* be the same. By contrast, -:func:`importlib.reload` will reuse the *same* module object, and simply -reinitialise the module contents by rerunning the module's code. - - -.. _finders-and-loaders: - -Finders and loaders -------------------- - -.. index:: - single: finder - single: loader - single: module spec - -If the named module is not found in :data:`sys.modules`, then Python's import -protocol is invoked to find and load the module. This protocol consists of -two conceptual objects, :term:`finders ` and :term:`loaders `. -A finder's job is to determine whether it can find the named module using -whatever strategy it knows about. Objects that implement both of these -interfaces are referred to as :term:`importers ` - they return -themselves when they find that they can load the requested module. - -Python includes a number of default finders and importers. The first one -knows how to locate built-in modules, and the second knows how to locate -frozen modules. A third default finder searches an :term:`import path` -for modules. The :term:`import path` is a list of locations that may -name file system paths or zip files. It can also be extended to search -for any locatable resource, such as those identified by URLs. - -The import machinery is extensible, so new finders can be added to extend the -range and scope of module searching. - -Finders do not actually load modules. If they can find the named module, they -return a :dfn:`module spec`, an encapsulation of the module's import-related -information, which the import machinery then uses when loading the module. - -The following sections describe the protocol for finders and loaders in more -detail, including how you can create and register new ones to extend the -import machinery. - -.. versionchanged:: 3.4 - In previous versions of Python, finders returned :term:`loaders ` - directly, whereas now they return module specs which *contain* loaders. - Loaders are still used during import but have fewer responsibilities. - -Import hooks ------------- - -.. index:: - single: import hooks - single: meta hooks - single: path hooks - pair: hooks; import - pair: hooks; meta - pair: hooks; path - -The import machinery is designed to be extensible; the primary mechanism for -this are the *import hooks*. There are two types of import hooks: *meta -hooks* and *import path hooks*. - -Meta hooks are called at the start of import processing, before any other -import processing has occurred, other than :data:`sys.modules` cache look up. -This allows meta hooks to override :data:`sys.path` processing, frozen -modules, or even built-in modules. Meta hooks are registered by adding new -finder objects to :data:`sys.meta_path`, as described below. - -Import path hooks are called as part of :data:`sys.path` (or -``package.__path__``) processing, at the point where their associated path -item is encountered. Import path hooks are registered by adding new callables -to :data:`sys.path_hooks` as described below. - - -The meta path -------------- - -.. index:: - single: sys.meta_path - pair: finder; find_spec - -When the named module is not found in :data:`sys.modules`, Python next -searches :data:`sys.meta_path`, which contains a list of meta path finder -objects. These finders are queried in order to see if they know how to handle -the named module. Meta path finders must implement a method called -:meth:`~importlib.abc.MetaPathFinder.find_spec()` which takes three arguments: -a name, an import path, and (optionally) a target module. The meta path -finder can use any strategy it wants to determine whether it can handle -the named module or not. - -If the meta path finder knows how to handle the named module, it returns a -spec object. If it cannot handle the named module, it returns ``None``. If -:data:`sys.meta_path` processing reaches the end of its list without returning -a spec, then a :exc:`ModuleNotFoundError` is raised. Any other exceptions -raised are simply propagated up, aborting the import process. - -The :meth:`~importlib.abc.MetaPathFinder.find_spec()` method of meta path -finders is called with two or three arguments. The first is the fully -qualified name of the module being imported, for example ``foo.bar.baz``. -The second argument is the path entries to use for the module search. For -top-level modules, the second argument is ``None``, but for submodules or -subpackages, the second argument is the value of the parent package's -``__path__`` attribute. If the appropriate ``__path__`` attribute cannot -be accessed, a :exc:`ModuleNotFoundError` is raised. The third argument -is an existing module object that will be the target of loading later. -The import system passes in a target module only during reload. - -The meta path may be traversed multiple times for a single import request. -For example, assuming none of the modules involved has already been cached, -importing ``foo.bar.baz`` will first perform a top level import, calling -``mpf.find_spec("foo", None, None)`` on each meta path finder (``mpf``). After -``foo`` has been imported, ``foo.bar`` will be imported by traversing the -meta path a second time, calling -``mpf.find_spec("foo.bar", foo.__path__, None)``. Once ``foo.bar`` has been -imported, the final traversal will call -``mpf.find_spec("foo.bar.baz", foo.bar.__path__, None)``. - -Some meta path finders only support top level imports. These importers will -always return ``None`` when anything other than ``None`` is passed as the -second argument. - -Python's default :data:`sys.meta_path` has three meta path finders, one that -knows how to import built-in modules, one that knows how to import frozen -modules, and one that knows how to import modules from an :term:`import path` -(i.e. the :term:`path based finder`). - -.. versionchanged:: 3.4 - The :meth:`~importlib.abc.MetaPathFinder.find_spec` method of meta path - finders replaced :meth:`~importlib.abc.MetaPathFinder.find_module`, which - is now deprecated. While it will continue to work without change, the - import machinery will try it only if the finder does not implement - ``find_spec()``. - -.. versionchanged:: 3.10 - Use of :meth:`~importlib.abc.MetaPathFinder.find_module` by the import system - now raises :exc:`ImportWarning`. - - -Loading -======= - -If and when a module spec is found, the import machinery will use it (and -the loader it contains) when loading the module. Here is an approximation -of what happens during the loading portion of import:: - - module = None - if spec.loader is not None and hasattr(spec.loader, 'create_module'): - # It is assumed 'exec_module' will also be defined on the loader. - module = spec.loader.create_module(spec) - if module is None: - module = ModuleType(spec.name) - # The import-related module attributes get set here: - _init_module_attrs(spec, module) - - if spec.loader is None: - # unsupported - raise ImportError - if spec.origin is None and spec.submodule_search_locations is not None: - # namespace package - sys.modules[spec.name] = module - elif not hasattr(spec.loader, 'exec_module'): - module = spec.loader.load_module(spec.name) - # Set __loader__ and __package__ if missing. - else: - sys.modules[spec.name] = module - try: - spec.loader.exec_module(module) - except BaseException: - try: - del sys.modules[spec.name] - except KeyError: - pass - raise - return sys.modules[spec.name] - -Note the following details: - -* If there is an existing module object with the given name in - :data:`sys.modules`, import will have already returned it. - -* The module will exist in :data:`sys.modules` before the loader - executes the module code. This is crucial because the module code may - (directly or indirectly) import itself; adding it to :data:`sys.modules` - beforehand prevents unbounded recursion in the worst case and multiple - loading in the best. - -* If loading fails, the failing module -- and only the failing module -- - gets removed from :data:`sys.modules`. Any module already in the - :data:`sys.modules` cache, and any module that was successfully loaded - as a side-effect, must remain in the cache. This contrasts with - reloading where even the failing module is left in :data:`sys.modules`. - -* After the module is created but before execution, the import machinery - sets the import-related module attributes ("_init_module_attrs" in - the pseudo-code example above), as summarized in a - :ref:`later section `. - -* Module execution is the key moment of loading in which the module's - namespace gets populated. Execution is entirely delegated to the - loader, which gets to decide what gets populated and how. - -* The module created during loading and passed to exec_module() may - not be the one returned at the end of import [#fnlo]_. - -.. versionchanged:: 3.4 - The import system has taken over the boilerplate responsibilities of - loaders. These were previously performed by the - :meth:`importlib.abc.Loader.load_module` method. - -Loaders -------- - -Module loaders provide the critical function of loading: module execution. -The import machinery calls the :meth:`importlib.abc.Loader.exec_module` -method with a single argument, the module object to execute. Any value -returned from :meth:`~importlib.abc.Loader.exec_module` is ignored. - -Loaders must satisfy the following requirements: - -* If the module is a Python module (as opposed to a built-in module or a - dynamically loaded extension), the loader should execute the module's code - in the module's global name space (``module.__dict__``). - -* If the loader cannot execute the module, it should raise an - :exc:`ImportError`, although any other exception raised during - :meth:`~importlib.abc.Loader.exec_module` will be propagated. - -In many cases, the finder and loader can be the same object; in such cases the -:meth:`~importlib.abc.MetaPathFinder.find_spec` method would just return a -spec with the loader set to ``self``. - -Module loaders may opt in to creating the module object during loading -by implementing a :meth:`~importlib.abc.Loader.create_module` method. -It takes one argument, the module spec, and returns the new module object -to use during loading. ``create_module()`` does not need to set any attributes -on the module object. If the method returns ``None``, the -import machinery will create the new module itself. - -.. versionadded:: 3.4 - The :meth:`~importlib.abc.Loader.create_module` method of loaders. - -.. versionchanged:: 3.4 - The :meth:`~importlib.abc.Loader.load_module` method was replaced by - :meth:`~importlib.abc.Loader.exec_module` and the import - machinery assumed all the boilerplate responsibilities of loading. - - For compatibility with existing loaders, the import machinery will use - the ``load_module()`` method of loaders if it exists and the loader does - not also implement ``exec_module()``. However, ``load_module()`` has been - deprecated and loaders should implement ``exec_module()`` instead. - - The ``load_module()`` method must implement all the boilerplate loading - functionality described above in addition to executing the module. All - the same constraints apply, with some additional clarification: - - * If there is an existing module object with the given name in - :data:`sys.modules`, the loader must use that existing module. - (Otherwise, :func:`importlib.reload` will not work correctly.) If the - named module does not exist in :data:`sys.modules`, the loader - must create a new module object and add it to :data:`sys.modules`. - - * The module *must* exist in :data:`sys.modules` before the loader - executes the module code, to prevent unbounded recursion or multiple - loading. - - * If loading fails, the loader must remove any modules it has inserted - into :data:`sys.modules`, but it must remove **only** the failing - module(s), and only if the loader itself has loaded the module(s) - explicitly. - -.. versionchanged:: 3.5 - A :exc:`DeprecationWarning` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. - -.. versionchanged:: 3.6 - An :exc:`ImportError` is raised when ``exec_module()`` is defined but - ``create_module()`` is not. - -.. versionchanged:: 3.10 - Use of ``load_module()`` will raise :exc:`ImportWarning`. - -Submodules ----------- - -When a submodule is loaded using any mechanism (e.g. ``importlib`` APIs, the -``import`` or ``import-from`` statements, or built-in ``__import__()``) a -binding is placed in the parent module's namespace to the submodule object. -For example, if package ``spam`` has a submodule ``foo``, after importing -``spam.foo``, ``spam`` will have an attribute ``foo`` which is bound to the -submodule. Let's say you have the following directory structure:: - - spam/ - __init__.py - foo.py - -and ``spam/__init__.py`` has the following line in it:: - - from .foo import Foo - -then executing the following puts name bindings for ``foo`` and ``Foo`` in the -``spam`` module:: - - >>> import spam - >>> spam.foo - - >>> spam.Foo - - -Given Python's familiar name binding rules this might seem surprising, but -it's actually a fundamental feature of the import system. The invariant -holding is that if you have ``sys.modules['spam']`` and -``sys.modules['spam.foo']`` (as you would after the above import), the latter -must appear as the ``foo`` attribute of the former. - -Module spec ------------ - -The import machinery uses a variety of information about each module -during import, especially before loading. Most of the information is -common to all modules. The purpose of a module's spec is to encapsulate -this import-related information on a per-module basis. - -Using a spec during import allows state to be transferred between import -system components, e.g. between the finder that creates the module spec -and the loader that executes it. Most importantly, it allows the -import machinery to perform the boilerplate operations of loading, -whereas without a module spec the loader had that responsibility. - -The module's spec is exposed as the ``__spec__`` attribute on a module object. -See :class:`~importlib.machinery.ModuleSpec` for details on the contents of -the module spec. - -.. versionadded:: 3.4 - -.. _import-mod-attrs: - -Import-related module attributes --------------------------------- - -The import machinery fills in these attributes on each module object -during loading, based on the module's spec, before the loader executes -the module. - -.. attribute:: __name__ - - The ``__name__`` attribute must be set to the fully qualified name of - the module. This name is used to uniquely identify the module in - the import system. - -.. attribute:: __loader__ - - The ``__loader__`` attribute must be set to the loader object that - the import machinery used when loading the module. This is mostly - for introspection, but can be used for additional loader-specific - functionality, for example getting data associated with a loader. - -.. attribute:: __package__ - - The module's ``__package__`` attribute must be set. Its value must - be a string, but it can be the same value as its ``__name__``. When - the module is a package, its ``__package__`` value should be set to - its ``__name__``. When the module is not a package, ``__package__`` - should be set to the empty string for top-level modules, or for - submodules, to the parent package's name. See :pep:`366` for further - details. - - This attribute is used instead of ``__name__`` to calculate explicit - relative imports for main modules, as defined in :pep:`366`. It is - expected to have the same value as ``__spec__.parent``. - - .. versionchanged:: 3.6 - The value of ``__package__`` is expected to be the same as - ``__spec__.parent``. - -.. attribute:: __spec__ - - The ``__spec__`` attribute must be set to the module spec that was - used when importing the module. Setting ``__spec__`` - appropriately applies equally to :ref:`modules initialized during - interpreter startup `. The one exception is ``__main__``, - where ``__spec__`` is :ref:`set to None in some cases `. - - When ``__package__`` is not defined, ``__spec__.parent`` is used as - a fallback. - - .. versionadded:: 3.4 - - .. versionchanged:: 3.6 - ``__spec__.parent`` is used as a fallback when ``__package__`` is - not defined. - -.. attribute:: __path__ - - If the module is a package (either regular or namespace), the module - object's ``__path__`` attribute must be set. The value must be - iterable, but may be empty if ``__path__`` has no further significance. - If ``__path__`` is not empty, it must produce strings when iterated - over. More details on the semantics of ``__path__`` are given - :ref:`below `. - - Non-package modules should not have a ``__path__`` attribute. - -.. attribute:: __file__ -.. attribute:: __cached__ - - ``__file__`` is optional (if set, value must be a string). It indicates - the pathname of the file from which the module was loaded (if - loaded from a file), or the pathname of the shared library file - for extension modules loaded dynamically from a shared library. - It might be missing for certain types of modules, such as C - modules that are statically linked into the interpreter, and the - import system may opt to leave it unset if it has no semantic - meaning (e.g. a module loaded from a database). - - If ``__file__`` is set then the ``__cached__`` attribute might also - be set, which is the path to any compiled version of - the code (e.g. byte-compiled file). The file does not need to exist - to set this attribute; the path can simply point to where the - compiled file would exist (see :pep:`3147`). - - Note that ``__cached__`` may be set even if ``__file__`` is not - set. However, that scenario is quite atypical. Ultimately, the - loader is what makes use of the module spec provided by the finder - (from which ``__file__`` and ``__cached__`` are derived). So - if a loader can load from a cached module but otherwise does not load - from a file, that atypical scenario may be appropriate. - -.. _package-path-rules: - -module.__path__ ---------------- - -By definition, if a module has a ``__path__`` attribute, it is a package. - -A package's ``__path__`` attribute is used during imports of its subpackages. -Within the import machinery, it functions much the same as :data:`sys.path`, -i.e. providing a list of locations to search for modules during import. -However, ``__path__`` is typically much more constrained than -:data:`sys.path`. - -``__path__`` must be an iterable of strings, but it may be empty. -The same rules used for :data:`sys.path` also apply to a package's -``__path__``, and :data:`sys.path_hooks` (described below) are -consulted when traversing a package's ``__path__``. - -A package's ``__init__.py`` file may set or alter the package's ``__path__`` -attribute, and this was typically the way namespace packages were implemented -prior to :pep:`420`. With the adoption of :pep:`420`, namespace packages no -longer need to supply ``__init__.py`` files containing only ``__path__`` -manipulation code; the import machinery automatically sets ``__path__`` -correctly for the namespace package. - -Module reprs ------------- - -By default, all modules have a usable repr, however depending on the -attributes set above, and in the module's spec, you can more explicitly -control the repr of module objects. - -If the module has a spec (``__spec__``), the import machinery will try -to generate a repr from it. If that fails or there is no spec, the import -system will craft a default repr using whatever information is available -on the module. It will try to use the ``module.__name__``, -``module.__file__``, and ``module.__loader__`` as input into the repr, -with defaults for whatever information is missing. - -Here are the exact rules used: - -* If the module has a ``__spec__`` attribute, the information in the spec - is used to generate the repr. The "name", "loader", "origin", and - "has_location" attributes are consulted. - -* If the module has a ``__file__`` attribute, this is used as part of the - module's repr. - -* If the module has no ``__file__`` but does have a ``__loader__`` that is not - ``None``, then the loader's repr is used as part of the module's repr. - -* Otherwise, just use the module's ``__name__`` in the repr. - -.. versionchanged:: 3.4 - Use of :meth:`loader.module_repr() ` - has been deprecated and the module spec is now used by the import - machinery to generate a module repr. - - For backward compatibility with Python 3.3, the module repr will be - generated by calling the loader's - :meth:`~importlib.abc.Loader.module_repr` method, if defined, before - trying either approach described above. However, the method is deprecated. - -.. versionchanged:: 3.10 - - Calling :meth:`~importlib.abc.Loader.module_repr` now occurs after trying to - use a module's ``__spec__`` attribute but before falling back on - ``__file__``. Use of :meth:`~importlib.abc.Loader.module_repr` is slated to - stop in Python 3.12. - -.. _pyc-invalidation: - -Cached bytecode invalidation ----------------------------- - -Before Python loads cached bytecode from a ``.pyc`` file, it checks whether the -cache is up-to-date with the source ``.py`` file. By default, Python does this -by storing the source's last-modified timestamp and size in the cache file when -writing it. At runtime, the import system then validates the cache file by -checking the stored metadata in the cache file against the source's -metadata. - -Python also supports "hash-based" cache files, which store a hash of the source -file's contents rather than its metadata. There are two variants of hash-based -``.pyc`` files: checked and unchecked. For checked hash-based ``.pyc`` files, -Python validates the cache file by hashing the source file and comparing the -resulting hash with the hash in the cache file. If a checked hash-based cache -file is found to be invalid, Python regenerates it and writes a new checked -hash-based cache file. For unchecked hash-based ``.pyc`` files, Python simply -assumes the cache file is valid if it exists. Hash-based ``.pyc`` files -validation behavior may be overridden with the :option:`--check-hash-based-pycs` -flag. - -.. versionchanged:: 3.7 - Added hash-based ``.pyc`` files. Previously, Python only supported - timestamp-based invalidation of bytecode caches. - - -The Path Based Finder -===================== - -.. index:: - single: path based finder - -As mentioned previously, Python comes with several default meta path finders. -One of these, called the :term:`path based finder` -(:class:`~importlib.machinery.PathFinder`), searches an :term:`import path`, -which contains a list of :term:`path entries `. Each path -entry names a location to search for modules. - -The path based finder itself doesn't know how to import anything. Instead, it -traverses the individual path entries, associating each of them with a -path entry finder that knows how to handle that particular kind of path. - -The default set of path entry finders implement all the semantics for finding -modules on the file system, handling special file types such as Python source -code (``.py`` files), Python byte code (``.pyc`` files) and -shared libraries (e.g. ``.so`` files). When supported by the :mod:`zipimport` -module in the standard library, the default path entry finders also handle -loading all of these file types (other than shared libraries) from zipfiles. - -Path entries need not be limited to file system locations. They can refer to -URLs, database queries, or any other location that can be specified as a -string. - -The path based finder provides additional hooks and protocols so that you -can extend and customize the types of searchable path entries. For example, -if you wanted to support path entries as network URLs, you could write a hook -that implements HTTP semantics to find modules on the web. This hook (a -callable) would return a :term:`path entry finder` supporting the protocol -described below, which was then used to get a loader for the module from the -web. - -A word of warning: this section and the previous both use the term *finder*, -distinguishing between them by using the terms :term:`meta path finder` and -:term:`path entry finder`. These two types of finders are very similar, -support similar protocols, and function in similar ways during the import -process, but it's important to keep in mind that they are subtly different. -In particular, meta path finders operate at the beginning of the import -process, as keyed off the :data:`sys.meta_path` traversal. - -By contrast, path entry finders are in a sense an implementation detail -of the path based finder, and in fact, if the path based finder were to be -removed from :data:`sys.meta_path`, none of the path entry finder semantics -would be invoked. - - -Path entry finders ------------------- - -.. index:: - single: sys.path - single: sys.path_hooks - single: sys.path_importer_cache - single: PYTHONPATH - -The :term:`path based finder` is responsible for finding and loading -Python modules and packages whose location is specified with a string -:term:`path entry`. Most path entries name locations in the file system, -but they need not be limited to this. - -As a meta path finder, the :term:`path based finder` implements the -:meth:`~importlib.abc.MetaPathFinder.find_spec` protocol previously -described, however it exposes additional hooks that can be used to -customize how modules are found and loaded from the :term:`import path`. - -Three variables are used by the :term:`path based finder`, :data:`sys.path`, -:data:`sys.path_hooks` and :data:`sys.path_importer_cache`. The ``__path__`` -attributes on package objects are also used. These provide additional ways -that the import machinery can be customized. - -:data:`sys.path` contains a list of strings providing search locations for -modules and packages. It is initialized from the :data:`PYTHONPATH` -environment variable and various other installation- and -implementation-specific defaults. Entries in :data:`sys.path` can name -directories on the file system, zip files, and potentially other "locations" -(see the :mod:`site` module) that should be searched for modules, such as -URLs, or database queries. Only strings should be present on -:data:`sys.path`; all other data types are ignored. - -The :term:`path based finder` is a :term:`meta path finder`, so the import -machinery begins the :term:`import path` search by calling the path -based finder's :meth:`~importlib.machinery.PathFinder.find_spec` method as -described previously. When the ``path`` argument to -:meth:`~importlib.machinery.PathFinder.find_spec` is given, it will be a -list of string paths to traverse - typically a package's ``__path__`` -attribute for an import within that package. If the ``path`` argument is -``None``, this indicates a top level import and :data:`sys.path` is used. - -The path based finder iterates over every entry in the search path, and -for each of these, looks for an appropriate :term:`path entry finder` -(:class:`~importlib.abc.PathEntryFinder`) for the -path entry. Because this can be an expensive operation (e.g. there may be -``stat()`` call overheads for this search), the path based finder maintains -a cache mapping path entries to path entry finders. This cache is maintained -in :data:`sys.path_importer_cache` (despite the name, this cache actually -stores finder objects rather than being limited to :term:`importer` objects). -In this way, the expensive search for a particular :term:`path entry` -location's :term:`path entry finder` need only be done once. User code is -free to remove cache entries from :data:`sys.path_importer_cache` forcing -the path based finder to perform the path entry search again [#fnpic]_. - -If the path entry is not present in the cache, the path based finder iterates -over every callable in :data:`sys.path_hooks`. Each of the :term:`path entry -hooks ` in this list is called with a single argument, the -path entry to be searched. This callable may either return a :term:`path -entry finder` that can handle the path entry, or it may raise -:exc:`ImportError`. An :exc:`ImportError` is used by the path based finder to -signal that the hook cannot find a :term:`path entry finder` -for that :term:`path entry`. The -exception is ignored and :term:`import path` iteration continues. The hook -should expect either a string or bytes object; the encoding of bytes objects -is up to the hook (e.g. it may be a file system encoding, UTF-8, or something -else), and if the hook cannot decode the argument, it should raise -:exc:`ImportError`. - -If :data:`sys.path_hooks` iteration ends with no :term:`path entry finder` -being returned, then the path based finder's -:meth:`~importlib.machinery.PathFinder.find_spec` method will store ``None`` -in :data:`sys.path_importer_cache` (to indicate that there is no finder for -this path entry) and return ``None``, indicating that this -:term:`meta path finder` could not find the module. - -If a :term:`path entry finder` *is* returned by one of the :term:`path entry -hook` callables on :data:`sys.path_hooks`, then the following protocol is used -to ask the finder for a module spec, which is then used when loading the -module. - -The current working directory -- denoted by an empty string -- is handled -slightly differently from other entries on :data:`sys.path`. First, if the -current working directory is found to not exist, no value is stored in -:data:`sys.path_importer_cache`. Second, the value for the current working -directory is looked up fresh for each module lookup. Third, the path used for -:data:`sys.path_importer_cache` and returned by -:meth:`importlib.machinery.PathFinder.find_spec` will be the actual current -working directory and not the empty string. - -Path entry finder protocol --------------------------- - -In order to support imports of modules and initialized packages and also to -contribute portions to namespace packages, path entry finders must implement -the :meth:`~importlib.abc.PathEntryFinder.find_spec` method. - -:meth:`~importlib.abc.PathEntryFinder.find_spec` takes two arguments: the -fully qualified name of the module being imported, and the (optional) target -module. ``find_spec()`` returns a fully populated spec for the module. -This spec will always have "loader" set (with one exception). - -To indicate to the import machinery that the spec represents a namespace -:term:`portion`, the path entry finder sets "submodule_search_locations" to -a list containing the portion. - -.. versionchanged:: 3.4 - :meth:`~importlib.abc.PathEntryFinder.find_spec` replaced - :meth:`~importlib.abc.PathEntryFinder.find_loader` and - :meth:`~importlib.abc.PathEntryFinder.find_module`, both of which - are now deprecated, but will be used if ``find_spec()`` is not defined. - - Older path entry finders may implement one of these two deprecated methods - instead of ``find_spec()``. The methods are still respected for the - sake of backward compatibility. However, if ``find_spec()`` is - implemented on the path entry finder, the legacy methods are ignored. - - :meth:`~importlib.abc.PathEntryFinder.find_loader` takes one argument, the - fully qualified name of the module being imported. ``find_loader()`` - returns a 2-tuple where the first item is the loader and the second item - is a namespace :term:`portion`. - - For backwards compatibility with other implementations of the import - protocol, many path entry finders also support the same, - traditional ``find_module()`` method that meta path finders support. - However path entry finder ``find_module()`` methods are never called - with a ``path`` argument (they are expected to record the appropriate - path information from the initial call to the path hook). - - The ``find_module()`` method on path entry finders is deprecated, - as it does not allow the path entry finder to contribute portions to - namespace packages. If both ``find_loader()`` and ``find_module()`` - exist on a path entry finder, the import system will always call - ``find_loader()`` in preference to ``find_module()``. - -.. versionchanged:: 3.10 - Calls to :meth:`~importlib.abc.PathEntryFinder.find_module` and - :meth:`~importlib.abc.PathEntryFinder.find_loader` by the import - system will raise :exc:`ImportWarning`. - - -Replacing the standard import system -==================================== - -The most reliable mechanism for replacing the entire import system is to -delete the default contents of :data:`sys.meta_path`, replacing them -entirely with a custom meta path hook. - -If it is acceptable to only alter the behaviour of import statements -without affecting other APIs that access the import system, then replacing -the builtin :func:`__import__` function may be sufficient. This technique -may also be employed at the module level to only alter the behaviour of -import statements within that module. - -To selectively prevent the import of some modules from a hook early on the -meta path (rather than disabling the standard import system entirely), -it is sufficient to raise :exc:`ModuleNotFoundError` directly from -:meth:`~importlib.abc.MetaPathFinder.find_spec` instead of returning -``None``. The latter indicates that the meta path search should continue, -while raising an exception terminates it immediately. - -.. _relativeimports: - -Package Relative Imports -======================== - -Relative imports use leading dots. A single leading dot indicates a relative -import, starting with the current package. Two or more leading dots indicate a -relative import to the parent(s) of the current package, one level per dot -after the first. For example, given the following package layout:: - - package/ - __init__.py - subpackage1/ - __init__.py - moduleX.py - moduleY.py - subpackage2/ - __init__.py - moduleZ.py - moduleA.py - -In either ``subpackage1/moduleX.py`` or ``subpackage1/__init__.py``, -the following are valid relative imports:: - - from .moduleY import spam - from .moduleY import spam as ham - from . import moduleY - from ..subpackage1 import moduleY - from ..subpackage2.moduleZ import eggs - from ..moduleA import foo - -Absolute imports may use either the ``import <>`` or ``from <> import <>`` -syntax, but relative imports may only use the second form; the reason -for this is that:: - - import XXX.YYY.ZZZ - -should expose ``XXX.YYY.ZZZ`` as a usable expression, but .moduleY is -not a valid expression. - - -.. _import-dunder-main: - -Special considerations for __main__ -=================================== - -The :mod:`__main__` module is a special case relative to Python's import -system. As noted :ref:`elsewhere `, the ``__main__`` module -is directly initialized at interpreter startup, much like :mod:`sys` and -:mod:`builtins`. However, unlike those two, it doesn't strictly -qualify as a built-in module. This is because the manner in which -``__main__`` is initialized depends on the flags and other options with -which the interpreter is invoked. - -.. _main_spec: - -__main__.__spec__ ------------------ - -Depending on how :mod:`__main__` is initialized, ``__main__.__spec__`` -gets set appropriately or to ``None``. - -When Python is started with the :option:`-m` option, ``__spec__`` is set -to the module spec of the corresponding module or package. ``__spec__`` is -also populated when the ``__main__`` module is loaded as part of executing a -directory, zipfile or other :data:`sys.path` entry. - -In :ref:`the remaining cases ` -``__main__.__spec__`` is set to ``None``, as the code used to populate the -:mod:`__main__` does not correspond directly with an importable module: - -- interactive prompt -- :option:`-c` option -- running from stdin -- running directly from a source or bytecode file - -Note that ``__main__.__spec__`` is always ``None`` in the last case, -*even if* the file could technically be imported directly as a module -instead. Use the :option:`-m` switch if valid module metadata is desired -in :mod:`__main__`. - -Note also that even when ``__main__`` corresponds with an importable module -and ``__main__.__spec__`` is set accordingly, they're still considered -*distinct* modules. This is due to the fact that blocks guarded by -``if __name__ == "__main__":`` checks only execute when the module is used -to populate the ``__main__`` namespace, and not during normal import. - - -References -========== - -The import machinery has evolved considerably since Python's early days. The -original `specification for packages -`_ is still available to read, -although some details have changed since the writing of that document. - -The original specification for :data:`sys.meta_path` was :pep:`302`, with -subsequent extension in :pep:`420`. - -:pep:`420` introduced :term:`namespace packages ` for -Python 3.3. :pep:`420` also introduced the :meth:`find_loader` protocol as an -alternative to :meth:`find_module`. - -:pep:`366` describes the addition of the ``__package__`` attribute for -explicit relative imports in main modules. - -:pep:`328` introduced absolute and explicit relative imports and initially -proposed ``__name__`` for semantics :pep:`366` would eventually specify for -``__package__``. - -:pep:`338` defines executing modules as scripts. - -:pep:`451` adds the encapsulation of per-module import state in spec -objects. It also off-loads most of the boilerplate responsibilities of -loaders back onto the import machinery. These changes allow the -deprecation of several APIs in the import system and also addition of new -methods to finders and loaders. - -.. rubric:: Footnotes - -.. [#fnmo] See :class:`types.ModuleType`. - -.. [#fnlo] The importlib implementation avoids using the return value - directly. Instead, it gets the module object by looking the module name up - in :data:`sys.modules`. The indirect effect of this is that an imported - module may replace itself in :data:`sys.modules`. This is - implementation-specific behavior that is not guaranteed to work in other - Python implementations. - -.. [#fnpic] In legacy code, it is possible to find instances of - :class:`imp.NullImporter` in the :data:`sys.path_importer_cache`. It - is recommended that code be changed to use ``None`` instead. See - :ref:`portingpythoncode` for more details. diff --git a/Doc/reference/index.rst b/Doc/reference/index.rst deleted file mode 100644 index a66673b17246d7..00000000000000 --- a/Doc/reference/index.rst +++ /dev/null @@ -1,29 +0,0 @@ -.. _reference-index: - -################################# - The Python Language Reference -################################# - -This reference manual describes the syntax and "core semantics" of the -language. It is terse, but attempts to be exact and complete. The semantics of -non-essential built-in object types and of the built-in functions and modules -are described in :ref:`library-index`. For an informal introduction to the -language, see :ref:`tutorial-index`. For C or C++ programmers, two additional -manuals exist: :ref:`extending-index` describes the high-level picture of how to -write a Python extension module, and the :ref:`c-api-index` describes the -interfaces available to C/C++ programmers in detail. - -.. toctree:: - :maxdepth: 2 - :numbered: - - introduction.rst - lexical_analysis.rst - datamodel.rst - executionmodel.rst - import.rst - expressions.rst - simple_stmts.rst - compound_stmts.rst - toplevel_components.rst - grammar.rst diff --git a/Doc/reference/introduction.rst b/Doc/reference/introduction.rst deleted file mode 100644 index cf186705e6e987..00000000000000 --- a/Doc/reference/introduction.rst +++ /dev/null @@ -1,133 +0,0 @@ - -.. _introduction: - -************ -Introduction -************ - -This reference manual describes the Python programming language. It is not -intended as a tutorial. - -While I am trying to be as precise as possible, I chose to use English rather -than formal specifications for everything except syntax and lexical analysis. -This should make the document more understandable to the average reader, but -will leave room for ambiguities. Consequently, if you were coming from Mars and -tried to re-implement Python from this document alone, you might have to guess -things and in fact you would probably end up implementing quite a different -language. On the other hand, if you are using Python and wonder what the precise -rules about a particular area of the language are, you should definitely be able -to find them here. If you would like to see a more formal definition of the -language, maybe you could volunteer your time --- or invent a cloning machine -:-). - -It is dangerous to add too many implementation details to a language reference -document --- the implementation may change, and other implementations of the -same language may work differently. On the other hand, CPython is the one -Python implementation in widespread use (although alternate implementations -continue to gain support), and its particular quirks are sometimes worth being -mentioned, especially where the implementation imposes additional limitations. -Therefore, you'll find short "implementation notes" sprinkled throughout the -text. - -Every Python implementation comes with a number of built-in and standard -modules. These are documented in :ref:`library-index`. A few built-in modules -are mentioned when they interact in a significant way with the language -definition. - - -.. _implementations: - -Alternate Implementations -========================= - -Though there is one Python implementation which is by far the most popular, -there are some alternate implementations which are of particular interest to -different audiences. - -Known implementations include: - -CPython - This is the original and most-maintained implementation of Python, written in C. - New language features generally appear here first. - -Jython - Python implemented in Java. This implementation can be used as a scripting - language for Java applications, or can be used to create applications using the - Java class libraries. It is also often used to create tests for Java libraries. - More information can be found at `the Jython website `_. - -Python for .NET - This implementation actually uses the CPython implementation, but is a managed - .NET application and makes .NET libraries available. It was created by Brian - Lloyd. For more information, see the `Python for .NET home page - `_. - -IronPython - An alternate Python for .NET. Unlike Python.NET, this is a complete Python - implementation that generates IL, and compiles Python code directly to .NET - assemblies. It was created by Jim Hugunin, the original creator of Jython. For - more information, see `the IronPython website `_. - -PyPy - An implementation of Python written completely in Python. It supports several - advanced features not found in other implementations like stackless support - and a Just in Time compiler. One of the goals of the project is to encourage - experimentation with the language itself by making it easier to modify the - interpreter (since it is written in Python). Additional information is - available on `the PyPy project's home page `_. - -Each of these implementations varies in some way from the language as documented -in this manual, or introduces specific information beyond what's covered in the -standard Python documentation. Please refer to the implementation-specific -documentation to determine what else you need to know about the specific -implementation you're using. - - -.. _notation: - -Notation -======== - -.. index:: BNF, grammar, syntax, notation - -The descriptions of lexical analysis and syntax use a modified -`Backus–Naur form (BNF) `_ grammar -notation. This uses the following style of definition: - -.. productionlist:: notation - name: `lc_letter` (`lc_letter` | "_")* - lc_letter: "a"..."z" - -The first line says that a ``name`` is an ``lc_letter`` followed by a sequence -of zero or more ``lc_letter``\ s and underscores. An ``lc_letter`` in turn is -any of the single characters ``'a'`` through ``'z'``. (This rule is actually -adhered to for the names defined in lexical and grammar rules in this document.) - -Each rule begins with a name (which is the name defined by the rule) and -``::=``. A vertical bar (``|``) is used to separate alternatives; it is the -least binding operator in this notation. A star (``*``) means zero or more -repetitions of the preceding item; likewise, a plus (``+``) means one or more -repetitions, and a phrase enclosed in square brackets (``[ ]``) means zero or -one occurrences (in other words, the enclosed phrase is optional). The ``*`` -and ``+`` operators bind as tightly as possible; parentheses are used for -grouping. Literal strings are enclosed in quotes. White space is only -meaningful to separate tokens. Rules are normally contained on a single line; -rules with many alternatives may be formatted alternatively with each line after -the first beginning with a vertical bar. - -.. index:: lexical definitions, ASCII - -In lexical definitions (as the example above), two more conventions are used: -Two literal characters separated by three dots mean a choice of any single -character in the given (inclusive) range of ASCII characters. A phrase between -angular brackets (``<...>``) gives an informal description of the symbol -defined; e.g., this could be used to describe the notion of 'control character' -if needed. - -Even though the notation used is almost the same, there is a big difference -between the meaning of lexical and syntactic definitions: a lexical definition -operates on the individual characters of the input source, while a syntax -definition operates on the stream of tokens generated by the lexical analysis. -All uses of BNF in the next chapter ("Lexical Analysis") are lexical -definitions; uses in subsequent chapters are syntactic definitions. - diff --git a/Doc/reference/lexical_analysis.rst b/Doc/reference/lexical_analysis.rst deleted file mode 100644 index b25d2c862927fe..00000000000000 --- a/Doc/reference/lexical_analysis.rst +++ /dev/null @@ -1,1012 +0,0 @@ - -.. _lexical: - -**************** -Lexical analysis -**************** - -.. index:: lexical analysis, parser, token - -A Python program is read by a *parser*. Input to the parser is a stream of -*tokens*, generated by the *lexical analyzer*. This chapter describes how the -lexical analyzer breaks a file into tokens. - -Python reads program text as Unicode code points; the encoding of a source file -can be given by an encoding declaration and defaults to UTF-8, see :pep:`3120` -for details. If the source file cannot be decoded, a :exc:`SyntaxError` is -raised. - - -.. _line-structure: - -Line structure -============== - -.. index:: line structure - -A Python program is divided into a number of *logical lines*. - - -.. _logical-lines: - -Logical lines -------------- - -.. index:: logical line, physical line, line joining, NEWLINE token - -The end of a logical line is represented by the token NEWLINE. Statements -cannot cross logical line boundaries except where NEWLINE is allowed by the -syntax (e.g., between statements in compound statements). A logical line is -constructed from one or more *physical lines* by following the explicit or -implicit *line joining* rules. - - -.. _physical-lines: - -Physical lines --------------- - -A physical line is a sequence of characters terminated by an end-of-line -sequence. In source files and strings, any of the standard platform line -termination sequences can be used - the Unix form using ASCII LF (linefeed), -the Windows form using the ASCII sequence CR LF (return followed by linefeed), -or the old Macintosh form using the ASCII CR (return) character. All of these -forms can be used equally, regardless of platform. The end of input also serves -as an implicit terminator for the final physical line. - -When embedding Python, source code strings should be passed to Python APIs using -the standard C conventions for newline characters (the ``\n`` character, -representing ASCII LF, is the line terminator). - - -.. _comments: - -Comments --------- - -.. index:: comment, hash character - single: # (hash); comment - -A comment starts with a hash character (``#``) that is not part of a string -literal, and ends at the end of the physical line. A comment signifies the end -of the logical line unless the implicit line joining rules are invoked. Comments -are ignored by the syntax. - - -.. _encodings: - -Encoding declarations ---------------------- - -.. index:: source character set, encoding declarations (source file) - single: # (hash); source encoding declaration - -If a comment in the first or second line of the Python script matches the -regular expression ``coding[=:]\s*([-\w.]+)``, this comment is processed as an -encoding declaration; the first group of this expression names the encoding of -the source code file. The encoding declaration must appear on a line of its -own. If it is the second line, the first line must also be a comment-only line. -The recommended forms of an encoding expression are :: - - # -*- coding: -*- - -which is recognized also by GNU Emacs, and :: - - # vim:fileencoding= - -which is recognized by Bram Moolenaar's VIM. - -If no encoding declaration is found, the default encoding is UTF-8. In -addition, if the first bytes of the file are the UTF-8 byte-order mark -(``b'\xef\xbb\xbf'``), the declared file encoding is UTF-8 (this is supported, -among others, by Microsoft's :program:`notepad`). - -If an encoding is declared, the encoding name must be recognized by Python -(see :ref:`standard-encodings`). The -encoding is used for all lexical analysis, including string literals, comments -and identifiers. - - -.. _explicit-joining: - -Explicit line joining ---------------------- - -.. index:: physical line, line joining, line continuation, backslash character - -Two or more physical lines may be joined into logical lines using backslash -characters (``\``), as follows: when a physical line ends in a backslash that is -not part of a string literal or comment, it is joined with the following forming -a single logical line, deleting the backslash and the following end-of-line -character. For example:: - - if 1900 < year < 2100 and 1 <= month <= 12 \ - and 1 <= day <= 31 and 0 <= hour < 24 \ - and 0 <= minute < 60 and 0 <= second < 60: # Looks like a valid date - return 1 - -A line ending in a backslash cannot carry a comment. A backslash does not -continue a comment. A backslash does not continue a token except for string -literals (i.e., tokens other than string literals cannot be split across -physical lines using a backslash). A backslash is illegal elsewhere on a line -outside a string literal. - - -.. _implicit-joining: - -Implicit line joining ---------------------- - -Expressions in parentheses, square brackets or curly braces can be split over -more than one physical line without using backslashes. For example:: - - month_names = ['Januari', 'Februari', 'Maart', # These are the - 'April', 'Mei', 'Juni', # Dutch names - 'Juli', 'Augustus', 'September', # for the months - 'Oktober', 'November', 'December'] # of the year - -Implicitly continued lines can carry comments. The indentation of the -continuation lines is not important. Blank continuation lines are allowed. -There is no NEWLINE token between implicit continuation lines. Implicitly -continued lines can also occur within triple-quoted strings (see below); in that -case they cannot carry comments. - - -.. _blank-lines: - -Blank lines ------------ - -.. index:: single: blank line - -A logical line that contains only spaces, tabs, formfeeds and possibly a -comment, is ignored (i.e., no NEWLINE token is generated). During interactive -input of statements, handling of a blank line may differ depending on the -implementation of the read-eval-print loop. In the standard interactive -interpreter, an entirely blank logical line (i.e. one containing not even -whitespace or a comment) terminates a multi-line statement. - - -.. _indentation: - -Indentation ------------ - -.. index:: indentation, leading whitespace, space, tab, grouping, statement grouping - -Leading whitespace (spaces and tabs) at the beginning of a logical line is used -to compute the indentation level of the line, which in turn is used to determine -the grouping of statements. - -Tabs are replaced (from left to right) by one to eight spaces such that the -total number of characters up to and including the replacement is a multiple of -eight (this is intended to be the same rule as used by Unix). The total number -of spaces preceding the first non-blank character then determines the line's -indentation. Indentation cannot be split over multiple physical lines using -backslashes; the whitespace up to the first backslash determines the -indentation. - -Indentation is rejected as inconsistent if a source file mixes tabs and spaces -in a way that makes the meaning dependent on the worth of a tab in spaces; a -:exc:`TabError` is raised in that case. - -**Cross-platform compatibility note:** because of the nature of text editors on -non-UNIX platforms, it is unwise to use a mixture of spaces and tabs for the -indentation in a single source file. It should also be noted that different -platforms may explicitly limit the maximum indentation level. - -A formfeed character may be present at the start of the line; it will be ignored -for the indentation calculations above. Formfeed characters occurring elsewhere -in the leading whitespace have an undefined effect (for instance, they may reset -the space count to zero). - -.. index:: INDENT token, DEDENT token - -The indentation levels of consecutive lines are used to generate INDENT and -DEDENT tokens, using a stack, as follows. - -Before the first line of the file is read, a single zero is pushed on the stack; -this will never be popped off again. The numbers pushed on the stack will -always be strictly increasing from bottom to top. At the beginning of each -logical line, the line's indentation level is compared to the top of the stack. -If it is equal, nothing happens. If it is larger, it is pushed on the stack, and -one INDENT token is generated. If it is smaller, it *must* be one of the -numbers occurring on the stack; all numbers on the stack that are larger are -popped off, and for each number popped off a DEDENT token is generated. At the -end of the file, a DEDENT token is generated for each number remaining on the -stack that is larger than zero. - -Here is an example of a correctly (though confusingly) indented piece of Python -code:: - - def perm(l): - # Compute the list of all permutations of l - if len(l) <= 1: - return [l] - r = [] - for i in range(len(l)): - s = l[:i] + l[i+1:] - p = perm(s) - for x in p: - r.append(l[i:i+1] + x) - return r - -The following example shows various indentation errors:: - - def perm(l): # error: first line indented - for i in range(len(l)): # error: not indented - s = l[:i] + l[i+1:] - p = perm(l[:i] + l[i+1:]) # error: unexpected indent - for x in p: - r.append(l[i:i+1] + x) - return r # error: inconsistent dedent - -(Actually, the first three errors are detected by the parser; only the last -error is found by the lexical analyzer --- the indentation of ``return r`` does -not match a level popped off the stack.) - - -.. _whitespace: - -Whitespace between tokens -------------------------- - -Except at the beginning of a logical line or in string literals, the whitespace -characters space, tab and formfeed can be used interchangeably to separate -tokens. Whitespace is needed between two tokens only if their concatenation -could otherwise be interpreted as a different token (e.g., ab is one token, but -a b is two tokens). - - -.. _other-tokens: - -Other tokens -============ - -Besides NEWLINE, INDENT and DEDENT, the following categories of tokens exist: -*identifiers*, *keywords*, *literals*, *operators*, and *delimiters*. Whitespace -characters (other than line terminators, discussed earlier) are not tokens, but -serve to delimit tokens. Where ambiguity exists, a token comprises the longest -possible string that forms a legal token, when read from left to right. - - -.. _identifiers: - -Identifiers and keywords -======================== - -.. index:: identifier, name - -Identifiers (also referred to as *names*) are described by the following lexical -definitions. - -The syntax of identifiers in Python is based on the Unicode standard annex -UAX-31, with elaboration and changes as defined below; see also :pep:`3131` for -further details. - -Within the ASCII range (U+0001..U+007F), the valid characters for identifiers -are the same as in Python 2.x: the uppercase and lowercase letters ``A`` through -``Z``, the underscore ``_`` and, except for the first character, the digits -``0`` through ``9``. - -Python 3.0 introduces additional characters from outside the ASCII range (see -:pep:`3131`). For these characters, the classification uses the version of the -Unicode Character Database as included in the :mod:`unicodedata` module. - -Identifiers are unlimited in length. Case is significant. - -.. productionlist:: python-grammar - identifier: `xid_start` `xid_continue`* - id_start: - id_continue: - xid_start: - xid_continue: - -The Unicode category codes mentioned above stand for: - -* *Lu* - uppercase letters -* *Ll* - lowercase letters -* *Lt* - titlecase letters -* *Lm* - modifier letters -* *Lo* - other letters -* *Nl* - letter numbers -* *Mn* - nonspacing marks -* *Mc* - spacing combining marks -* *Nd* - decimal numbers -* *Pc* - connector punctuations -* *Other_ID_Start* - explicit list of characters in `PropList.txt - `_ to support backwards - compatibility -* *Other_ID_Continue* - likewise - -All identifiers are converted into the normal form NFKC while parsing; comparison -of identifiers is based on NFKC. - -A non-normative HTML file listing all valid identifier characters for Unicode -14.0.0 can be found at -https://www.unicode.org/Public/14.0.0/ucd/DerivedCoreProperties.txt - - -.. _keywords: - -Keywords --------- - -.. index:: - single: keyword - single: reserved word - -The following identifiers are used as reserved words, or *keywords* of the -language, and cannot be used as ordinary identifiers. They must be spelled -exactly as written here: - -.. sourcecode:: text - - False await else import pass - None break except in raise - True class finally is return - and continue for lambda try - as def from nonlocal while - assert del global not with - async elif if or yield - - -.. _soft-keywords: - -Soft Keywords -------------- - -.. index:: soft keyword, keyword - -.. versionadded:: 3.10 - -Some identifiers are only reserved under specific contexts. These are known as -*soft keywords*. The identifiers ``match``, ``case`` and ``_`` can -syntactically act as keywords in contexts related to the pattern matching -statement, but this distinction is done at the parser level, not when -tokenizing. - -As soft keywords, their use with pattern matching is possible while still -preserving compatibility with existing code that uses ``match``, ``case`` and ``_`` as -identifier names. - - -.. index:: - single: _, identifiers - single: __, identifiers -.. _id-classes: - -Reserved classes of identifiers -------------------------------- - -Certain classes of identifiers (besides keywords) have special meanings. These -classes are identified by the patterns of leading and trailing underscore -characters: - -``_*`` - Not imported by ``from module import *``. - -``_`` - In a ``case`` pattern within a :keyword:`match` statement, ``_`` is a - :ref:`soft keyword ` that denotes a - :ref:`wildcard `. - - Separately, the interactive interpreter makes the result of the last evaluation - available in the variable ``_``. - (It is stored in the :mod:`builtins` module, alongside built-in - functions like ``print``.) - - Elsewhere, ``_`` is a regular identifier. It is often used to name - "special" items, but it is not special to Python itself. - - .. note:: - - The name ``_`` is often used in conjunction with internationalization; - refer to the documentation for the :mod:`gettext` module for more - information on this convention. - - It is also commonly used for unused variables. - -``__*__`` - System-defined names, informally known as "dunder" names. These names are - defined by the interpreter and its implementation (including the standard library). - Current system names are discussed in the :ref:`specialnames` section and elsewhere. - More will likely be defined in future versions of Python. *Any* use of ``__*__`` names, - in any context, that does not follow explicitly documented use, is subject to - breakage without warning. - -``__*`` - Class-private names. Names in this category, when used within the context of a - class definition, are re-written to use a mangled form to help avoid name - clashes between "private" attributes of base and derived classes. See section - :ref:`atom-identifiers`. - - -.. _literals: - -Literals -======== - -.. index:: literal, constant - -Literals are notations for constant values of some built-in types. - - -.. index:: string literal, bytes literal, ASCII - single: ' (single quote); string literal - single: " (double quote); string literal - single: u'; string literal - single: u"; string literal -.. _strings: - -String and Bytes literals -------------------------- - -String literals are described by the following lexical definitions: - -.. productionlist:: python-grammar - stringliteral: [`stringprefix`](`shortstring` | `longstring`) - stringprefix: "r" | "u" | "R" | "U" | "f" | "F" - : | "fr" | "Fr" | "fR" | "FR" | "rf" | "rF" | "Rf" | "RF" - shortstring: "'" `shortstringitem`* "'" | '"' `shortstringitem`* '"' - longstring: "'''" `longstringitem`* "'''" | '"""' `longstringitem`* '"""' - shortstringitem: `shortstringchar` | `stringescapeseq` - longstringitem: `longstringchar` | `stringescapeseq` - shortstringchar: - longstringchar: - stringescapeseq: "\" - -.. productionlist:: python-grammar - bytesliteral: `bytesprefix`(`shortbytes` | `longbytes`) - bytesprefix: "b" | "B" | "br" | "Br" | "bR" | "BR" | "rb" | "rB" | "Rb" | "RB" - shortbytes: "'" `shortbytesitem`* "'" | '"' `shortbytesitem`* '"' - longbytes: "'''" `longbytesitem`* "'''" | '"""' `longbytesitem`* '"""' - shortbytesitem: `shortbyteschar` | `bytesescapeseq` - longbytesitem: `longbyteschar` | `bytesescapeseq` - shortbyteschar: - longbyteschar: - bytesescapeseq: "\" - -One syntactic restriction not indicated by these productions is that whitespace -is not allowed between the :token:`~python-grammar:stringprefix` or -:token:`~python-grammar:bytesprefix` and the rest of the literal. The source -character set is defined by the encoding declaration; it is UTF-8 if no encoding -declaration is given in the source file; see section :ref:`encodings`. - -.. index:: triple-quoted string, Unicode Consortium, raw string - single: """; string literal - single: '''; string literal - -In plain English: Both types of literals can be enclosed in matching single quotes -(``'``) or double quotes (``"``). They can also be enclosed in matching groups -of three single or double quotes (these are generally referred to as -*triple-quoted strings*). The backslash (``\``) character is used to escape -characters that otherwise have a special meaning, such as newline, backslash -itself, or the quote character. - -.. index:: - single: b'; bytes literal - single: b"; bytes literal - -Bytes literals are always prefixed with ``'b'`` or ``'B'``; they produce an -instance of the :class:`bytes` type instead of the :class:`str` type. They -may only contain ASCII characters; bytes with a numeric value of 128 or greater -must be expressed with escapes. - -.. index:: - single: r'; raw string literal - single: r"; raw string literal - -Both string and bytes literals may optionally be prefixed with a letter ``'r'`` -or ``'R'``; such strings are called :dfn:`raw strings` and treat backslashes as -literal characters. As a result, in string literals, ``'\U'`` and ``'\u'`` -escapes in raw strings are not treated specially. Given that Python 2.x's raw -unicode literals behave differently than Python 3.x's the ``'ur'`` syntax -is not supported. - -.. versionadded:: 3.3 - The ``'rb'`` prefix of raw bytes literals has been added as a synonym - of ``'br'``. - -.. versionadded:: 3.3 - Support for the unicode legacy literal (``u'value'``) was reintroduced - to simplify the maintenance of dual Python 2.x and 3.x codebases. - See :pep:`414` for more information. - -.. index:: - single: f'; formatted string literal - single: f"; formatted string literal - -A string literal with ``'f'`` or ``'F'`` in its prefix is a -:dfn:`formatted string literal`; see :ref:`f-strings`. The ``'f'`` may be -combined with ``'r'``, but not with ``'b'`` or ``'u'``, therefore raw -formatted strings are possible, but formatted bytes literals are not. - -In triple-quoted literals, unescaped newlines and quotes are allowed (and are -retained), except that three unescaped quotes in a row terminate the literal. (A -"quote" is the character used to open the literal, i.e. either ``'`` or ``"``.) - -.. index:: physical line, escape sequence, Standard C, C - single: \ (backslash); escape sequence - single: \\; escape sequence - single: \a; escape sequence - single: \b; escape sequence - single: \f; escape sequence - single: \n; escape sequence - single: \r; escape sequence - single: \t; escape sequence - single: \v; escape sequence - single: \x; escape sequence - single: \N; escape sequence - single: \u; escape sequence - single: \U; escape sequence - -Unless an ``'r'`` or ``'R'`` prefix is present, escape sequences in string and -bytes literals are interpreted according to rules similar to those used by -Standard C. The recognized escape sequences are: - -+-------------------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=========================+=================================+=======+ -| ``\``\ | Backslash and newline ignored | \(1) | -+-------------------------+---------------------------------+-------+ -| ``\\`` | Backslash (``\``) | | -+-------------------------+---------------------------------+-------+ -| ``\'`` | Single quote (``'``) | | -+-------------------------+---------------------------------+-------+ -| ``\"`` | Double quote (``"``) | | -+-------------------------+---------------------------------+-------+ -| ``\a`` | ASCII Bell (BEL) | | -+-------------------------+---------------------------------+-------+ -| ``\b`` | ASCII Backspace (BS) | | -+-------------------------+---------------------------------+-------+ -| ``\f`` | ASCII Formfeed (FF) | | -+-------------------------+---------------------------------+-------+ -| ``\n`` | ASCII Linefeed (LF) | | -+-------------------------+---------------------------------+-------+ -| ``\r`` | ASCII Carriage Return (CR) | | -+-------------------------+---------------------------------+-------+ -| ``\t`` | ASCII Horizontal Tab (TAB) | | -+-------------------------+---------------------------------+-------+ -| ``\v`` | ASCII Vertical Tab (VT) | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\\\{ooo}` | Character with octal value | (2,4) | -| | *ooo* | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\x{hh}` | Character with hex value *hh* | (3,4) | -+-------------------------+---------------------------------+-------+ - -Escape sequences only recognized in string literals are: - -+-------------------------+---------------------------------+-------+ -| Escape Sequence | Meaning | Notes | -+=========================+=================================+=======+ -| :samp:`\\N\\{{name}\\}` | Character named *name* in the | \(5) | -| | Unicode database | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\u{xxxx}` | Character with 16-bit hex value | \(6) | -| | *xxxx* | | -+-------------------------+---------------------------------+-------+ -| :samp:`\\U{xxxxxxxx}` | Character with 32-bit hex value | \(7) | -| | *xxxxxxxx* | | -+-------------------------+---------------------------------+-------+ - -Notes: - -(1) - A backslash can be added at the end of a line to ignore the newline:: - - >>> 'This string will not include \ - ... backslashes or newline characters.' - 'This string will not include backslashes or newline characters.' - - The same result can be achieved using :ref:`triple-quoted strings `, - or parentheses and :ref:`string literal concatenation `. - - -(2) - As in Standard C, up to three octal digits are accepted. - - .. versionchanged:: 3.11 - Octal escapes with value larger than ``0o377`` produce a :exc:`DeprecationWarning`. - In a future Python version they will be a :exc:`SyntaxWarning` and - eventually a :exc:`SyntaxError`. - -(3) - Unlike in Standard C, exactly two hex digits are required. - -(4) - In a bytes literal, hexadecimal and octal escapes denote the byte with the - given value. In a string literal, these escapes denote a Unicode character - with the given value. - -(5) - .. versionchanged:: 3.3 - Support for name aliases [#]_ has been added. - -(6) - Exactly four hex digits are required. - -(7) - Any Unicode character can be encoded this way. Exactly eight hex digits - are required. - - -.. index:: unrecognized escape sequence - -Unlike Standard C, all unrecognized escape sequences are left in the string -unchanged, i.e., *the backslash is left in the result*. (This behavior is -useful when debugging: if an escape sequence is mistyped, the resulting output -is more easily recognized as broken.) It is also important to note that the -escape sequences only recognized in string literals fall into the category of -unrecognized escapes for bytes literals. - -.. versionchanged:: 3.6 - Unrecognized escape sequences produce a :exc:`DeprecationWarning`. In - a future Python version they will be a :exc:`SyntaxWarning` and - eventually a :exc:`SyntaxError`. - -Even in a raw literal, quotes can be escaped with a backslash, but the -backslash remains in the result; for example, ``r"\""`` is a valid string -literal consisting of two characters: a backslash and a double quote; ``r"\"`` -is not a valid string literal (even a raw string cannot end in an odd number of -backslashes). Specifically, *a raw literal cannot end in a single backslash* -(since the backslash would escape the following quote character). Note also -that a single backslash followed by a newline is interpreted as those two -characters as part of the literal, *not* as a line continuation. - - -.. _string-concatenation: - -String literal concatenation ----------------------------- - -Multiple adjacent string or bytes literals (delimited by whitespace), possibly -using different quoting conventions, are allowed, and their meaning is the same -as their concatenation. Thus, ``"hello" 'world'`` is equivalent to -``"helloworld"``. This feature can be used to reduce the number of backslashes -needed, to split long strings conveniently across long lines, or even to add -comments to parts of strings, for example:: - - re.compile("[A-Za-z_]" # letter or underscore - "[A-Za-z0-9_]*" # letter, digit or underscore - ) - -Note that this feature is defined at the syntactical level, but implemented at -compile time. The '+' operator must be used to concatenate string expressions -at run time. Also note that literal concatenation can use different quoting -styles for each component (even mixing raw strings and triple quoted strings), -and formatted string literals may be concatenated with plain string literals. - - -.. index:: - single: formatted string literal - single: interpolated string literal - single: string; formatted literal - single: string; interpolated literal - single: f-string - single: fstring - single: {} (curly brackets); in formatted string literal - single: ! (exclamation); in formatted string literal - single: : (colon); in formatted string literal - single: = (equals); for help in debugging using string literals -.. _f-strings: - -Formatted string literals -------------------------- - -.. versionadded:: 3.6 - -A :dfn:`formatted string literal` or :dfn:`f-string` is a string literal -that is prefixed with ``'f'`` or ``'F'``. These strings may contain -replacement fields, which are expressions delimited by curly braces ``{}``. -While other string literals always have a constant value, formatted strings -are really expressions evaluated at run time. - -Escape sequences are decoded like in ordinary string literals (except when -a literal is also marked as a raw string). After decoding, the grammar -for the contents of the string is: - -.. productionlist:: python-grammar - f_string: (`literal_char` | "{{" | "}}" | `replacement_field`)* - replacement_field: "{" `f_expression` ["="] ["!" `conversion`] [":" `format_spec`] "}" - f_expression: (`conditional_expression` | "*" `or_expr`) - : ("," `conditional_expression` | "," "*" `or_expr`)* [","] - : | `yield_expression` - conversion: "s" | "r" | "a" - format_spec: (`literal_char` | NULL | `replacement_field`)* - literal_char: - -The parts of the string outside curly braces are treated literally, -except that any doubled curly braces ``'{{'`` or ``'}}'`` are replaced -with the corresponding single curly brace. A single opening curly -bracket ``'{'`` marks a replacement field, which starts with a -Python expression. To display both the expression text and its value after -evaluation, (useful in debugging), an equal sign ``'='`` may be added after the -expression. A conversion field, introduced by an exclamation point ``'!'`` may -follow. A format specifier may also be appended, introduced by a colon ``':'``. -A replacement field ends with a closing curly bracket ``'}'``. - -Expressions in formatted string literals are treated like regular -Python expressions surrounded by parentheses, with a few exceptions. -An empty expression is not allowed, and both :keyword:`lambda` and -assignment expressions ``:=`` must be surrounded by explicit parentheses. -Replacement expressions can contain line breaks (e.g. in triple-quoted -strings), but they cannot contain comments. Each expression is evaluated -in the context where the formatted string literal appears, in order from -left to right. - -.. versionchanged:: 3.7 - Prior to Python 3.7, an :keyword:`await` expression and comprehensions - containing an :keyword:`async for` clause were illegal in the expressions - in formatted string literals due to a problem with the implementation. - -When the equal sign ``'='`` is provided, the output will have the expression -text, the ``'='`` and the evaluated value. Spaces after the opening brace -``'{'``, within the expression and after the ``'='`` are all retained in the -output. By default, the ``'='`` causes the :func:`repr` of the expression to be -provided, unless there is a format specified. When a format is specified it -defaults to the :func:`str` of the expression unless a conversion ``'!r'`` is -declared. - -.. versionadded:: 3.8 - The equal sign ``'='``. - -If a conversion is specified, the result of evaluating the expression -is converted before formatting. Conversion ``'!s'`` calls :func:`str` on -the result, ``'!r'`` calls :func:`repr`, and ``'!a'`` calls :func:`ascii`. - -The result is then formatted using the :func:`format` protocol. The -format specifier is passed to the :meth:`~object.__format__` method of the -expression or conversion result. An empty string is passed when the -format specifier is omitted. The formatted result is then included in -the final value of the whole string. - -Top-level format specifiers may include nested replacement fields. These nested -fields may include their own conversion fields and :ref:`format specifiers -`, but may not include more deeply nested replacement fields. The -:ref:`format specifier mini-language ` is the same as that used by -the :meth:`str.format` method. - -Formatted string literals may be concatenated, but replacement fields -cannot be split across literals. - -Some examples of formatted string literals:: - - >>> name = "Fred" - >>> f"He said his name is {name!r}." - "He said his name is 'Fred'." - >>> f"He said his name is {repr(name)}." # repr() is equivalent to !r - "He said his name is 'Fred'." - >>> width = 10 - >>> precision = 4 - >>> value = decimal.Decimal("12.34567") - >>> f"result: {value:{width}.{precision}}" # nested fields - 'result: 12.35' - >>> today = datetime(year=2017, month=1, day=27) - >>> f"{today:%B %d, %Y}" # using date format specifier - 'January 27, 2017' - >>> f"{today=:%B %d, %Y}" # using date format specifier and debugging - 'today=January 27, 2017' - >>> number = 1024 - >>> f"{number:#0x}" # using integer format specifier - '0x400' - >>> foo = "bar" - >>> f"{ foo = }" # preserves whitespace - " foo = 'bar'" - >>> line = "The mill's closed" - >>> f"{line = }" - 'line = "The mill\'s closed"' - >>> f"{line = :20}" - "line = The mill's closed " - >>> f"{line = !r:20}" - 'line = "The mill\'s closed" ' - - -A consequence of sharing the same syntax as regular string literals is -that characters in the replacement fields must not conflict with the -quoting used in the outer formatted string literal:: - - f"abc {a["x"]} def" # error: outer string literal ended prematurely - f"abc {a['x']} def" # workaround: use different quoting - -Backslashes are not allowed in format expressions and will raise -an error:: - - f"newline: {ord('\n')}" # raises SyntaxError - -To include a value in which a backslash escape is required, create -a temporary variable. - - >>> newline = ord('\n') - >>> f"newline: {newline}" - 'newline: 10' - -Formatted string literals cannot be used as docstrings, even if they do not -include expressions. - -:: - - >>> def foo(): - ... f"Not a docstring" - ... - >>> foo.__doc__ is None - True - -See also :pep:`498` for the proposal that added formatted string literals, -and :meth:`str.format`, which uses a related format string mechanism. - - -.. _numbers: - -Numeric literals ----------------- - -.. index:: number, numeric literal, integer literal - floating point literal, hexadecimal literal - octal literal, binary literal, decimal literal, imaginary literal, complex literal - -There are three types of numeric literals: integers, floating point numbers, and -imaginary numbers. There are no complex literals (complex numbers can be formed -by adding a real number and an imaginary number). - -Note that numeric literals do not include a sign; a phrase like ``-1`` is -actually an expression composed of the unary operator '``-``' and the literal -``1``. - - -.. index:: - single: 0b; integer literal - single: 0o; integer literal - single: 0x; integer literal - single: _ (underscore); in numeric literal - -.. _integers: - -Integer literals ----------------- - -Integer literals are described by the following lexical definitions: - -.. productionlist:: python-grammar - integer: `decinteger` | `bininteger` | `octinteger` | `hexinteger` - decinteger: `nonzerodigit` (["_"] `digit`)* | "0"+ (["_"] "0")* - bininteger: "0" ("b" | "B") (["_"] `bindigit`)+ - octinteger: "0" ("o" | "O") (["_"] `octdigit`)+ - hexinteger: "0" ("x" | "X") (["_"] `hexdigit`)+ - nonzerodigit: "1"..."9" - digit: "0"..."9" - bindigit: "0" | "1" - octdigit: "0"..."7" - hexdigit: `digit` | "a"..."f" | "A"..."F" - -There is no limit for the length of integer literals apart from what can be -stored in available memory. - -Underscores are ignored for determining the numeric value of the literal. They -can be used to group digits for enhanced readability. One underscore can occur -between digits, and after base specifiers like ``0x``. - -Note that leading zeros in a non-zero decimal number are not allowed. This is -for disambiguation with C-style octal literals, which Python used before version -3.0. - -Some examples of integer literals:: - - 7 2147483647 0o177 0b100110111 - 3 79228162514264337593543950336 0o377 0xdeadbeef - 100_000_000_000 0b_1110_0101 - -.. versionchanged:: 3.6 - Underscores are now allowed for grouping purposes in literals. - - -.. index:: - single: . (dot); in numeric literal - single: e; in numeric literal - single: _ (underscore); in numeric literal -.. _floating: - -Floating point literals ------------------------ - -Floating point literals are described by the following lexical definitions: - -.. productionlist:: python-grammar - floatnumber: `pointfloat` | `exponentfloat` - pointfloat: [`digitpart`] `fraction` | `digitpart` "." - exponentfloat: (`digitpart` | `pointfloat`) `exponent` - digitpart: `digit` (["_"] `digit`)* - fraction: "." `digitpart` - exponent: ("e" | "E") ["+" | "-"] `digitpart` - -Note that the integer and exponent parts are always interpreted using radix 10. -For example, ``077e010`` is legal, and denotes the same number as ``77e10``. The -allowed range of floating point literals is implementation-dependent. As in -integer literals, underscores are supported for digit grouping. - -Some examples of floating point literals:: - - 3.14 10. .001 1e100 3.14e-10 0e0 3.14_15_93 - -.. versionchanged:: 3.6 - Underscores are now allowed for grouping purposes in literals. - - -.. index:: - single: j; in numeric literal -.. _imaginary: - -Imaginary literals ------------------- - -Imaginary literals are described by the following lexical definitions: - -.. productionlist:: python-grammar - imagnumber: (`floatnumber` | `digitpart`) ("j" | "J") - -An imaginary literal yields a complex number with a real part of 0.0. Complex -numbers are represented as a pair of floating point numbers and have the same -restrictions on their range. To create a complex number with a nonzero real -part, add a floating point number to it, e.g., ``(3+4j)``. Some examples of -imaginary literals:: - - 3.14j 10.j 10j .001j 1e100j 3.14e-10j 3.14_15_93j - - -.. _operators: - -Operators -========= - -.. index:: single: operators - -The following tokens are operators: - -.. code-block:: none - - - + - * ** / // % @ - << >> & | ^ ~ := - < > <= >= == != - - -.. _delimiters: - -Delimiters -========== - -.. index:: single: delimiters - -The following tokens serve as delimiters in the grammar: - -.. code-block:: none - - ( ) [ ] { } - , : . ; @ = -> - += -= *= /= //= %= @= - &= |= ^= >>= <<= **= - -The period can also occur in floating-point and imaginary literals. A sequence -of three periods has a special meaning as an ellipsis literal. The second half -of the list, the augmented assignment operators, serve lexically as delimiters, -but also perform an operation. - -The following printing ASCII characters have special meaning as part of other -tokens or are otherwise significant to the lexical analyzer: - -.. code-block:: none - - ' " # \ - -The following printing ASCII characters are not used in Python. Their -occurrence outside string literals and comments is an unconditional error: - -.. code-block:: none - - $ ? ` - - -.. rubric:: Footnotes - -.. [#] https://www.unicode.org/Public/11.0.0/ucd/NameAliases.txt diff --git a/Doc/reference/simple_stmts.rst b/Doc/reference/simple_stmts.rst deleted file mode 100644 index bf9a4e7d2a021b..00000000000000 --- a/Doc/reference/simple_stmts.rst +++ /dev/null @@ -1,1014 +0,0 @@ - -.. _simple: - -***************** -Simple statements -***************** - -.. index:: pair: simple; statement - -A simple statement is comprised within a single logical line. Several simple -statements may occur on a single line separated by semicolons. The syntax for -simple statements is: - -.. productionlist:: python-grammar - simple_stmt: `expression_stmt` - : | `assert_stmt` - : | `assignment_stmt` - : | `augmented_assignment_stmt` - : | `annotated_assignment_stmt` - : | `pass_stmt` - : | `del_stmt` - : | `return_stmt` - : | `yield_stmt` - : | `raise_stmt` - : | `break_stmt` - : | `continue_stmt` - : | `import_stmt` - : | `future_stmt` - : | `global_stmt` - : | `nonlocal_stmt` - - -.. _exprstmts: - -Expression statements -===================== - -.. index:: - pair: expression; statement - pair: expression; list -.. index:: pair: expression; list - -Expression statements are used (mostly interactively) to compute and write a -value, or (usually) to call a procedure (a function that returns no meaningful -result; in Python, procedures return the value ``None``). Other uses of -expression statements are allowed and occasionally useful. The syntax for an -expression statement is: - -.. productionlist:: python-grammar - expression_stmt: `starred_expression` - -An expression statement evaluates the expression list (which may be a single -expression). - -.. index:: - pair: built-in function; repr - pair: object; None - pair: string; conversion - single: output - pair: standard; output - pair: writing; values - pair: procedure; call - -In interactive mode, if the value is not ``None``, it is converted to a string -using the built-in :func:`repr` function and the resulting string is written to -standard output on a line by itself (except if the result is ``None``, so that -procedure calls do not cause any output.) - -.. _assignment: - -Assignment statements -===================== - -.. index:: - single: = (equals); assignment statement - pair: assignment; statement - pair: binding; name - pair: rebinding; name - pair: object; mutable - pair: attribute; assignment - -Assignment statements are used to (re)bind names to values and to modify -attributes or items of mutable objects: - -.. productionlist:: python-grammar - assignment_stmt: (`target_list` "=")+ (`starred_expression` | `yield_expression`) - target_list: `target` ("," `target`)* [","] - target: `identifier` - : | "(" [`target_list`] ")" - : | "[" [`target_list`] "]" - : | `attributeref` - : | `subscription` - : | `slicing` - : | "*" `target` - -(See section :ref:`primaries` for the syntax definitions for *attributeref*, -*subscription*, and *slicing*.) - -An assignment statement evaluates the expression list (remember that this can be -a single expression or a comma-separated list, the latter yielding a tuple) and -assigns the single resulting object to each of the target lists, from left to -right. - -.. index:: - single: target - pair: target; list - -Assignment is defined recursively depending on the form of the target (list). -When a target is part of a mutable object (an attribute reference, subscription -or slicing), the mutable object must ultimately perform the assignment and -decide about its validity, and may raise an exception if the assignment is -unacceptable. The rules observed by various types and the exceptions raised are -given with the definition of the object types (see section :ref:`types`). - -.. index:: triple: target; list; assignment - single: , (comma); in target list - single: * (asterisk); in assignment target list - single: [] (square brackets); in assignment target list - single: () (parentheses); in assignment target list - -Assignment of an object to a target list, optionally enclosed in parentheses or -square brackets, is recursively defined as follows. - -* If the target list is a single target with no trailing comma, - optionally in parentheses, the object is assigned to that target. - -* Else: - - * If the target list contains one target prefixed with an asterisk, called a - "starred" target: The object must be an iterable with at least as many items - as there are targets in the target list, minus one. The first items of the - iterable are assigned, from left to right, to the targets before the starred - target. The final items of the iterable are assigned to the targets after - the starred target. A list of the remaining items in the iterable is then - assigned to the starred target (the list can be empty). - - * Else: The object must be an iterable with the same number of items as there - are targets in the target list, and the items are assigned, from left to - right, to the corresponding targets. - -Assignment of an object to a single target is recursively defined as follows. - -* If the target is an identifier (name): - - * If the name does not occur in a :keyword:`global` or :keyword:`nonlocal` - statement in the current code block: the name is bound to the object in the - current local namespace. - - * Otherwise: the name is bound to the object in the global namespace or the - outer namespace determined by :keyword:`nonlocal`, respectively. - - .. index:: single: destructor - - The name is rebound if it was already bound. This may cause the reference - count for the object previously bound to the name to reach zero, causing the - object to be deallocated and its destructor (if it has one) to be called. - - .. index:: pair: attribute; assignment - -* If the target is an attribute reference: The primary expression in the - reference is evaluated. It should yield an object with assignable attributes; - if this is not the case, :exc:`TypeError` is raised. That object is then - asked to assign the assigned object to the given attribute; if it cannot - perform the assignment, it raises an exception (usually but not necessarily - :exc:`AttributeError`). - - .. _attr-target-note: - - Note: If the object is a class instance and the attribute reference occurs on - both sides of the assignment operator, the right-hand side expression, ``a.x`` can access - either an instance attribute or (if no instance attribute exists) a class - attribute. The left-hand side target ``a.x`` is always set as an instance attribute, - creating it if necessary. Thus, the two occurrences of ``a.x`` do not - necessarily refer to the same attribute: if the right-hand side expression refers to a - class attribute, the left-hand side creates a new instance attribute as the target of the - assignment:: - - class Cls: - x = 3 # class variable - inst = Cls() - inst.x = inst.x + 1 # writes inst.x as 4 leaving Cls.x as 3 - - This description does not necessarily apply to descriptor attributes, such as - properties created with :func:`property`. - - .. index:: - pair: subscription; assignment - pair: object; mutable - -* If the target is a subscription: The primary expression in the reference is - evaluated. It should yield either a mutable sequence object (such as a list) - or a mapping object (such as a dictionary). Next, the subscript expression is - evaluated. - - .. index:: - pair: object; sequence - pair: object; list - - If the primary is a mutable sequence object (such as a list), the subscript - must yield an integer. If it is negative, the sequence's length is added to - it. The resulting value must be a nonnegative integer less than the - sequence's length, and the sequence is asked to assign the assigned object to - its item with that index. If the index is out of range, :exc:`IndexError` is - raised (assignment to a subscripted sequence cannot add new items to a list). - - .. index:: - pair: object; mapping - pair: object; dictionary - - If the primary is a mapping object (such as a dictionary), the subscript must - have a type compatible with the mapping's key type, and the mapping is then - asked to create a key/value pair which maps the subscript to the assigned - object. This can either replace an existing key/value pair with the same key - value, or insert a new key/value pair (if no key with the same value existed). - - For user-defined objects, the :meth:`__setitem__` method is called with - appropriate arguments. - - .. index:: pair: slicing; assignment - -* If the target is a slicing: The primary expression in the reference is - evaluated. It should yield a mutable sequence object (such as a list). The - assigned object should be a sequence object of the same type. Next, the lower - and upper bound expressions are evaluated, insofar they are present; defaults - are zero and the sequence's length. The bounds should evaluate to integers. - If either bound is negative, the sequence's length is added to it. The - resulting bounds are clipped to lie between zero and the sequence's length, - inclusive. Finally, the sequence object is asked to replace the slice with - the items of the assigned sequence. The length of the slice may be different - from the length of the assigned sequence, thus changing the length of the - target sequence, if the target sequence allows it. - -.. impl-detail:: - - In the current implementation, the syntax for targets is taken to be the same - as for expressions, and invalid syntax is rejected during the code generation - phase, causing less detailed error messages. - -Although the definition of assignment implies that overlaps between the -left-hand side and the right-hand side are 'simultaneous' (for example ``a, b = -b, a`` swaps two variables), overlaps *within* the collection of assigned-to -variables occur left-to-right, sometimes resulting in confusion. For instance, -the following program prints ``[0, 2]``:: - - x = [0, 1] - i = 0 - i, x[i] = 1, 2 # i is updated, then x[i] is updated - print(x) - - -.. seealso:: - - :pep:`3132` - Extended Iterable Unpacking - The specification for the ``*target`` feature. - - -.. _augassign: - -Augmented assignment statements -------------------------------- - -.. index:: - pair: augmented; assignment - single: statement; assignment, augmented - single: +=; augmented assignment - single: -=; augmented assignment - single: *=; augmented assignment - single: /=; augmented assignment - single: %=; augmented assignment - single: &=; augmented assignment - single: ^=; augmented assignment - single: |=; augmented assignment - single: **=; augmented assignment - single: //=; augmented assignment - single: >>=; augmented assignment - single: <<=; augmented assignment - -Augmented assignment is the combination, in a single statement, of a binary -operation and an assignment statement: - -.. productionlist:: python-grammar - augmented_assignment_stmt: `augtarget` `augop` (`expression_list` | `yield_expression`) - augtarget: `identifier` | `attributeref` | `subscription` | `slicing` - augop: "+=" | "-=" | "*=" | "@=" | "/=" | "//=" | "%=" | "**=" - : | ">>=" | "<<=" | "&=" | "^=" | "|=" - -(See section :ref:`primaries` for the syntax definitions of the last three -symbols.) - -An augmented assignment evaluates the target (which, unlike normal assignment -statements, cannot be an unpacking) and the expression list, performs the binary -operation specific to the type of assignment on the two operands, and assigns -the result to the original target. The target is only evaluated once. - -An augmented assignment expression like ``x += 1`` can be rewritten as ``x = x + -1`` to achieve a similar, but not exactly equal effect. In the augmented -version, ``x`` is only evaluated once. Also, when possible, the actual operation -is performed *in-place*, meaning that rather than creating a new object and -assigning that to the target, the old object is modified instead. - -Unlike normal assignments, augmented assignments evaluate the left-hand side -*before* evaluating the right-hand side. For example, ``a[i] += f(x)`` first -looks-up ``a[i]``, then it evaluates ``f(x)`` and performs the addition, and -lastly, it writes the result back to ``a[i]``. - -With the exception of assigning to tuples and multiple targets in a single -statement, the assignment done by augmented assignment statements is handled the -same way as normal assignments. Similarly, with the exception of the possible -*in-place* behavior, the binary operation performed by augmented assignment is -the same as the normal binary operations. - -For targets which are attribute references, the same :ref:`caveat about class -and instance attributes ` applies as for regular assignments. - - -.. _annassign: - -Annotated assignment statements -------------------------------- - -.. index:: - pair: annotated; assignment - single: statement; assignment, annotated - single: : (colon); annotated variable - -:term:`Annotation ` assignment is the combination, in a single -statement, of a variable or attribute annotation and an optional assignment statement: - -.. productionlist:: python-grammar - annotated_assignment_stmt: `augtarget` ":" `expression` - : ["=" (`starred_expression` | `yield_expression`)] - -The difference from normal :ref:`assignment` is that only a single target is allowed. - -For simple names as assignment targets, if in class or module scope, -the annotations are evaluated and stored in a special class or module -attribute :attr:`__annotations__` -that is a dictionary mapping from variable names (mangled if private) to -evaluated annotations. This attribute is writable and is automatically -created at the start of class or module body execution, if annotations -are found statically. - -For expressions as assignment targets, the annotations are evaluated if -in class or module scope, but not stored. - -If a name is annotated in a function scope, then this name is local for -that scope. Annotations are never evaluated and stored in function scopes. - -If the right hand side is present, an annotated -assignment performs the actual assignment before evaluating annotations -(where applicable). If the right hand side is not present for an expression -target, then the interpreter evaluates the target except for the last -:meth:`__setitem__` or :meth:`__setattr__` call. - -.. seealso:: - - :pep:`526` - Syntax for Variable Annotations - The proposal that added syntax for annotating the types of variables - (including class variables and instance variables), instead of expressing - them through comments. - - :pep:`484` - Type hints - The proposal that added the :mod:`typing` module to provide a standard - syntax for type annotations that can be used in static analysis tools and - IDEs. - -.. versionchanged:: 3.8 - Now annotated assignments allow the same expressions in the right hand side as - regular assignments. Previously, some expressions (like un-parenthesized - tuple expressions) caused a syntax error. - - -.. _assert: - -The :keyword:`!assert` statement -================================ - -.. index:: - ! pair: statement; assert - pair: debugging; assertions - single: , (comma); expression list - -Assert statements are a convenient way to insert debugging assertions into a -program: - -.. productionlist:: python-grammar - assert_stmt: "assert" `expression` ["," `expression`] - -The simple form, ``assert expression``, is equivalent to :: - - if __debug__: - if not expression: raise AssertionError - -The extended form, ``assert expression1, expression2``, is equivalent to :: - - if __debug__: - if not expression1: raise AssertionError(expression2) - -.. index:: - single: __debug__ - pair: exception; AssertionError - -These equivalences assume that :const:`__debug__` and :exc:`AssertionError` refer to -the built-in variables with those names. In the current implementation, the -built-in variable :const:`__debug__` is ``True`` under normal circumstances, -``False`` when optimization is requested (command line option :option:`-O`). The current -code generator emits no code for an assert statement when optimization is -requested at compile time. Note that it is unnecessary to include the source -code for the expression that failed in the error message; it will be displayed -as part of the stack trace. - -Assignments to :const:`__debug__` are illegal. The value for the built-in variable -is determined when the interpreter starts. - - -.. _pass: - -The :keyword:`!pass` statement -============================== - -.. index:: - pair: statement; pass - pair: null; operation - pair: null; operation - -.. productionlist:: python-grammar - pass_stmt: "pass" - -:keyword:`pass` is a null operation --- when it is executed, nothing happens. -It is useful as a placeholder when a statement is required syntactically, but no -code needs to be executed, for example:: - - def f(arg): pass # a function that does nothing (yet) - - class C: pass # a class with no methods (yet) - - -.. _del: - -The :keyword:`!del` statement -============================= - -.. index:: - ! pair: statement; del - pair: deletion; target - triple: deletion; target; list - -.. productionlist:: python-grammar - del_stmt: "del" `target_list` - -Deletion is recursively defined very similar to the way assignment is defined. -Rather than spelling it out in full details, here are some hints. - -Deletion of a target list recursively deletes each target, from left to right. - -.. index:: - pair: statement; global - pair: unbinding; name - -Deletion of a name removes the binding of that name from the local or global -namespace, depending on whether the name occurs in a :keyword:`global` statement -in the same code block. If the name is unbound, a :exc:`NameError` exception -will be raised. - -.. index:: pair: attribute; deletion - -Deletion of attribute references, subscriptions and slicings is passed to the -primary object involved; deletion of a slicing is in general equivalent to -assignment of an empty slice of the right type (but even this is determined by -the sliced object). - -.. versionchanged:: 3.2 - Previously it was illegal to delete a name from the local namespace if it - occurs as a free variable in a nested block. - - -.. _return: - -The :keyword:`!return` statement -================================ - -.. index:: - ! pair: statement; return - pair: function; definition - pair: class; definition - -.. productionlist:: python-grammar - return_stmt: "return" [`expression_list`] - -:keyword:`return` may only occur syntactically nested in a function definition, -not within a nested class definition. - -If an expression list is present, it is evaluated, else ``None`` is substituted. - -:keyword:`return` leaves the current function call with the expression list (or -``None``) as return value. - -.. index:: pair: keyword; finally - -When :keyword:`return` passes control out of a :keyword:`try` statement with a -:keyword:`finally` clause, that :keyword:`!finally` clause is executed before -really leaving the function. - -In a generator function, the :keyword:`return` statement indicates that the -generator is done and will cause :exc:`StopIteration` to be raised. The returned -value (if any) is used as an argument to construct :exc:`StopIteration` and -becomes the :attr:`StopIteration.value` attribute. - -In an asynchronous generator function, an empty :keyword:`return` statement -indicates that the asynchronous generator is done and will cause -:exc:`StopAsyncIteration` to be raised. A non-empty :keyword:`!return` -statement is a syntax error in an asynchronous generator function. - -.. _yield: - -The :keyword:`!yield` statement -=============================== - -.. index:: - pair: statement; yield - single: generator; function - single: generator; iterator - single: function; generator - pair: exception; StopIteration - -.. productionlist:: python-grammar - yield_stmt: `yield_expression` - -A :keyword:`yield` statement is semantically equivalent to a :ref:`yield -expression `. The yield statement can be used to omit the parentheses -that would otherwise be required in the equivalent yield expression -statement. For example, the yield statements :: - - yield - yield from - -are equivalent to the yield expression statements :: - - (yield ) - (yield from ) - -Yield expressions and statements are only used when defining a :term:`generator` -function, and are only used in the body of the generator function. Using yield -in a function definition is sufficient to cause that definition to create a -generator function instead of a normal function. - -For full details of :keyword:`yield` semantics, refer to the -:ref:`yieldexpr` section. - -.. _raise: - -The :keyword:`!raise` statement -=============================== - -.. index:: - ! pair: statement; raise - single: exception - pair: raising; exception - single: __traceback__ (exception attribute) - -.. productionlist:: python-grammar - raise_stmt: "raise" [`expression` ["from" `expression`]] - -If no expressions are present, :keyword:`raise` re-raises the -exception that is currently being handled, which is also known as the *active exception*. -If there isn't currently an active exception, a :exc:`RuntimeError` exception is raised -indicating that this is an error. - -Otherwise, :keyword:`raise` evaluates the first expression as the exception -object. It must be either a subclass or an instance of :class:`BaseException`. -If it is a class, the exception instance will be obtained when needed by -instantiating the class with no arguments. - -The :dfn:`type` of the exception is the exception instance's class, the -:dfn:`value` is the instance itself. - -.. index:: pair: object; traceback - -A traceback object is normally created automatically when an exception is raised -and attached to it as the :attr:`__traceback__` attribute, which is writable. -You can create an exception and set your own traceback in one step using the -:meth:`~BaseException.with_traceback` exception method (which returns the -same exception instance, with its traceback set to its argument), like so:: - - raise Exception("foo occurred").with_traceback(tracebackobj) - -.. index:: pair: exception; chaining - __cause__ (exception attribute) - __context__ (exception attribute) - -The ``from`` clause is used for exception chaining: if given, the second -*expression* must be another exception class or instance. If the second -expression is an exception instance, it will be attached to the raised -exception as the :attr:`__cause__` attribute (which is writable). If the -expression is an exception class, the class will be instantiated and the -resulting exception instance will be attached to the raised exception as the -:attr:`__cause__` attribute. If the raised exception is not handled, both -exceptions will be printed:: - - >>> try: - ... print(1 / 0) - ... except Exception as exc: - ... raise RuntimeError("Something bad happened") from exc - ... - Traceback (most recent call last): - File "", line 2, in - ZeroDivisionError: division by zero - - The above exception was the direct cause of the following exception: - - Traceback (most recent call last): - File "", line 4, in - RuntimeError: Something bad happened - -A similar mechanism works implicitly if a new exception is raised when -an exception is already being handled. An exception may be handled -when an :keyword:`except` or :keyword:`finally` clause, or a -:keyword:`with` statement, is used. The previous exception is then -attached as the new exception's :attr:`__context__` attribute:: - - >>> try: - ... print(1 / 0) - ... except: - ... raise RuntimeError("Something bad happened") - ... - Traceback (most recent call last): - File "", line 2, in - ZeroDivisionError: division by zero - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "", line 4, in - RuntimeError: Something bad happened - -Exception chaining can be explicitly suppressed by specifying :const:`None` in -the ``from`` clause:: - - >>> try: - ... print(1 / 0) - ... except: - ... raise RuntimeError("Something bad happened") from None - ... - Traceback (most recent call last): - File "", line 4, in - RuntimeError: Something bad happened - -Additional information on exceptions can be found in section :ref:`exceptions`, -and information about handling exceptions is in section :ref:`try`. - -.. versionchanged:: 3.3 - :const:`None` is now permitted as ``Y`` in ``raise X from Y``. - -.. versionadded:: 3.3 - The ``__suppress_context__`` attribute to suppress automatic display of the - exception context. - -.. versionchanged:: 3.11 - If the traceback of the active exception is modified in an :keyword:`except` - clause, a subsequent ``raise`` statement re-raises the exception with the - modified traceback. Previously, the exception was re-raised with the - traceback it had when it was caught. - -.. _break: - -The :keyword:`!break` statement -=============================== - -.. index:: - ! pair: statement; break - pair: statement; for - pair: statement; while - pair: loop; statement - -.. productionlist:: python-grammar - break_stmt: "break" - -:keyword:`break` may only occur syntactically nested in a :keyword:`for` or -:keyword:`while` loop, but not nested in a function or class definition within -that loop. - -.. index:: pair: keyword; else - pair: loop control; target - -It terminates the nearest enclosing loop, skipping the optional :keyword:`!else` -clause if the loop has one. - -If a :keyword:`for` loop is terminated by :keyword:`break`, the loop control -target keeps its current value. - -.. index:: pair: keyword; finally - -When :keyword:`break` passes control out of a :keyword:`try` statement with a -:keyword:`finally` clause, that :keyword:`!finally` clause is executed before -really leaving the loop. - - -.. _continue: - -The :keyword:`!continue` statement -================================== - -.. index:: - ! pair: statement; continue - pair: statement; for - pair: statement; while - pair: loop; statement - pair: keyword; finally - -.. productionlist:: python-grammar - continue_stmt: "continue" - -:keyword:`continue` may only occur syntactically nested in a :keyword:`for` or -:keyword:`while` loop, but not nested in a function or class definition within -that loop. It continues with the next cycle of the nearest enclosing loop. - -When :keyword:`continue` passes control out of a :keyword:`try` statement with a -:keyword:`finally` clause, that :keyword:`!finally` clause is executed before -really starting the next loop cycle. - - -.. _import: -.. _from: - -The :keyword:`!import` statement -================================ - -.. index:: - ! pair: statement; import - single: module; importing - pair: name; binding - pair: keyword; from - pair: keyword; as - pair: exception; ImportError - single: , (comma); import statement - -.. productionlist:: python-grammar - import_stmt: "import" `module` ["as" `identifier`] ("," `module` ["as" `identifier`])* - : | "from" `relative_module` "import" `identifier` ["as" `identifier`] - : ("," `identifier` ["as" `identifier`])* - : | "from" `relative_module` "import" "(" `identifier` ["as" `identifier`] - : ("," `identifier` ["as" `identifier`])* [","] ")" - : | "from" `relative_module` "import" "*" - module: (`identifier` ".")* `identifier` - relative_module: "."* `module` | "."+ - -The basic import statement (no :keyword:`from` clause) is executed in two -steps: - -#. find a module, loading and initializing it if necessary -#. define a name or names in the local namespace for the scope where - the :keyword:`import` statement occurs. - -When the statement contains multiple clauses (separated by -commas) the two steps are carried out separately for each clause, just -as though the clauses had been separated out into individual import -statements. - -The details of the first step, finding and loading modules, are described in -greater detail in the section on the :ref:`import system `, -which also describes the various types of packages and modules that can -be imported, as well as all the hooks that can be used to customize -the import system. Note that failures in this step may indicate either -that the module could not be located, *or* that an error occurred while -initializing the module, which includes execution of the module's code. - -If the requested module is retrieved successfully, it will be made -available in the local namespace in one of three ways: - -.. index:: single: as; import statement - -* If the module name is followed by :keyword:`!as`, then the name - following :keyword:`!as` is bound directly to the imported module. -* If no other name is specified, and the module being imported is a top - level module, the module's name is bound in the local namespace as a - reference to the imported module -* If the module being imported is *not* a top level module, then the name - of the top level package that contains the module is bound in the local - namespace as a reference to the top level package. The imported module - must be accessed using its full qualified name rather than directly - - -.. index:: - pair: name; binding - single: from; import statement - -The :keyword:`from` form uses a slightly more complex process: - -#. find the module specified in the :keyword:`from` clause, loading and - initializing it if necessary; -#. for each of the identifiers specified in the :keyword:`import` clauses: - - #. check if the imported module has an attribute by that name - #. if not, attempt to import a submodule with that name and then - check the imported module again for that attribute - #. if the attribute is not found, :exc:`ImportError` is raised. - #. otherwise, a reference to that value is stored in the local namespace, - using the name in the :keyword:`!as` clause if it is present, - otherwise using the attribute name - -Examples:: - - import foo # foo imported and bound locally - import foo.bar.baz # foo, foo.bar, and foo.bar.baz imported, foo bound locally - import foo.bar.baz as fbb # foo, foo.bar, and foo.bar.baz imported, foo.bar.baz bound as fbb - from foo.bar import baz # foo, foo.bar, and foo.bar.baz imported, foo.bar.baz bound as baz - from foo import attr # foo imported and foo.attr bound as attr - -.. index:: single: * (asterisk); import statement - -If the list of identifiers is replaced by a star (``'*'``), all public -names defined in the module are bound in the local namespace for the scope -where the :keyword:`import` statement occurs. - -.. index:: single: __all__ (optional module attribute) - -The *public names* defined by a module are determined by checking the module's -namespace for a variable named ``__all__``; if defined, it must be a sequence -of strings which are names defined or imported by that module. The names -given in ``__all__`` are all considered public and are required to exist. If -``__all__`` is not defined, the set of public names includes all names found -in the module's namespace which do not begin with an underscore character -(``'_'``). ``__all__`` should contain the entire public API. It is intended -to avoid accidentally exporting items that are not part of the API (such as -library modules which were imported and used within the module). - -The wild card form of import --- ``from module import *`` --- is only allowed at -the module level. Attempting to use it in class or function definitions will -raise a :exc:`SyntaxError`. - -.. index:: - single: relative; import - -When specifying what module to import you do not have to specify the absolute -name of the module. When a module or package is contained within another -package it is possible to make a relative import within the same top package -without having to mention the package name. By using leading dots in the -specified module or package after :keyword:`from` you can specify how high to -traverse up the current package hierarchy without specifying exact names. One -leading dot means the current package where the module making the import -exists. Two dots means up one package level. Three dots is up two levels, etc. -So if you execute ``from . import mod`` from a module in the ``pkg`` package -then you will end up importing ``pkg.mod``. If you execute ``from ..subpkg2 -import mod`` from within ``pkg.subpkg1`` you will import ``pkg.subpkg2.mod``. -The specification for relative imports is contained in -the :ref:`relativeimports` section. - -:func:`importlib.import_module` is provided to support applications that -determine dynamically the modules to be loaded. - -.. audit-event:: import module,filename,sys.path,sys.meta_path,sys.path_hooks import - -.. _future: - -Future statements ------------------ - -.. index:: - pair: future; statement - single: __future__; future statement - -A :dfn:`future statement` is a directive to the compiler that a particular -module should be compiled using syntax or semantics that will be available in a -specified future release of Python where the feature becomes standard. - -The future statement is intended to ease migration to future versions of Python -that introduce incompatible changes to the language. It allows use of the new -features on a per-module basis before the release in which the feature becomes -standard. - -.. productionlist:: python-grammar - future_stmt: "from" "__future__" "import" `feature` ["as" `identifier`] - : ("," `feature` ["as" `identifier`])* - : | "from" "__future__" "import" "(" `feature` ["as" `identifier`] - : ("," `feature` ["as" `identifier`])* [","] ")" - feature: `identifier` - -A future statement must appear near the top of the module. The only lines that -can appear before a future statement are: - -* the module docstring (if any), -* comments, -* blank lines, and -* other future statements. - -The only feature that requires using the future statement is -``annotations`` (see :pep:`563`). - -All historical features enabled by the future statement are still recognized -by Python 3. The list includes ``absolute_import``, ``division``, -``generators``, ``generator_stop``, ``unicode_literals``, -``print_function``, ``nested_scopes`` and ``with_statement``. They are -all redundant because they are always enabled, and only kept for -backwards compatibility. - -A future statement is recognized and treated specially at compile time: Changes -to the semantics of core constructs are often implemented by generating -different code. It may even be the case that a new feature introduces new -incompatible syntax (such as a new reserved word), in which case the compiler -may need to parse the module differently. Such decisions cannot be pushed off -until runtime. - -For any given release, the compiler knows which feature names have been defined, -and raises a compile-time error if a future statement contains a feature not -known to it. - -The direct runtime semantics are the same as for any import statement: there is -a standard module :mod:`__future__`, described later, and it will be imported in -the usual way at the time the future statement is executed. - -The interesting runtime semantics depend on the specific feature enabled by the -future statement. - -Note that there is nothing special about the statement:: - - import __future__ [as name] - -That is not a future statement; it's an ordinary import statement with no -special semantics or syntax restrictions. - -Code compiled by calls to the built-in functions :func:`exec` and :func:`compile` -that occur in a module :mod:`M` containing a future statement will, by default, -use the new syntax or semantics associated with the future statement. This can -be controlled by optional arguments to :func:`compile` --- see the documentation -of that function for details. - -A future statement typed at an interactive interpreter prompt will take effect -for the rest of the interpreter session. If an interpreter is started with the -:option:`-i` option, is passed a script name to execute, and the script includes -a future statement, it will be in effect in the interactive session started -after the script is executed. - -.. seealso:: - - :pep:`236` - Back to the __future__ - The original proposal for the __future__ mechanism. - - -.. _global: - -The :keyword:`!global` statement -================================ - -.. index:: - ! pair: statement; global - triple: global; name; binding - single: , (comma); identifier list - -.. productionlist:: python-grammar - global_stmt: "global" `identifier` ("," `identifier`)* - -The :keyword:`global` statement is a declaration which holds for the entire -current code block. It means that the listed identifiers are to be interpreted -as globals. It would be impossible to assign to a global variable without -:keyword:`!global`, although free variables may refer to globals without being -declared global. - -Names listed in a :keyword:`global` statement must not be used in the same code -block textually preceding that :keyword:`!global` statement. - -Names listed in a :keyword:`global` statement must not be defined as formal -parameters, or as targets in :keyword:`with` statements or :keyword:`except` clauses, or in a :keyword:`for` target list, :keyword:`class` -definition, function definition, :keyword:`import` statement, or variable -annotation. - -.. impl-detail:: - - The current implementation does not enforce some of these restrictions, but - programs should not abuse this freedom, as future implementations may enforce - them or silently change the meaning of the program. - -.. index:: - pair: built-in function; exec - pair: built-in function; eval - pair: built-in function; compile - -**Programmer's note:** :keyword:`global` is a directive to the parser. It -applies only to code parsed at the same time as the :keyword:`!global` statement. -In particular, a :keyword:`!global` statement contained in a string or code -object supplied to the built-in :func:`exec` function does not affect the code -block *containing* the function call, and code contained in such a string is -unaffected by :keyword:`!global` statements in the code containing the function -call. The same applies to the :func:`eval` and :func:`compile` functions. - - -.. _nonlocal: - -The :keyword:`!nonlocal` statement -================================== - -.. index:: pair: statement; nonlocal - single: , (comma); identifier list - -.. productionlist:: python-grammar - nonlocal_stmt: "nonlocal" `identifier` ("," `identifier`)* - -The :keyword:`nonlocal` statement causes the listed identifiers to refer to -previously bound variables in the nearest enclosing scope excluding globals. -This is important because the default behavior for binding is to search the -local namespace first. The statement allows encapsulated code to rebind -variables outside of the local scope besides the global (module) scope. - -Names listed in a :keyword:`nonlocal` statement, unlike those listed in a -:keyword:`global` statement, must refer to pre-existing bindings in an -enclosing scope (the scope in which a new binding should be created cannot -be determined unambiguously). - -Names listed in a :keyword:`nonlocal` statement must not collide with -pre-existing bindings in the local scope. - -.. seealso:: - - :pep:`3104` - Access to Names in Outer Scopes - The specification for the :keyword:`nonlocal` statement. diff --git a/Doc/reference/toplevel_components.rst b/Doc/reference/toplevel_components.rst deleted file mode 100644 index dd3d3d6878e289..00000000000000 --- a/Doc/reference/toplevel_components.rst +++ /dev/null @@ -1,107 +0,0 @@ - -.. _top-level: - -******************** -Top-level components -******************** - -.. index:: single: interpreter - -The Python interpreter can get its input from a number of sources: from a script -passed to it as standard input or as program argument, typed in interactively, -from a module source file, etc. This chapter gives the syntax used in these -cases. - - -.. _programs: - -Complete Python programs -======================== - -.. index:: single: program - -.. index:: - pair: module; sys - pair: module; __main__ - pair: module; builtins - -While a language specification need not prescribe how the language interpreter -is invoked, it is useful to have a notion of a complete Python program. A -complete Python program is executed in a minimally initialized environment: all -built-in and standard modules are available, but none have been initialized, -except for :mod:`sys` (various system services), :mod:`builtins` (built-in -functions, exceptions and ``None``) and :mod:`__main__`. The latter is used to -provide the local and global namespace for execution of the complete program. - -The syntax for a complete Python program is that for file input, described in -the next section. - -.. index:: - single: interactive mode - pair: module; __main__ - -The interpreter may also be invoked in interactive mode; in this case, it does -not read and execute a complete program but reads and executes one statement -(possibly compound) at a time. The initial environment is identical to that of -a complete program; each statement is executed in the namespace of -:mod:`__main__`. - -.. index:: - single: UNIX - single: Windows - single: command line - single: standard input - -A complete program can be passed to the interpreter -in three forms: with the :option:`-c` *string* command line option, as a file -passed as the first command line argument, or as standard input. If the file -or standard input is a tty device, the interpreter enters interactive mode; -otherwise, it executes the file as a complete program. - - -.. _file-input: - -File input -========== - -All input read from non-interactive files has the same form: - -.. productionlist:: python-grammar - file_input: (NEWLINE | `statement`)* - -This syntax is used in the following situations: - -* when parsing a complete Python program (from a file or from a string); - -* when parsing a module; - -* when parsing a string passed to the :func:`exec` function; - - -.. _interactive: - -Interactive input -================= - -Input in interactive mode is parsed using the following grammar: - -.. productionlist:: python-grammar - interactive_input: [`stmt_list`] NEWLINE | `compound_stmt` NEWLINE - -Note that a (top-level) compound statement must be followed by a blank line in -interactive mode; this is needed to help the parser detect the end of the input. - - -.. _expression-input: - -Expression input -================ - -.. index:: single: input -.. index:: pair: built-in function; eval - -:func:`eval` is used for expression input. It ignores leading whitespace. The -string argument to :func:`eval` must have the following form: - -.. productionlist:: python-grammar - eval_input: `expression_list` NEWLINE* diff --git a/Doc/requirements.txt b/Doc/requirements.txt deleted file mode 100644 index df79ae63840471..00000000000000 --- a/Doc/requirements.txt +++ /dev/null @@ -1,19 +0,0 @@ -# Requirements to build the Python documentation -# -# Note that when updating this file, you will likely also have to update -# the Doc/constraints.txt file. - -# Sphinx version is pinned so that new versions that introduce new warnings -# won't suddenly cause build failures. Updating the version is fine as long -# as no warnings are raised by doing so. -sphinx==4.5.0 - -blurb - -sphinxext-opengraph>=0.7.1 - -# The theme used by the documentation is stored separately, so we need -# to install that as well. -python-docs-theme>=2023.3.1,!=2023.7 - --c constraints.txt diff --git a/Doc/tools/extensions/asdl_highlight.py b/Doc/tools/extensions/asdl_highlight.py deleted file mode 100644 index b1989e53957072..00000000000000 --- a/Doc/tools/extensions/asdl_highlight.py +++ /dev/null @@ -1,54 +0,0 @@ -import os -import sys -from pathlib import Path - -CPYTHON_ROOT = Path(__file__).resolve().parent.parent.parent.parent -sys.path.append(str(CPYTHON_ROOT / "Parser")) - -from pygments.lexer import RegexLexer, bygroups, include, words -from pygments.token import (Comment, Generic, Keyword, Name, Operator, - Punctuation, Text) - -from asdl import builtin_types -from sphinx.highlighting import lexers - -class ASDLLexer(RegexLexer): - name = "ASDL" - aliases = ["asdl"] - filenames = ["*.asdl"] - _name = r"([^\W\d]\w*)" - _text_ws = r"(\s*)" - - tokens = { - "ws": [ - (r"\n", Text), - (r"\s+", Text), - (r"--.*?$", Comment.Singleline), - ], - "root": [ - include("ws"), - ( - r"(module)" + _text_ws + _name, - bygroups(Keyword, Text, Name.Tag), - ), - ( - r"(\w+)(\*\s|\?\s|\s)(\w+)", - bygroups(Name.Builtin.Pseudo, Operator, Name), - ), - (words(builtin_types), Name.Builtin), - (r"attributes", Name.Builtin), - ( - _name + _text_ws + "(=)", - bygroups(Name, Text, Operator), - ), - (_name, Name.Class), - (r"\|", Operator), - (r"{|}|\(|\)", Punctuation), - (r".", Text), - ], - } - - -def setup(app): - lexers["asdl"] = ASDLLexer() - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/c_annotations.py b/Doc/tools/extensions/c_annotations.py deleted file mode 100644 index 7cb6bd87103f64..00000000000000 --- a/Doc/tools/extensions/c_annotations.py +++ /dev/null @@ -1,218 +0,0 @@ -# -*- coding: utf-8 -*- -""" - c_annotations.py - ~~~~~~~~~~~~~~~~ - - Supports annotations for C API elements: - - * reference count annotations for C API functions. Based on - refcount.py and anno-api.py in the old Python documentation tools. - - * stable API annotations - - Usage: - * Set the `refcount_file` config value to the path to the reference - count data file. - * Set the `stable_abi_file` config value to the path to stable ABI list. - - :copyright: Copyright 2007-2014 by Georg Brandl. - :license: Python license. -""" - -from os import path -import docutils -from docutils import nodes -from docutils.parsers.rst import directives -from docutils.parsers.rst import Directive -from docutils.statemachine import StringList -from sphinx.locale import _ as sphinx_gettext -import csv - -from sphinx import addnodes -from sphinx.domains.c import CObject - - -REST_ROLE_MAP = { - 'function': 'func', - 'var': 'data', - 'type': 'type', - 'macro': 'macro', - 'type': 'type', - 'member': 'member', -} - - -# Monkeypatch nodes.Node.findall for forwards compatability -# This patch can be dropped when the minimum Sphinx version is 4.4.0 -# or the minimum Docutils version is 0.18.1. -if docutils.__version_info__ < (0, 18, 1): - def findall(self, *args, **kwargs): - return iter(self.traverse(*args, **kwargs)) - - nodes.Node.findall = findall - - -class RCEntry: - def __init__(self, name): - self.name = name - self.args = [] - self.result_type = '' - self.result_refs = None - - -class Annotations: - def __init__(self, refcount_filename, stable_abi_file): - self.refcount_data = {} - with open(refcount_filename, 'r') as fp: - for line in fp: - line = line.strip() - if line[:1] in ("", "#"): - # blank lines and comments - continue - parts = line.split(":", 4) - if len(parts) != 5: - raise ValueError("Wrong field count in %r" % line) - function, type, arg, refcount, comment = parts - # Get the entry, creating it if needed: - try: - entry = self.refcount_data[function] - except KeyError: - entry = self.refcount_data[function] = RCEntry(function) - if not refcount or refcount == "null": - refcount = None - else: - refcount = int(refcount) - # Update the entry with the new parameter or the result - # information. - if arg: - entry.args.append((arg, type, refcount)) - else: - entry.result_type = type - entry.result_refs = refcount - - self.stable_abi_data = {} - with open(stable_abi_file, 'r') as fp: - for record in csv.DictReader(fp): - role = record['role'] - name = record['name'] - self.stable_abi_data[name] = record - - def add_annotations(self, app, doctree): - for node in doctree.findall(addnodes.desc_content): - par = node.parent - if par['domain'] != 'c': - continue - if not par[0].has_key('ids') or not par[0]['ids']: - continue - name = par[0]['ids'][0] - if name.startswith("c."): - name = name[2:] - - objtype = par['objtype'] - - # Stable ABI annotation. These have two forms: - # Part of the [Stable ABI](link). - # Part of the [Stable ABI](link) since version X.Y. - # For structs, there's some more info in the message: - # Part of the [Limited API](link) (as an opaque struct). - # Part of the [Stable ABI](link) (including all members). - # Part of the [Limited API](link) (Only some members are part - # of the stable ABI.). - # ... all of which can have "since version X.Y" appended. - record = self.stable_abi_data.get(name) - if record: - if record['role'] != objtype: - raise ValueError( - f"Object type mismatch in limited API annotation " - f"for {name}: {record['role']!r} != {objtype!r}") - stable_added = record['added'] - message = ' Part of the ' - emph_node = nodes.emphasis(message, message, - classes=['stableabi']) - ref_node = addnodes.pending_xref( - 'Stable ABI', refdomain="std", reftarget='stable', - reftype='ref', refexplicit="False") - struct_abi_kind = record['struct_abi_kind'] - if struct_abi_kind in {'opaque', 'members'}: - ref_node += nodes.Text('Limited API') - else: - ref_node += nodes.Text('Stable ABI') - emph_node += ref_node - if struct_abi_kind == 'opaque': - emph_node += nodes.Text(' (as an opaque struct)') - elif struct_abi_kind == 'full-abi': - emph_node += nodes.Text(' (including all members)') - if record['ifdef_note']: - emph_node += nodes.Text(' ' + record['ifdef_note']) - if stable_added == '3.2': - # Stable ABI was introduced in 3.2. - pass - else: - emph_node += nodes.Text(f' since version {stable_added}') - emph_node += nodes.Text('.') - if struct_abi_kind == 'members': - emph_node += nodes.Text( - ' (Only some members are part of the stable ABI.)') - node.insert(0, emph_node) - - # Return value annotation - if objtype != 'function': - continue - entry = self.refcount_data.get(name) - if not entry: - continue - elif not entry.result_type.endswith("Object*"): - continue - if entry.result_refs is None: - rc = sphinx_gettext('Return value: Always NULL.') - elif entry.result_refs: - rc = sphinx_gettext('Return value: New reference.') - else: - rc = sphinx_gettext('Return value: Borrowed reference.') - node.insert(0, nodes.emphasis(rc, rc, classes=['refcount'])) - - -def init_annotations(app): - annotations = Annotations( - path.join(app.srcdir, app.config.refcount_file), - path.join(app.srcdir, app.config.stable_abi_file), - ) - app.connect('doctree-read', annotations.add_annotations) - - class LimitedAPIList(Directive): - - has_content = False - required_arguments = 0 - optional_arguments = 0 - final_argument_whitespace = True - - def run(self): - content = [] - for record in annotations.stable_abi_data.values(): - role = REST_ROLE_MAP[record['role']] - name = record['name'] - content.append(f'* :c:{role}:`{name}`') - - pnode = nodes.paragraph() - self.state.nested_parse(StringList(content), 0, pnode) - return [pnode] - - app.add_directive('limited-api-list', LimitedAPIList) - - -def setup(app): - app.add_config_value('refcount_file', '', True) - app.add_config_value('stable_abi_file', '', True) - app.connect('builder-inited', init_annotations) - - # monkey-patch C object... - CObject.option_spec = { - 'noindex': directives.flag, - 'stableabi': directives.flag, - } - old_handle_signature = CObject.handle_signature - def new_handle_signature(self, sig, signode): - signode.parent['stableabi'] = 'stableabi' in self.options - return old_handle_signature(self, sig, signode) - CObject.handle_signature = new_handle_signature - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/escape4chm.py b/Doc/tools/extensions/escape4chm.py deleted file mode 100644 index 89970975b9032b..00000000000000 --- a/Doc/tools/extensions/escape4chm.py +++ /dev/null @@ -1,58 +0,0 @@ -""" -Escape the `body` part of .chm source file to 7-bit ASCII, to fix visual -effect on some MBCS Windows systems. - -https://bugs.python.org/issue32174 -""" - -import pathlib -import re -from html.entities import codepoint2name - -from sphinx.util.logging import getLogger - -# escape the characters which codepoint > 0x7F -def _process(string): - def escape(matchobj): - codepoint = ord(matchobj.group(0)) - - name = codepoint2name.get(codepoint) - if name is None: - return '&#%d;' % codepoint - else: - return '&%s;' % name - - return re.sub(r'[^\x00-\x7F]', escape, string) - -def escape_for_chm(app, pagename, templatename, context, doctree): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp': - return - - # escape the `body` part to 7-bit ASCII - body = context.get('body') - if body is not None: - context['body'] = _process(body) - -def fixup_keywords(app, exception): - # only works for .chm output - if getattr(app.builder, 'name', '') != 'htmlhelp' or exception: - return - - getLogger(__name__).info('fixing HTML escapes in keywords file...') - outdir = pathlib.Path(app.builder.outdir) - outname = app.builder.config.htmlhelp_basename - with open(outdir / (outname + '.hhk'), 'rb') as f: - index = f.read() - with open(outdir / (outname + '.hhk'), 'wb') as f: - f.write(index.replace(b''', b''')) - -def setup(app): - # `html-page-context` event emitted when the HTML builder has - # created a context dictionary to render a template with. - app.connect('html-page-context', escape_for_chm) - # `build-finished` event emitted when all the files have been - # output. - app.connect('build-finished', fixup_keywords) - - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/patchlevel.py b/Doc/tools/extensions/patchlevel.py deleted file mode 100644 index 617f28c2527ddf..00000000000000 --- a/Doc/tools/extensions/patchlevel.py +++ /dev/null @@ -1,68 +0,0 @@ -# -*- coding: utf-8 -*- -""" - patchlevel.py - ~~~~~~~~~~~~~ - - Extract version info from Include/patchlevel.h. - Adapted from Doc/tools/getversioninfo. - - :copyright: 2007-2008 by Georg Brandl. - :license: Python license. -""" - -from __future__ import print_function - -import os -import re -import sys - -def get_header_version_info(srcdir): - patchlevel_h = os.path.join(srcdir, '..', 'Include', 'patchlevel.h') - - # This won't pick out all #defines, but it will pick up the ones we - # care about. - rx = re.compile(r'\s*#define\s+([a-zA-Z][a-zA-Z_0-9]*)\s+([a-zA-Z_0-9]+)') - - d = {} - with open(patchlevel_h) as f: - for line in f: - m = rx.match(line) - if m is not None: - name, value = m.group(1, 2) - d[name] = value - - release = version = '%s.%s' % (d['PY_MAJOR_VERSION'], d['PY_MINOR_VERSION']) - micro = int(d['PY_MICRO_VERSION']) - release += '.' + str(micro) - - level = d['PY_RELEASE_LEVEL'] - suffixes = { - 'PY_RELEASE_LEVEL_ALPHA': 'a', - 'PY_RELEASE_LEVEL_BETA': 'b', - 'PY_RELEASE_LEVEL_GAMMA': 'rc', - } - if level != 'PY_RELEASE_LEVEL_FINAL': - release += suffixes[level] + str(int(d['PY_RELEASE_SERIAL'])) - return version, release - - -def get_sys_version_info(): - major, minor, micro, level, serial = sys.version_info - release = version = '%s.%s' % (major, minor) - release += '.%s' % micro - if level != 'final': - release += '%s%s' % (level[0], serial) - return version, release - - -def get_version_info(): - try: - return get_header_version_info('.') - except (IOError, OSError): - version, release = get_sys_version_info() - print('Can\'t get version info from Include/patchlevel.h, ' \ - 'using version of this interpreter (%s).' % release, file=sys.stderr) - return version, release - -if __name__ == '__main__': - print(get_header_version_info('.')[1]) diff --git a/Doc/tools/extensions/peg_highlight.py b/Doc/tools/extensions/peg_highlight.py deleted file mode 100644 index 27f54cdf593c87..00000000000000 --- a/Doc/tools/extensions/peg_highlight.py +++ /dev/null @@ -1,86 +0,0 @@ -from pygments.lexer import RegexLexer, bygroups, include -from pygments.token import Comment, Generic, Keyword, Name, Operator, Punctuation, Text - -from sphinx.highlighting import lexers - - -class PEGLexer(RegexLexer): - """Pygments Lexer for PEG grammar (.gram) files - - This lexer strips the following elements from the grammar: - - - Meta-tags - - Variable assignments - - Actions - - Lookaheads - - Rule types - - Rule options - - Rules named `invalid_*` or `incorrect_*` - """ - - name = "PEG" - aliases = ["peg"] - filenames = ["*.gram"] - _name = r"([^\W\d]\w*)" - _text_ws = r"(\s*)" - - tokens = { - "ws": [(r"\n", Text), (r"\s+", Text), (r"#.*$", Comment.Singleline),], - "lookaheads": [ - # Forced tokens - (r"(&&)(?=\w+\s?)", bygroups(None)), - (r"(&&)(?='.+'\s?)", bygroups(None)), - (r'(&&)(?=".+"\s?)', bygroups(None)), - (r"(&&)(?=\(.+\)\s?)", bygroups(None)), - - (r"(?<=\|\s)(&\w+\s?)", bygroups(None)), - (r"(?<=\|\s)(&'.+'\s?)", bygroups(None)), - (r'(?<=\|\s)(&".+"\s?)', bygroups(None)), - (r"(?<=\|\s)(&\(.+\)\s?)", bygroups(None)), - ], - "metas": [ - (r"(@\w+ '''(.|\n)+?''')", bygroups(None)), - (r"^(@.*)$", bygroups(None)), - ], - "actions": [ - (r"{(.|\n)+?}", bygroups(None)), - ], - "strings": [ - (r"'\w+?'", Keyword), - (r'"\w+?"', Keyword), - (r"'\W+?'", Text), - (r'"\W+?"', Text), - ], - "variables": [ - (_name + _text_ws + "(=)", bygroups(None, None, None),), - (_name + _text_ws + r"(\[[\w\d_\*]+?\])" + _text_ws + "(=)", bygroups(None, None, None, None, None),), - ], - "invalids": [ - (r"^(\s+\|\s+.*invalid_\w+.*\n)", bygroups(None)), - (r"^(\s+\|\s+.*incorrect_\w+.*\n)", bygroups(None)), - (r"^(#.*invalid syntax.*(?:.|\n)*)", bygroups(None),), - ], - "root": [ - include("invalids"), - include("ws"), - include("lookaheads"), - include("metas"), - include("actions"), - include("strings"), - include("variables"), - (r"\b(?!(NULL|EXTRA))([A-Z_]+)\b\s*(?!\()", Text,), - ( - r"^\s*" + _name + r"\s*" + r"(\[.*\])?" + r"\s*" + r"(\(.+\))?" + r"\s*(:)", - bygroups(Name.Function, None, None, Punctuation), - ), - (_name, Name.Function), - (r"[\||\.|\+|\*|\?]", Operator), - (r"{|}|\(|\)|\[|\]", Punctuation), - (r".", Text), - ], - } - - -def setup(app): - lexers["peg"] = PEGLexer() - return {"version": "1.0", "parallel_read_safe": True} diff --git a/Doc/tools/extensions/pyspecific.py b/Doc/tools/extensions/pyspecific.py deleted file mode 100644 index 31c42e9d9d3a63..00000000000000 --- a/Doc/tools/extensions/pyspecific.py +++ /dev/null @@ -1,729 +0,0 @@ -# -*- coding: utf-8 -*- -""" - pyspecific.py - ~~~~~~~~~~~~~ - - Sphinx extension with Python doc-specific markup. - - :copyright: 2008-2014 by Georg Brandl. - :license: Python license. -""" - -import re -import io -from os import getenv, path -from time import asctime -from pprint import pformat - -from docutils import nodes, utils -from docutils.io import StringOutput -from docutils.parsers.rst import Directive -from docutils.utils import new_document -from sphinx import addnodes -from sphinx.builders import Builder -from sphinx.domains.python import PyFunction, PyMethod -from sphinx.errors import NoUri -from sphinx.locale import _ as sphinx_gettext -from sphinx.util import logging -from sphinx.util.docutils import SphinxDirective -from sphinx.util.nodes import split_explicit_title -from sphinx.writers.text import TextWriter, TextTranslator -from sphinx.writers.latex import LaTeXTranslator - -try: - # Sphinx 6+ - from sphinx.util.display import status_iterator -except ImportError: - # Deprecated in Sphinx 6.1, will be removed in Sphinx 8 - from sphinx.util import status_iterator - -# Support for checking for suspicious markup - -import suspicious - - -ISSUE_URI = 'https://bugs.python.org/issue?@action=redirect&bpo=%s' -GH_ISSUE_URI = 'https://github.com/python/cpython/issues/%s' -SOURCE_URI = 'https://github.com/python/cpython/tree/3.11/%s' - -# monkey-patch reST parser to disable alphabetic and roman enumerated lists -from docutils.parsers.rst.states import Body -Body.enum.converters['loweralpha'] = \ - Body.enum.converters['upperalpha'] = \ - Body.enum.converters['lowerroman'] = \ - Body.enum.converters['upperroman'] = lambda x: None - - -# Support for marking up and linking to bugs.python.org issues - -def issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) - # sanity check: there are no bpo issues within these two values - if 47261 < int(issue) < 400000: - msg = inliner.reporter.error(f'The BPO ID {text!r} seems too high -- ' - 'use :gh:`...` for GitHub IDs', line=lineno) - prb = inliner.problematic(rawtext, rawtext, msg) - return [prb], [msg] - text = 'bpo-' + issue - refnode = nodes.reference(text, text, refuri=ISSUE_URI % issue) - return [refnode], [] - - -# Support for marking up and linking to GitHub issues - -def gh_issue_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - issue = utils.unescape(text) - # sanity check: all GitHub issues have ID >= 32426 - # even though some of them are also valid BPO IDs - if int(issue) < 32426: - msg = inliner.reporter.error(f'The GitHub ID {text!r} seems too low -- ' - 'use :issue:`...` for BPO IDs', line=lineno) - prb = inliner.problematic(rawtext, rawtext, msg) - return [prb], [msg] - text = 'gh-' + issue - refnode = nodes.reference(text, text, refuri=GH_ISSUE_URI % issue) - return [refnode], [] - - -# Support for linking to Python source files easily - -def source_role(typ, rawtext, text, lineno, inliner, options={}, content=[]): - has_t, title, target = split_explicit_title(text) - title = utils.unescape(title) - target = utils.unescape(target) - refnode = nodes.reference(title, title, refuri=SOURCE_URI % target) - return [refnode], [] - - -# Support for marking up implementation details - -class ImplementationDetail(Directive): - - has_content = True - final_argument_whitespace = True - - # This text is copied to templates/dummy.html - label_text = sphinx_gettext('CPython implementation detail:') - - def run(self): - self.assert_has_content() - pnode = nodes.compound(classes=['impl-detail']) - content = self.content - add_text = nodes.strong(self.label_text, self.label_text) - self.state.nested_parse(content, self.content_offset, pnode) - content = nodes.inline(pnode[0].rawsource, translatable=True) - content.source = pnode[0].source - content.line = pnode[0].line - content += pnode[0].children - pnode[0].replace_self(nodes.paragraph( - '', '', add_text, nodes.Text(' '), content, translatable=False)) - return [pnode] - - -# Support for documenting platform availability - -class Availability(SphinxDirective): - - has_content = True - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = True - - # known platform, libc, and threading implementations - known_platforms = frozenset({ - "AIX", "Android", "BSD", "DragonFlyBSD", "Emscripten", "FreeBSD", - "Linux", "NetBSD", "OpenBSD", "POSIX", "Solaris", "Unix", "VxWorks", - "WASI", "Windows", "macOS", - # libc - "BSD libc", "glibc", "musl", - # POSIX platforms with pthreads - "pthreads", - }) - - def run(self): - availability_ref = ':ref:`Availability `: ' - avail_nodes, avail_msgs = self.state.inline_text( - availability_ref + self.arguments[0], - self.lineno) - pnode = nodes.paragraph(availability_ref + self.arguments[0], - '', *avail_nodes, *avail_msgs) - self.set_source_info(pnode) - cnode = nodes.container("", pnode, classes=["availability"]) - self.set_source_info(cnode) - if self.content: - self.state.nested_parse(self.content, self.content_offset, cnode) - self.parse_platforms() - - return [cnode] - - def parse_platforms(self): - """Parse platform information from arguments - - Arguments is a comma-separated string of platforms. A platform may - be prefixed with "not " to indicate that a feature is not available. - - Example:: - - .. availability:: Windows, Linux >= 4.2, not Emscripten, not WASI - - Arguments like "Linux >= 3.17 with glibc >= 2.27" are currently not - parsed into separate tokens. - """ - platforms = {} - for arg in self.arguments[0].rstrip(".").split(","): - arg = arg.strip() - platform, _, version = arg.partition(" >= ") - if platform.startswith("not "): - version = False - platform = platform[4:] - elif not version: - version = True - platforms[platform] = version - - unknown = set(platforms).difference(self.known_platforms) - if unknown: - cls = type(self) - logger = logging.getLogger(cls.__qualname__) - logger.warning( - f"Unknown platform(s) or syntax '{' '.join(sorted(unknown))}' " - f"in '.. availability:: {self.arguments[0]}', see " - f"{__file__}:{cls.__qualname__}.known_platforms for a set " - "known platforms." - ) - - return platforms - - - -# Support for documenting audit event - -def audit_events_purge(app, env, docname): - """This is to remove from env.all_audit_events old traces of removed - documents. - """ - if not hasattr(env, 'all_audit_events'): - return - fresh_all_audit_events = {} - for name, event in env.all_audit_events.items(): - event["source"] = [(d, t) for d, t in event["source"] if d != docname] - if event["source"]: - # Only keep audit_events that have at least one source. - fresh_all_audit_events[name] = event - env.all_audit_events = fresh_all_audit_events - - -def audit_events_merge(app, env, docnames, other): - """In Sphinx parallel builds, this merges env.all_audit_events from - subprocesses. - - all_audit_events is a dict of names, with values like: - {'source': [(docname, target), ...], 'args': args} - """ - if not hasattr(other, 'all_audit_events'): - return - if not hasattr(env, 'all_audit_events'): - env.all_audit_events = {} - for name, value in other.all_audit_events.items(): - if name in env.all_audit_events: - env.all_audit_events[name]["source"].extend(value["source"]) - else: - env.all_audit_events[name] = value - - -class AuditEvent(Directive): - - has_content = True - required_arguments = 1 - optional_arguments = 2 - final_argument_whitespace = True - - _label = [ - sphinx_gettext("Raises an :ref:`auditing event ` {name} with no arguments."), - sphinx_gettext("Raises an :ref:`auditing event ` {name} with argument {args}."), - sphinx_gettext("Raises an :ref:`auditing event ` {name} with arguments {args}."), - ] - - @property - def logger(self): - cls = type(self) - return logging.getLogger(cls.__module__ + "." + cls.__name__) - - def run(self): - name = self.arguments[0] - if len(self.arguments) >= 2 and self.arguments[1]: - args = (a.strip() for a in self.arguments[1].strip("'\"").split(",")) - args = [a for a in args if a] - else: - args = [] - - label = self._label[min(2, len(args))] - text = label.format(name="``{}``".format(name), - args=", ".join("``{}``".format(a) for a in args if a)) - - env = self.state.document.settings.env - if not hasattr(env, 'all_audit_events'): - env.all_audit_events = {} - - new_info = { - 'source': [], - 'args': args - } - info = env.all_audit_events.setdefault(name, new_info) - if info is not new_info: - if not self._do_args_match(info['args'], new_info['args']): - self.logger.warning( - "Mismatched arguments for audit-event {}: {!r} != {!r}" - .format(name, info['args'], new_info['args']) - ) - - ids = [] - try: - target = self.arguments[2].strip("\"'") - except (IndexError, TypeError): - target = None - if not target: - target = "audit_event_{}_{}".format( - re.sub(r'\W', '_', name), - len(info['source']), - ) - ids.append(target) - - info['source'].append((env.docname, target)) - - pnode = nodes.paragraph(text, classes=["audit-hook"], ids=ids) - pnode.line = self.lineno - if self.content: - self.state.nested_parse(self.content, self.content_offset, pnode) - else: - n, m = self.state.inline_text(text, self.lineno) - pnode.extend(n + m) - - return [pnode] - - # This list of sets are allowable synonyms for event argument names. - # If two names are in the same set, they are treated as equal for the - # purposes of warning. This won't help if number of arguments is - # different! - _SYNONYMS = [ - {"file", "path", "fd"}, - ] - - def _do_args_match(self, args1, args2): - if args1 == args2: - return True - if len(args1) != len(args2): - return False - for a1, a2 in zip(args1, args2): - if a1 == a2: - continue - if any(a1 in s and a2 in s for s in self._SYNONYMS): - continue - return False - return True - - -class audit_event_list(nodes.General, nodes.Element): - pass - - -class AuditEventListDirective(Directive): - - def run(self): - return [audit_event_list('')] - - -# Support for documenting decorators - -class PyDecoratorMixin(object): - def handle_signature(self, sig, signode): - ret = super(PyDecoratorMixin, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_addname('@', '@')) - return ret - - def needs_arglist(self): - return False - - -class PyDecoratorFunction(PyDecoratorMixin, PyFunction): - def run(self): - # a decorator function is a function after all - self.name = 'py:function' - return PyFunction.run(self) - - -# TODO: Use sphinx.domains.python.PyDecoratorMethod when possible -class PyDecoratorMethod(PyDecoratorMixin, PyMethod): - def run(self): - self.name = 'py:method' - return PyMethod.run(self) - - -class PyCoroutineMixin(object): - def handle_signature(self, sig, signode): - ret = super(PyCoroutineMixin, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_annotation('coroutine ', 'coroutine ')) - return ret - - -class PyAwaitableMixin(object): - def handle_signature(self, sig, signode): - ret = super(PyAwaitableMixin, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_annotation('awaitable ', 'awaitable ')) - return ret - - -class PyCoroutineFunction(PyCoroutineMixin, PyFunction): - def run(self): - self.name = 'py:function' - return PyFunction.run(self) - - -class PyCoroutineMethod(PyCoroutineMixin, PyMethod): - def run(self): - self.name = 'py:method' - return PyMethod.run(self) - - -class PyAwaitableFunction(PyAwaitableMixin, PyFunction): - def run(self): - self.name = 'py:function' - return PyFunction.run(self) - - -class PyAwaitableMethod(PyAwaitableMixin, PyMethod): - def run(self): - self.name = 'py:method' - return PyMethod.run(self) - - -class PyAbstractMethod(PyMethod): - - def handle_signature(self, sig, signode): - ret = super(PyAbstractMethod, self).handle_signature(sig, signode) - signode.insert(0, addnodes.desc_annotation('abstractmethod ', - 'abstractmethod ')) - return ret - - def run(self): - self.name = 'py:method' - return PyMethod.run(self) - - -# Support for documenting version of removal in deprecations - -class DeprecatedRemoved(Directive): - has_content = True - required_arguments = 2 - optional_arguments = 1 - final_argument_whitespace = True - option_spec = {} - - _deprecated_label = sphinx_gettext('Deprecated since version {deprecated}, will be removed in version {removed}') - _removed_label = sphinx_gettext('Deprecated since version {deprecated}, removed in version {removed}') - - def run(self): - node = addnodes.versionmodified() - node.document = self.state.document - node['type'] = 'deprecated-removed' - version = (self.arguments[0], self.arguments[1]) - node['version'] = version - env = self.state.document.settings.env - current_version = tuple(int(e) for e in env.config.version.split('.')) - removed_version = tuple(int(e) for e in self.arguments[1].split('.')) - if current_version < removed_version: - label = self._deprecated_label - else: - label = self._removed_label - - text = label.format(deprecated=self.arguments[0], removed=self.arguments[1]) - if len(self.arguments) == 3: - inodes, messages = self.state.inline_text(self.arguments[2], - self.lineno+1) - para = nodes.paragraph(self.arguments[2], '', *inodes, translatable=False) - node.append(para) - else: - messages = [] - if self.content: - self.state.nested_parse(self.content, self.content_offset, node) - if len(node): - if isinstance(node[0], nodes.paragraph) and node[0].rawsource: - content = nodes.inline(node[0].rawsource, translatable=True) - content.source = node[0].source - content.line = node[0].line - content += node[0].children - node[0].replace_self(nodes.paragraph('', '', content, translatable=False)) - node[0].insert(0, nodes.inline('', '%s: ' % text, - classes=['versionmodified'])) - else: - para = nodes.paragraph('', '', - nodes.inline('', '%s.' % text, - classes=['versionmodified']), - translatable=False) - node.append(para) - env = self.state.document.settings.env - env.get_domain('changeset').note_changeset(node) - return [node] + messages - - -# Support for including Misc/NEWS - -issue_re = re.compile('(?:[Ii]ssue #|bpo-)([0-9]+)', re.I) -gh_issue_re = re.compile('(?:gh-issue-|gh-)([0-9]+)', re.I) -whatsnew_re = re.compile(r"(?im)^what's new in (.*?)\??$") - - -class MiscNews(Directive): - has_content = False - required_arguments = 1 - optional_arguments = 0 - final_argument_whitespace = False - option_spec = {} - - def run(self): - fname = self.arguments[0] - source = self.state_machine.input_lines.source( - self.lineno - self.state_machine.input_offset - 1) - source_dir = getenv('PY_MISC_NEWS_DIR') - if not source_dir: - source_dir = path.dirname(path.abspath(source)) - fpath = path.join(source_dir, fname) - self.state.document.settings.record_dependencies.add(fpath) - try: - with io.open(fpath, encoding='utf-8') as fp: - content = fp.read() - except Exception: - text = 'The NEWS file is not available.' - node = nodes.strong(text, text) - return [node] - content = issue_re.sub(r':issue:`\1`', content) - # Fallback handling for the GitHub issue - content = gh_issue_re.sub(r':gh:`\1`', content) - content = whatsnew_re.sub(r'\1', content) - # remove first 3 lines as they are the main heading - lines = ['.. default-role:: obj', ''] + content.splitlines()[3:] - self.state_machine.insert_input(lines, fname) - return [] - - -# Support for building "topic help" for pydoc - -pydoc_topic_labels = [ - 'assert', 'assignment', 'async', 'atom-identifiers', 'atom-literals', - 'attribute-access', 'attribute-references', 'augassign', 'await', - 'binary', 'bitwise', 'bltin-code-objects', 'bltin-ellipsis-object', - 'bltin-null-object', 'bltin-type-objects', 'booleans', - 'break', 'callable-types', 'calls', 'class', 'comparisons', 'compound', - 'context-managers', 'continue', 'conversions', 'customization', 'debugger', - 'del', 'dict', 'dynamic-features', 'else', 'exceptions', 'execmodel', - 'exprlists', 'floating', 'for', 'formatstrings', 'function', 'global', - 'id-classes', 'identifiers', 'if', 'imaginary', 'import', 'in', 'integers', - 'lambda', 'lists', 'naming', 'nonlocal', 'numbers', 'numeric-types', - 'objects', 'operator-summary', 'pass', 'power', 'raise', 'return', - 'sequence-types', 'shifting', 'slicings', 'specialattrs', 'specialnames', - 'string-methods', 'strings', 'subscriptions', 'truth', 'try', 'types', - 'typesfunctions', 'typesmapping', 'typesmethods', 'typesmodules', - 'typesseq', 'typesseq-mutable', 'unary', 'while', 'with', 'yield' -] - - -class PydocTopicsBuilder(Builder): - name = 'pydoc-topics' - - default_translator_class = TextTranslator - - def init(self): - self.topics = {} - self.secnumbers = {} - - def get_outdated_docs(self): - return 'all pydoc topics' - - def get_target_uri(self, docname, typ=None): - return '' # no URIs - - def write(self, *ignored): - writer = TextWriter(self) - for label in status_iterator(pydoc_topic_labels, - 'building topics... ', - length=len(pydoc_topic_labels)): - if label not in self.env.domaindata['std']['labels']: - self.env.logger.warning(f'label {label!r} not in documentation') - continue - docname, labelid, sectname = self.env.domaindata['std']['labels'][label] - doctree = self.env.get_and_resolve_doctree(docname, self) - document = new_document('
    ') - document.append(doctree.ids[labelid]) - destination = StringOutput(encoding='utf-8') - writer.write(document, destination) - self.topics[label] = writer.output - - def finish(self): - f = open(path.join(self.outdir, 'topics.py'), 'wb') - try: - f.write('# -*- coding: utf-8 -*-\n'.encode('utf-8')) - f.write(('# Autogenerated by Sphinx on %s\n' % asctime()).encode('utf-8')) - f.write('# as part of the release process.\n'.encode('utf-8')) - f.write(('topics = ' + pformat(self.topics) + '\n').encode('utf-8')) - finally: - f.close() - - -# Support for documenting Opcodes - -opcode_sig_re = re.compile(r'(\w+(?:\+\d)?)(?:\s*\((.*)\))?') - - -def parse_opcode_signature(env, sig, signode): - """Transform an opcode signature into RST nodes.""" - m = opcode_sig_re.match(sig) - if m is None: - raise ValueError - opname, arglist = m.groups() - signode += addnodes.desc_name(opname, opname) - if arglist is not None: - paramlist = addnodes.desc_parameterlist() - signode += paramlist - paramlist += addnodes.desc_parameter(arglist, arglist) - return opname.strip() - - -# Support for documenting pdb commands - -pdbcmd_sig_re = re.compile(r'([a-z()!]+)\s*(.*)') - -# later... -# pdbargs_tokens_re = re.compile(r'''[a-zA-Z]+ | # identifiers -# [.,:]+ | # punctuation -# [\[\]()] | # parens -# \s+ # whitespace -# ''', re.X) - - -def parse_pdb_command(env, sig, signode): - """Transform a pdb command signature into RST nodes.""" - m = pdbcmd_sig_re.match(sig) - if m is None: - raise ValueError - name, args = m.groups() - fullname = name.replace('(', '').replace(')', '') - signode += addnodes.desc_name(name, name) - if args: - signode += addnodes.desc_addname(' '+args, ' '+args) - return fullname - - -def process_audit_events(app, doctree, fromdocname): - for node in doctree.traverse(audit_event_list): - break - else: - return - - env = app.builder.env - - table = nodes.table(cols=3) - group = nodes.tgroup( - '', - nodes.colspec(colwidth=30), - nodes.colspec(colwidth=55), - nodes.colspec(colwidth=15), - cols=3, - ) - head = nodes.thead() - body = nodes.tbody() - - table += group - group += head - group += body - - row = nodes.row() - row += nodes.entry('', nodes.paragraph('', nodes.Text('Audit event'))) - row += nodes.entry('', nodes.paragraph('', nodes.Text('Arguments'))) - row += nodes.entry('', nodes.paragraph('', nodes.Text('References'))) - head += row - - for name in sorted(getattr(env, "all_audit_events", ())): - audit_event = env.all_audit_events[name] - - row = nodes.row() - node = nodes.paragraph('', nodes.Text(name)) - row += nodes.entry('', node) - - node = nodes.paragraph() - for i, a in enumerate(audit_event['args']): - if i: - node += nodes.Text(", ") - node += nodes.literal(a, nodes.Text(a)) - row += nodes.entry('', node) - - node = nodes.paragraph() - backlinks = enumerate(sorted(set(audit_event['source'])), start=1) - for i, (doc, label) in backlinks: - if isinstance(label, str): - ref = nodes.reference("", nodes.Text("[{}]".format(i)), internal=True) - try: - ref['refuri'] = "{}#{}".format( - app.builder.get_relative_uri(fromdocname, doc), - label, - ) - except NoUri: - continue - node += ref - row += nodes.entry('', node) - - body += row - - for node in doctree.traverse(audit_event_list): - node.replace_self(table) - - -def patch_pairindextypes(app, _env) -> None: - """Remove all entries from ``pairindextypes`` before writing POT files. - - We want to run this just before writing output files, as the check to - circumvent is in ``I18nBuilder.write_doc()``. - As such, we link this to ``env-check-consistency``, even though it has - nothing to do with the environment consistency check. - """ - if app.builder.name != 'gettext': - return - - # allow translating deprecated index entries - try: - from sphinx.domains.python import pairindextypes - except ImportError: - pass - else: - # Sphinx checks if a 'pair' type entry on an index directive is one of - # the Sphinx-translated pairindextypes values. As we intend to move - # away from this, we need Sphinx to believe that these values don't - # exist, by deleting them when using the gettext builder. - pairindextypes.clear() - - -def setup(app): - app.add_role('issue', issue_role) - app.add_role('gh', gh_issue_role) - app.add_role('source', source_role) - app.add_directive('impl-detail', ImplementationDetail) - app.add_directive('availability', Availability) - app.add_directive('audit-event', AuditEvent) - app.add_directive('audit-event-table', AuditEventListDirective) - app.add_directive('deprecated-removed', DeprecatedRemoved) - app.add_builder(PydocTopicsBuilder) - app.add_builder(suspicious.CheckSuspiciousMarkupBuilder) - app.add_object_type('opcode', 'opcode', '%s (opcode)', parse_opcode_signature) - app.add_object_type('pdbcommand', 'pdbcmd', '%s (pdb command)', parse_pdb_command) - app.add_object_type('2to3fixer', '2to3fixer', '%s (2to3 fixer)') - app.add_directive_to_domain('py', 'decorator', PyDecoratorFunction) - app.add_directive_to_domain('py', 'decoratormethod', PyDecoratorMethod) - app.add_directive_to_domain('py', 'coroutinefunction', PyCoroutineFunction) - app.add_directive_to_domain('py', 'coroutinemethod', PyCoroutineMethod) - app.add_directive_to_domain('py', 'awaitablefunction', PyAwaitableFunction) - app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod) - app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod) - app.add_directive('miscnews', MiscNews) - app.connect('env-check-consistency', patch_pairindextypes) - app.connect('doctree-resolved', process_audit_events) - app.connect('env-merge-info', audit_events_merge) - app.connect('env-purge-doc', audit_events_purge) - return {'version': '1.0', 'parallel_read_safe': True} diff --git a/Doc/tools/extensions/suspicious.py b/Doc/tools/extensions/suspicious.py deleted file mode 100644 index 2d581a8a6c3db6..00000000000000 --- a/Doc/tools/extensions/suspicious.py +++ /dev/null @@ -1,251 +0,0 @@ -""" -Try to detect suspicious constructs, resembling markup -that has leaked into the final output. - -Suspicious lines are reported in a comma-separated-file, -``suspicious.csv``, located in the output directory. - -The file is utf-8 encoded, and each line contains four fields: - - * document name (normalized) - * line number in the source document - * problematic text - * complete line showing the problematic text in context - -It is common to find many false positives. To avoid reporting them -again and again, they may be added to the ``ignored.csv`` file -(located in the configuration directory). The file has the same -format as ``suspicious.csv`` with a few differences: - - - each line defines a rule; if the rule matches, the issue - is ignored. - - line number may be empty (that is, nothing between the - commas: ",,"). In this case, line numbers are ignored (the - rule matches anywhere in the file). - - the last field does not have to be a complete line; some - surrounding text (never more than a line) is enough for - context. - -Rules are processed sequentially. A rule matches when: - - * document names are the same - * problematic texts are the same - * line numbers are close to each other (5 lines up or down) - * the rule text is completely contained into the source line - -The simplest way to create the ignored.csv file is by copying -undesired entries from suspicious.csv (possibly trimming the last -field.) - -Copyright 2009 Gabriel A. Genellina - -""" - -import os -import re -import csv - -from docutils import nodes -from sphinx.builders import Builder -import sphinx.util - -detect_all = re.compile(r''' - ::(?=[^=])| # two :: (but NOT ::=) - :[a-zA-Z][a-zA-Z0-9]+| # :foo - `| # ` (seldom used by itself) - (? don't care - self.issue = issue # the markup fragment that triggered this rule - self.line = line # text of the container element (single line only) - self.used = False - - def __repr__(self): - return '{0.docname},,{0.issue},{0.line}'.format(self) - - - -class dialect(csv.excel): - """Our dialect: uses only linefeed as newline.""" - lineterminator = '\n' - - -class CheckSuspiciousMarkupBuilder(Builder): - """ - Checks for possibly invalid markup that may leak into the output. - """ - name = 'suspicious' - logger = sphinx.util.logging.getLogger("CheckSuspiciousMarkupBuilder") - - def init(self): - # create output file - self.log_file_name = os.path.join(self.outdir, 'suspicious.csv') - open(self.log_file_name, 'w').close() - # load database of previously ignored issues - self.load_rules(os.path.join(os.path.dirname(__file__), '..', - 'susp-ignored.csv')) - - def get_outdated_docs(self): - return self.env.found_docs - - def get_target_uri(self, docname, typ=None): - return '' - - def prepare_writing(self, docnames): - pass - - def write_doc(self, docname, doctree): - # set when any issue is encountered in this document - self.any_issue = False - self.docname = docname - visitor = SuspiciousVisitor(doctree, self) - doctree.walk(visitor) - - def finish(self): - unused_rules = [rule for rule in self.rules if not rule.used] - if unused_rules: - self.logger.warning( - 'Found %s/%s unused rules: %s' % ( - len(unused_rules), len(self.rules), - '\n'.join(repr(rule) for rule in unused_rules), - ) - ) - return - - def check_issue(self, line, lineno, issue): - if not self.is_ignored(line, lineno, issue): - self.report_issue(line, lineno, issue) - - def is_ignored(self, line, lineno, issue): - """Determine whether this issue should be ignored.""" - docname = self.docname - for rule in self.rules: - if rule.docname != docname: continue - if rule.issue != issue: continue - # Both lines must match *exactly*. This is rather strict, - # and probably should be improved. - # Doing fuzzy matches with levenshtein distance could work, - # but that means bringing other libraries... - # Ok, relax that requirement: just check if the rule fragment - # is contained in the document line - if rule.line not in line: continue - # Check both line numbers. If they're "near" - # this rule matches. (lineno=None means "don't care") - if (rule.lineno is not None) and \ - abs(rule.lineno - lineno) > 5: continue - # if it came this far, the rule matched - rule.used = True - return True - return False - - def report_issue(self, text, lineno, issue): - self.any_issue = True - self.write_log_entry(lineno, issue, text) - self.logger.warning('[%s:%d] "%s" found in "%-.120s"' % - (self.docname, lineno, issue, text)) - self.app.statuscode = 1 - - def write_log_entry(self, lineno, issue, text): - f = open(self.log_file_name, 'a') - writer = csv.writer(f, dialect) - writer.writerow([self.docname, lineno, issue, text.strip()]) - f.close() - - def load_rules(self, filename): - """Load database of previously ignored issues. - - A csv file, with exactly the same format as suspicious.csv - Fields: document name (normalized), line number, issue, surrounding text - """ - self.logger.info("loading ignore rules... ", nonl=1) - self.rules = rules = [] - try: - f = open(filename, 'r') - except IOError: - return - for i, row in enumerate(csv.reader(f)): - if len(row) != 4: - raise ValueError( - "wrong format in %s, line %d: %s" % (filename, i+1, row)) - docname, lineno, issue, text = row - if lineno: - lineno = int(lineno) - else: - lineno = None - rule = Rule(docname, lineno, issue, text) - rules.append(rule) - f.close() - self.logger.info('done, %d rules loaded' % len(self.rules)) - - -def get_lineno(node): - """Obtain line number information for a node.""" - lineno = None - while lineno is None and node: - node = node.parent - lineno = node.line - return lineno - - -def extract_line(text, index): - """text may be a multiline string; extract - only the line containing the given character index. - - >>> extract_line("abc\ndefgh\ni", 6) - >>> 'defgh' - >>> for i in (0, 2, 3, 4, 10): - ... print extract_line("abc\ndefgh\ni", i) - abc - abc - abc - defgh - defgh - i - """ - p = text.rfind('\n', 0, index) + 1 - q = text.find('\n', index) - if q < 0: - q = len(text) - return text[p:q] - - -class SuspiciousVisitor(nodes.GenericNodeVisitor): - - lastlineno = 0 - - def __init__(self, document, builder): - nodes.GenericNodeVisitor.__init__(self, document) - self.builder = builder - - def default_visit(self, node): - if isinstance(node, (nodes.Text, nodes.image)): # direct text containers - text = node.astext() - # lineno seems to go backwards sometimes (?) - self.lastlineno = lineno = max(get_lineno(node) or 0, self.lastlineno) - seen = set() # don't report the same issue more than only once per line - for match in detect_all(text): - issue = match.group() - line = extract_line(text, match.start()) - if (issue, line) not in seen: - self.builder.check_issue(line, lineno, issue) - seen.add((issue, line)) - - unknown_visit = default_visit - - def visit_document(self, node): - self.lastlineno = 0 - - def visit_comment(self, node): - # ignore comments -- too much false positives. - # (although doing this could miss some errors; - # there were two sections "commented-out" by mistake - # in the Python docs that would not be caught) - raise nodes.SkipNode diff --git a/Doc/tools/rstlint.py b/Doc/tools/rstlint.py deleted file mode 100644 index 4ea68ef3b030c8..00000000000000 --- a/Doc/tools/rstlint.py +++ /dev/null @@ -1,408 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Check for stylistic and formal issues in .rst and .py -# files included in the documentation. -# -# 01/2009, Georg Brandl - -# TODO: - wrong versions in versionadded/changed -# - wrong markup after versionchanged directive - -import os -import re -import sys -import getopt -from string import ascii_letters -from os.path import join, splitext, abspath, exists -from collections import defaultdict - -directives = [ - # standard docutils ones - 'admonition', 'attention', 'caution', 'class', 'compound', 'container', - 'contents', 'csv-table', 'danger', 'date', 'default-role', 'epigraph', - 'error', 'figure', 'footer', 'header', 'highlights', 'hint', 'image', - 'important', 'include', 'line-block', 'list-table', 'meta', 'note', - 'parsed-literal', 'pull-quote', 'raw', 'replace', - 'restructuredtext-test-directive', 'role', 'rubric', 'sectnum', 'sidebar', - 'table', 'target-notes', 'tip', 'title', 'topic', 'unicode', 'warning', - # Sphinx and Python docs custom ones - 'acks', 'attribute', 'autoattribute', 'autoclass', 'autodata', - 'autoexception', 'autofunction', 'automethod', 'automodule', - 'availability', 'centered', 'cfunction', 'class', 'classmethod', 'cmacro', - 'cmdoption', 'cmember', 'code-block', 'confval', 'cssclass', 'ctype', - 'currentmodule', 'cvar', 'data', 'decorator', 'decoratormethod', - 'deprecated-removed', 'deprecated(?!-removed)', 'describe', 'directive', - 'doctest', 'envvar', 'event', 'exception', 'function', 'glossary', - 'highlight', 'highlightlang', 'impl-detail', 'index', 'literalinclude', - 'method', 'miscnews', 'module', 'moduleauthor', 'opcode', 'pdbcommand', - 'productionlist', 'program', 'role', 'sectionauthor', 'seealso', - 'sourcecode', 'staticmethod', 'tabularcolumns', 'testcode', 'testoutput', - 'testsetup', 'toctree', 'todo', 'todolist', 'versionadded', - 'versionchanged' -] - -roles = [ - "(? 81: - # don't complain about tables, links and function signatures - if line.lstrip()[0] not in '+|' and \ - 'http://' not in line and \ - not line.lstrip().startswith(('.. function', - '.. method', - '.. cfunction')): - yield lno+1, "line too long" - - -@checker('.html', severity=2, falsepositives=True) -def check_leaked_markup(fn, lines): - """Check HTML files for leaked reST markup; this only works if - the HTML files have been built. - """ - for lno, line in enumerate(lines): - if leaked_markup_re.search(line): - yield lno+1, 'possibly leaked markup: %r' % line - - -def hide_literal_blocks(lines): - """Tool to remove literal blocks from given lines. - - It yields empty lines in place of blocks, so line numbers are - still meaningful. - """ - in_block = False - for line in lines: - if line.endswith("::\n"): - in_block = True - elif in_block: - if line == "\n" or line.startswith(" "): - line = "\n" - else: - in_block = False - yield line - - -def type_of_explicit_markup(line): - if re.match(fr'\.\. {all_directives}::', line): - return 'directive' - if re.match(r'\.\. \[[0-9]+\] ', line): - return 'footnote' - if re.match(r'\.\. \[[^\]]+\] ', line): - return 'citation' - if re.match(r'\.\. _.*[^_]: ', line): - return 'target' - if re.match(r'\.\. \|[^\|]*\| ', line): - return 'substitution_definition' - return 'comment' - - -def hide_comments(lines): - """Tool to remove comments from given lines. - - It yields empty lines in place of comments, so line numbers are - still meaningful. - """ - in_multiline_comment = False - for line in lines: - if line == "..\n": - in_multiline_comment = True - elif in_multiline_comment: - if line == "\n" or line.startswith(" "): - line = "\n" - else: - in_multiline_comment = False - if line.startswith(".. ") and type_of_explicit_markup(line) == 'comment': - line = "\n" - yield line - - - -@checker(".rst", severity=2) -def check_missing_surrogate_space_on_plural(fn, lines): - r"""Check for missing 'backslash-space' between a code sample a letter. - - Good: ``Point``\ s - Bad: ``Point``s - """ - in_code_sample = False - check_next_one = False - for lno, line in enumerate(hide_comments(hide_literal_blocks(lines))): - tokens = line.split("``") - for token_no, token in enumerate(tokens): - if check_next_one: - if token[0] in ascii_letters: - yield lno + 1, f"Missing backslash-space between code sample and {token!r}." - check_next_one = False - if token_no == len(tokens) - 1: - continue - if in_code_sample: - check_next_one = True - in_code_sample = not in_code_sample - -def main(argv): - usage = '''\ -Usage: %s [-v] [-f] [-s sev] [-i path]* [path] - -Options: -v verbose (print all checked file names) - -f enable checkers that yield many false positives - -s sev only show problems with severity >= sev - -i path ignore subdir or file path -''' % argv[0] - try: - gopts, args = getopt.getopt(argv[1:], 'vfs:i:') - except getopt.GetoptError: - print(usage) - return 2 - - verbose = False - severity = 1 - ignore = [] - falsepos = False - for opt, val in gopts: - if opt == '-v': - verbose = True - elif opt == '-f': - falsepos = True - elif opt == '-s': - severity = int(val) - elif opt == '-i': - ignore.append(abspath(val)) - - if len(args) == 0: - path = '.' - elif len(args) == 1: - path = args[0] - else: - print(usage) - return 2 - - if not exists(path): - print('Error: path %s does not exist' % path) - return 2 - - count = defaultdict(int) - - print("""⚠ rstlint.py is no longer maintained here and will be removed -⚠ in a future release. -⚠ Please use https://pypi.org/p/sphinx-lint instead. -""") - - for root, dirs, files in os.walk(path): - # ignore subdirs in ignore list - if abspath(root) in ignore: - del dirs[:] - continue - - for fn in files: - fn = join(root, fn) - if fn[:2] == './': - fn = fn[2:] - - # ignore files in ignore list - if abspath(fn) in ignore: - continue - - ext = splitext(fn)[1] - checkerlist = checkers.get(ext, None) - if not checkerlist: - continue - - if verbose: - print('Checking %s...' % fn) - - try: - with open(fn, 'r', encoding='utf-8') as f: - lines = list(f) - except (IOError, OSError) as err: - print('%s: cannot open: %s' % (fn, err)) - count[4] += 1 - continue - - for checker in checkerlist: - if checker.falsepositives and not falsepos: - continue - csev = checker.severity - if csev >= severity: - for lno, msg in checker(fn, lines): - print('[%d] %s:%d: %s' % (csev, fn, lno, msg)) - count[csev] += 1 - if verbose: - print() - if not count: - if severity > 1: - print('No problems with severity >= %d found.' % severity) - else: - print('No problems found.') - else: - for severity in sorted(count): - number = count[severity] - print('%d problem%s with severity %d found.' % - (number, number > 1 and 's' or '', severity)) - return int(bool(count)) - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/Doc/tools/static/changelog_search.js b/Doc/tools/static/changelog_search.js deleted file mode 100644 index c881a9bd4c84a7..00000000000000 --- a/Doc/tools/static/changelog_search.js +++ /dev/null @@ -1,53 +0,0 @@ -$(document).ready(function() { - // add the search form and bind the events - $('h1').after([ - '

    Filter entries by content:', - '', - '

    ' - ].join('\n')); - - function dofilter() { - try { - var query = new RegExp($('#searchbox').val(), 'i'); - } - catch (e) { - return; // not a valid regex (yet) - } - // find headers for the versions (What's new in Python X.Y.Z?) - $('#changelog h2').each(function(index1, h2) { - var h2_parent = $(h2).parent(); - var sections_found = 0; - // find headers for the sections (Core, Library, etc.) - h2_parent.find('h3').each(function(index2, h3) { - var h3_parent = $(h3).parent(); - var entries_found = 0; - // find all the entries - h3_parent.find('li').each(function(index3, li) { - var li = $(li); - // check if the query matches the entry - if (query.test(li.text())) { - li.show(); - entries_found++; - } - else { - li.hide(); - } - }); - // if there are entries, show the section, otherwise hide it - if (entries_found > 0) { - h3_parent.show(); - sections_found++; - } - else { - h3_parent.hide(); - } - }); - if (sections_found > 0) - h2_parent.show(); - else - h2_parent.hide(); - }); - } - $('#searchbox').keyup(dofilter); - $('#searchbox-submit').click(dofilter); -}); diff --git a/Doc/tools/susp-ignored.csv b/Doc/tools/susp-ignored.csv deleted file mode 100644 index 29d65cd7207fe7..00000000000000 --- a/Doc/tools/susp-ignored.csv +++ /dev/null @@ -1,398 +0,0 @@ -c-api/arg,,:ref,"PyArg_ParseTuple(args, ""O|O:ref"", &object, &callback)" -c-api/list,,:high,list[low:high] -c-api/sequence,,:i2,del o[i1:i2] -c-api/sequence,,:i2,o[i1:i2] -c-api/tuple,,:high,p[low:high] -c-api/unicode,,:end,str[start:end] -c-api/unicode,,:start,unicode[start:start+length] -distutils/examples,,`,This is the description of the ``foobar`` package. -distutils/setupscript,,::, -extending/embedding,,:numargs,"if(!PyArg_ParseTuple(args, "":numargs""))" -extending/extending,,:myfunction,"PyArg_ParseTuple(args, ""D:myfunction"", &c);" -extending/extending,,:set,"if (PyArg_ParseTuple(args, ""O:set_callback"", &temp)) {" -extending/newtypes,,:call,"if (!PyArg_ParseTuple(args, ""sss:call"", &arg1, &arg2, &arg3)) {" -faq/programming,,:chr,">=4.0) or 1+f(xc,yc,x*x-y*y+xc,2.0*x*y+yc,k-1,f):f(xc,yc,x,y,k,f):chr(" -faq/programming,,:reduce,"print((lambda Ru,Ro,Iu,Io,IM,Sx,Sy:reduce(lambda x,y:x+y,map(lambda y," -faq/programming,,:reduce,"Sx=Sx,Sy=Sy:reduce(lambda x,y:x+y,map(lambda x,xc=Ru,yc=yc,Ru=Ru,Ro=Ro," -faq/windows,,:d48eceb,"Python 3.6.4 (v3.6.4:d48eceb, Dec 19 2017, 06:04:45) [MSC v.1900 32 bit (Intel)] on win32" -howto/curses,,:black,"colors when it activates color mode. They are: 0:black, 1:red," -howto/curses,,:red,"colors when it activates color mode. They are: 0:black, 1:red," -howto/curses,,:green,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:yellow,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:blue,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:magenta,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:cyan,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/curses,,:white,"2:green, 3:yellow, 4:blue, 5:magenta, 6:cyan, and 7:white. The" -howto/descriptor,,:root,"INFO:root" -howto/descriptor,,:Updating,"root:Updating" -howto/descriptor,,:Accessing,"root:Accessing" -howto/instrumentation,,::,python$target:::function-entry -howto/instrumentation,,:function,python$target:::function-entry -howto/instrumentation,,::,python$target:::function-return -howto/instrumentation,,:function,python$target:::function-return -howto/instrumentation,,:call,156641360502280 function-entry:call_stack.py:start:23 -howto/instrumentation,,:start,156641360502280 function-entry:call_stack.py:start:23 -howto/instrumentation,,:function,156641360518804 function-entry: call_stack.py:function_1:1 -howto/instrumentation,,:function,156641360532797 function-entry: call_stack.py:function_3:9 -howto/instrumentation,,:function,156641360546807 function-return: call_stack.py:function_3:10 -howto/instrumentation,,:function,156641360563367 function-return: call_stack.py:function_1:2 -howto/instrumentation,,:function,156641360578365 function-entry: call_stack.py:function_2:5 -howto/instrumentation,,:function,156641360591757 function-entry: call_stack.py:function_1:1 -howto/instrumentation,,:function,156641360605556 function-entry: call_stack.py:function_3:9 -howto/instrumentation,,:function,156641360617482 function-return: call_stack.py:function_3:10 -howto/instrumentation,,:function,156641360629814 function-return: call_stack.py:function_1:2 -howto/instrumentation,,:function,156641360642285 function-return: call_stack.py:function_2:6 -howto/instrumentation,,:function,156641360656770 function-entry: call_stack.py:function_3:9 -howto/instrumentation,,:function,156641360669707 function-return: call_stack.py:function_3:10 -howto/instrumentation,,:function,156641360687853 function-entry: call_stack.py:function_4:13 -howto/instrumentation,,:function,156641360700719 function-return: call_stack.py:function_4:14 -howto/instrumentation,,:function,156641360719640 function-entry: call_stack.py:function_5:18 -howto/instrumentation,,:function,156641360732567 function-return: call_stack.py:function_5:21 -howto/instrumentation,,:call,156641360747370 function-return:call_stack.py:start:28 -howto/instrumentation,,:start,156641360747370 function-return:call_stack.py:start:28 -howto/ipaddress,,:DB8,>>> ipaddress.ip_address('2001:DB8::1') -howto/ipaddress,,::,>>> ipaddress.ip_address('2001:DB8::1') -howto/ipaddress,,:db8,IPv6Address('2001:db8::1') -howto/ipaddress,,::,IPv6Address('2001:db8::1') -howto/ipaddress,,::,IPv6Address('::1') -howto/ipaddress,,:db8,>>> ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,::,>>> ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,:db8,IPv6Network('2001:db8::/96') -howto/ipaddress,,::,IPv6Network('2001:db8::/96') -howto/ipaddress,,:db8,IPv6Network('2001:db8::/128') -howto/ipaddress,,::,IPv6Network('2001:db8::/128') -howto/ipaddress,,:db8,IPv6Interface('2001:db8::1/96') -howto/ipaddress,,::,IPv6Interface('2001:db8::1/96') -howto/ipaddress,,:db8,>>> addr6 = ipaddress.ip_address('2001:db8::1') -howto/ipaddress,,::,>>> addr6 = ipaddress.ip_address('2001:db8::1') -howto/ipaddress,,:db8,>>> host6 = ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,::,>>> host6 = ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,:db8,>>> net6 = ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,::,>>> net6 = ipaddress.ip_network('2001:db8::0/96') -howto/ipaddress,,:ffff,IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::') -howto/ipaddress,,::,IPv6Address('ffff:ffff:ffff:ffff:ffff:ffff::') -howto/ipaddress,,::,IPv6Address('::ffff:ffff') -howto/ipaddress,,:ffff,IPv6Address('::ffff:ffff') -howto/ipaddress,,:db8,'2001:db8::/96' -howto/ipaddress,,::,'2001:db8::/96' -howto/ipaddress,,:db8,>>> ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,::,>>> ipaddress.ip_interface('2001:db8::1/96') -howto/ipaddress,,:db8,'2001:db8::1' -howto/ipaddress,,::,'2001:db8::1' -howto/ipaddress,,:db8,IPv6Address('2001:db8::ffff:ffff') -howto/ipaddress,,::,IPv6Address('2001:db8::ffff:ffff') -howto/ipaddress,,:ffff,IPv6Address('2001:db8::ffff:ffff') -howto/logging,,:And,"WARNING:And this, too" -howto/logging,,:And,"WARNING:root:And this, too" -howto/logging,,:And,"ERROR:root:And non-ASCII stuff, too, like " -howto/logging,,:Doing,INFO:root:Doing something -howto/logging,,:Finished,INFO:root:Finished -howto/logging,,:logger,severity:logger name:message -howto/logging,,:Look,WARNING:root:Look before you leap! -howto/logging,,:message,severity:logger name:message -howto/logging,,:root,DEBUG:root:This message should go to the log file -howto/logging,,:root,INFO:root:Doing something -howto/logging,,:root,INFO:root:Finished -howto/logging,,:root,INFO:root:So should this -howto/logging,,:root,"ERROR:root:And non-ASCII stuff, too, like " -howto/logging,,:root,INFO:root:Started -howto/logging,,:root,"WARNING:root:And this, too" -howto/logging,,:root,WARNING:root:Look before you leap! -howto/logging,,:root,WARNING:root:Watch out! -howto/logging,,:So,INFO:root:So should this -howto/logging,,:So,INFO:So should this -howto/logging,,:Started,INFO:root:Started -howto/logging,,:This,DEBUG:root:This message should go to the log file -howto/logging,,:This,DEBUG:This message should appear on the console -howto/logging,,:Watch,WARNING:root:Watch out! -howto/pyporting,,::,Programming Language :: Python :: 2 -howto/pyporting,,::,Programming Language :: Python :: 3 -howto/regex,,::, -howto/regex,,:foo,(?:foo) -howto/urllib2,,:password,"""joe:password@example.com""" -library/__main__,,`, -library/ast,,:upper,lower:upper -library/ast,,:step,lower:upper:step -library/audioop,,:ipos,"# factor = audioop.findfactor(in_test[ipos*2:ipos*2+len(out_test)]," -library/configparser,,:home,my_dir: ${Common:home_dir}/twosheds -library/configparser,,:option,${section:option} -library/configparser,,:path,python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} -library/configparser,,:Python,python_dir: ${Frameworks:path}/Python/Versions/${Frameworks:Python} -library/configparser,,:system,path: ${Common:system_dir}/Library/Frameworks/ -library/datetime,,:MM, -library/datetime,,:SS, -library/decimal,,:optional,"trailneg:optional trailing minus indicator" -library/difflib,,:ahi,a[alo:ahi] -library/difflib,,:bhi,b[blo:bhi] -library/difflib,,:i1, -library/difflib,,:i2, -library/difflib,,:j2, -library/doctest,,`,``factorial`` from the ``example`` module: -library/doctest,,`,The ``example`` module -library/doctest,,`,Using ``factorial`` -library/exceptions,,:err,err.object[err.start:err.end] -library/functions,,:step,a[start:stop:step] -library/functions,,:stop,"a[start:stop, i]" -library/functions,,:stop,a[start:stop:step] -library/hashlib,,:LEAF,"h00 = blake2b(buf[0:LEAF_SIZE], fanout=FANOUT, depth=DEPTH," -library/http.client,,:port,host:port -library/http.cookies,,`,!#$%&'*+-.^_`|~: -library/imaplib,,:MM,"""DD-Mmm-YYYY HH:MM:SS" -library/imaplib,,:SS,"""DD-Mmm-YYYY HH:MM:SS" -library/inspect,,:int,">>> def foo(a, *, b:int, **kwargs):" -library/inspect,,:int,"'(a, *, b:int, **kwargs)'" -library/inspect,,:int,'b:int' -library/ipaddress,,:db8,>>> ipaddress.ip_address('2001:db8::') -library/ipaddress,,::,>>> ipaddress.ip_address('2001:db8::') -library/ipaddress,,:db8,IPv6Address('2001:db8::') -library/ipaddress,,::,IPv6Address('2001:db8::') -library/ipaddress,,:db8,>>> ipaddress.IPv6Address('2001:db8::1000') -library/ipaddress,,::,>>> ipaddress.IPv6Address('2001:db8::1000') -library/ipaddress,,:db8,'2001:db8::1000' -library/ipaddress,,::,'2001:db8::1000' -library/ipaddress,,:db8,">>> f'{ipaddress.IPv6Address(""2001:db8::1000""):s}'" -library/ipaddress,,::,">>> f'{ipaddress.IPv6Address(""2001:db8::1000""):s}'" -library/ipaddress,,::,IPv6Address('ff02::5678%1') -library/ipaddress,,::,fe80::1234 -library/ipaddress,,:db8,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" -library/ipaddress,,::,">>> ipaddress.ip_address(""2001:db8::1"").reverse_pointer" -library/ipaddress,,::,"""::abc:7:def""" -library/ipaddress,,:def,"""::abc:7:def""" -library/ipaddress,,::,::FFFF/96 -library/ipaddress,,::,2002::/16 -library/ipaddress,,::,2001::/32 -library/ipaddress,,::,>>> str(ipaddress.IPv6Address('::1')) -library/ipaddress,,::,'::1' -library/ipaddress,,:ff00,ffff:ff00:: -library/ipaddress,,:db00,2001:db00::0/24 -library/ipaddress,,::,2001:db00::0/24 -library/ipaddress,,:db00,2001:db00::0/ffff:ff00:: -library/ipaddress,,::,2001:db00::0/ffff:ff00:: -library/itertools,,:step,elements from seq[start:stop:step] -library/itertools,,::,kernel = tuple(kernel)[::-1] -library/itertools,,:stop,elements from seq[start:stop:step] -library/logging.handlers,,:port,host:port -library/logging,,:root,WARNING:root:Watch out! -library/logging,,:Watch,WARNING:root:Watch out! -library/mmap,,:i2,obj[i1:i2] -library/multiprocessing,,`,# Add more tasks using `put()` -library/multiprocessing,,:queue,">>> QueueManager.register('get_queue', callable=lambda:queue)" -library/multiprocessing,,`,# register the Foo class; make `f()` and `g()` accessible via proxy -library/multiprocessing,,`,# register the Foo class; make `g()` and `_h()` accessible via proxy -library/multiprocessing,,`,# register the generator function baz; use `GeneratorProxy` to make proxies -library/nntplib,,:bytes,:bytes -library/nntplib,,:lines,:lines -library/optparse,,:len,"del parser.rargs[:len(value)]" -library/os.path,,:foo,c:foo -library/pathlib,,:bar,">>> PureWindowsPath('c:/Windows', 'd:bar')" -library/pathlib,,:bar,PureWindowsPath('d:bar') -library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').root -library/pathlib,,:Program,>>> PureWindowsPath('c:Program Files/').anchor -library/pdb,,:lineno,filename:lineno -library/pickle,,:memory,"conn = sqlite3.connect("":memory:"")" -library/posix,,`,"CFLAGS=""`getconf LFS_CFLAGS`"" OPT=""-g -O2 $CFLAGS""" -library/pprint,,::,"'Programming Language :: Python :: 2.6'," -library/pprint,,::,"'Programming Language :: Python :: 2.7'," -library/pprint,,::,"'classifiers': ['Development Status :: 3 - Alpha'," -library/pprint,,::,"'Intended Audience :: Developers'," -library/pprint,,::,"'License :: OSI Approved :: MIT License'," -library/pprint,,::,"'Programming Language :: Python :: 2'," -library/pprint,,::,"'Programming Language :: Python :: 3'," -library/pprint,,::,"'Programming Language :: Python :: 3.2'," -library/pprint,,::,"'Programming Language :: Python :: 3.3'," -library/pprint,,::,"'Programming Language :: Python :: 3.4'," -library/pprint,,::,"'Topic :: Software Development :: Build Tools']," -library/profile,,:lineno,filename:lineno(function) -library/pyexpat,,:elem1, -library/pyexpat,,:py,"xmlns:py = ""http://www.python.org/ns/"">" -library/random,,:len,new_diff = mean(combined[:len(drug)]) - mean(combined[len(drug):]) -library/readline,,:bind,"python:bind -v" -library/readline,,:bind,"python:bind ^I rl_complete" -library/smtplib,,:port,method must support that as well as a regular host:port -library/socket,,::,'5aef:2b::8' -library/socket,,:can,"return (can_id, can_dlc, data[:can_dlc])" -library/socket,,:len,fds.frombytes(cmsg_data[:len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) -library/sqlite3,,:year,"cur.execute(""select * from lang where first_appeared=:year"", {""year"": 1972})" -library/sqlite3,,:memory, -library/sqlite3,,:template,"con = sqlite3.connect(""file:template.db?mode=ro"", uri=True)" -library/sqlite3,,:nosuchdb,"con = sqlite3.connect(""file:nosuchdb.db?mode=rw"", uri=True)" -library/sqlite3,,:mem1,"con1 = sqlite3.connect(""file:mem1?mode=memory&cache=shared"", uri=True)" -library/sqlite3,,:mem1,"con2 = sqlite3.connect(""file:mem1?mode=memory&cache=shared"", uri=True)" -library/ssl,,:My,"Organizational Unit Name (eg, section) []:My Group" -library/ssl,,:My,"Organization Name (eg, company) [Internet Widgits Pty Ltd]:My Organization, Inc." -library/ssl,,:myserver,"Common Name (eg, YOUR name) []:myserver.mygroup.myorganization.com" -library/ssl,,:MyState,State or Province Name (full name) [Some-State]:MyState -library/ssl,,:ops,Email Address []:ops@myserver.mygroup.myorganization.com -library/ssl,,:Some,"Locality Name (eg, city) []:Some City" -library/ssl,,:US,Country Name (2 letter code) [AU]:US -library/stdtypes,,:end,s[start:end] -library/stdtypes,,::,>>> hash(v[::-2]) == hash(b'abcefg'[::-2]) -library/stdtypes,,:len,s[len(s):len(s)] -library/stdtypes,,::,>>> y = m[::2] -library/stdtypes,,::,>>> z = y[::-2] -library/string,,`,"!""#$%&'()*+,-./:;<=>?@[\]^_`{|}~" -library/tarfile,,:bz2, -library/tarfile,,:compression,filemode[:compression] -library/tarfile,,:gz, -library/tarfile,,:xz,'a:xz' -library/tarfile,,:xz,'r:xz' -library/tarfile,,:xz,'w:xz' -library/time,,:mm, -library/time,,:ss, -library/tracemalloc,,:limit,"for index, stat in enumerate(top_stats[:limit], 1):" -library/turtle,,::,Example:: -library/unittest,,:foo,"self.assertEqual(cm.output, ['INFO:foo:first message'," -library/unittest,,:first,"self.assertEqual(cm.output, ['INFO:foo:first message'," -library/unittest,,:foo,'ERROR:foo.bar:second message']) -library/unittest,,:second,'ERROR:foo.bar:second message']) -library/urllib.request,,:close,Connection:close -library/urllib.request,,:port,:port -library/urllib.request,,:lang,"xmlns=""http://www.w3.org/1999/xhtml"" xml:lang=""en"" lang=""en"">\n\n\n" -library/urllib.request,,:password,"""joe:password@python.org""" -library/urllib.parse,,:scheme, -library/urllib.parse,,:scheme,URL:scheme://host/path -library/uuid,,:uuid,urn:uuid:12345678-1234-5678-1234-567812345678 -library/venv,,:param,":param nodist: If true, setuptools and pip are not installed into the" -library/venv,,:param,":param progress: If setuptools or pip are installed, the progress of the" -library/venv,,:param,":param nopip: If true, pip is not installed into the created" -library/venv,,:param,:param context: The information for the virtual environment -library/xmlrpc.client,,:nil,ex:nil -library/xmlrpc.client,,:pass,http://user:pass@host:port/path -library/xmlrpc.client,,:pass,user:pass -library/xmlrpc.client,,:port,http://user:pass@host:port/path -license,,`,"``Software''), to deal in the Software without restriction, including" -license,,`,"THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND," -license,,`,* THIS SOFTWARE IS PROVIDED BY ERIC YOUNG ``AS IS'' AND -license,,`,THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND -license,,`,* THIS SOFTWARE IS PROVIDED BY THE OpenSSL PROJECT ``AS IS'' AND ANY -license,,`,THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND -license,,:zooko,mailto:zooko@zooko.com -reference/expressions,,:index,x[index:index] -reference/lexical_analysis,,`,$ ? ` -reference/lexical_analysis,,:fileencoding,# vim:fileencoding= -tutorial/datastructures,,:value,It is also possible to delete a key:value -tutorial/datastructures,,:value,key:value pairs within the braces adds initial key:value pairs -tutorial/stdlib2,,:config,"logging.warning('Warning:config file %s not found', 'server.conf')" -tutorial/stdlib2,,:config,WARNING:root:Warning:config file server.conf not found -tutorial/stdlib2,,:Critical,CRITICAL:root:Critical error -- shutting down -tutorial/stdlib2,,:Error,ERROR:root:Error occurred -tutorial/stdlib2,,:root,CRITICAL:root:Critical error -- shutting down -tutorial/stdlib2,,:root,ERROR:root:Error occurred -tutorial/stdlib2,,:root,WARNING:root:Warning:config file server.conf not found -tutorial/stdlib2,,:start,extra = data[start:start+extra_size] -tutorial/stdlib2,,:start,"fields = struct.unpack('>> urlparse.urlparse('http://[1080::8:800:200C:417A]/foo') -whatsnew/2.7,,:Sunday,'2009:4:Sunday' -whatsnew/2.7,,:Cookie,"export PYTHONWARNINGS=all,error:::Cookie:0" -whatsnew/2.7,,::,"export PYTHONWARNINGS=all,error:::Cookie:0" -whatsnew/3.2,,:affe,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:affe,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:beef,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:beef,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:cafe,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:cafe,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:deaf,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:deaf,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:directory,${buildout:directory}/downloads/dist -whatsnew/3.2,,::,"$ export PYTHONWARNINGS='ignore::RuntimeWarning::,once::UnicodeWarning::'" -whatsnew/3.2,,:feed,"netloc='[dead:beef:cafe:5417:affe:8FA3:deaf:feed]'," -whatsnew/3.2,,:feed,>>> urllib.parse.urlparse('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/') -whatsnew/3.2,,:gz,">>> with tarfile.open(name='myarchive.tar.gz', mode='w:gz') as tf:" -whatsnew/3.2,,:location,zope9-location = ${zope9:location} -whatsnew/3.2,,:prefix,zope-conf = ${custom:prefix}/etc/zope.conf -library/re,,`,!#$%&'*+-.^_`|~: -library/re,,`,!\#\$%\&'\*\+\-\.\^_`\|\~: -library/tarfile,,:xz,'x:xz' -library/warnings,,:message,action:message:category:module:line -library/warnings,,:category,action:message:category:module:line -library/warnings,,:module,action:message:category:module:line -library/warnings,,:line,action:message:category:module:line -library/warnings,,::,error::ResourceWarning -library/warnings,,::,default::DeprecationWarning -library/warnings,,::,default:::mymodule -library/warnings,,:mymodule,default:::mymodule -library/warnings,,::,error:::mymodule -library/warnings,,:mymodule,error:::mymodule -library/warnings,,::,ignore::DeprecationWarning -library/warnings,,::,ignore::PendingDeprecationWarning -library/warnings,,::,ignore::ImportWarning -library/warnings,,::,ignore::ResourceWarning -library/xml.etree.elementtree,,:sometag,prefix:sometag -library/xml.etree.elementtree,,:fictional,"Lancelot -library/xml.etree.elementtree,,:character,Archie Leach -library/xml.etree.elementtree,,:character,Sir Robin -library/xml.etree.elementtree,,:character,Gunther -library/xml.etree.elementtree,,:character,Commander Clement -library/xml.etree.elementtree,,:actor,"for actor in root.findall('real_person:actor', ns):" -library/xml.etree.elementtree,,:name,"name = actor.find('real_person:name', ns)" -library/xml.etree.elementtree,,:character,"for char in actor.findall('role:character', ns):" -library/xml.etree.elementtree,,:xi, -library/xml.etree.elementtree,,:include, -library/xml.etree.elementtree,,:include, Copyright (c) . -library/zipapp,,:main,"$ python -m zipapp myapp -m ""myapp:main""" -library/zipapp,,:fn,"pkg.mod:fn" -library/zipapp,,:callable,"pkg.module:callable" -library/stdtypes,,::,>>> m[::2].tolist() -whatsnew/3.5,,:root,'WARNING:root:warning\n' -whatsnew/3.5,,:warning,'WARNING:root:warning\n' -whatsnew/3.5,,::,>>> addr6 = ipaddress.IPv6Address('::1') -whatsnew/3.5,,:root,ERROR:root:exception -whatsnew/3.5,,:exception,ERROR:root:exception -whatsnew/changelog,,`,'`' -whatsnew/changelog,,:end,str[start:end] -library/binascii,,`,'`' -library/uu,,`,'`' -whatsnew/3.7,,`,'`' -whatsnew/3.7,,::,error::BytesWarning -whatsnew/changelog,,::,error::BytesWarning -whatsnew/changelog,,::,default::BytesWarning -whatsnew/changelog,,::,default::DeprecationWarning -library/importlib.metadata,,:main,"EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts')" -library/importlib.metadata,,`,loading the metadata for packages for the indicated ``context``. -library/re,,`,"`" -library/typing,,`,# Type of ``val`` is narrowed to ``str`` -library/typing,,`,"# Else, type of ``val`` is narrowed to ``float``." -library/typing,,`,# Type of ``val`` is narrowed to ``list[str]``. -library/typing,,`,# Type of ``val`` remains as ``list[object]``. -library/tkinter,,::,ttk::frame .frm -padding 10 -library/tkinter,,::,"grid [ttk::label .frm.lbl -text ""Hello World!""] -column 0 -row 0" -library/tkinter,,::,"grid [ttk::button .frm.btn -text ""Quit"" -command ""destroy .""] -column 1 -row 0" -library/tkinter,,::,ttk::frame -library/tkinter,,::,ttk::button -library/tkinter,,::,ttk::widget -reference/compound_stmts,,:exc,subclass of :exc:`BaseExceptionGroup`. It is not possible to mix except -reference/compound_stmts,,`,subclass of :exc:`BaseExceptionGroup`. It is not possible to mix except -reference/compound_stmts,,:keyword,"and except* in the same :keyword:`try`. :keyword:`break`," -reference/compound_stmts,,`,"and except* in the same :keyword:`try`. :keyword:`break`," -reference/compound_stmts,,:keyword,:keyword:`continue` and :keyword:`return` cannot appear in an except* -reference/compound_stmts,,`,:keyword:`continue` and :keyword:`return` cannot appear in an except* -whatsnew/changelog,,:CON,": os.path.abspath(“C:CON”) is now fixed to return “\.CON”, not" -whatsnew/changelog,,::,Lib/email/mime/nonmultipart.py::MIMENonMultipart -library/typing,,`,"assert_type(name, str) # OK, inferred type of `name` is `str`" -library/typing,,`,# after which we hope the inferred type will be `int` -whatsnew/changelog,,:company,-V:company/tag -library/typing,,`,# are located in the `typing_extensions` backports package. diff --git a/Doc/tools/templates/customsourcelink.html b/Doc/tools/templates/customsourcelink.html deleted file mode 100644 index 21af621efce95b..00000000000000 --- a/Doc/tools/templates/customsourcelink.html +++ /dev/null @@ -1,13 +0,0 @@ -{%- if show_source and has_source and sourcename %} - -{%- endif %} diff --git a/Doc/tools/templates/download.html b/Doc/tools/templates/download.html deleted file mode 100644 index b5353d6fb77ab4..00000000000000 --- a/Doc/tools/templates/download.html +++ /dev/null @@ -1,64 +0,0 @@ -{% extends "layout.html" %} -{% set title = 'Download' %} -{% if daily is defined %} - {% set dlbase = pathto('archives', 1) %} -{% else %} - {# - The link below returns HTTP 404 until the first related alpha release. - This is expected; use daily documentation builds for CPython development. - #} - {% set dlbase = 'https://docs.python.org/ftp/python/doc/' + release %} -{% endif %} - -{% block body %} -

    Download Python {{ release }} Documentation

    - -{% if last_updated %}

    Last updated on: {{ last_updated }}.

    {% endif %} - -

    To download an archive containing all the documents for this version of -Python in one of various formats, follow one of links in this table.

    - - - - - - - - - - - - - - - - - - - - - - - -
    FormatPacked as .zipPacked as .tar.bz2
    PDF (US-Letter paper size)Download (ca. 13 MiB)Download (ca. 13 MiB)
    PDF (A4 paper size)Download (ca. 13 MiB)Download (ca. 13 MiB)
    HTMLDownload (ca. 9 MiB)Download (ca. 6 MiB)
    Plain TextDownload (ca. 3 MiB)Download (ca. 2 MiB)
    EPUBDownload (ca. 5 MiB)
    - -

    These archives contain all the content in the documentation.

    - - -

    Unpacking

    - -

    Unix users should download the .tar.bz2 archives; these are bzipped tar -archives and can be handled in the usual way using tar and the bzip2 -program. The Info-ZIP unzip program can be -used to handle the ZIP archives if desired. The .tar.bz2 archives provide the -best compression and fastest download times.

    - -

    Windows users can use the ZIP archives since those are customary on that -platform. These are created on Unix using the Info-ZIP zip program.

    - - -

    Problems

    - -

    If you have comments or suggestions for the Python documentation, please send -email to docs@python.org.

    -{% endblock %} diff --git a/Doc/tools/templates/dummy.html b/Doc/tools/templates/dummy.html deleted file mode 100644 index bab4aaeb4604b8..00000000000000 --- a/Doc/tools/templates/dummy.html +++ /dev/null @@ -1,22 +0,0 @@ -This file is not an actual template, but used to add some -texts in extensions to sphinx.pot file. - -In extensions/pyspecific.py: - -{% trans %}CPython implementation detail:{% endtrans %} -{% trans %}Deprecated since version {deprecated}, will be removed in version {removed}{% endtrans %} -{% trans %}Deprecated since version {deprecated}, removed in version {removed}{% endtrans %} - -In extensions/c_annotations.py: - -{% trans %}Return value: Always NULL.{% endtrans %} -{% trans %}Return value: New reference.{% endtrans %} -{% trans %}Return value: Borrowed reference.{% endtrans %} - -In docsbuild-scripts, when rewriting indexsidebar.html with actual versions: - -{% trans %}in development{% endtrans %} -{% trans %}pre-release{% endtrans %} -{% trans %}stable{% endtrans %} -{% trans %}security-fixes{% endtrans %} -{% trans %}EOL{% endtrans %} diff --git a/Doc/tools/templates/indexcontent.html b/Doc/tools/templates/indexcontent.html deleted file mode 100644 index 1e3ab7cfe02fee..00000000000000 --- a/Doc/tools/templates/indexcontent.html +++ /dev/null @@ -1,68 +0,0 @@ -{% extends "layout.html" %} -{%- block htmltitle -%} -{{ shorttitle }} -{%- endblock -%} -{% block body %} -

    {{ docstitle|e }}

    -

    - {% trans %}Welcome! This is the official documentation for Python {{ release }}.{% endtrans %} -

    -

    {% trans %}Parts of the documentation:{% endtrans %}

    - - -
    -

    {% trans %}What's new in Python {{ version }}?{% endtrans %}
    - {% trans whatsnew_index=pathto("whatsnew/index") %}or all "What's new" documents since 2.0{% endtrans %}

    -

    {% trans %}Tutorial{% endtrans %}
    - {% trans %}start here{% endtrans %}

    -

    {% trans %}Library Reference{% endtrans %}
    - {% trans %}keep this under your pillow{% endtrans %}

    -

    {% trans %}Language Reference{% endtrans %}
    - {% trans %}describes syntax and language elements{% endtrans %}

    -

    {% trans %}Python Setup and Usage{% endtrans %}
    - {% trans %}how to use Python on different platforms{% endtrans %}

    -

    {% trans %}Python HOWTOs{% endtrans %}
    - {% trans %}in-depth documents on specific topics{% endtrans %}

    -     		-

    {% trans %}Installing Python Modules{% endtrans %}
    - {% trans %}installing from the Python Package Index & other sources{% endtrans %}

    -

    {% trans %}Distributing Python Modules{% endtrans %}
    - {% trans %}publishing modules for installation by others{% endtrans %}

    -

    {% trans %}Extending and Embedding{% endtrans %}
    - {% trans %}tutorial for C/C++ programmers{% endtrans %}

    -

    {% trans %}Python/C API{% endtrans %}
    - {% trans %}reference for C/C++ programmers{% endtrans %}

    -

    {% trans %}FAQs{% endtrans %}
    - {% trans %}frequently asked questions (with answers!){% endtrans %}

    -
    - -

    {% trans %}Indices and tables:{% endtrans %}

    - - -
    -

    {% trans %}Global Module Index{% endtrans %}
    - {% trans %}quick access to all modules{% endtrans %}

    -

    {% trans %}General Index{% endtrans %}
    - {% trans %}all functions, classes, terms{% endtrans %}

    -

    {% trans %}Glossary{% endtrans %}
    - {% trans %}the most important terms explained{% endtrans %}

    -     		-

    {% trans %}Search page{% endtrans %}
    - {% trans %}search this documentation{% endtrans %}

    -

    {% trans %}Complete Table of Contents{% endtrans %}
    - {% trans %}lists all sections and subsections{% endtrans %}

    -
    - -

    {% trans %}Meta information:{% endtrans %}

    - - -
    -

    {% trans %}Reporting bugs{% endtrans %}

    -

    {% trans %}Contributing to Docs{% endtrans %}

    -

    {% trans %}About the documentation{% endtrans %}

    -     		-

    {% trans %}History and License of Python{% endtrans %}

    -

    {% trans %}Copyright{% endtrans %}

    -

    {% trans %}Download the documentation{% endtrans %}

    -
    -{% endblock %} diff --git a/Doc/tools/templates/indexsidebar.html b/Doc/tools/templates/indexsidebar.html deleted file mode 100644 index 5986204256f4bc..00000000000000 --- a/Doc/tools/templates/indexsidebar.html +++ /dev/null @@ -1,18 +0,0 @@ -

    {% trans %}Download{% endtrans %}

    -

    {% trans %}Download these documents{% endtrans %}

    -

    {% trans %}Docs by version{% endtrans %}

    - - -

    {% trans %}Other resources{% endtrans %}

    - diff --git a/Doc/tools/templates/layout.html b/Doc/tools/templates/layout.html deleted file mode 100644 index 9498b2ccc5af92..00000000000000 --- a/Doc/tools/templates/layout.html +++ /dev/null @@ -1,44 +0,0 @@ -{% extends "!layout.html" %} - -{% block header %} -{%- if outdated %} -
    - {% trans %}This document is for an old version of Python that is no longer supported. - You should upgrade, and read the{% endtrans %} - {% trans %}Python documentation for the current stable release{% endtrans %}. -
    -{%- endif %} - -{%- if is_deployment_preview %} -
    - {% trans %}This is a deploy preview created from a pull request. - For authoritative documentation, see{% endtrans %} - {% trans %}the current stable release{% endtrans %}. -
    -{%- endif %} -{% endblock %} - -{% block rootrellink %} -{{ super() }} -
  • - {{ shorttitle }}{{ reldelim1 }} -
    • -{% endblock %} - -{% block extrahead %} - - {% if builder != "htmlhelp" %} - {% if pagename == 'whatsnew/changelog' and not embedded %} - {% endif %} - {% endif %} - - {# custom CSS; used in asyncio docs! #} - -{{ super() }} -{% endblock %} diff --git a/Doc/tools/templates/opensearch.xml b/Doc/tools/templates/opensearch.xml deleted file mode 100644 index 7a5cdddd277123..00000000000000 --- a/Doc/tools/templates/opensearch.xml +++ /dev/null @@ -1,4 +0,0 @@ -{% extends "!opensearch.xml" %} -{% block extra -%} -https://www.python.org/images/favicon16x16.ico -{%- endblock %} diff --git a/Doc/tutorial/appendix.rst b/Doc/tutorial/appendix.rst deleted file mode 100644 index 588591fcdb726f..00000000000000 --- a/Doc/tutorial/appendix.rst +++ /dev/null @@ -1,124 +0,0 @@ -.. _tut-appendix: - -******** -Appendix -******** - - -.. _tut-interac: - -Interactive Mode -================ - -.. _tut-error: - -Error Handling --------------- - -When an error occurs, the interpreter prints an error message and a stack trace. -In interactive mode, it then returns to the primary prompt; when input came from -a file, it exits with a nonzero exit status after printing the stack trace. -(Exceptions handled by an :keyword:`except` clause in a :keyword:`try` statement -are not errors in this context.) Some errors are unconditionally fatal and -cause an exit with a nonzero exit; this applies to internal inconsistencies and -some cases of running out of memory. All error messages are written to the -standard error stream; normal output from executed commands is written to -standard output. - -Typing the interrupt character (usually :kbd:`Control-C` or :kbd:`Delete`) to the primary or -secondary prompt cancels the input and returns to the primary prompt. [#]_ -Typing an interrupt while a command is executing raises the -:exc:`KeyboardInterrupt` exception, which may be handled by a :keyword:`try` -statement. - - -.. _tut-scripts: - -Executable Python Scripts -------------------------- - -On BSD'ish Unix systems, Python scripts can be made directly executable, like -shell scripts, by putting the line :: - - #!/usr/bin/env python3.5 - -(assuming that the interpreter is on the user's :envvar:`PATH`) at the beginning -of the script and giving the file an executable mode. The ``#!`` must be the -first two characters of the file. On some platforms, this first line must end -with a Unix-style line ending (``'\n'``), not a Windows (``'\r\n'``) line -ending. Note that the hash, or pound, character, ``'#'``, is used to start a -comment in Python. - -The script can be given an executable mode, or permission, using the -:program:`chmod` command. - -.. code-block:: shell-session - - $ chmod +x myscript.py - -On Windows systems, there is no notion of an "executable mode". The Python -installer automatically associates ``.py`` files with ``python.exe`` so that -a double-click on a Python file will run it as a script. The extension can -also be ``.pyw``, in that case, the console window that normally appears is -suppressed. - - -.. _tut-startup: - -The Interactive Startup File ----------------------------- - -When you use Python interactively, it is frequently handy to have some standard -commands executed every time the interpreter is started. You can do this by -setting an environment variable named :envvar:`PYTHONSTARTUP` to the name of a -file containing your start-up commands. This is similar to the :file:`.profile` -feature of the Unix shells. - -This file is only read in interactive sessions, not when Python reads commands -from a script, and not when :file:`/dev/tty` is given as the explicit source of -commands (which otherwise behaves like an interactive session). It is executed -in the same namespace where interactive commands are executed, so that objects -that it defines or imports can be used without qualification in the interactive -session. You can also change the prompts ``sys.ps1`` and ``sys.ps2`` in this -file. - -If you want to read an additional start-up file from the current directory, you -can program this in the global start-up file using code like ``if -os.path.isfile('.pythonrc.py'): exec(open('.pythonrc.py').read())``. -If you want to use the startup file in a script, you must do this explicitly -in the script:: - - import os - filename = os.environ.get('PYTHONSTARTUP') - if filename and os.path.isfile(filename): - with open(filename) as fobj: - startup_file = fobj.read() - exec(startup_file) - - -.. _tut-customize: - -The Customization Modules -------------------------- - -Python provides two hooks to let you customize it: :index:`sitecustomize` and -:index:`usercustomize`. To see how it works, you need first to find the location -of your user site-packages directory. Start Python and run this code:: - - >>> import site - >>> site.getusersitepackages() - '/home/user/.local/lib/python3.5/site-packages' - -Now you can create a file named :file:`usercustomize.py` in that directory and -put anything you want in it. It will affect every invocation of Python, unless -it is started with the :option:`-s` option to disable the automatic import. - -:index:`sitecustomize` works in the same way, but is typically created by an -administrator of the computer in the global site-packages directory, and is -imported before :index:`usercustomize`. See the documentation of the :mod:`site` -module for more details. - - -.. rubric:: Footnotes - -.. [#] A problem with the GNU Readline package may prevent this. diff --git a/Doc/tutorial/appetite.rst b/Doc/tutorial/appetite.rst deleted file mode 100644 index 3fa68097ee3665..00000000000000 --- a/Doc/tutorial/appetite.rst +++ /dev/null @@ -1,87 +0,0 @@ -.. _tut-intro: - -********************** -Whetting Your Appetite -********************** - -If you do much work on computers, eventually you find that there's some task -you'd like to automate. For example, you may wish to perform a -search-and-replace over a large number of text files, or rename and rearrange a -bunch of photo files in a complicated way. Perhaps you'd like to write a small -custom database, or a specialized GUI application, or a simple game. - -If you're a professional software developer, you may have to work with several -C/C++/Java libraries but find the usual write/compile/test/re-compile cycle is -too slow. Perhaps you're writing a test suite for such a library and find -writing the testing code a tedious task. Or maybe you've written a program that -could use an extension language, and you don't want to design and implement a -whole new language for your application. - -Python is just the language for you. - -You could write a Unix shell script or Windows batch files for some of these -tasks, but shell scripts are best at moving around files and changing text data, -not well-suited for GUI applications or games. You could write a C/C++/Java -program, but it can take a lot of development time to get even a first-draft -program. Python is simpler to use, available on Windows, macOS, and Unix -operating systems, and will help you get the job done more quickly. - -Python is simple to use, but it is a real programming language, offering much -more structure and support for large programs than shell scripts or batch files -can offer. On the other hand, Python also offers much more error checking than -C, and, being a *very-high-level language*, it has high-level data types built -in, such as flexible arrays and dictionaries. Because of its more general data -types Python is applicable to a much larger problem domain than Awk or even -Perl, yet many things are at least as easy in Python as in those languages. - -Python allows you to split your program into modules that can be reused in other -Python programs. It comes with a large collection of standard modules that you -can use as the basis of your programs --- or as examples to start learning to -program in Python. Some of these modules provide things like file I/O, system -calls, sockets, and even interfaces to graphical user interface toolkits like -Tk. - -Python is an interpreted language, which can save you considerable time during -program development because no compilation and linking is necessary. The -interpreter can be used interactively, which makes it easy to experiment with -features of the language, to write throw-away programs, or to test functions -during bottom-up program development. It is also a handy desk calculator. - -Python enables programs to be written compactly and readably. Programs written -in Python are typically much shorter than equivalent C, C++, or Java programs, -for several reasons: - -* the high-level data types allow you to express complex operations in a single - statement; - -* statement grouping is done by indentation instead of beginning and ending - brackets; - -* no variable or argument declarations are necessary. - -Python is *extensible*: if you know how to program in C it is easy to add a new -built-in function or module to the interpreter, either to perform critical -operations at maximum speed, or to link Python programs to libraries that may -only be available in binary form (such as a vendor-specific graphics library). -Once you are really hooked, you can link the Python interpreter into an -application written in C and use it as an extension or command language for that -application. - -By the way, the language is named after the BBC show "Monty Python's Flying -Circus" and has nothing to do with reptiles. Making references to Monty -Python skits in documentation is not only allowed, it is encouraged! - -Now that you are all excited about Python, you'll want to examine it in some -more detail. Since the best way to learn a language is to use it, the tutorial -invites you to play with the Python interpreter as you read. - -In the next chapter, the mechanics of using the interpreter are explained. This -is rather mundane information, but essential for trying out the examples shown -later. - -The rest of the tutorial introduces various features of the Python language and -system through examples, beginning with simple expressions, statements and data -types, through functions and modules, and finally touching upon advanced -concepts like exceptions and user-defined classes. - - diff --git a/Doc/tutorial/classes.rst b/Doc/tutorial/classes.rst deleted file mode 100644 index 46dfb378590ffb..00000000000000 --- a/Doc/tutorial/classes.rst +++ /dev/null @@ -1,934 +0,0 @@ -.. _tut-classes: - -******* -Classes -******* - -Classes provide a means of bundling data and functionality together. Creating -a new class creates a new *type* of object, allowing new *instances* of that -type to be made. Each class instance can have attributes attached to it for -maintaining its state. Class instances can also have methods (defined by its -class) for modifying its state. - -Compared with other programming languages, Python's class mechanism adds classes -with a minimum of new syntax and semantics. It is a mixture of the class -mechanisms found in C++ and Modula-3. Python classes provide all the standard -features of Object Oriented Programming: the class inheritance mechanism allows -multiple base classes, a derived class can override any methods of its base -class or classes, and a method can call the method of a base class with the same -name. Objects can contain arbitrary amounts and kinds of data. As is true for -modules, classes partake of the dynamic nature of Python: they are created at -runtime, and can be modified further after creation. - -In C++ terminology, normally class members (including the data members) are -*public* (except see below :ref:`tut-private`), and all member functions are -*virtual*. As in Modula-3, there are no shorthands for referencing the object's -members from its methods: the method function is declared with an explicit first -argument representing the object, which is provided implicitly by the call. As -in Smalltalk, classes themselves are objects. This provides semantics for -importing and renaming. Unlike C++ and Modula-3, built-in types can be used as -base classes for extension by the user. Also, like in C++, most built-in -operators with special syntax (arithmetic operators, subscripting etc.) can be -redefined for class instances. - -(Lacking universally accepted terminology to talk about classes, I will make -occasional use of Smalltalk and C++ terms. I would use Modula-3 terms, since -its object-oriented semantics are closer to those of Python than C++, but I -expect that few readers have heard of it.) - - -.. _tut-object: - -A Word About Names and Objects -============================== - -Objects have individuality, and multiple names (in multiple scopes) can be bound -to the same object. This is known as aliasing in other languages. This is -usually not appreciated on a first glance at Python, and can be safely ignored -when dealing with immutable basic types (numbers, strings, tuples). However, -aliasing has a possibly surprising effect on the semantics of Python code -involving mutable objects such as lists, dictionaries, and most other types. -This is usually used to the benefit of the program, since aliases behave like -pointers in some respects. For example, passing an object is cheap since only a -pointer is passed by the implementation; and if a function modifies an object -passed as an argument, the caller will see the change --- this eliminates the -need for two different argument passing mechanisms as in Pascal. - - -.. _tut-scopes: - -Python Scopes and Namespaces -============================ - -Before introducing classes, I first have to tell you something about Python's -scope rules. Class definitions play some neat tricks with namespaces, and you -need to know how scopes and namespaces work to fully understand what's going on. -Incidentally, knowledge about this subject is useful for any advanced Python -programmer. - -Let's begin with some definitions. - -A *namespace* is a mapping from names to objects. Most namespaces are currently -implemented as Python dictionaries, but that's normally not noticeable in any -way (except for performance), and it may change in the future. Examples of -namespaces are: the set of built-in names (containing functions such as :func:`abs`, and -built-in exception names); the global names in a module; and the local names in -a function invocation. In a sense the set of attributes of an object also form -a namespace. The important thing to know about namespaces is that there is -absolutely no relation between names in different namespaces; for instance, two -different modules may both define a function ``maximize`` without confusion --- -users of the modules must prefix it with the module name. - -By the way, I use the word *attribute* for any name following a dot --- for -example, in the expression ``z.real``, ``real`` is an attribute of the object -``z``. Strictly speaking, references to names in modules are attribute -references: in the expression ``modname.funcname``, ``modname`` is a module -object and ``funcname`` is an attribute of it. In this case there happens to be -a straightforward mapping between the module's attributes and the global names -defined in the module: they share the same namespace! [#]_ - -Attributes may be read-only or writable. In the latter case, assignment to -attributes is possible. Module attributes are writable: you can write -``modname.the_answer = 42``. Writable attributes may also be deleted with the -:keyword:`del` statement. For example, ``del modname.the_answer`` will remove -the attribute :attr:`!the_answer` from the object named by ``modname``. - -Namespaces are created at different moments and have different lifetimes. The -namespace containing the built-in names is created when the Python interpreter -starts up, and is never deleted. The global namespace for a module is created -when the module definition is read in; normally, module namespaces also last -until the interpreter quits. The statements executed by the top-level -invocation of the interpreter, either read from a script file or interactively, -are considered part of a module called :mod:`__main__`, so they have their own -global namespace. (The built-in names actually also live in a module; this is -called :mod:`builtins`.) - -The local namespace for a function is created when the function is called, and -deleted when the function returns or raises an exception that is not handled -within the function. (Actually, forgetting would be a better way to describe -what actually happens.) Of course, recursive invocations each have their own -local namespace. - -A *scope* is a textual region of a Python program where a namespace is directly -accessible. "Directly accessible" here means that an unqualified reference to a -name attempts to find the name in the namespace. - -Although scopes are determined statically, they are used dynamically. At any -time during execution, there are 3 or 4 nested scopes whose namespaces are -directly accessible: - -* the innermost scope, which is searched first, contains the local names -* the scopes of any enclosing functions, which are searched starting with the - nearest enclosing scope, contain non-local, but also non-global names -* the next-to-last scope contains the current module's global names -* the outermost scope (searched last) is the namespace containing built-in names - -If a name is declared global, then all references and assignments go directly to -the next-to-last scope containing the module's global names. To rebind variables -found outside of the innermost scope, the :keyword:`nonlocal` statement can be -used; if not declared nonlocal, those variables are read-only (an attempt to -write to such a variable will simply create a *new* local variable in the -innermost scope, leaving the identically named outer variable unchanged). - -Usually, the local scope references the local names of the (textually) current -function. Outside functions, the local scope references the same namespace as -the global scope: the module's namespace. Class definitions place yet another -namespace in the local scope. - -It is important to realize that scopes are determined textually: the global -scope of a function defined in a module is that module's namespace, no matter -from where or by what alias the function is called. On the other hand, the -actual search for names is done dynamically, at run time --- however, the -language definition is evolving towards static name resolution, at "compile" -time, so don't rely on dynamic name resolution! (In fact, local variables are -already determined statically.) - -A special quirk of Python is that -- if no :keyword:`global` or :keyword:`nonlocal` -statement is in effect -- assignments to names always go into the innermost scope. -Assignments do not copy data --- they just bind names to objects. The same is true -for deletions: the statement ``del x`` removes the binding of ``x`` from the -namespace referenced by the local scope. In fact, all operations that introduce -new names use the local scope: in particular, :keyword:`import` statements and -function definitions bind the module or function name in the local scope. - -The :keyword:`global` statement can be used to indicate that particular -variables live in the global scope and should be rebound there; the -:keyword:`nonlocal` statement indicates that particular variables live in -an enclosing scope and should be rebound there. - -.. _tut-scopeexample: - -Scopes and Namespaces Example ------------------------------ - -This is an example demonstrating how to reference the different scopes and -namespaces, and how :keyword:`global` and :keyword:`nonlocal` affect variable -binding:: - - def scope_test(): - def do_local(): - spam = "local spam" - - def do_nonlocal(): - nonlocal spam - spam = "nonlocal spam" - - def do_global(): - global spam - spam = "global spam" - - spam = "test spam" - do_local() - print("After local assignment:", spam) - do_nonlocal() - print("After nonlocal assignment:", spam) - do_global() - print("After global assignment:", spam) - - scope_test() - print("In global scope:", spam) - -The output of the example code is: - -.. code-block:: none - - After local assignment: test spam - After nonlocal assignment: nonlocal spam - After global assignment: nonlocal spam - In global scope: global spam - -Note how the *local* assignment (which is default) didn't change *scope_test*\'s -binding of *spam*. The :keyword:`nonlocal` assignment changed *scope_test*\'s -binding of *spam*, and the :keyword:`global` assignment changed the module-level -binding. - -You can also see that there was no previous binding for *spam* before the -:keyword:`global` assignment. - - -.. _tut-firstclasses: - -A First Look at Classes -======================= - -Classes introduce a little bit of new syntax, three new object types, and some -new semantics. - - -.. _tut-classdefinition: - -Class Definition Syntax ------------------------ - -The simplest form of class definition looks like this:: - - class ClassName: - - . - . - . - - -Class definitions, like function definitions (:keyword:`def` statements) must be -executed before they have any effect. (You could conceivably place a class -definition in a branch of an :keyword:`if` statement, or inside a function.) - -In practice, the statements inside a class definition will usually be function -definitions, but other statements are allowed, and sometimes useful --- we'll -come back to this later. The function definitions inside a class normally have -a peculiar form of argument list, dictated by the calling conventions for -methods --- again, this is explained later. - -When a class definition is entered, a new namespace is created, and used as the -local scope --- thus, all assignments to local variables go into this new -namespace. In particular, function definitions bind the name of the new -function here. - -When a class definition is left normally (via the end), a *class object* is -created. This is basically a wrapper around the contents of the namespace -created by the class definition; we'll learn more about class objects in the -next section. The original local scope (the one in effect just before the class -definition was entered) is reinstated, and the class object is bound here to the -class name given in the class definition header (:class:`!ClassName` in the -example). - - -.. _tut-classobjects: - -Class Objects -------------- - -Class objects support two kinds of operations: attribute references and -instantiation. - -*Attribute references* use the standard syntax used for all attribute references -in Python: ``obj.name``. Valid attribute names are all the names that were in -the class's namespace when the class object was created. So, if the class -definition looked like this:: - - class MyClass: - """A simple example class""" - i = 12345 - - def f(self): - return 'hello world' - -then ``MyClass.i`` and ``MyClass.f`` are valid attribute references, returning -an integer and a function object, respectively. Class attributes can also be -assigned to, so you can change the value of ``MyClass.i`` by assignment. -:attr:`!__doc__` is also a valid attribute, returning the docstring belonging to -the class: ``"A simple example class"``. - -Class *instantiation* uses function notation. Just pretend that the class -object is a parameterless function that returns a new instance of the class. -For example (assuming the above class):: - - x = MyClass() - -creates a new *instance* of the class and assigns this object to the local -variable ``x``. - -The instantiation operation ("calling" a class object) creates an empty object. -Many classes like to create objects with instances customized to a specific -initial state. Therefore a class may define a special method named -:meth:`~object.__init__`, like this:: - - def __init__(self): - self.data = [] - -When a class defines an :meth:`~object.__init__` method, class instantiation -automatically invokes :meth:`!__init__` for the newly created class instance. So -in this example, a new, initialized instance can be obtained by:: - - x = MyClass() - -Of course, the :meth:`~object.__init__` method may have arguments for greater -flexibility. In that case, arguments given to the class instantiation operator -are passed on to :meth:`!__init__`. For example, :: - - >>> class Complex: - ... def __init__(self, realpart, imagpart): - ... self.r = realpart - ... self.i = imagpart - ... - >>> x = Complex(3.0, -4.5) - >>> x.r, x.i - (3.0, -4.5) - - -.. _tut-instanceobjects: - -Instance Objects ----------------- - -Now what can we do with instance objects? The only operations understood by -instance objects are attribute references. There are two kinds of valid -attribute names: data attributes and methods. - -*data attributes* correspond to "instance variables" in Smalltalk, and to "data -members" in C++. Data attributes need not be declared; like local variables, -they spring into existence when they are first assigned to. For example, if -``x`` is the instance of :class:`!MyClass` created above, the following piece of -code will print the value ``16``, without leaving a trace:: - - x.counter = 1 - while x.counter < 10: - x.counter = x.counter * 2 - print(x.counter) - del x.counter - -The other kind of instance attribute reference is a *method*. A method is a -function that "belongs to" an object. (In Python, the term method is not unique -to class instances: other object types can have methods as well. For example, -list objects have methods called append, insert, remove, sort, and so on. -However, in the following discussion, we'll use the term method exclusively to -mean methods of class instance objects, unless explicitly stated otherwise.) - -.. index:: pair: object; method - -Valid method names of an instance object depend on its class. By definition, -all attributes of a class that are function objects define corresponding -methods of its instances. So in our example, ``x.f`` is a valid method -reference, since ``MyClass.f`` is a function, but ``x.i`` is not, since -``MyClass.i`` is not. But ``x.f`` is not the same thing as ``MyClass.f`` --- it -is a *method object*, not a function object. - - -.. _tut-methodobjects: - -Method Objects --------------- - -Usually, a method is called right after it is bound:: - - x.f() - -In the :class:`!MyClass` example, this will return the string ``'hello world'``. -However, it is not necessary to call a method right away: ``x.f`` is a method -object, and can be stored away and called at a later time. For example:: - - xf = x.f - while True: - print(xf()) - -will continue to print ``hello world`` until the end of time. - -What exactly happens when a method is called? You may have noticed that -``x.f()`` was called without an argument above, even though the function -definition for :meth:`!f` specified an argument. What happened to the argument? -Surely Python raises an exception when a function that requires an argument is -called without any --- even if the argument isn't actually used... - -Actually, you may have guessed the answer: the special thing about methods is -that the instance object is passed as the first argument of the function. In our -example, the call ``x.f()`` is exactly equivalent to ``MyClass.f(x)``. In -general, calling a method with a list of *n* arguments is equivalent to calling -the corresponding function with an argument list that is created by inserting -the method's instance object before the first argument. - -If you still don't understand how methods work, a look at the implementation can -perhaps clarify matters. When a non-data attribute of an instance is -referenced, the instance's class is searched. If the name denotes a valid class -attribute that is a function object, a method object is created by packing -(pointers to) the instance object and the function object just found together in -an abstract object: this is the method object. When the method object is called -with an argument list, a new argument list is constructed from the instance -object and the argument list, and the function object is called with this new -argument list. - - -.. _tut-class-and-instance-variables: - -Class and Instance Variables ----------------------------- - -Generally speaking, instance variables are for data unique to each instance -and class variables are for attributes and methods shared by all instances -of the class:: - - class Dog: - - kind = 'canine' # class variable shared by all instances - - def __init__(self, name): - self.name = name # instance variable unique to each instance - - >>> d = Dog('Fido') - >>> e = Dog('Buddy') - >>> d.kind # shared by all dogs - 'canine' - >>> e.kind # shared by all dogs - 'canine' - >>> d.name # unique to d - 'Fido' - >>> e.name # unique to e - 'Buddy' - -As discussed in :ref:`tut-object`, shared data can have possibly surprising -effects with involving :term:`mutable` objects such as lists and dictionaries. -For example, the *tricks* list in the following code should not be used as a -class variable because just a single list would be shared by all *Dog* -instances:: - - class Dog: - - tricks = [] # mistaken use of a class variable - - def __init__(self, name): - self.name = name - - def add_trick(self, trick): - self.tricks.append(trick) - - >>> d = Dog('Fido') - >>> e = Dog('Buddy') - >>> d.add_trick('roll over') - >>> e.add_trick('play dead') - >>> d.tricks # unexpectedly shared by all dogs - ['roll over', 'play dead'] - -Correct design of the class should use an instance variable instead:: - - class Dog: - - def __init__(self, name): - self.name = name - self.tricks = [] # creates a new empty list for each dog - - def add_trick(self, trick): - self.tricks.append(trick) - - >>> d = Dog('Fido') - >>> e = Dog('Buddy') - >>> d.add_trick('roll over') - >>> e.add_trick('play dead') - >>> d.tricks - ['roll over'] - >>> e.tricks - ['play dead'] - - -.. _tut-remarks: - -Random Remarks -============== - -.. These should perhaps be placed more carefully... - -If the same attribute name occurs in both an instance and in a class, -then attribute lookup prioritizes the instance:: - - >>> class Warehouse: - ... purpose = 'storage' - ... region = 'west' - ... - >>> w1 = Warehouse() - >>> print(w1.purpose, w1.region) - storage west - >>> w2 = Warehouse() - >>> w2.region = 'east' - >>> print(w2.purpose, w2.region) - storage east - -Data attributes may be referenced by methods as well as by ordinary users -("clients") of an object. In other words, classes are not usable to implement -pure abstract data types. In fact, nothing in Python makes it possible to -enforce data hiding --- it is all based upon convention. (On the other hand, -the Python implementation, written in C, can completely hide implementation -details and control access to an object if necessary; this can be used by -extensions to Python written in C.) - -Clients should use data attributes with care --- clients may mess up invariants -maintained by the methods by stamping on their data attributes. Note that -clients may add data attributes of their own to an instance object without -affecting the validity of the methods, as long as name conflicts are avoided --- -again, a naming convention can save a lot of headaches here. - -There is no shorthand for referencing data attributes (or other methods!) from -within methods. I find that this actually increases the readability of methods: -there is no chance of confusing local variables and instance variables when -glancing through a method. - -Often, the first argument of a method is called ``self``. This is nothing more -than a convention: the name ``self`` has absolutely no special meaning to -Python. Note, however, that by not following the convention your code may be -less readable to other Python programmers, and it is also conceivable that a -*class browser* program might be written that relies upon such a convention. - -Any function object that is a class attribute defines a method for instances of -that class. It is not necessary that the function definition is textually -enclosed in the class definition: assigning a function object to a local -variable in the class is also ok. For example:: - - # Function defined outside the class - def f1(self, x, y): - return min(x, x+y) - - class C: - f = f1 - - def g(self): - return 'hello world' - - h = g - -Now ``f``, ``g`` and ``h`` are all attributes of class :class:`!C` that refer to -function objects, and consequently they are all methods of instances of -:class:`!C` --- ``h`` being exactly equivalent to ``g``. Note that this practice -usually only serves to confuse the reader of a program. - -Methods may call other methods by using method attributes of the ``self`` -argument:: - - class Bag: - def __init__(self): - self.data = [] - - def add(self, x): - self.data.append(x) - - def addtwice(self, x): - self.add(x) - self.add(x) - -Methods may reference global names in the same way as ordinary functions. The -global scope associated with a method is the module containing its -definition. (A class is never used as a global scope.) While one -rarely encounters a good reason for using global data in a method, there are -many legitimate uses of the global scope: for one thing, functions and modules -imported into the global scope can be used by methods, as well as functions and -classes defined in it. Usually, the class containing the method is itself -defined in this global scope, and in the next section we'll find some good -reasons why a method would want to reference its own class. - -Each value is an object, and therefore has a *class* (also called its *type*). -It is stored as ``object.__class__``. - - -.. _tut-inheritance: - -Inheritance -=========== - -Of course, a language feature would not be worthy of the name "class" without -supporting inheritance. The syntax for a derived class definition looks like -this:: - - class DerivedClassName(BaseClassName): - - . - . - . - - -The name :class:`!BaseClassName` must be defined in a scope containing the -derived class definition. In place of a base class name, other arbitrary -expressions are also allowed. This can be useful, for example, when the base -class is defined in another module:: - - class DerivedClassName(modname.BaseClassName): - -Execution of a derived class definition proceeds the same as for a base class. -When the class object is constructed, the base class is remembered. This is -used for resolving attribute references: if a requested attribute is not found -in the class, the search proceeds to look in the base class. This rule is -applied recursively if the base class itself is derived from some other class. - -There's nothing special about instantiation of derived classes: -``DerivedClassName()`` creates a new instance of the class. Method references -are resolved as follows: the corresponding class attribute is searched, -descending down the chain of base classes if necessary, and the method reference -is valid if this yields a function object. - -Derived classes may override methods of their base classes. Because methods -have no special privileges when calling other methods of the same object, a -method of a base class that calls another method defined in the same base class -may end up calling a method of a derived class that overrides it. (For C++ -programmers: all methods in Python are effectively ``virtual``.) - -An overriding method in a derived class may in fact want to extend rather than -simply replace the base class method of the same name. There is a simple way to -call the base class method directly: just call ``BaseClassName.methodname(self, -arguments)``. This is occasionally useful to clients as well. (Note that this -only works if the base class is accessible as ``BaseClassName`` in the global -scope.) - -Python has two built-in functions that work with inheritance: - -* Use :func:`isinstance` to check an instance's type: ``isinstance(obj, int)`` - will be ``True`` only if ``obj.__class__`` is :class:`int` or some class - derived from :class:`int`. - -* Use :func:`issubclass` to check class inheritance: ``issubclass(bool, int)`` - is ``True`` since :class:`bool` is a subclass of :class:`int`. However, - ``issubclass(float, int)`` is ``False`` since :class:`float` is not a - subclass of :class:`int`. - - - -.. _tut-multiple: - -Multiple Inheritance --------------------- - -Python supports a form of multiple inheritance as well. A class definition with -multiple base classes looks like this:: - - class DerivedClassName(Base1, Base2, Base3): - - . - . - . - - -For most purposes, in the simplest cases, you can think of the search for -attributes inherited from a parent class as depth-first, left-to-right, not -searching twice in the same class where there is an overlap in the hierarchy. -Thus, if an attribute is not found in :class:`!DerivedClassName`, it is searched -for in :class:`!Base1`, then (recursively) in the base classes of :class:`!Base1`, -and if it was not found there, it was searched for in :class:`!Base2`, and so on. - -In fact, it is slightly more complex than that; the method resolution order -changes dynamically to support cooperative calls to :func:`super`. This -approach is known in some other multiple-inheritance languages as -call-next-method and is more powerful than the super call found in -single-inheritance languages. - -Dynamic ordering is necessary because all cases of multiple inheritance exhibit -one or more diamond relationships (where at least one of the parent classes -can be accessed through multiple paths from the bottommost class). For example, -all classes inherit from :class:`object`, so any case of multiple inheritance -provides more than one path to reach :class:`object`. To keep the base classes -from being accessed more than once, the dynamic algorithm linearizes the search -order in a way that preserves the left-to-right ordering specified in each -class, that calls each parent only once, and that is monotonic (meaning that a -class can be subclassed without affecting the precedence order of its parents). -Taken together, these properties make it possible to design reliable and -extensible classes with multiple inheritance. For more detail, see -https://www.python.org/download/releases/2.3/mro/. - - -.. _tut-private: - -Private Variables -================= - -"Private" instance variables that cannot be accessed except from inside an -object don't exist in Python. However, there is a convention that is followed -by most Python code: a name prefixed with an underscore (e.g. ``_spam``) should -be treated as a non-public part of the API (whether it is a function, a method -or a data member). It should be considered an implementation detail and subject -to change without notice. - -.. index:: - pair: name; mangling - -Since there is a valid use-case for class-private members (namely to avoid name -clashes of names with names defined by subclasses), there is limited support for -such a mechanism, called :dfn:`name mangling`. Any identifier of the form -``__spam`` (at least two leading underscores, at most one trailing underscore) -is textually replaced with ``_classname__spam``, where ``classname`` is the -current class name with leading underscore(s) stripped. This mangling is done -without regard to the syntactic position of the identifier, as long as it -occurs within the definition of a class. - -Name mangling is helpful for letting subclasses override methods without -breaking intraclass method calls. For example:: - - class Mapping: - def __init__(self, iterable): - self.items_list = [] - self.__update(iterable) - - def update(self, iterable): - for item in iterable: - self.items_list.append(item) - - __update = update # private copy of original update() method - - class MappingSubclass(Mapping): - - def update(self, keys, values): - # provides new signature for update() - # but does not break __init__() - for item in zip(keys, values): - self.items_list.append(item) - -The above example would work even if ``MappingSubclass`` were to introduce a -``__update`` identifier since it is replaced with ``_Mapping__update`` in the -``Mapping`` class and ``_MappingSubclass__update`` in the ``MappingSubclass`` -class respectively. - -Note that the mangling rules are designed mostly to avoid accidents; it still is -possible to access or modify a variable that is considered private. This can -even be useful in special circumstances, such as in the debugger. - -Notice that code passed to ``exec()`` or ``eval()`` does not consider the -classname of the invoking class to be the current class; this is similar to the -effect of the ``global`` statement, the effect of which is likewise restricted -to code that is byte-compiled together. The same restriction applies to -``getattr()``, ``setattr()`` and ``delattr()``, as well as when referencing -``__dict__`` directly. - - -.. _tut-odds: - -Odds and Ends -============= - -Sometimes it is useful to have a data type similar to the Pascal "record" or C -"struct", bundling together a few named data items. The idiomatic approach -is to use :mod:`dataclasses` for this purpose:: - - from dataclasses import dataclass - - @dataclass - class Employee: - name: str - dept: str - salary: int - -:: - - >>> john = Employee('john', 'computer lab', 1000) - >>> john.dept - 'computer lab' - >>> john.salary - 1000 - -A piece of Python code that expects a particular abstract data type can often be -passed a class that emulates the methods of that data type instead. For -instance, if you have a function that formats some data from a file object, you -can define a class with methods :meth:`~io.TextIOBase.read` and -:meth:`~io.TextIOBase.readline` that get the -data from a string buffer instead, and pass it as an argument. - -.. (Unfortunately, this technique has its limitations: a class can't define - operations that are accessed by special syntax such as sequence subscripting - or arithmetic operators, and assigning such a "pseudo-file" to sys.stdin will - not cause the interpreter to read further input from it.) - -Instance method objects have attributes, too: ``m.__self__`` is the instance -object with the method :meth:`!m`, and ``m.__func__`` is the function object -corresponding to the method. - - -.. _tut-iterators: - -Iterators -========= - -By now you have probably noticed that most container objects can be looped over -using a :keyword:`for` statement:: - - for element in [1, 2, 3]: - print(element) - for element in (1, 2, 3): - print(element) - for key in {'one':1, 'two':2}: - print(key) - for char in "123": - print(char) - for line in open("myfile.txt"): - print(line, end='') - -This style of access is clear, concise, and convenient. The use of iterators -pervades and unifies Python. Behind the scenes, the :keyword:`for` statement -calls :func:`iter` on the container object. The function returns an iterator -object that defines the method :meth:`~iterator.__next__` which accesses -elements in the container one at a time. When there are no more elements, -:meth:`~iterator.__next__` raises a :exc:`StopIteration` exception which tells the -:keyword:`!for` loop to terminate. You can call the :meth:`~iterator.__next__` method -using the :func:`next` built-in function; this example shows how it all works:: - - >>> s = 'abc' - >>> it = iter(s) - >>> it - - >>> next(it) - 'a' - >>> next(it) - 'b' - >>> next(it) - 'c' - >>> next(it) - Traceback (most recent call last): - File "", line 1, in - next(it) - StopIteration - -Having seen the mechanics behind the iterator protocol, it is easy to add -iterator behavior to your classes. Define an :meth:`~container.__iter__` method which -returns an object with a :meth:`~iterator.__next__` method. If the class -defines :meth:`!__next__`, then :meth:`!__iter__` can just return ``self``:: - - class Reverse: - """Iterator for looping over a sequence backwards.""" - def __init__(self, data): - self.data = data - self.index = len(data) - - def __iter__(self): - return self - - def __next__(self): - if self.index == 0: - raise StopIteration - self.index = self.index - 1 - return self.data[self.index] - -:: - - >>> rev = Reverse('spam') - >>> iter(rev) - <__main__.Reverse object at 0x00A1DB50> - >>> for char in rev: - ... print(char) - ... - m - a - p - s - - -.. _tut-generators: - -Generators -========== - -:term:`Generators ` are a simple and powerful tool for creating iterators. They -are written like regular functions but use the :keyword:`yield` statement -whenever they want to return data. Each time :func:`next` is called on it, the -generator resumes where it left off (it remembers all the data values and which -statement was last executed). An example shows that generators can be trivially -easy to create:: - - def reverse(data): - for index in range(len(data)-1, -1, -1): - yield data[index] - -:: - - >>> for char in reverse('golf'): - ... print(char) - ... - f - l - o - g - -Anything that can be done with generators can also be done with class-based -iterators as described in the previous section. What makes generators so -compact is that the :meth:`~iterator.__iter__` and :meth:`~generator.__next__` methods -are created automatically. - -Another key feature is that the local variables and execution state are -automatically saved between calls. This made the function easier to write and -much more clear than an approach using instance variables like ``self.index`` -and ``self.data``. - -In addition to automatic method creation and saving program state, when -generators terminate, they automatically raise :exc:`StopIteration`. In -combination, these features make it easy to create iterators with no more effort -than writing a regular function. - - -.. _tut-genexps: - -Generator Expressions -===================== - -Some simple generators can be coded succinctly as expressions using a syntax -similar to list comprehensions but with parentheses instead of square brackets. -These expressions are designed for situations where the generator is used right -away by an enclosing function. Generator expressions are more compact but less -versatile than full generator definitions and tend to be more memory friendly -than equivalent list comprehensions. - -Examples:: - - >>> sum(i*i for i in range(10)) # sum of squares - 285 - - >>> xvec = [10, 20, 30] - >>> yvec = [7, 5, 3] - >>> sum(x*y for x,y in zip(xvec, yvec)) # dot product - 260 - - >>> unique_words = set(word for line in page for word in line.split()) - - >>> valedictorian = max((student.gpa, student.name) for student in graduates) - - >>> data = 'golf' - >>> list(data[i] for i in range(len(data)-1, -1, -1)) - ['f', 'l', 'o', 'g'] - - - -.. rubric:: Footnotes - -.. [#] Except for one thing. Module objects have a secret read-only attribute called - :attr:`~object.__dict__` which returns the dictionary used to implement the module's - namespace; the name :attr:`~object.__dict__` is an attribute but not a global name. - Obviously, using this violates the abstraction of namespace implementation, and - should be restricted to things like post-mortem debuggers. diff --git a/Doc/tutorial/controlflow.rst b/Doc/tutorial/controlflow.rst deleted file mode 100644 index aa9caa101da40a..00000000000000 --- a/Doc/tutorial/controlflow.rst +++ /dev/null @@ -1,1126 +0,0 @@ -.. _tut-morecontrol: - -*********************** -More Control Flow Tools -*********************** - -As well as the :keyword:`while` statement just introduced, Python uses a few more -that we will encounter in this chapter. - - -.. _tut-if: - -:keyword:`!if` Statements -========================= - -Perhaps the most well-known statement type is the :keyword:`if` statement. For -example:: - - >>> x = int(input("Please enter an integer: ")) - Please enter an integer: 42 - >>> if x < 0: - ... x = 0 - ... print('Negative changed to zero') - ... elif x == 0: - ... print('Zero') - ... elif x == 1: - ... print('Single') - ... else: - ... print('More') - ... - More - -There can be zero or more :keyword:`elif` parts, and the :keyword:`else` part is -optional. The keyword ':keyword:`!elif`' is short for 'else if', and is useful -to avoid excessive indentation. An :keyword:`!if` ... :keyword:`!elif` ... -:keyword:`!elif` ... sequence is a substitute for the ``switch`` or -``case`` statements found in other languages. - -If you're comparing the same value to several constants, or checking for specific types or -attributes, you may also find the :keyword:`!match` statement useful. For more -details see :ref:`tut-match`. - -.. _tut-for: - -:keyword:`!for` Statements -========================== - -.. index:: - pair: statement; for - -The :keyword:`for` statement in Python differs a bit from what you may be used -to in C or Pascal. Rather than always iterating over an arithmetic progression -of numbers (like in Pascal), or giving the user the ability to define both the -iteration step and halting condition (as C), Python's :keyword:`!for` statement -iterates over the items of any sequence (a list or a string), in the order that -they appear in the sequence. For example (no pun intended): - -.. One suggestion was to give a real C example here, but that may only serve to - confuse non-C programmers. - -:: - - >>> # Measure some strings: - ... words = ['cat', 'window', 'defenestrate'] - >>> for w in words: - ... print(w, len(w)) - ... - cat 3 - window 6 - defenestrate 12 - -Code that modifies a collection while iterating over that same collection can -be tricky to get right. Instead, it is usually more straight-forward to loop -over a copy of the collection or to create a new collection:: - - # Create a sample collection - users = {'Hans': 'active', 'Éléonore': 'inactive', '景太郎': 'active'} - - # Strategy: Iterate over a copy - for user, status in users.copy().items(): - if status == 'inactive': - del users[user] - - # Strategy: Create a new collection - active_users = {} - for user, status in users.items(): - if status == 'active': - active_users[user] = status - - -.. _tut-range: - -The :func:`range` Function -========================== - -If you do need to iterate over a sequence of numbers, the built-in function -:func:`range` comes in handy. It generates arithmetic progressions:: - - >>> for i in range(5): - ... print(i) - ... - 0 - 1 - 2 - 3 - 4 - -The given end point is never part of the generated sequence; ``range(10)`` generates -10 values, the legal indices for items of a sequence of length 10. It -is possible to let the range start at another number, or to specify a different -increment (even negative; sometimes this is called the 'step'):: - - >>> list(range(5, 10)) - [5, 6, 7, 8, 9] - - >>> list(range(0, 10, 3)) - [0, 3, 6, 9] - - >>> list(range(-10, -100, -30)) - [-10, -40, -70] - -To iterate over the indices of a sequence, you can combine :func:`range` and -:func:`len` as follows:: - - >>> a = ['Mary', 'had', 'a', 'little', 'lamb'] - >>> for i in range(len(a)): - ... print(i, a[i]) - ... - 0 Mary - 1 had - 2 a - 3 little - 4 lamb - -In most such cases, however, it is convenient to use the :func:`enumerate` -function, see :ref:`tut-loopidioms`. - -A strange thing happens if you just print a range:: - - >>> range(10) - range(0, 10) - -In many ways the object returned by :func:`range` behaves as if it is a list, -but in fact it isn't. It is an object which returns the successive items of -the desired sequence when you iterate over it, but it doesn't really make -the list, thus saving space. - -We say such an object is :term:`iterable`, that is, suitable as a target for -functions and constructs that expect something from which they can -obtain successive items until the supply is exhausted. We have seen that -the :keyword:`for` statement is such a construct, while an example of a function -that takes an iterable is :func:`sum`:: - - >>> sum(range(4)) # 0 + 1 + 2 + 3 - 6 - -Later we will see more functions that return iterables and take iterables as -arguments. In chapter :ref:`tut-structures`, we will discuss in more detail about -:func:`list`. - -.. _tut-break: - -:keyword:`!break` and :keyword:`!continue` Statements, and :keyword:`!else` Clauses on Loops -============================================================================================ - -The :keyword:`break` statement breaks out of the innermost enclosing -:keyword:`for` or :keyword:`while` loop. - -A :keyword:`!for` or :keyword:`!while` loop can include an :keyword:`!else` clause. - -In a :keyword:`for` loop, the :keyword:`!else` clause is executed -after the loop reaches its final iteration. - -In a :keyword:`while` loop, it's executed after the loop's condition becomes false. - -In either kind of loop, the :keyword:`!else` clause is **not** executed -if the loop was terminated by a :keyword:`break`. - -This is exemplified in the following :keyword:`!for` loop, -which searches for prime numbers:: - - >>> for n in range(2, 10): - ... for x in range(2, n): - ... if n % x == 0: - ... print(n, 'equals', x, '*', n//x) - ... break - ... else: - ... # loop fell through without finding a factor - ... print(n, 'is a prime number') - ... - 2 is a prime number - 3 is a prime number - 4 equals 2 * 2 - 5 is a prime number - 6 equals 2 * 3 - 7 is a prime number - 8 equals 2 * 4 - 9 equals 3 * 3 - -(Yes, this is the correct code. Look closely: the ``else`` clause belongs to -the :keyword:`for` loop, **not** the :keyword:`if` statement.) - -When used with a loop, the ``else`` clause has more in common with the -``else`` clause of a :keyword:`try` statement than it does with that of -:keyword:`if` statements: a :keyword:`try` statement's ``else`` clause runs -when no exception occurs, and a loop's ``else`` clause runs when no ``break`` -occurs. For more on the :keyword:`!try` statement and exceptions, see -:ref:`tut-handling`. - -The :keyword:`continue` statement, also borrowed from C, continues with the next -iteration of the loop:: - - >>> for num in range(2, 10): - ... if num % 2 == 0: - ... print("Found an even number", num) - ... continue - ... print("Found an odd number", num) - ... - Found an even number 2 - Found an odd number 3 - Found an even number 4 - Found an odd number 5 - Found an even number 6 - Found an odd number 7 - Found an even number 8 - Found an odd number 9 - -.. _tut-pass: - -:keyword:`!pass` Statements -=========================== - -The :keyword:`pass` statement does nothing. It can be used when a statement is -required syntactically but the program requires no action. For example:: - - >>> while True: - ... pass # Busy-wait for keyboard interrupt (Ctrl+C) - ... - -This is commonly used for creating minimal classes:: - - >>> class MyEmptyClass: - ... pass - ... - -Another place :keyword:`pass` can be used is as a place-holder for a function or -conditional body when you are working on new code, allowing you to keep thinking -at a more abstract level. The :keyword:`!pass` is silently ignored:: - - >>> def initlog(*args): - ... pass # Remember to implement this! - ... - - -.. _tut-match: - -:keyword:`!match` Statements -============================ - -A :keyword:`match` statement takes an expression and compares its value to successive -patterns given as one or more case blocks. This is superficially -similar to a switch statement in C, Java or JavaScript (and many -other languages), but it's more similar to pattern matching in -languages like Rust or Haskell. Only the first pattern that matches -gets executed and it can also extract components (sequence elements -or object attributes) from the value into variables. - -The simplest form compares a subject value against one or more literals:: - - def http_error(status): - match status: - case 400: - return "Bad request" - case 404: - return "Not found" - case 418: - return "I'm a teapot" - case _: - return "Something's wrong with the internet" - -Note the last block: the "variable name" ``_`` acts as a *wildcard* and -never fails to match. If no case matches, none of the branches is executed. - -You can combine several literals in a single pattern using ``|`` ("or"):: - - case 401 | 403 | 404: - return "Not allowed" - -Patterns can look like unpacking assignments, and can be used to bind -variables:: - - # point is an (x, y) tuple - match point: - case (0, 0): - print("Origin") - case (0, y): - print(f"Y={y}") - case (x, 0): - print(f"X={x}") - case (x, y): - print(f"X={x}, Y={y}") - case _: - raise ValueError("Not a point") - -Study that one carefully! The first pattern has two literals, and can -be thought of as an extension of the literal pattern shown above. But -the next two patterns combine a literal and a variable, and the -variable *binds* a value from the subject (``point``). The fourth -pattern captures two values, which makes it conceptually similar to -the unpacking assignment ``(x, y) = point``. - -If you are using classes to structure your data -you can use the class name followed by an argument list resembling a -constructor, but with the ability to capture attributes into variables:: - - class Point: - def __init__(self, x, y): - self.x = x - self.y = y - - def where_is(point): - match point: - case Point(x=0, y=0): - print("Origin") - case Point(x=0, y=y): - print(f"Y={y}") - case Point(x=x, y=0): - print(f"X={x}") - case Point(): - print("Somewhere else") - case _: - print("Not a point") - -You can use positional parameters with some builtin classes that provide an -ordering for their attributes (e.g. dataclasses). You can also define a specific -position for attributes in patterns by setting the ``__match_args__`` special -attribute in your classes. If it's set to ("x", "y"), the following patterns are all -equivalent (and all bind the ``y`` attribute to the ``var`` variable):: - - Point(1, var) - Point(1, y=var) - Point(x=1, y=var) - Point(y=var, x=1) - -A recommended way to read patterns is to look at them as an extended form of what you -would put on the left of an assignment, to understand which variables would be set to -what. -Only the standalone names (like ``var`` above) are assigned to by a match statement. -Dotted names (like ``foo.bar``), attribute names (the ``x=`` and ``y=`` above) or class names -(recognized by the "(...)" next to them like ``Point`` above) are never assigned to. - -Patterns can be arbitrarily nested. For example, if we have a short -list of Points, with ``__match_args__`` added, we could match it like this:: - - class Point: - __match_args__ = ('x', 'y') - def __init__(self, x, y): - self.x = x - self.y = y - - match points: - case []: - print("No points") - case [Point(0, 0)]: - print("The origin") - case [Point(x, y)]: - print(f"Single point {x}, {y}") - case [Point(0, y1), Point(0, y2)]: - print(f"Two on the Y axis at {y1}, {y2}") - case _: - print("Something else") - -We can add an ``if`` clause to a pattern, known as a "guard". If the -guard is false, ``match`` goes on to try the next case block. Note -that value capture happens before the guard is evaluated:: - - match point: - case Point(x, y) if x == y: - print(f"Y=X at {x}") - case Point(x, y): - print(f"Not on the diagonal") - -Several other key features of this statement: - -- Like unpacking assignments, tuple and list patterns have exactly the - same meaning and actually match arbitrary sequences. An important - exception is that they don't match iterators or strings. - -- Sequence patterns support extended unpacking: ``[x, y, *rest]`` and ``(x, y, - *rest)`` work similar to unpacking assignments. The - name after ``*`` may also be ``_``, so ``(x, y, *_)`` matches a sequence - of at least two items without binding the remaining items. - -- Mapping patterns: ``{"bandwidth": b, "latency": l}`` captures the - ``"bandwidth"`` and ``"latency"`` values from a dictionary. Unlike sequence - patterns, extra keys are ignored. An unpacking like ``**rest`` is also - supported. (But ``**_`` would be redundant, so it is not allowed.) - -- Subpatterns may be captured using the ``as`` keyword:: - - case (Point(x1, y1), Point(x2, y2) as p2): ... - - will capture the second element of the input as ``p2`` (as long as the input is - a sequence of two points) - -- Most literals are compared by equality, however the singletons ``True``, - ``False`` and ``None`` are compared by identity. - -- Patterns may use named constants. These must be dotted names - to prevent them from being interpreted as capture variable:: - - from enum import Enum - class Color(Enum): - RED = 'red' - GREEN = 'green' - BLUE = 'blue' - - color = Color(input("Enter your choice of 'red', 'blue' or 'green': ")) - - match color: - case Color.RED: - print("I see red!") - case Color.GREEN: - print("Grass is green") - case Color.BLUE: - print("I'm feeling the blues :(") - -For a more detailed explanation and additional examples, you can look into -:pep:`636` which is written in a tutorial format. - -.. _tut-functions: - -Defining Functions -================== - -We can create a function that writes the Fibonacci series to an arbitrary -boundary:: - - >>> def fib(n): # write Fibonacci series up to n - ... """Print a Fibonacci series up to n.""" - ... a, b = 0, 1 - ... while a < n: - ... print(a, end=' ') - ... a, b = b, a+b - ... print() - ... - >>> # Now call the function we just defined: - ... fib(2000) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987 1597 - -.. index:: - single: documentation strings - single: docstrings - single: strings, documentation - -The keyword :keyword:`def` introduces a function *definition*. It must be -followed by the function name and the parenthesized list of formal parameters. -The statements that form the body of the function start at the next line, and -must be indented. - -The first statement of the function body can optionally be a string literal; -this string literal is the function's documentation string, or :dfn:`docstring`. -(More about docstrings can be found in the section :ref:`tut-docstrings`.) -There are tools which use docstrings to automatically produce online or printed -documentation, or to let the user interactively browse through code; it's good -practice to include docstrings in code that you write, so make a habit of it. - -The *execution* of a function introduces a new symbol table used for the local -variables of the function. More precisely, all variable assignments in a -function store the value in the local symbol table; whereas variable references -first look in the local symbol table, then in the local symbol tables of -enclosing functions, then in the global symbol table, and finally in the table -of built-in names. Thus, global variables and variables of enclosing functions -cannot be directly assigned a value within a function (unless, for global -variables, named in a :keyword:`global` statement, or, for variables of enclosing -functions, named in a :keyword:`nonlocal` statement), although they may be -referenced. - -The actual parameters (arguments) to a function call are introduced in the local -symbol table of the called function when it is called; thus, arguments are -passed using *call by value* (where the *value* is always an object *reference*, -not the value of the object). [#]_ When a function calls another function, -or calls itself recursively, a new -local symbol table is created for that call. - -A function definition associates the function name with the function object in -the current symbol table. The interpreter recognizes the object pointed to by -that name as a user-defined function. Other names can also point to that same -function object and can also be used to access the function:: - - >>> fib - - >>> f = fib - >>> f(100) - 0 1 1 2 3 5 8 13 21 34 55 89 - -Coming from other languages, you might object that ``fib`` is not a function but -a procedure since it doesn't return a value. In fact, even functions without a -:keyword:`return` statement do return a value, albeit a rather boring one. This -value is called ``None`` (it's a built-in name). Writing the value ``None`` is -normally suppressed by the interpreter if it would be the only value written. -You can see it if you really want to using :func:`print`:: - - >>> fib(0) - >>> print(fib(0)) - None - -It is simple to write a function that returns a list of the numbers of the -Fibonacci series, instead of printing it:: - - >>> def fib2(n): # return Fibonacci series up to n - ... """Return a list containing the Fibonacci series up to n.""" - ... result = [] - ... a, b = 0, 1 - ... while a < n: - ... result.append(a) # see below - ... a, b = b, a+b - ... return result - ... - >>> f100 = fib2(100) # call it - >>> f100 # write the result - [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89] - -This example, as usual, demonstrates some new Python features: - -* The :keyword:`return` statement returns with a value from a function. - :keyword:`!return` without an expression argument returns ``None``. Falling off - the end of a function also returns ``None``. - -* The statement ``result.append(a)`` calls a *method* of the list object - ``result``. A method is a function that 'belongs' to an object and is named - ``obj.methodname``, where ``obj`` is some object (this may be an expression), - and ``methodname`` is the name of a method that is defined by the object's type. - Different types define different methods. Methods of different types may have - the same name without causing ambiguity. (It is possible to define your own - object types and methods, using *classes*, see :ref:`tut-classes`) - The method :meth:`!append` shown in the example is defined for list objects; it - adds a new element at the end of the list. In this example it is equivalent to - ``result = result + [a]``, but more efficient. - - -.. _tut-defining: - -More on Defining Functions -========================== - -It is also possible to define functions with a variable number of arguments. -There are three forms, which can be combined. - - -.. _tut-defaultargs: - -Default Argument Values ------------------------ - -The most useful form is to specify a default value for one or more arguments. -This creates a function that can be called with fewer arguments than it is -defined to allow. For example:: - - def ask_ok(prompt, retries=4, reminder='Please try again!'): - while True: - ok = input(prompt) - if ok in ('y', 'ye', 'yes'): - return True - if ok in ('n', 'no', 'nop', 'nope'): - return False - retries = retries - 1 - if retries < 0: - raise ValueError('invalid user response') - print(reminder) - -This function can be called in several ways: - -* giving only the mandatory argument: - ``ask_ok('Do you really want to quit?')`` -* giving one of the optional arguments: - ``ask_ok('OK to overwrite the file?', 2)`` -* or even giving all arguments: - ``ask_ok('OK to overwrite the file?', 2, 'Come on, only yes or no!')`` - -This example also introduces the :keyword:`in` keyword. This tests whether or -not a sequence contains a certain value. - -The default values are evaluated at the point of function definition in the -*defining* scope, so that :: - - i = 5 - - def f(arg=i): - print(arg) - - i = 6 - f() - -will print ``5``. - -**Important warning:** The default value is evaluated only once. This makes a -difference when the default is a mutable object such as a list, dictionary, or -instances of most classes. For example, the following function accumulates the -arguments passed to it on subsequent calls:: - - def f(a, L=[]): - L.append(a) - return L - - print(f(1)) - print(f(2)) - print(f(3)) - -This will print :: - - [1] - [1, 2] - [1, 2, 3] - -If you don't want the default to be shared between subsequent calls, you can -write the function like this instead:: - - def f(a, L=None): - if L is None: - L = [] - L.append(a) - return L - - -.. _tut-keywordargs: - -Keyword Arguments ------------------ - -Functions can also be called using :term:`keyword arguments ` -of the form ``kwarg=value``. For instance, the following function:: - - def parrot(voltage, state='a stiff', action='voom', type='Norwegian Blue'): - print("-- This parrot wouldn't", action, end=' ') - print("if you put", voltage, "volts through it.") - print("-- Lovely plumage, the", type) - print("-- It's", state, "!") - -accepts one required argument (``voltage``) and three optional arguments -(``state``, ``action``, and ``type``). This function can be called in any -of the following ways:: - - parrot(1000) # 1 positional argument - parrot(voltage=1000) # 1 keyword argument - parrot(voltage=1000000, action='VOOOOOM') # 2 keyword arguments - parrot(action='VOOOOOM', voltage=1000000) # 2 keyword arguments - parrot('a million', 'bereft of life', 'jump') # 3 positional arguments - parrot('a thousand', state='pushing up the daisies') # 1 positional, 1 keyword - -but all the following calls would be invalid:: - - parrot() # required argument missing - parrot(voltage=5.0, 'dead') # non-keyword argument after a keyword argument - parrot(110, voltage=220) # duplicate value for the same argument - parrot(actor='John Cleese') # unknown keyword argument - -In a function call, keyword arguments must follow positional arguments. -All the keyword arguments passed must match one of the arguments -accepted by the function (e.g. ``actor`` is not a valid argument for the -``parrot`` function), and their order is not important. This also includes -non-optional arguments (e.g. ``parrot(voltage=1000)`` is valid too). -No argument may receive a value more than once. -Here's an example that fails due to this restriction:: - - >>> def function(a): - ... pass - ... - >>> function(0, a=0) - Traceback (most recent call last): - File "", line 1, in - TypeError: function() got multiple values for argument 'a' - -When a final formal parameter of the form ``**name`` is present, it receives a -dictionary (see :ref:`typesmapping`) containing all keyword arguments except for -those corresponding to a formal parameter. This may be combined with a formal -parameter of the form ``*name`` (described in the next subsection) which -receives a :ref:`tuple ` containing the positional -arguments beyond the formal parameter list. (``*name`` must occur -before ``**name``.) For example, if we define a function like this:: - - def cheeseshop(kind, *arguments, **keywords): - print("-- Do you have any", kind, "?") - print("-- I'm sorry, we're all out of", kind) - for arg in arguments: - print(arg) - print("-" * 40) - for kw in keywords: - print(kw, ":", keywords[kw]) - -It could be called like this:: - - cheeseshop("Limburger", "It's very runny, sir.", - "It's really very, VERY runny, sir.", - shopkeeper="Michael Palin", - client="John Cleese", - sketch="Cheese Shop Sketch") - -and of course it would print: - -.. code-block:: none - - -- Do you have any Limburger ? - -- I'm sorry, we're all out of Limburger - It's very runny, sir. - It's really very, VERY runny, sir. - ---------------------------------------- - shopkeeper : Michael Palin - client : John Cleese - sketch : Cheese Shop Sketch - -Note that the order in which the keyword arguments are printed is guaranteed -to match the order in which they were provided in the function call. - -Special parameters ------------------- - -By default, arguments may be passed to a Python function either by position -or explicitly by keyword. For readability and performance, it makes sense to -restrict the way arguments can be passed so that a developer need only look -at the function definition to determine if items are passed by position, by -position or keyword, or by keyword. - -A function definition may look like: - -.. code-block:: none - - def f(pos1, pos2, /, pos_or_kwd, *, kwd1, kwd2): - ----------- ---------- ---------- - | | | - | Positional or keyword | - | - Keyword only - -- Positional only - -where ``/`` and ``*`` are optional. If used, these symbols indicate the kind of -parameter by how the arguments may be passed to the function: -positional-only, positional-or-keyword, and keyword-only. Keyword parameters -are also referred to as named parameters. - -------------------------------- -Positional-or-Keyword Arguments -------------------------------- - -If ``/`` and ``*`` are not present in the function definition, arguments may -be passed to a function by position or by keyword. - --------------------------- -Positional-Only Parameters --------------------------- - -Looking at this in a bit more detail, it is possible to mark certain parameters -as *positional-only*. If *positional-only*, the parameters' order matters, and -the parameters cannot be passed by keyword. Positional-only parameters are -placed before a ``/`` (forward-slash). The ``/`` is used to logically -separate the positional-only parameters from the rest of the parameters. -If there is no ``/`` in the function definition, there are no positional-only -parameters. - -Parameters following the ``/`` may be *positional-or-keyword* or *keyword-only*. - ----------------------- -Keyword-Only Arguments ----------------------- - -To mark parameters as *keyword-only*, indicating the parameters must be passed -by keyword argument, place an ``*`` in the arguments list just before the first -*keyword-only* parameter. - ------------------ -Function Examples ------------------ - -Consider the following example function definitions paying close attention to the -markers ``/`` and ``*``:: - - >>> def standard_arg(arg): - ... print(arg) - ... - >>> def pos_only_arg(arg, /): - ... print(arg) - ... - >>> def kwd_only_arg(*, arg): - ... print(arg) - ... - >>> def combined_example(pos_only, /, standard, *, kwd_only): - ... print(pos_only, standard, kwd_only) - - -The first function definition, ``standard_arg``, the most familiar form, -places no restrictions on the calling convention and arguments may be -passed by position or keyword:: - - >>> standard_arg(2) - 2 - - >>> standard_arg(arg=2) - 2 - -The second function ``pos_only_arg`` is restricted to only use positional -parameters as there is a ``/`` in the function definition:: - - >>> pos_only_arg(1) - 1 - - >>> pos_only_arg(arg=1) - Traceback (most recent call last): - File "", line 1, in - TypeError: pos_only_arg() got some positional-only arguments passed as keyword arguments: 'arg' - -The third function ``kwd_only_args`` only allows keyword arguments as indicated -by a ``*`` in the function definition:: - - >>> kwd_only_arg(3) - Traceback (most recent call last): - File "", line 1, in - TypeError: kwd_only_arg() takes 0 positional arguments but 1 was given - - >>> kwd_only_arg(arg=3) - 3 - -And the last uses all three calling conventions in the same function -definition:: - - >>> combined_example(1, 2, 3) - Traceback (most recent call last): - File "", line 1, in - TypeError: combined_example() takes 2 positional arguments but 3 were given - - >>> combined_example(1, 2, kwd_only=3) - 1 2 3 - - >>> combined_example(1, standard=2, kwd_only=3) - 1 2 3 - - >>> combined_example(pos_only=1, standard=2, kwd_only=3) - Traceback (most recent call last): - File "", line 1, in - TypeError: combined_example() got some positional-only arguments passed as keyword arguments: 'pos_only' - - -Finally, consider this function definition which has a potential collision between the positional argument ``name`` and ``**kwds`` which has ``name`` as a key:: - - def foo(name, **kwds): - return 'name' in kwds - -There is no possible call that will make it return ``True`` as the keyword ``'name'`` -will always bind to the first parameter. For example:: - - >>> foo(1, **{'name': 2}) - Traceback (most recent call last): - File "", line 1, in - TypeError: foo() got multiple values for argument 'name' - >>> - -But using ``/`` (positional only arguments), it is possible since it allows ``name`` as a positional argument and ``'name'`` as a key in the keyword arguments:: - - >>> def foo(name, /, **kwds): - ... return 'name' in kwds - ... - >>> foo(1, **{'name': 2}) - True - -In other words, the names of positional-only parameters can be used in -``**kwds`` without ambiguity. - ------ -Recap ------ - -The use case will determine which parameters to use in the function definition:: - - def f(pos1, pos2, /, pos_or_kwd, *, kwd1, kwd2): - -As guidance: - -* Use positional-only if you want the name of the parameters to not be - available to the user. This is useful when parameter names have no real - meaning, if you want to enforce the order of the arguments when the function - is called or if you need to take some positional parameters and arbitrary - keywords. -* Use keyword-only when names have meaning and the function definition is - more understandable by being explicit with names or you want to prevent - users relying on the position of the argument being passed. -* For an API, use positional-only to prevent breaking API changes - if the parameter's name is modified in the future. - -.. _tut-arbitraryargs: - -Arbitrary Argument Lists ------------------------- - -.. index:: - single: * (asterisk); in function calls - -Finally, the least frequently used option is to specify that a function can be -called with an arbitrary number of arguments. These arguments will be wrapped -up in a tuple (see :ref:`tut-tuples`). Before the variable number of arguments, -zero or more normal arguments may occur. :: - - def write_multiple_items(file, separator, *args): - file.write(separator.join(args)) - - -Normally, these *variadic* arguments will be last in the list of formal -parameters, because they scoop up all remaining input arguments that are -passed to the function. Any formal parameters which occur after the ``*args`` -parameter are 'keyword-only' arguments, meaning that they can only be used as -keywords rather than positional arguments. :: - - >>> def concat(*args, sep="/"): - ... return sep.join(args) - ... - >>> concat("earth", "mars", "venus") - 'earth/mars/venus' - >>> concat("earth", "mars", "venus", sep=".") - 'earth.mars.venus' - -.. _tut-unpacking-arguments: - -Unpacking Argument Lists ------------------------- - -The reverse situation occurs when the arguments are already in a list or tuple -but need to be unpacked for a function call requiring separate positional -arguments. For instance, the built-in :func:`range` function expects separate -*start* and *stop* arguments. If they are not available separately, write the -function call with the ``*``\ -operator to unpack the arguments out of a list -or tuple:: - - >>> list(range(3, 6)) # normal call with separate arguments - [3, 4, 5] - >>> args = [3, 6] - >>> list(range(*args)) # call with arguments unpacked from a list - [3, 4, 5] - -.. index:: - single: **; in function calls - -In the same fashion, dictionaries can deliver keyword arguments with the -``**``\ -operator:: - - >>> def parrot(voltage, state='a stiff', action='voom'): - ... print("-- This parrot wouldn't", action, end=' ') - ... print("if you put", voltage, "volts through it.", end=' ') - ... print("E's", state, "!") - ... - >>> d = {"voltage": "four million", "state": "bleedin' demised", "action": "VOOM"} - >>> parrot(**d) - -- This parrot wouldn't VOOM if you put four million volts through it. E's bleedin' demised ! - - -.. _tut-lambda: - -Lambda Expressions ------------------- - -Small anonymous functions can be created with the :keyword:`lambda` keyword. -This function returns the sum of its two arguments: ``lambda a, b: a+b``. -Lambda functions can be used wherever function objects are required. They are -syntactically restricted to a single expression. Semantically, they are just -syntactic sugar for a normal function definition. Like nested function -definitions, lambda functions can reference variables from the containing -scope:: - - >>> def make_incrementor(n): - ... return lambda x: x + n - ... - >>> f = make_incrementor(42) - >>> f(0) - 42 - >>> f(1) - 43 - -The above example uses a lambda expression to return a function. Another use -is to pass a small function as an argument:: - - >>> pairs = [(1, 'one'), (2, 'two'), (3, 'three'), (4, 'four')] - >>> pairs.sort(key=lambda pair: pair[1]) - >>> pairs - [(4, 'four'), (1, 'one'), (3, 'three'), (2, 'two')] - - -.. _tut-docstrings: - -Documentation Strings ---------------------- - -.. index:: - single: docstrings - single: documentation strings - single: strings, documentation - -Here are some conventions about the content and formatting of documentation -strings. - -The first line should always be a short, concise summary of the object's -purpose. For brevity, it should not explicitly state the object's name or type, -since these are available by other means (except if the name happens to be a -verb describing a function's operation). This line should begin with a capital -letter and end with a period. - -If there are more lines in the documentation string, the second line should be -blank, visually separating the summary from the rest of the description. The -following lines should be one or more paragraphs describing the object's calling -conventions, its side effects, etc. - -The Python parser does not strip indentation from multi-line string literals in -Python, so tools that process documentation have to strip indentation if -desired. This is done using the following convention. The first non-blank line -*after* the first line of the string determines the amount of indentation for -the entire documentation string. (We can't use the first line since it is -generally adjacent to the string's opening quotes so its indentation is not -apparent in the string literal.) Whitespace "equivalent" to this indentation is -then stripped from the start of all lines of the string. Lines that are -indented less should not occur, but if they occur all their leading whitespace -should be stripped. Equivalence of whitespace should be tested after expansion -of tabs (to 8 spaces, normally). - -Here is an example of a multi-line docstring:: - - >>> def my_function(): - ... """Do nothing, but document it. - ... - ... No, really, it doesn't do anything. - ... """ - ... pass - ... - >>> print(my_function.__doc__) - Do nothing, but document it. - - No, really, it doesn't do anything. - - -.. _tut-annotations: - -Function Annotations --------------------- - -.. sectionauthor:: Zachary Ware -.. index:: - pair: function; annotations - single: ->; function annotations - single: : (colon); function annotations - -:ref:`Function annotations ` are completely optional metadata -information about the types used by user-defined functions (see :pep:`3107` and -:pep:`484` for more information). - -:term:`Annotations ` are stored in the :attr:`!__annotations__` -attribute of the function as a dictionary and have no effect on any other part of the -function. Parameter annotations are defined by a colon after the parameter name, followed -by an expression evaluating to the value of the annotation. Return annotations are -defined by a literal ``->``, followed by an expression, between the parameter -list and the colon denoting the end of the :keyword:`def` statement. The -following example has a required argument, an optional argument, and the return -value annotated:: - - >>> def f(ham: str, eggs: str = 'eggs') -> str: - ... print("Annotations:", f.__annotations__) - ... print("Arguments:", ham, eggs) - ... return ham + ' and ' + eggs - ... - >>> f('spam') - Annotations: {'ham': , 'return': , 'eggs': } - Arguments: spam eggs - 'spam and eggs' - -.. _tut-codingstyle: - -Intermezzo: Coding Style -======================== - -.. sectionauthor:: Georg Brandl -.. index:: pair: coding; style - -Now that you are about to write longer, more complex pieces of Python, it is a -good time to talk about *coding style*. Most languages can be written (or more -concise, *formatted*) in different styles; some are more readable than others. -Making it easy for others to read your code is always a good idea, and adopting -a nice coding style helps tremendously for that. - -For Python, :pep:`8` has emerged as the style guide that most projects adhere to; -it promotes a very readable and eye-pleasing coding style. Every Python -developer should read it at some point; here are the most important points -extracted for you: - -* Use 4-space indentation, and no tabs. - - 4 spaces are a good compromise between small indentation (allows greater - nesting depth) and large indentation (easier to read). Tabs introduce - confusion, and are best left out. - -* Wrap lines so that they don't exceed 79 characters. - - This helps users with small displays and makes it possible to have several - code files side-by-side on larger displays. - -* Use blank lines to separate functions and classes, and larger blocks of - code inside functions. - -* When possible, put comments on a line of their own. - -* Use docstrings. - -* Use spaces around operators and after commas, but not directly inside - bracketing constructs: ``a = f(1, 2) + g(3, 4)``. - -* Name your classes and functions consistently; the convention is to use - ``UpperCamelCase`` for classes and ``lowercase_with_underscores`` for functions - and methods. Always use ``self`` as the name for the first method argument - (see :ref:`tut-firstclasses` for more on classes and methods). - -* Don't use fancy encodings if your code is meant to be used in international - environments. Python's default, UTF-8, or even plain ASCII work best in any - case. - -* Likewise, don't use non-ASCII characters in identifiers if there is only the - slightest chance people speaking a different language will read or maintain - the code. - - -.. rubric:: Footnotes - -.. [#] Actually, *call by object reference* would be a better description, - since if a mutable object is passed, the caller will see any changes the - callee makes to it (items inserted into a list). diff --git a/Doc/tutorial/datastructures.rst b/Doc/tutorial/datastructures.rst deleted file mode 100644 index 87614d082a1d4e..00000000000000 --- a/Doc/tutorial/datastructures.rst +++ /dev/null @@ -1,735 +0,0 @@ -.. _tut-structures: - -*************** -Data Structures -*************** - -This chapter describes some things you've learned about already in more detail, -and adds some new things as well. - -.. _tut-morelists: - -More on Lists -============= - -The list data type has some more methods. Here are all of the methods of list -objects: - - -.. method:: list.append(x) - :noindex: - - Add an item to the end of the list. Equivalent to ``a[len(a):] = [x]``. - - -.. method:: list.extend(iterable) - :noindex: - - Extend the list by appending all the items from the iterable. Equivalent to - ``a[len(a):] = iterable``. - - -.. method:: list.insert(i, x) - :noindex: - - Insert an item at a given position. The first argument is the index of the - element before which to insert, so ``a.insert(0, x)`` inserts at the front of - the list, and ``a.insert(len(a), x)`` is equivalent to ``a.append(x)``. - - -.. method:: list.remove(x) - :noindex: - - Remove the first item from the list whose value is equal to *x*. It raises a - :exc:`ValueError` if there is no such item. - - -.. method:: list.pop([i]) - :noindex: - - Remove the item at the given position in the list, and return it. If no index - is specified, ``a.pop()`` removes and returns the last item in the list. (The - square brackets around the *i* in the method signature denote that the parameter - is optional, not that you should type square brackets at that position. You - will see this notation frequently in the Python Library Reference.) - - -.. method:: list.clear() - :noindex: - - Remove all items from the list. Equivalent to ``del a[:]``. - - -.. method:: list.index(x[, start[, end]]) - :noindex: - - Return zero-based index in the list of the first item whose value is equal to *x*. - Raises a :exc:`ValueError` if there is no such item. - - The optional arguments *start* and *end* are interpreted as in the slice - notation and are used to limit the search to a particular subsequence of - the list. The returned index is computed relative to the beginning of the full - sequence rather than the *start* argument. - - -.. method:: list.count(x) - :noindex: - - Return the number of times *x* appears in the list. - - -.. method:: list.sort(*, key=None, reverse=False) - :noindex: - - Sort the items of the list in place (the arguments can be used for sort - customization, see :func:`sorted` for their explanation). - - -.. method:: list.reverse() - :noindex: - - Reverse the elements of the list in place. - - -.. method:: list.copy() - :noindex: - - Return a shallow copy of the list. Equivalent to ``a[:]``. - - -An example that uses most of the list methods:: - - >>> fruits = ['orange', 'apple', 'pear', 'banana', 'kiwi', 'apple', 'banana'] - >>> fruits.count('apple') - 2 - >>> fruits.count('tangerine') - 0 - >>> fruits.index('banana') - 3 - >>> fruits.index('banana', 4) # Find next banana starting at position 4 - 6 - >>> fruits.reverse() - >>> fruits - ['banana', 'apple', 'kiwi', 'banana', 'pear', 'apple', 'orange'] - >>> fruits.append('grape') - >>> fruits - ['banana', 'apple', 'kiwi', 'banana', 'pear', 'apple', 'orange', 'grape'] - >>> fruits.sort() - >>> fruits - ['apple', 'apple', 'banana', 'banana', 'grape', 'kiwi', 'orange', 'pear'] - >>> fruits.pop() - 'pear' - -You might have noticed that methods like ``insert``, ``remove`` or ``sort`` that -only modify the list have no return value printed -- they return the default -``None``. [#]_ This is a design principle for all mutable data structures in -Python. - -Another thing you might notice is that not all data can be sorted or -compared. For instance, ``[None, 'hello', 10]`` doesn't sort because -integers can't be compared to strings and *None* can't be compared to -other types. Also, there are some types that don't have a defined -ordering relation. For example, ``3+4j < 5+7j`` isn't a valid -comparison. - - -.. _tut-lists-as-stacks: - -Using Lists as Stacks ---------------------- - -.. sectionauthor:: Ka-Ping Yee - - -The list methods make it very easy to use a list as a stack, where the last -element added is the first element retrieved ("last-in, first-out"). To add an -item to the top of the stack, use :meth:`~list.append`. To retrieve an item from the -top of the stack, use :meth:`~list.pop` without an explicit index. For example:: - - >>> stack = [3, 4, 5] - >>> stack.append(6) - >>> stack.append(7) - >>> stack - [3, 4, 5, 6, 7] - >>> stack.pop() - 7 - >>> stack - [3, 4, 5, 6] - >>> stack.pop() - 6 - >>> stack.pop() - 5 - >>> stack - [3, 4] - - -.. _tut-lists-as-queues: - -Using Lists as Queues ---------------------- - -.. sectionauthor:: Ka-Ping Yee - -It is also possible to use a list as a queue, where the first element added is -the first element retrieved ("first-in, first-out"); however, lists are not -efficient for this purpose. While appends and pops from the end of list are -fast, doing inserts or pops from the beginning of a list is slow (because all -of the other elements have to be shifted by one). - -To implement a queue, use :class:`collections.deque` which was designed to -have fast appends and pops from both ends. For example:: - - >>> from collections import deque - >>> queue = deque(["Eric", "John", "Michael"]) - >>> queue.append("Terry") # Terry arrives - >>> queue.append("Graham") # Graham arrives - >>> queue.popleft() # The first to arrive now leaves - 'Eric' - >>> queue.popleft() # The second to arrive now leaves - 'John' - >>> queue # Remaining queue in order of arrival - deque(['Michael', 'Terry', 'Graham']) - - -.. _tut-listcomps: - -List Comprehensions -------------------- - -List comprehensions provide a concise way to create lists. -Common applications are to make new lists where each element is the result of -some operations applied to each member of another sequence or iterable, or to -create a subsequence of those elements that satisfy a certain condition. - -For example, assume we want to create a list of squares, like:: - - >>> squares = [] - >>> for x in range(10): - ... squares.append(x**2) - ... - >>> squares - [0, 1, 4, 9, 16, 25, 36, 49, 64, 81] - -Note that this creates (or overwrites) a variable named ``x`` that still exists -after the loop completes. We can calculate the list of squares without any -side effects using:: - - squares = list(map(lambda x: x**2, range(10))) - -or, equivalently:: - - squares = [x**2 for x in range(10)] - -which is more concise and readable. - -A list comprehension consists of brackets containing an expression followed -by a :keyword:`!for` clause, then zero or more :keyword:`!for` or :keyword:`!if` -clauses. The result will be a new list resulting from evaluating the expression -in the context of the :keyword:`!for` and :keyword:`!if` clauses which follow it. -For example, this listcomp combines the elements of two lists if they are not -equal:: - - >>> [(x, y) for x in [1,2,3] for y in [3,1,4] if x != y] - [(1, 3), (1, 4), (2, 3), (2, 1), (2, 4), (3, 1), (3, 4)] - -and it's equivalent to:: - - >>> combs = [] - >>> for x in [1,2,3]: - ... for y in [3,1,4]: - ... if x != y: - ... combs.append((x, y)) - ... - >>> combs - [(1, 3), (1, 4), (2, 3), (2, 1), (2, 4), (3, 1), (3, 4)] - -Note how the order of the :keyword:`for` and :keyword:`if` statements is the -same in both these snippets. - -If the expression is a tuple (e.g. the ``(x, y)`` in the previous example), -it must be parenthesized. :: - - >>> vec = [-4, -2, 0, 2, 4] - >>> # create a new list with the values doubled - >>> [x*2 for x in vec] - [-8, -4, 0, 4, 8] - >>> # filter the list to exclude negative numbers - >>> [x for x in vec if x >= 0] - [0, 2, 4] - >>> # apply a function to all the elements - >>> [abs(x) for x in vec] - [4, 2, 0, 2, 4] - >>> # call a method on each element - >>> freshfruit = [' banana', ' loganberry ', 'passion fruit '] - >>> [weapon.strip() for weapon in freshfruit] - ['banana', 'loganberry', 'passion fruit'] - >>> # create a list of 2-tuples like (number, square) - >>> [(x, x**2) for x in range(6)] - [(0, 0), (1, 1), (2, 4), (3, 9), (4, 16), (5, 25)] - >>> # the tuple must be parenthesized, otherwise an error is raised - >>> [x, x**2 for x in range(6)] - File "", line 1 - [x, x**2 for x in range(6)] - ^^^^^^^ - SyntaxError: did you forget parentheses around the comprehension target? - >>> # flatten a list using a listcomp with two 'for' - >>> vec = [[1,2,3], [4,5,6], [7,8,9]] - >>> [num for elem in vec for num in elem] - [1, 2, 3, 4, 5, 6, 7, 8, 9] - -List comprehensions can contain complex expressions and nested functions:: - - >>> from math import pi - >>> [str(round(pi, i)) for i in range(1, 6)] - ['3.1', '3.14', '3.142', '3.1416', '3.14159'] - -Nested List Comprehensions --------------------------- - -The initial expression in a list comprehension can be any arbitrary expression, -including another list comprehension. - -Consider the following example of a 3x4 matrix implemented as a list of -3 lists of length 4:: - - >>> matrix = [ - ... [1, 2, 3, 4], - ... [5, 6, 7, 8], - ... [9, 10, 11, 12], - ... ] - -The following list comprehension will transpose rows and columns:: - - >>> [[row[i] for row in matrix] for i in range(4)] - [[1, 5, 9], [2, 6, 10], [3, 7, 11], [4, 8, 12]] - -As we saw in the previous section, the inner list comprehension is evaluated in -the context of the :keyword:`for` that follows it, so this example is -equivalent to:: - - >>> transposed = [] - >>> for i in range(4): - ... transposed.append([row[i] for row in matrix]) - ... - >>> transposed - [[1, 5, 9], [2, 6, 10], [3, 7, 11], [4, 8, 12]] - -which, in turn, is the same as:: - - >>> transposed = [] - >>> for i in range(4): - ... # the following 3 lines implement the nested listcomp - ... transposed_row = [] - ... for row in matrix: - ... transposed_row.append(row[i]) - ... transposed.append(transposed_row) - ... - >>> transposed - [[1, 5, 9], [2, 6, 10], [3, 7, 11], [4, 8, 12]] - -In the real world, you should prefer built-in functions to complex flow statements. -The :func:`zip` function would do a great job for this use case:: - - >>> list(zip(*matrix)) - [(1, 5, 9), (2, 6, 10), (3, 7, 11), (4, 8, 12)] - -See :ref:`tut-unpacking-arguments` for details on the asterisk in this line. - -.. _tut-del: - -The :keyword:`!del` statement -============================= - -There is a way to remove an item from a list given its index instead of its -value: the :keyword:`del` statement. This differs from the :meth:`~list.pop` method -which returns a value. The :keyword:`!del` statement can also be used to remove -slices from a list or clear the entire list (which we did earlier by assignment -of an empty list to the slice). For example:: - - >>> a = [-1, 1, 66.25, 333, 333, 1234.5] - >>> del a[0] - >>> a - [1, 66.25, 333, 333, 1234.5] - >>> del a[2:4] - >>> a - [1, 66.25, 1234.5] - >>> del a[:] - >>> a - [] - -:keyword:`del` can also be used to delete entire variables:: - - >>> del a - -Referencing the name ``a`` hereafter is an error (at least until another value -is assigned to it). We'll find other uses for :keyword:`del` later. - - -.. _tut-tuples: - -Tuples and Sequences -==================== - -We saw that lists and strings have many common properties, such as indexing and -slicing operations. They are two examples of *sequence* data types (see -:ref:`typesseq`). Since Python is an evolving language, other sequence data -types may be added. There is also another standard sequence data type: the -*tuple*. - -A tuple consists of a number of values separated by commas, for instance:: - - >>> t = 12345, 54321, 'hello!' - >>> t[0] - 12345 - >>> t - (12345, 54321, 'hello!') - >>> # Tuples may be nested: - ... u = t, (1, 2, 3, 4, 5) - >>> u - ((12345, 54321, 'hello!'), (1, 2, 3, 4, 5)) - >>> # Tuples are immutable: - ... t[0] = 88888 - Traceback (most recent call last): - File "", line 1, in - TypeError: 'tuple' object does not support item assignment - >>> # but they can contain mutable objects: - ... v = ([1, 2, 3], [3, 2, 1]) - >>> v - ([1, 2, 3], [3, 2, 1]) - - -As you see, on output tuples are always enclosed in parentheses, so that nested -tuples are interpreted correctly; they may be input with or without surrounding -parentheses, although often parentheses are necessary anyway (if the tuple is -part of a larger expression). It is not possible to assign to the individual -items of a tuple, however it is possible to create tuples which contain mutable -objects, such as lists. - -Though tuples may seem similar to lists, they are often used in different -situations and for different purposes. -Tuples are :term:`immutable`, and usually contain a heterogeneous sequence of -elements that are accessed via unpacking (see later in this section) or indexing -(or even by attribute in the case of :func:`namedtuples `). -Lists are :term:`mutable`, and their elements are usually homogeneous and are -accessed by iterating over the list. - -A special problem is the construction of tuples containing 0 or 1 items: the -syntax has some extra quirks to accommodate these. Empty tuples are constructed -by an empty pair of parentheses; a tuple with one item is constructed by -following a value with a comma (it is not sufficient to enclose a single value -in parentheses). Ugly, but effective. For example:: - - >>> empty = () - >>> singleton = 'hello', # <-- note trailing comma - >>> len(empty) - 0 - >>> len(singleton) - 1 - >>> singleton - ('hello',) - -The statement ``t = 12345, 54321, 'hello!'`` is an example of *tuple packing*: -the values ``12345``, ``54321`` and ``'hello!'`` are packed together in a tuple. -The reverse operation is also possible:: - - >>> x, y, z = t - -This is called, appropriately enough, *sequence unpacking* and works for any -sequence on the right-hand side. Sequence unpacking requires that there are as -many variables on the left side of the equals sign as there are elements in the -sequence. Note that multiple assignment is really just a combination of tuple -packing and sequence unpacking. - - -.. _tut-sets: - -Sets -==== - -Python also includes a data type for *sets*. A set is an unordered collection -with no duplicate elements. Basic uses include membership testing and -eliminating duplicate entries. Set objects also support mathematical operations -like union, intersection, difference, and symmetric difference. - -Curly braces or the :func:`set` function can be used to create sets. Note: to -create an empty set you have to use ``set()``, not ``{}``; the latter creates an -empty dictionary, a data structure that we discuss in the next section. - -Here is a brief demonstration:: - - >>> basket = {'apple', 'orange', 'apple', 'pear', 'orange', 'banana'} - >>> print(basket) # show that duplicates have been removed - {'orange', 'banana', 'pear', 'apple'} - >>> 'orange' in basket # fast membership testing - True - >>> 'crabgrass' in basket - False - - >>> # Demonstrate set operations on unique letters from two words - ... - >>> a = set('abracadabra') - >>> b = set('alacazam') - >>> a # unique letters in a - {'a', 'r', 'b', 'c', 'd'} - >>> a - b # letters in a but not in b - {'r', 'd', 'b'} - >>> a | b # letters in a or b or both - {'a', 'c', 'r', 'd', 'b', 'm', 'z', 'l'} - >>> a & b # letters in both a and b - {'a', 'c'} - >>> a ^ b # letters in a or b but not both - {'r', 'd', 'b', 'm', 'z', 'l'} - -Similarly to :ref:`list comprehensions `, set comprehensions -are also supported:: - - >>> a = {x for x in 'abracadabra' if x not in 'abc'} - >>> a - {'r', 'd'} - - -.. _tut-dictionaries: - -Dictionaries -============ - -Another useful data type built into Python is the *dictionary* (see -:ref:`typesmapping`). Dictionaries are sometimes found in other languages as -"associative memories" or "associative arrays". Unlike sequences, which are -indexed by a range of numbers, dictionaries are indexed by *keys*, which can be -any immutable type; strings and numbers can always be keys. Tuples can be used -as keys if they contain only strings, numbers, or tuples; if a tuple contains -any mutable object either directly or indirectly, it cannot be used as a key. -You can't use lists as keys, since lists can be modified in place using index -assignments, slice assignments, or methods like :meth:`~list.append` and -:meth:`~list.extend`. - -It is best to think of a dictionary as a set of *key: value* pairs, -with the requirement that the keys are unique (within one dictionary). A pair of -braces creates an empty dictionary: ``{}``. Placing a comma-separated list of -key:value pairs within the braces adds initial key:value pairs to the -dictionary; this is also the way dictionaries are written on output. - -The main operations on a dictionary are storing a value with some key and -extracting the value given the key. It is also possible to delete a key:value -pair with ``del``. If you store using a key that is already in use, the old -value associated with that key is forgotten. It is an error to extract a value -using a non-existent key. - -Performing ``list(d)`` on a dictionary returns a list of all the keys -used in the dictionary, in insertion order (if you want it sorted, just use -``sorted(d)`` instead). To check whether a single key is in the -dictionary, use the :keyword:`in` keyword. - -Here is a small example using a dictionary:: - - >>> tel = {'jack': 4098, 'sape': 4139} - >>> tel['guido'] = 4127 - >>> tel - {'jack': 4098, 'sape': 4139, 'guido': 4127} - >>> tel['jack'] - 4098 - >>> del tel['sape'] - >>> tel['irv'] = 4127 - >>> tel - {'jack': 4098, 'guido': 4127, 'irv': 4127} - >>> list(tel) - ['jack', 'guido', 'irv'] - >>> sorted(tel) - ['guido', 'irv', 'jack'] - >>> 'guido' in tel - True - >>> 'jack' not in tel - False - -The :func:`dict` constructor builds dictionaries directly from sequences of -key-value pairs:: - - >>> dict([('sape', 4139), ('guido', 4127), ('jack', 4098)]) - {'sape': 4139, 'guido': 4127, 'jack': 4098} - -In addition, dict comprehensions can be used to create dictionaries from -arbitrary key and value expressions:: - - >>> {x: x**2 for x in (2, 4, 6)} - {2: 4, 4: 16, 6: 36} - -When the keys are simple strings, it is sometimes easier to specify pairs using -keyword arguments:: - - >>> dict(sape=4139, guido=4127, jack=4098) - {'sape': 4139, 'guido': 4127, 'jack': 4098} - - -.. _tut-loopidioms: - -Looping Techniques -================== - -When looping through dictionaries, the key and corresponding value can be -retrieved at the same time using the :meth:`~dict.items` method. :: - - >>> knights = {'gallahad': 'the pure', 'robin': 'the brave'} - >>> for k, v in knights.items(): - ... print(k, v) - ... - gallahad the pure - robin the brave - -When looping through a sequence, the position index and corresponding value can -be retrieved at the same time using the :func:`enumerate` function. :: - - >>> for i, v in enumerate(['tic', 'tac', 'toe']): - ... print(i, v) - ... - 0 tic - 1 tac - 2 toe - -To loop over two or more sequences at the same time, the entries can be paired -with the :func:`zip` function. :: - - >>> questions = ['name', 'quest', 'favorite color'] - >>> answers = ['lancelot', 'the holy grail', 'blue'] - >>> for q, a in zip(questions, answers): - ... print('What is your {0}? It is {1}.'.format(q, a)) - ... - What is your name? It is lancelot. - What is your quest? It is the holy grail. - What is your favorite color? It is blue. - -To loop over a sequence in reverse, first specify the sequence in a forward -direction and then call the :func:`reversed` function. :: - - >>> for i in reversed(range(1, 10, 2)): - ... print(i) - ... - 9 - 7 - 5 - 3 - 1 - -To loop over a sequence in sorted order, use the :func:`sorted` function which -returns a new sorted list while leaving the source unaltered. :: - - >>> basket = ['apple', 'orange', 'apple', 'pear', 'orange', 'banana'] - >>> for i in sorted(basket): - ... print(i) - ... - apple - apple - banana - orange - orange - pear - -Using :func:`set` on a sequence eliminates duplicate elements. The use of -:func:`sorted` in combination with :func:`set` over a sequence is an idiomatic -way to loop over unique elements of the sequence in sorted order. :: - - >>> basket = ['apple', 'orange', 'apple', 'pear', 'orange', 'banana'] - >>> for f in sorted(set(basket)): - ... print(f) - ... - apple - banana - orange - pear - -It is sometimes tempting to change a list while you are looping over it; -however, it is often simpler and safer to create a new list instead. :: - - >>> import math - >>> raw_data = [56.2, float('NaN'), 51.7, 55.3, 52.5, float('NaN'), 47.8] - >>> filtered_data = [] - >>> for value in raw_data: - ... if not math.isnan(value): - ... filtered_data.append(value) - ... - >>> filtered_data - [56.2, 51.7, 55.3, 52.5, 47.8] - - -.. _tut-conditions: - -More on Conditions -================== - -The conditions used in ``while`` and ``if`` statements can contain any -operators, not just comparisons. - - -The comparison operators ``in`` and ``not in`` are membership tests that -determine whether a value is in (or not in) a container. The operators ``is`` -and ``is not`` compare whether two objects are really the same object. All -comparison operators have the same priority, which is lower than that of all -numerical operators. - -Comparisons can be chained. For example, ``a < b == c`` tests whether ``a`` is -less than ``b`` and moreover ``b`` equals ``c``. - -Comparisons may be combined using the Boolean operators ``and`` and ``or``, and -the outcome of a comparison (or of any other Boolean expression) may be negated -with ``not``. These have lower priorities than comparison operators; between -them, ``not`` has the highest priority and ``or`` the lowest, so that ``A and -not B or C`` is equivalent to ``(A and (not B)) or C``. As always, parentheses -can be used to express the desired composition. - -The Boolean operators ``and`` and ``or`` are so-called *short-circuit* -operators: their arguments are evaluated from left to right, and evaluation -stops as soon as the outcome is determined. For example, if ``A`` and ``C`` are -true but ``B`` is false, ``A and B and C`` does not evaluate the expression -``C``. When used as a general value and not as a Boolean, the return value of a -short-circuit operator is the last evaluated argument. - -It is possible to assign the result of a comparison or other Boolean expression -to a variable. For example, :: - - >>> string1, string2, string3 = '', 'Trondheim', 'Hammer Dance' - >>> non_null = string1 or string2 or string3 - >>> non_null - 'Trondheim' - -Note that in Python, unlike C, assignment inside expressions must be done -explicitly with the -:ref:`walrus operator ` ``:=``. -This avoids a common class of problems encountered in C programs: typing ``=`` -in an expression when ``==`` was intended. - - -.. _tut-comparing: - -Comparing Sequences and Other Types -=================================== -Sequence objects typically may be compared to other objects with the same sequence -type. The comparison uses *lexicographical* ordering: first the first two -items are compared, and if they differ this determines the outcome of the -comparison; if they are equal, the next two items are compared, and so on, until -either sequence is exhausted. If two items to be compared are themselves -sequences of the same type, the lexicographical comparison is carried out -recursively. If all items of two sequences compare equal, the sequences are -considered equal. If one sequence is an initial sub-sequence of the other, the -shorter sequence is the smaller (lesser) one. Lexicographical ordering for -strings uses the Unicode code point number to order individual characters. -Some examples of comparisons between sequences of the same type:: - - (1, 2, 3) < (1, 2, 4) - [1, 2, 3] < [1, 2, 4] - 'ABC' < 'C' < 'Pascal' < 'Python' - (1, 2, 3, 4) < (1, 2, 4) - (1, 2) < (1, 2, -1) - (1, 2, 3) == (1.0, 2.0, 3.0) - (1, 2, ('aa', 'ab')) < (1, 2, ('abc', 'a'), 4) - -Note that comparing objects of different types with ``<`` or ``>`` is legal -provided that the objects have appropriate comparison methods. For example, -mixed numeric types are compared according to their numeric value, so 0 equals -0.0, etc. Otherwise, rather than providing an arbitrary ordering, the -interpreter will raise a :exc:`TypeError` exception. - - -.. rubric:: Footnotes - -.. [#] Other languages may return the mutated object, which allows method - chaining, such as ``d->insert("a")->remove("b")->sort();``. diff --git a/Doc/tutorial/errors.rst b/Doc/tutorial/errors.rst deleted file mode 100644 index 1ec59767e9ce12..00000000000000 --- a/Doc/tutorial/errors.rst +++ /dev/null @@ -1,654 +0,0 @@ -.. _tut-errors: - -********************* -Errors and Exceptions -********************* - -Until now error messages haven't been more than mentioned, but if you have tried -out the examples you have probably seen some. There are (at least) two -distinguishable kinds of errors: *syntax errors* and *exceptions*. - - -.. _tut-syntaxerrors: - -Syntax Errors -============= - -Syntax errors, also known as parsing errors, are perhaps the most common kind of -complaint you get while you are still learning Python:: - - >>> while True print('Hello world') - File "", line 1 - while True print('Hello world') - ^ - SyntaxError: invalid syntax - -The parser repeats the offending line and displays a little 'arrow' pointing at -the earliest point in the line where the error was detected. The error is -caused by (or at least detected at) the token *preceding* the arrow: in the -example, the error is detected at the function :func:`print`, since a colon -(``':'``) is missing before it. File name and line number are printed so you -know where to look in case the input came from a script. - - -.. _tut-exceptions: - -Exceptions -========== - -Even if a statement or expression is syntactically correct, it may cause an -error when an attempt is made to execute it. Errors detected during execution -are called *exceptions* and are not unconditionally fatal: you will soon learn -how to handle them in Python programs. Most exceptions are not handled by -programs, however, and result in error messages as shown here:: - - >>> 10 * (1/0) - Traceback (most recent call last): - File "", line 1, in - ZeroDivisionError: division by zero - >>> 4 + spam*3 - Traceback (most recent call last): - File "", line 1, in - NameError: name 'spam' is not defined - >>> '2' + 2 - Traceback (most recent call last): - File "", line 1, in - TypeError: can only concatenate str (not "int") to str - -The last line of the error message indicates what happened. Exceptions come in -different types, and the type is printed as part of the message: the types in -the example are :exc:`ZeroDivisionError`, :exc:`NameError` and :exc:`TypeError`. -The string printed as the exception type is the name of the built-in exception -that occurred. This is true for all built-in exceptions, but need not be true -for user-defined exceptions (although it is a useful convention). Standard -exception names are built-in identifiers (not reserved keywords). - -The rest of the line provides detail based on the type of exception and what -caused it. - -The preceding part of the error message shows the context where the exception -occurred, in the form of a stack traceback. In general it contains a stack -traceback listing source lines; however, it will not display lines read from -standard input. - -:ref:`bltin-exceptions` lists the built-in exceptions and their meanings. - - -.. _tut-handling: - -Handling Exceptions -=================== - -It is possible to write programs that handle selected exceptions. Look at the -following example, which asks the user for input until a valid integer has been -entered, but allows the user to interrupt the program (using :kbd:`Control-C` or -whatever the operating system supports); note that a user-generated interruption -is signalled by raising the :exc:`KeyboardInterrupt` exception. :: - - >>> while True: - ... try: - ... x = int(input("Please enter a number: ")) - ... break - ... except ValueError: - ... print("Oops! That was no valid number. Try again...") - ... - -The :keyword:`try` statement works as follows. - -* First, the *try clause* (the statement(s) between the :keyword:`try` and - :keyword:`except` keywords) is executed. - -* If no exception occurs, the *except clause* is skipped and execution of the - :keyword:`try` statement is finished. - -* If an exception occurs during execution of the :keyword:`try` clause, the rest of the - clause is skipped. Then, if its type matches the exception named after the - :keyword:`except` keyword, the *except clause* is executed, and then execution - continues after the try/except block. - -* If an exception occurs which does not match the exception named in the *except - clause*, it is passed on to outer :keyword:`try` statements; if no handler is - found, it is an *unhandled exception* and execution stops with a message as - shown above. - -A :keyword:`try` statement may have more than one *except clause*, to specify -handlers for different exceptions. At most one handler will be executed. -Handlers only handle exceptions that occur in the corresponding *try clause*, -not in other handlers of the same :keyword:`!try` statement. An *except clause* -may name multiple exceptions as a parenthesized tuple, for example:: - - ... except (RuntimeError, TypeError, NameError): - ... pass - -A class in an :keyword:`except` clause is compatible with an exception if it is -the same class or a base class thereof (but not the other way around --- an -*except clause* listing a derived class is not compatible with a base class). -For example, the following code will print B, C, D in that order:: - - class B(Exception): - pass - - class C(B): - pass - - class D(C): - pass - - for cls in [B, C, D]: - try: - raise cls() - except D: - print("D") - except C: - print("C") - except B: - print("B") - -Note that if the *except clauses* were reversed (with ``except B`` first), it -would have printed B, B, B --- the first matching *except clause* is triggered. - -When an exception occurs, it may have associated values, also known as the -exception's *arguments*. The presence and types of the arguments depend on the -exception type. - -The *except clause* may specify a variable after the exception name. The -variable is bound to the exception instance which typically has an ``args`` -attribute that stores the arguments. For convenience, builtin exception -types define :meth:`~object.__str__` to print all the arguments without explicitly -accessing ``.args``. :: - - >>> try: - ... raise Exception('spam', 'eggs') - ... except Exception as inst: - ... print(type(inst)) # the exception type - ... print(inst.args) # arguments stored in .args - ... print(inst) # __str__ allows args to be printed directly, - ... # but may be overridden in exception subclasses - ... x, y = inst.args # unpack args - ... print('x =', x) - ... print('y =', y) - ... - - ('spam', 'eggs') - ('spam', 'eggs') - x = spam - y = eggs - -The exception's :meth:`~object.__str__` output is printed as the last part ('detail') -of the message for unhandled exceptions. - -:exc:`BaseException` is the common base class of all exceptions. One of its -subclasses, :exc:`Exception`, is the base class of all the non-fatal exceptions. -Exceptions which are not subclasses of :exc:`Exception` are not typically -handled, because they are used to indicate that the program should terminate. -They include :exc:`SystemExit` which is raised by :meth:`sys.exit` and -:exc:`KeyboardInterrupt` which is raised when a user wishes to interrupt -the program. - -:exc:`Exception` can be used as a wildcard that catches (almost) everything. -However, it is good practice to be as specific as possible with the types -of exceptions that we intend to handle, and to allow any unexpected -exceptions to propagate on. - -The most common pattern for handling :exc:`Exception` is to print or log -the exception and then re-raise it (allowing a caller to handle the -exception as well):: - - import sys - - try: - f = open('myfile.txt') - s = f.readline() - i = int(s.strip()) - except OSError as err: - print("OS error:", err) - except ValueError: - print("Could not convert data to an integer.") - except Exception as err: - print(f"Unexpected {err=}, {type(err)=}") - raise - -The :keyword:`try` ... :keyword:`except` statement has an optional *else -clause*, which, when present, must follow all *except clauses*. It is useful -for code that must be executed if the *try clause* does not raise an exception. -For example:: - - for arg in sys.argv[1:]: - try: - f = open(arg, 'r') - except OSError: - print('cannot open', arg) - else: - print(arg, 'has', len(f.readlines()), 'lines') - f.close() - -The use of the :keyword:`!else` clause is better than adding additional code to -the :keyword:`try` clause because it avoids accidentally catching an exception -that wasn't raised by the code being protected by the :keyword:`!try` ... -:keyword:`!except` statement. - -Exception handlers do not handle only exceptions that occur immediately in the -*try clause*, but also those that occur inside functions that are called (even -indirectly) in the *try clause*. For example:: - - >>> def this_fails(): - ... x = 1/0 - ... - >>> try: - ... this_fails() - ... except ZeroDivisionError as err: - ... print('Handling run-time error:', err) - ... - Handling run-time error: division by zero - - -.. _tut-raising: - -Raising Exceptions -================== - -The :keyword:`raise` statement allows the programmer to force a specified -exception to occur. For example:: - - >>> raise NameError('HiThere') - Traceback (most recent call last): - File "", line 1, in - NameError: HiThere - -The sole argument to :keyword:`raise` indicates the exception to be raised. -This must be either an exception instance or an exception class (a class that -derives from :class:`BaseException`, such as :exc:`Exception` or one of its -subclasses). If an exception class is passed, it will be implicitly -instantiated by calling its constructor with no arguments:: - - raise ValueError # shorthand for 'raise ValueError()' - -If you need to determine whether an exception was raised but don't intend to -handle it, a simpler form of the :keyword:`raise` statement allows you to -re-raise the exception:: - - >>> try: - ... raise NameError('HiThere') - ... except NameError: - ... print('An exception flew by!') - ... raise - ... - An exception flew by! - Traceback (most recent call last): - File "", line 2, in - NameError: HiThere - - -.. _tut-exception-chaining: - -Exception Chaining -================== - -If an unhandled exception occurs inside an :keyword:`except` section, it will -have the exception being handled attached to it and included in the error -message:: - - >>> try: - ... open("database.sqlite") - ... except OSError: - ... raise RuntimeError("unable to handle error") - ... - Traceback (most recent call last): - File "", line 2, in - FileNotFoundError: [Errno 2] No such file or directory: 'database.sqlite' - - During handling of the above exception, another exception occurred: - - Traceback (most recent call last): - File "", line 4, in - RuntimeError: unable to handle error - -To indicate that an exception is a direct consequence of another, the -:keyword:`raise` statement allows an optional :keyword:`from` clause:: - - # exc must be exception instance or None. - raise RuntimeError from exc - -This can be useful when you are transforming exceptions. For example:: - - >>> def func(): - ... raise ConnectionError - ... - >>> try: - ... func() - ... except ConnectionError as exc: - ... raise RuntimeError('Failed to open database') from exc - ... - Traceback (most recent call last): - File "", line 2, in - File "", line 2, in func - ConnectionError - - The above exception was the direct cause of the following exception: - - Traceback (most recent call last): - File "", line 4, in - RuntimeError: Failed to open database - -It also allows disabling automatic exception chaining using the ``from None`` -idiom:: - - >>> try: - ... open('database.sqlite') - ... except OSError: - ... raise RuntimeError from None - ... - Traceback (most recent call last): - File "", line 4, in - RuntimeError - -For more information about chaining mechanics, see :ref:`bltin-exceptions`. - - -.. _tut-userexceptions: - -User-defined Exceptions -======================= - -Programs may name their own exceptions by creating a new exception class (see -:ref:`tut-classes` for more about Python classes). Exceptions should typically -be derived from the :exc:`Exception` class, either directly or indirectly. - -Exception classes can be defined which do anything any other class can do, but -are usually kept simple, often only offering a number of attributes that allow -information about the error to be extracted by handlers for the exception. - -Most exceptions are defined with names that end in "Error", similar to the -naming of the standard exceptions. - -Many standard modules define their own exceptions to report errors that may -occur in functions they define. - - -.. _tut-cleanup: - -Defining Clean-up Actions -========================= - -The :keyword:`try` statement has another optional clause which is intended to -define clean-up actions that must be executed under all circumstances. For -example:: - - >>> try: - ... raise KeyboardInterrupt - ... finally: - ... print('Goodbye, world!') - ... - Goodbye, world! - Traceback (most recent call last): - File "", line 2, in - KeyboardInterrupt - -If a :keyword:`finally` clause is present, the :keyword:`!finally` -clause will execute as the last task before the :keyword:`try` -statement completes. The :keyword:`!finally` clause runs whether or -not the :keyword:`!try` statement produces an exception. The following -points discuss more complex cases when an exception occurs: - -* If an exception occurs during execution of the :keyword:`!try` - clause, the exception may be handled by an :keyword:`except` - clause. If the exception is not handled by an :keyword:`!except` - clause, the exception is re-raised after the :keyword:`!finally` - clause has been executed. - -* An exception could occur during execution of an :keyword:`!except` - or :keyword:`!else` clause. Again, the exception is re-raised after - the :keyword:`!finally` clause has been executed. - -* If the :keyword:`!finally` clause executes a :keyword:`break`, - :keyword:`continue` or :keyword:`return` statement, exceptions are not - re-raised. - -* If the :keyword:`!try` statement reaches a :keyword:`break`, - :keyword:`continue` or :keyword:`return` statement, the - :keyword:`!finally` clause will execute just prior to the - :keyword:`!break`, :keyword:`!continue` or :keyword:`!return` - statement's execution. - -* If a :keyword:`!finally` clause includes a :keyword:`!return` - statement, the returned value will be the one from the - :keyword:`!finally` clause's :keyword:`!return` statement, not the - value from the :keyword:`!try` clause's :keyword:`!return` - statement. - -For example:: - - >>> def bool_return(): - ... try: - ... return True - ... finally: - ... return False - ... - >>> bool_return() - False - -A more complicated example:: - - >>> def divide(x, y): - ... try: - ... result = x / y - ... except ZeroDivisionError: - ... print("division by zero!") - ... else: - ... print("result is", result) - ... finally: - ... print("executing finally clause") - ... - >>> divide(2, 1) - result is 2.0 - executing finally clause - >>> divide(2, 0) - division by zero! - executing finally clause - >>> divide("2", "1") - executing finally clause - Traceback (most recent call last): - File "", line 1, in - File "", line 3, in divide - TypeError: unsupported operand type(s) for /: 'str' and 'str' - -As you can see, the :keyword:`finally` clause is executed in any event. The -:exc:`TypeError` raised by dividing two strings is not handled by the -:keyword:`except` clause and therefore re-raised after the :keyword:`!finally` -clause has been executed. - -In real world applications, the :keyword:`finally` clause is useful for -releasing external resources (such as files or network connections), regardless -of whether the use of the resource was successful. - - -.. _tut-cleanup-with: - -Predefined Clean-up Actions -=========================== - -Some objects define standard clean-up actions to be undertaken when the object -is no longer needed, regardless of whether or not the operation using the object -succeeded or failed. Look at the following example, which tries to open a file -and print its contents to the screen. :: - - for line in open("myfile.txt"): - print(line, end="") - -The problem with this code is that it leaves the file open for an indeterminate -amount of time after this part of the code has finished executing. -This is not an issue in simple scripts, but can be a problem for larger -applications. The :keyword:`with` statement allows objects like files to be -used in a way that ensures they are always cleaned up promptly and correctly. :: - - with open("myfile.txt") as f: - for line in f: - print(line, end="") - -After the statement is executed, the file *f* is always closed, even if a -problem was encountered while processing the lines. Objects which, like files, -provide predefined clean-up actions will indicate this in their documentation. - - -.. _tut-exception-groups: - -Raising and Handling Multiple Unrelated Exceptions -================================================== - -There are situations where it is necessary to report several exceptions that -have occurred. This is often the case in concurrency frameworks, when several -tasks may have failed in parallel, but there are also other use cases where -it is desirable to continue execution and collect multiple errors rather than -raise the first exception. - -The builtin :exc:`ExceptionGroup` wraps a list of exception instances so -that they can be raised together. It is an exception itself, so it can be -caught like any other exception. :: - - >>> def f(): - ... excs = [OSError('error 1'), SystemError('error 2')] - ... raise ExceptionGroup('there were problems', excs) - ... - >>> f() - + Exception Group Traceback (most recent call last): - | File "", line 1, in - | File "", line 3, in f - | ExceptionGroup: there were problems - +-+---------------- 1 ---------------- - | OSError: error 1 - +---------------- 2 ---------------- - | SystemError: error 2 - +------------------------------------ - >>> try: - ... f() - ... except Exception as e: - ... print(f'caught {type(e)}: e') - ... - caught : e - >>> - -By using ``except*`` instead of ``except``, we can selectively -handle only the exceptions in the group that match a certain -type. In the following example, which shows a nested exception -group, each ``except*`` clause extracts from the group exceptions -of a certain type while letting all other exceptions propagate to -other clauses and eventually to be reraised. :: - - >>> def f(): - ... raise ExceptionGroup( - ... "group1", - ... [ - ... OSError(1), - ... SystemError(2), - ... ExceptionGroup( - ... "group2", - ... [ - ... OSError(3), - ... RecursionError(4) - ... ] - ... ) - ... ] - ... ) - ... - >>> try: - ... f() - ... except* OSError as e: - ... print("There were OSErrors") - ... except* SystemError as e: - ... print("There were SystemErrors") - ... - There were OSErrors - There were SystemErrors - + Exception Group Traceback (most recent call last): - | File "", line 2, in - | File "", line 2, in f - | ExceptionGroup: group1 - +-+---------------- 1 ---------------- - | ExceptionGroup: group2 - +-+---------------- 1 ---------------- - | RecursionError: 4 - +------------------------------------ - >>> - -Note that the exceptions nested in an exception group must be instances, -not types. This is because in practice the exceptions would typically -be ones that have already been raised and caught by the program, along -the following pattern:: - - >>> excs = [] - ... for test in tests: - ... try: - ... test.run() - ... except Exception as e: - ... excs.append(e) - ... - >>> if excs: - ... raise ExceptionGroup("Test Failures", excs) - ... - - -.. _tut-exception-notes: - -Enriching Exceptions with Notes -=============================== - -When an exception is created in order to be raised, it is usually initialized -with information that describes the error that has occurred. There are cases -where it is useful to add information after the exception was caught. For this -purpose, exceptions have a method ``add_note(note)`` that accepts a string and -adds it to the exception's notes list. The standard traceback rendering -includes all notes, in the order they were added, after the exception. :: - - >>> try: - ... raise TypeError('bad type') - ... except Exception as e: - ... e.add_note('Add some information') - ... e.add_note('Add some more information') - ... raise - ... - Traceback (most recent call last): - File "", line 2, in - TypeError: bad type - Add some information - Add some more information - >>> - -For example, when collecting exceptions into an exception group, we may want -to add context information for the individual errors. In the following each -exception in the group has a note indicating when this error has occurred. :: - - >>> def f(): - ... raise OSError('operation failed') - ... - >>> excs = [] - >>> for i in range(3): - ... try: - ... f() - ... except Exception as e: - ... e.add_note(f'Happened in Iteration {i+1}') - ... excs.append(e) - ... - >>> raise ExceptionGroup('We have some problems', excs) - + Exception Group Traceback (most recent call last): - | File "", line 1, in - | ExceptionGroup: We have some problems (3 sub-exceptions) - +-+---------------- 1 ---------------- - | Traceback (most recent call last): - | File "", line 3, in - | File "", line 2, in f - | OSError: operation failed - | Happened in Iteration 1 - +---------------- 2 ---------------- - | Traceback (most recent call last): - | File "", line 3, in - | File "", line 2, in f - | OSError: operation failed - | Happened in Iteration 2 - +---------------- 3 ---------------- - | Traceback (most recent call last): - | File "", line 3, in - | File "", line 2, in f - | OSError: operation failed - | Happened in Iteration 3 - +------------------------------------ - >>> diff --git a/Doc/tutorial/floatingpoint.rst b/Doc/tutorial/floatingpoint.rst deleted file mode 100644 index 40c38be2fcd1a3..00000000000000 --- a/Doc/tutorial/floatingpoint.rst +++ /dev/null @@ -1,301 +0,0 @@ -.. testsetup:: - - import math - -.. _tut-fp-issues: - -************************************************** -Floating Point Arithmetic: Issues and Limitations -************************************************** - -.. sectionauthor:: Tim Peters - - -Floating-point numbers are represented in computer hardware as base 2 (binary) -fractions. For example, the **decimal** fraction ``0.125`` -has value 1/10 + 2/100 + 5/1000, and in the same way the **binary** fraction ``0.001`` -has value 0/2 + 0/4 + 1/8. These two fractions have identical values, the only -real difference being that the first is written in base 10 fractional notation, -and the second in base 2. - -Unfortunately, most decimal fractions cannot be represented exactly as binary -fractions. A consequence is that, in general, the decimal floating-point -numbers you enter are only approximated by the binary floating-point numbers -actually stored in the machine. - -The problem is easier to understand at first in base 10. Consider the fraction -1/3. You can approximate that as a base 10 fraction:: - - 0.3 - -or, better, :: - - 0.33 - -or, better, :: - - 0.333 - -and so on. No matter how many digits you're willing to write down, the result -will never be exactly 1/3, but will be an increasingly better approximation of -1/3. - -In the same way, no matter how many base 2 digits you're willing to use, the -decimal value 0.1 cannot be represented exactly as a base 2 fraction. In base -2, 1/10 is the infinitely repeating fraction :: - - 0.0001100110011001100110011001100110011001100110011... - -Stop at any finite number of bits, and you get an approximation. On most -machines today, floats are approximated using a binary fraction with -the numerator using the first 53 bits starting with the most significant bit and -with the denominator as a power of two. In the case of 1/10, the binary fraction -is ``3602879701896397 / 2 ** 55`` which is close to but not exactly -equal to the true value of 1/10. - -Many users are not aware of the approximation because of the way values are -displayed. Python only prints a decimal approximation to the true decimal -value of the binary approximation stored by the machine. On most machines, if -Python were to print the true decimal value of the binary approximation stored -for 0.1, it would have to display :: - - >>> 0.1 - 0.1000000000000000055511151231257827021181583404541015625 - -That is more digits than most people find useful, so Python keeps the number -of digits manageable by displaying a rounded value instead :: - - >>> 1 / 10 - 0.1 - -Just remember, even though the printed result looks like the exact value -of 1/10, the actual stored value is the nearest representable binary fraction. - -Interestingly, there are many different decimal numbers that share the same -nearest approximate binary fraction. For example, the numbers ``0.1`` and -``0.10000000000000001`` and -``0.1000000000000000055511151231257827021181583404541015625`` are all -approximated by ``3602879701896397 / 2 ** 55``. Since all of these decimal -values share the same approximation, any one of them could be displayed -while still preserving the invariant ``eval(repr(x)) == x``. - -Historically, the Python prompt and built-in :func:`repr` function would choose -the one with 17 significant digits, ``0.10000000000000001``. Starting with -Python 3.1, Python (on most systems) is now able to choose the shortest of -these and simply display ``0.1``. - -Note that this is in the very nature of binary floating-point: this is not a bug -in Python, and it is not a bug in your code either. You'll see the same kind of -thing in all languages that support your hardware's floating-point arithmetic -(although some languages may not *display* the difference by default, or in all -output modes). - -For more pleasant output, you may wish to use string formatting to produce a limited number of significant digits:: - - >>> format(math.pi, '.12g') # give 12 significant digits - '3.14159265359' - - >>> format(math.pi, '.2f') # give 2 digits after the point - '3.14' - - >>> repr(math.pi) - '3.141592653589793' - - -It's important to realize that this is, in a real sense, an illusion: you're -simply rounding the *display* of the true machine value. - -One illusion may beget another. For example, since 0.1 is not exactly 1/10, -summing three values of 0.1 may not yield exactly 0.3, either:: - - >>> .1 + .1 + .1 == .3 - False - -Also, since the 0.1 cannot get any closer to the exact value of 1/10 and -0.3 cannot get any closer to the exact value of 3/10, then pre-rounding with -:func:`round` function cannot help:: - - >>> round(.1, 1) + round(.1, 1) + round(.1, 1) == round(.3, 1) - False - -Though the numbers cannot be made closer to their intended exact values, -the :func:`round` function can be useful for post-rounding so that results -with inexact values become comparable to one another:: - - >>> round(.1 + .1 + .1, 10) == round(.3, 10) - True - -Binary floating-point arithmetic holds many surprises like this. The problem -with "0.1" is explained in precise detail below, in the "Representation Error" -section. See `Examples of Floating Point Problems -`_ for -a pleasant summary of how binary floating-point works and the kinds of -problems commonly encountered in practice. Also see -`The Perils of Floating Point `_ -for a more complete account of other common surprises. - -As that says near the end, "there are no easy answers." Still, don't be unduly -wary of floating-point! The errors in Python float operations are inherited -from the floating-point hardware, and on most machines are on the order of no -more than 1 part in 2\*\*53 per operation. That's more than adequate for most -tasks, but you do need to keep in mind that it's not decimal arithmetic and -that every float operation can suffer a new rounding error. - -While pathological cases do exist, for most casual use of floating-point -arithmetic you'll see the result you expect in the end if you simply round the -display of your final results to the number of decimal digits you expect. -:func:`str` usually suffices, and for finer control see the :meth:`str.format` -method's format specifiers in :ref:`formatstrings`. - -For use cases which require exact decimal representation, try using the -:mod:`decimal` module which implements decimal arithmetic suitable for -accounting applications and high-precision applications. - -Another form of exact arithmetic is supported by the :mod:`fractions` module -which implements arithmetic based on rational numbers (so the numbers like -1/3 can be represented exactly). - -If you are a heavy user of floating-point operations you should take a look -at the NumPy package and many other packages for mathematical and -statistical operations supplied by the SciPy project. See . - -Python provides tools that may help on those rare occasions when you really -*do* want to know the exact value of a float. The -:meth:`float.as_integer_ratio` method expresses the value of a float as a -fraction:: - - >>> x = 3.14159 - >>> x.as_integer_ratio() - (3537115888337719, 1125899906842624) - -Since the ratio is exact, it can be used to losslessly recreate the -original value:: - - >>> x == 3537115888337719 / 1125899906842624 - True - -The :meth:`float.hex` method expresses a float in hexadecimal (base -16), again giving the exact value stored by your computer:: - - >>> x.hex() - '0x1.921f9f01b866ep+1' - -This precise hexadecimal representation can be used to reconstruct -the float value exactly:: - - >>> x == float.fromhex('0x1.921f9f01b866ep+1') - True - -Since the representation is exact, it is useful for reliably porting values -across different versions of Python (platform independence) and exchanging -data with other languages that support the same format (such as Java and C99). - -Another helpful tool is the :func:`math.fsum` function which helps mitigate -loss-of-precision during summation. It tracks "lost digits" as values are -added onto a running total. That can make a difference in overall accuracy -so that the errors do not accumulate to the point where they affect the -final total: - - >>> sum([0.1] * 10) == 1.0 - False - >>> math.fsum([0.1] * 10) == 1.0 - True - -.. _tut-fp-error: - -Representation Error -==================== - -This section explains the "0.1" example in detail, and shows how you can perform -an exact analysis of cases like this yourself. Basic familiarity with binary -floating-point representation is assumed. - -:dfn:`Representation error` refers to the fact that some (most, actually) -decimal fractions cannot be represented exactly as binary (base 2) fractions. -This is the chief reason why Python (or Perl, C, C++, Java, Fortran, and many -others) often won't display the exact decimal number you expect. - -Why is that? 1/10 is not exactly representable as a binary fraction. Since at -least 2000, almost all machines use IEEE 754 binary floating-point arithmetic, -and almost all platforms map Python floats to IEEE 754 binary64 "double -precision" values. IEEE 754 binary64 values contain 53 bits of precision, so -on input the computer strives to convert 0.1 to the closest fraction it can of -the form *J*/2**\ *N* where *J* is an integer containing exactly 53 bits. -Rewriting -:: - - 1 / 10 ~= J / (2**N) - -as :: - - J ~= 2**N / 10 - -and recalling that *J* has exactly 53 bits (is ``>= 2**52`` but ``< 2**53``), -the best value for *N* is 56:: - - >>> 2**52 <= 2**56 // 10 < 2**53 - True - -That is, 56 is the only value for *N* that leaves *J* with exactly 53 bits. The -best possible value for *J* is then that quotient rounded:: - - >>> q, r = divmod(2**56, 10) - >>> r - 6 - -Since the remainder is more than half of 10, the best approximation is obtained -by rounding up:: - - >>> q+1 - 7205759403792794 - -Therefore the best possible approximation to 1/10 in IEEE 754 double precision -is:: - - 7205759403792794 / 2 ** 56 - -Dividing both the numerator and denominator by two reduces the fraction to:: - - 3602879701896397 / 2 ** 55 - -Note that since we rounded up, this is actually a little bit larger than 1/10; -if we had not rounded up, the quotient would have been a little bit smaller than -1/10. But in no case can it be *exactly* 1/10! - -So the computer never "sees" 1/10: what it sees is the exact fraction given -above, the best IEEE 754 double approximation it can get: - - >>> 0.1 * 2 ** 55 - 3602879701896397.0 - -If we multiply that fraction by 10\*\*55, we can see the value out to -55 decimal digits:: - - >>> 3602879701896397 * 10 ** 55 // 2 ** 55 - 1000000000000000055511151231257827021181583404541015625 - -meaning that the exact number stored in the computer is equal to -the decimal value 0.1000000000000000055511151231257827021181583404541015625. -Instead of displaying the full decimal value, many languages (including -older versions of Python), round the result to 17 significant digits:: - - >>> format(0.1, '.17f') - '0.10000000000000001' - -The :mod:`fractions` and :mod:`decimal` modules make these calculations -easy:: - - >>> from decimal import Decimal - >>> from fractions import Fraction - - >>> Fraction.from_float(0.1) - Fraction(3602879701896397, 36028797018963968) - - >>> (0.1).as_integer_ratio() - (3602879701896397, 36028797018963968) - - >>> Decimal.from_float(0.1) - Decimal('0.1000000000000000055511151231257827021181583404541015625') - - >>> format(Decimal.from_float(0.1), '.17') - '0.10000000000000001' diff --git a/Doc/tutorial/index.rst b/Doc/tutorial/index.rst deleted file mode 100644 index 96791f88c867ab..00000000000000 --- a/Doc/tutorial/index.rst +++ /dev/null @@ -1,60 +0,0 @@ -.. _tutorial-index: - -###################### - The Python Tutorial -###################### - -Python is an easy to learn, powerful programming language. It has efficient -high-level data structures and a simple but effective approach to -object-oriented programming. Python's elegant syntax and dynamic typing, -together with its interpreted nature, make it an ideal language for scripting -and rapid application development in many areas on most platforms. - -The Python interpreter and the extensive standard library are freely available -in source or binary form for all major platforms from the Python web site, -https://www.python.org/, and may be freely distributed. The same site also -contains distributions of and pointers to many free third party Python modules, -programs and tools, and additional documentation. - -The Python interpreter is easily extended with new functions and data types -implemented in C or C++ (or other languages callable from C). Python is also -suitable as an extension language for customizable applications. - -This tutorial introduces the reader informally to the basic concepts and -features of the Python language and system. It helps to have a Python -interpreter handy for hands-on experience, but all examples are self-contained, -so the tutorial can be read off-line as well. - -For a description of standard objects and modules, see :ref:`library-index`. -:ref:`reference-index` gives a more formal definition of the language. To write -extensions in C or C++, read :ref:`extending-index` and -:ref:`c-api-index`. There are also several books covering Python in depth. - -This tutorial does not attempt to be comprehensive and cover every single -feature, or even every commonly used feature. Instead, it introduces many of -Python's most noteworthy features, and will give you a good idea of the -language's flavor and style. After reading it, you will be able to read and -write Python modules and programs, and you will be ready to learn more about the -various Python library modules described in :ref:`library-index`. - -The :ref:`glossary` is also worth going through. - -.. toctree:: - :numbered: - - appetite.rst - interpreter.rst - introduction.rst - controlflow.rst - datastructures.rst - modules.rst - inputoutput.rst - errors.rst - classes.rst - stdlib.rst - stdlib2.rst - venv.rst - whatnow.rst - interactive.rst - floatingpoint.rst - appendix.rst diff --git a/Doc/tutorial/inputoutput.rst b/Doc/tutorial/inputoutput.rst deleted file mode 100644 index fe9ca9ccb9c7e0..00000000000000 --- a/Doc/tutorial/inputoutput.rst +++ /dev/null @@ -1,529 +0,0 @@ -.. _tut-io: - -**************** -Input and Output -**************** - -There are several ways to present the output of a program; data can be printed -in a human-readable form, or written to a file for future use. This chapter will -discuss some of the possibilities. - - -.. _tut-formatting: - -Fancier Output Formatting -========================= - -So far we've encountered two ways of writing values: *expression statements* and -the :func:`print` function. (A third way is using the :meth:`~io.TextIOBase.write` method -of file objects; the standard output file can be referenced as ``sys.stdout``. -See the Library Reference for more information on this.) - -Often you'll want more control over the formatting of your output than simply -printing space-separated values. There are several ways to format output. - -* To use :ref:`formatted string literals `, begin a string - with ``f`` or ``F`` before the opening quotation mark or triple quotation mark. - Inside this string, you can write a Python expression between ``{`` and ``}`` - characters that can refer to variables or literal values. - - :: - - >>> year = 2016 - >>> event = 'Referendum' - >>> f'Results of the {year} {event}' - 'Results of the 2016 Referendum' - -* The :meth:`str.format` method of strings requires more manual - effort. You'll still use ``{`` and ``}`` to mark where a variable - will be substituted and can provide detailed formatting directives, - but you'll also need to provide the information to be formatted. - - :: - - >>> yes_votes = 42_572_654 - >>> no_votes = 43_132_495 - >>> percentage = yes_votes / (yes_votes + no_votes) - >>> '{:-9} YES votes {:2.2%}'.format(yes_votes, percentage) - ' 42572654 YES votes 49.67%' - -* Finally, you can do all the string handling yourself by using string slicing and - concatenation operations to create any layout you can imagine. The - string type has some methods that perform useful operations for padding - strings to a given column width. - -When you don't need fancy output but just want a quick display of some -variables for debugging purposes, you can convert any value to a string with -the :func:`repr` or :func:`str` functions. - -The :func:`str` function is meant to return representations of values which are -fairly human-readable, while :func:`repr` is meant to generate representations -which can be read by the interpreter (or will force a :exc:`SyntaxError` if -there is no equivalent syntax). For objects which don't have a particular -representation for human consumption, :func:`str` will return the same value as -:func:`repr`. Many values, such as numbers or structures like lists and -dictionaries, have the same representation using either function. Strings, in -particular, have two distinct representations. - -Some examples:: - - >>> s = 'Hello, world.' - >>> str(s) - 'Hello, world.' - >>> repr(s) - "'Hello, world.'" - >>> str(1/7) - '0.14285714285714285' - >>> x = 10 * 3.25 - >>> y = 200 * 200 - >>> s = 'The value of x is ' + repr(x) + ', and y is ' + repr(y) + '...' - >>> print(s) - The value of x is 32.5, and y is 40000... - >>> # The repr() of a string adds string quotes and backslashes: - ... hello = 'hello, world\n' - >>> hellos = repr(hello) - >>> print(hellos) - 'hello, world\n' - >>> # The argument to repr() may be any Python object: - ... repr((x, y, ('spam', 'eggs'))) - "(32.5, 40000, ('spam', 'eggs'))" - -The :mod:`string` module contains a :class:`~string.Template` class that offers -yet another way to substitute values into strings, using placeholders like -``$x`` and replacing them with values from a dictionary, but offers much less -control of the formatting. - - -.. _tut-f-strings: - -Formatted String Literals -------------------------- - -:ref:`Formatted string literals ` (also called f-strings for -short) let you include the value of Python expressions inside a string by -prefixing the string with ``f`` or ``F`` and writing expressions as -``{expression}``. - -An optional format specifier can follow the expression. This allows greater -control over how the value is formatted. The following example rounds pi to -three places after the decimal:: - - >>> import math - >>> print(f'The value of pi is approximately {math.pi:.3f}.') - The value of pi is approximately 3.142. - -Passing an integer after the ``':'`` will cause that field to be a minimum -number of characters wide. This is useful for making columns line up. :: - - >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 7678} - >>> for name, phone in table.items(): - ... print(f'{name:10} ==> {phone:10d}') - ... - Sjoerd ==> 4127 - Jack ==> 4098 - Dcab ==> 7678 - -Other modifiers can be used to convert the value before it is formatted. -``'!a'`` applies :func:`ascii`, ``'!s'`` applies :func:`str`, and ``'!r'`` -applies :func:`repr`:: - - >>> animals = 'eels' - >>> print(f'My hovercraft is full of {animals}.') - My hovercraft is full of eels. - >>> print(f'My hovercraft is full of {animals!r}.') - My hovercraft is full of 'eels'. - -The ``=`` specifier can be used to expand an expression to the text of the -expression, an equal sign, then the representation of the evaluated expression: - - >>> bugs = 'roaches' - >>> count = 13 - >>> area = 'living room' - >>> print(f'Debugging {bugs=} {count=} {area=}') - Debugging bugs='roaches' count=13 area='living room' - -See :ref:`self-documenting expressions ` for more information -on the ``=`` specifier. For a reference on these format specifications, see -the reference guide for the :ref:`formatspec`. - -.. _tut-string-format: - -The String format() Method --------------------------- - -Basic usage of the :meth:`str.format` method looks like this:: - - >>> print('We are the {} who say "{}!"'.format('knights', 'Ni')) - We are the knights who say "Ni!" - -The brackets and characters within them (called format fields) are replaced with -the objects passed into the :meth:`str.format` method. A number in the -brackets can be used to refer to the position of the object passed into the -:meth:`str.format` method. :: - - >>> print('{0} and {1}'.format('spam', 'eggs')) - spam and eggs - >>> print('{1} and {0}'.format('spam', 'eggs')) - eggs and spam - -If keyword arguments are used in the :meth:`str.format` method, their values -are referred to by using the name of the argument. :: - - >>> print('This {food} is {adjective}.'.format( - ... food='spam', adjective='absolutely horrible')) - This spam is absolutely horrible. - -Positional and keyword arguments can be arbitrarily combined:: - - >>> print('The story of {0}, {1}, and {other}.'.format('Bill', 'Manfred', - ... other='Georg')) - The story of Bill, Manfred, and Georg. - -If you have a really long format string that you don't want to split up, it -would be nice if you could reference the variables to be formatted by name -instead of by position. This can be done by simply passing the dict and using -square brackets ``'[]'`` to access the keys. :: - - >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 8637678} - >>> print('Jack: {0[Jack]:d}; Sjoerd: {0[Sjoerd]:d}; ' - ... 'Dcab: {0[Dcab]:d}'.format(table)) - Jack: 4098; Sjoerd: 4127; Dcab: 8637678 - -This could also be done by passing the ``table`` dictionary as keyword arguments with the ``**`` -notation. :: - - >>> table = {'Sjoerd': 4127, 'Jack': 4098, 'Dcab': 8637678} - >>> print('Jack: {Jack:d}; Sjoerd: {Sjoerd:d}; Dcab: {Dcab:d}'.format(**table)) - Jack: 4098; Sjoerd: 4127; Dcab: 8637678 - -This is particularly useful in combination with the built-in function -:func:`vars`, which returns a dictionary containing all local variables. - -As an example, the following lines produce a tidily aligned -set of columns giving integers and their squares and cubes:: - - >>> for x in range(1, 11): - ... print('{0:2d} {1:3d} {2:4d}'.format(x, x*x, x*x*x)) - ... - 1 1 1 - 2 4 8 - 3 9 27 - 4 16 64 - 5 25 125 - 6 36 216 - 7 49 343 - 8 64 512 - 9 81 729 - 10 100 1000 - -For a complete overview of string formatting with :meth:`str.format`, see -:ref:`formatstrings`. - - -Manual String Formatting ------------------------- - -Here's the same table of squares and cubes, formatted manually:: - - >>> for x in range(1, 11): - ... print(repr(x).rjust(2), repr(x*x).rjust(3), end=' ') - ... # Note use of 'end' on previous line - ... print(repr(x*x*x).rjust(4)) - ... - 1 1 1 - 2 4 8 - 3 9 27 - 4 16 64 - 5 25 125 - 6 36 216 - 7 49 343 - 8 64 512 - 9 81 729 - 10 100 1000 - -(Note that the one space between each column was added by the -way :func:`print` works: it always adds spaces between its arguments.) - -The :meth:`str.rjust` method of string objects right-justifies a string in a -field of a given width by padding it with spaces on the left. There are -similar methods :meth:`str.ljust` and :meth:`str.center`. These methods do -not write anything, they just return a new string. If the input string is too -long, they don't truncate it, but return it unchanged; this will mess up your -column lay-out but that's usually better than the alternative, which would be -lying about a value. (If you really want truncation you can always add a -slice operation, as in ``x.ljust(n)[:n]``.) - -There is another method, :meth:`str.zfill`, which pads a numeric string on the -left with zeros. It understands about plus and minus signs:: - - >>> '12'.zfill(5) - '00012' - >>> '-3.14'.zfill(7) - '-003.14' - >>> '3.14159265359'.zfill(5) - '3.14159265359' - - -Old string formatting ---------------------- - -The % operator (modulo) can also be used for string formatting. Given ``'string' -% values``, instances of ``%`` in ``string`` are replaced with zero or more -elements of ``values``. This operation is commonly known as string -interpolation. For example:: - - >>> import math - >>> print('The value of pi is approximately %5.3f.' % math.pi) - The value of pi is approximately 3.142. - -More information can be found in the :ref:`old-string-formatting` section. - - -.. _tut-files: - -Reading and Writing Files -========================= - -.. index:: - pair: built-in function; open - pair: object; file - -:func:`open` returns a :term:`file object`, and is most commonly used with -two positional arguments and one keyword argument: -``open(filename, mode, encoding=None)`` - -:: - - >>> f = open('workfile', 'w', encoding="utf-8") - -.. XXX str(f) is - - >>> print(f) - - -The first argument is a string containing the filename. The second argument is -another string containing a few characters describing the way in which the file -will be used. *mode* can be ``'r'`` when the file will only be read, ``'w'`` -for only writing (an existing file with the same name will be erased), and -``'a'`` opens the file for appending; any data written to the file is -automatically added to the end. ``'r+'`` opens the file for both reading and -writing. The *mode* argument is optional; ``'r'`` will be assumed if it's -omitted. - -Normally, files are opened in :dfn:`text mode`, that means, you read and write -strings from and to the file, which are encoded in a specific *encoding*. -If *encoding* is not specified, the default is platform dependent -(see :func:`open`). -Because UTF-8 is the modern de-facto standard, ``encoding="utf-8"`` is -recommended unless you know that you need to use a different encoding. -Appending a ``'b'`` to the mode opens the file in :dfn:`binary mode`. -Binary mode data is read and written as :class:`bytes` objects. -You can not specify *encoding* when opening file in binary mode. - -In text mode, the default when reading is to convert platform-specific line -endings (``\n`` on Unix, ``\r\n`` on Windows) to just ``\n``. When writing in -text mode, the default is to convert occurrences of ``\n`` back to -platform-specific line endings. This behind-the-scenes modification -to file data is fine for text files, but will corrupt binary data like that in -:file:`JPEG` or :file:`EXE` files. Be very careful to use binary mode when -reading and writing such files. - -It is good practice to use the :keyword:`with` keyword when dealing -with file objects. The advantage is that the file is properly closed -after its suite finishes, even if an exception is raised at some -point. Using :keyword:`!with` is also much shorter than writing -equivalent :keyword:`try`\ -\ :keyword:`finally` blocks:: - - >>> with open('workfile', encoding="utf-8") as f: - ... read_data = f.read() - - >>> # We can check that the file has been automatically closed. - >>> f.closed - True - -If you're not using the :keyword:`with` keyword, then you should call -``f.close()`` to close the file and immediately free up any system -resources used by it. - -.. warning:: - Calling ``f.write()`` without using the :keyword:`!with` keyword or calling - ``f.close()`` **might** result in the arguments - of ``f.write()`` not being completely written to the disk, even if the - program exits successfully. - -.. - See also https://bugs.python.org/issue17852 - -After a file object is closed, either by a :keyword:`with` statement -or by calling ``f.close()``, attempts to use the file object will -automatically fail. :: - - >>> f.close() - >>> f.read() - Traceback (most recent call last): - File "", line 1, in - ValueError: I/O operation on closed file. - - -.. _tut-filemethods: - -Methods of File Objects ------------------------ - -The rest of the examples in this section will assume that a file object called -``f`` has already been created. - -To read a file's contents, call ``f.read(size)``, which reads some quantity of -data and returns it as a string (in text mode) or bytes object (in binary mode). -*size* is an optional numeric argument. When *size* is omitted or negative, the -entire contents of the file will be read and returned; it's your problem if the -file is twice as large as your machine's memory. Otherwise, at most *size* -characters (in text mode) or *size* bytes (in binary mode) are read and returned. -If the end of the file has been reached, ``f.read()`` will return an empty -string (``''``). :: - - >>> f.read() - 'This is the entire file.\n' - >>> f.read() - '' - -``f.readline()`` reads a single line from the file; a newline character (``\n``) -is left at the end of the string, and is only omitted on the last line of the -file if the file doesn't end in a newline. This makes the return value -unambiguous; if ``f.readline()`` returns an empty string, the end of the file -has been reached, while a blank line is represented by ``'\n'``, a string -containing only a single newline. :: - - >>> f.readline() - 'This is the first line of the file.\n' - >>> f.readline() - 'Second line of the file\n' - >>> f.readline() - '' - -For reading lines from a file, you can loop over the file object. This is memory -efficient, fast, and leads to simple code:: - - >>> for line in f: - ... print(line, end='') - ... - This is the first line of the file. - Second line of the file - -If you want to read all the lines of a file in a list you can also use -``list(f)`` or ``f.readlines()``. - -``f.write(string)`` writes the contents of *string* to the file, returning -the number of characters written. :: - - >>> f.write('This is a test\n') - 15 - -Other types of objects need to be converted -- either to a string (in text mode) -or a bytes object (in binary mode) -- before writing them:: - - >>> value = ('the answer', 42) - >>> s = str(value) # convert the tuple to string - >>> f.write(s) - 18 - -``f.tell()`` returns an integer giving the file object's current position in the file -represented as number of bytes from the beginning of the file when in binary mode and -an opaque number when in text mode. - -To change the file object's position, use ``f.seek(offset, whence)``. The position is computed -from adding *offset* to a reference point; the reference point is selected by -the *whence* argument. A *whence* value of 0 measures from the beginning -of the file, 1 uses the current file position, and 2 uses the end of the file as -the reference point. *whence* can be omitted and defaults to 0, using the -beginning of the file as the reference point. :: - - >>> f = open('workfile', 'rb+') - >>> f.write(b'0123456789abcdef') - 16 - >>> f.seek(5) # Go to the 6th byte in the file - 5 - >>> f.read(1) - b'5' - >>> f.seek(-3, 2) # Go to the 3rd byte before the end - 13 - >>> f.read(1) - b'd' - -In text files (those opened without a ``b`` in the mode string), only seeks -relative to the beginning of the file are allowed (the exception being seeking -to the very file end with ``seek(0, 2)``) and the only valid *offset* values are -those returned from the ``f.tell()``, or zero. Any other *offset* value produces -undefined behaviour. - -File objects have some additional methods, such as :meth:`~io.IOBase.isatty` and -:meth:`~io.IOBase.truncate` which are less frequently used; consult the Library -Reference for a complete guide to file objects. - - -.. _tut-json: - -Saving structured data with :mod:`json` ---------------------------------------- - -.. index:: pair: module; json - -Strings can easily be written to and read from a file. Numbers take a bit more -effort, since the :meth:`~io.TextIOBase.read` method only returns strings, which will have to -be passed to a function like :func:`int`, which takes a string like ``'123'`` -and returns its numeric value 123. When you want to save more complex data -types like nested lists and dictionaries, parsing and serializing by hand -becomes complicated. - -Rather than having users constantly writing and debugging code to save -complicated data types to files, Python allows you to use the popular data -interchange format called `JSON (JavaScript Object Notation) -`_. The standard module called :mod:`json` can take Python -data hierarchies, and convert them to string representations; this process is -called :dfn:`serializing`. Reconstructing the data from the string representation -is called :dfn:`deserializing`. Between serializing and deserializing, the -string representing the object may have been stored in a file or data, or -sent over a network connection to some distant machine. - -.. note:: - The JSON format is commonly used by modern applications to allow for data - exchange. Many programmers are already familiar with it, which makes - it a good choice for interoperability. - -If you have an object ``x``, you can view its JSON string representation with a -simple line of code:: - - >>> import json - >>> x = [1, 'simple', 'list'] - >>> json.dumps(x) - '[1, "simple", "list"]' - -Another variant of the :func:`~json.dumps` function, called :func:`~json.dump`, -simply serializes the object to a :term:`text file`. So if ``f`` is a -:term:`text file` object opened for writing, we can do this:: - - json.dump(x, f) - -To decode the object again, if ``f`` is a :term:`binary file` or -:term:`text file` object which has been opened for reading:: - - x = json.load(f) - -.. note:: - JSON files must be encoded in UTF-8. Use ``encoding="utf-8"`` when opening - JSON file as a :term:`text file` for both of reading and writing. - -This simple serialization technique can handle lists and dictionaries, but -serializing arbitrary class instances in JSON requires a bit of extra effort. -The reference for the :mod:`json` module contains an explanation of this. - -.. seealso:: - - :mod:`pickle` - the pickle module - - Contrary to :ref:`JSON `, *pickle* is a protocol which allows - the serialization of arbitrarily complex Python objects. As such, it is - specific to Python and cannot be used to communicate with applications - written in other languages. It is also insecure by default: - deserializing pickle data coming from an untrusted source can execute - arbitrary code, if the data was crafted by a skilled attacker. diff --git a/Doc/tutorial/interactive.rst b/Doc/tutorial/interactive.rst deleted file mode 100644 index 4e054c4e6c2c32..00000000000000 --- a/Doc/tutorial/interactive.rst +++ /dev/null @@ -1,54 +0,0 @@ -.. _tut-interacting: - -************************************************** -Interactive Input Editing and History Substitution -************************************************** - -Some versions of the Python interpreter support editing of the current input -line and history substitution, similar to facilities found in the Korn shell and -the GNU Bash shell. This is implemented using the `GNU Readline`_ library, -which supports various styles of editing. This library has its own -documentation which we won't duplicate here. - - -.. _tut-keybindings: - -Tab Completion and History Editing -================================== - -Completion of variable and module names is -:ref:`automatically enabled ` at interpreter startup so -that the :kbd:`Tab` key invokes the completion function; it looks at -Python statement names, the current local variables, and the available -module names. For dotted expressions such as ``string.a``, it will evaluate -the expression up to the final ``'.'`` and then suggest completions from -the attributes of the resulting object. Note that this may execute -application-defined code if an object with a :meth:`~object.__getattr__` method -is part of the expression. The default configuration also saves your -history into a file named :file:`.python_history` in your user directory. -The history will be available again during the next interactive interpreter -session. - - -.. _tut-commentary: - -Alternatives to the Interactive Interpreter -=========================================== - -This facility is an enormous step forward compared to earlier versions of the -interpreter; however, some wishes are left: It would be nice if the proper -indentation were suggested on continuation lines (the parser knows if an indent -token is required next). The completion mechanism might use the interpreter's -symbol table. A command to check (or even suggest) matching parentheses, -quotes, etc., would also be useful. - -One alternative enhanced interactive interpreter that has been around for quite -some time is IPython_, which features tab completion, object exploration and -advanced history management. It can also be thoroughly customized and embedded -into other applications. Another similar enhanced interactive environment is -bpython_. - - -.. _GNU Readline: https://tiswww.case.edu/php/chet/readline/rltop.html -.. _IPython: https://ipython.org/ -.. _bpython: https://bpython-interpreter.org/ diff --git a/Doc/tutorial/interpreter.rst b/Doc/tutorial/interpreter.rst deleted file mode 100644 index 08851cb8fda012..00000000000000 --- a/Doc/tutorial/interpreter.rst +++ /dev/null @@ -1,163 +0,0 @@ -.. _tut-using: - -**************************** -Using the Python Interpreter -**************************** - - -.. _tut-invoking: - -Invoking the Interpreter -======================== - -The Python interpreter is usually installed as :file:`/usr/local/bin/python3.11` -on those machines where it is available; putting :file:`/usr/local/bin` in your -Unix shell's search path makes it possible to start it by typing the command: - -.. code-block:: text - - python3.11 - -to the shell. [#]_ Since the choice of the directory where the interpreter lives -is an installation option, other places are possible; check with your local -Python guru or system administrator. (E.g., :file:`/usr/local/python` is a -popular alternative location.) - -On Windows machines where you have installed Python from the :ref:`Microsoft Store -`, the :file:`python3.11` command will be available. If you have -the :ref:`py.exe launcher ` installed, you can use the :file:`py` -command. See :ref:`setting-envvars` for other ways to launch Python. - -Typing an end-of-file character (:kbd:`Control-D` on Unix, :kbd:`Control-Z` on -Windows) at the primary prompt causes the interpreter to exit with a zero exit -status. If that doesn't work, you can exit the interpreter by typing the -following command: ``quit()``. - -The interpreter's line-editing features include interactive editing, history -substitution and code completion on systems that support the `GNU Readline -`_ library. -Perhaps the quickest check to see whether command line editing is supported is -typing :kbd:`Control-P` to the first Python prompt you get. If it beeps, you -have command line editing; see Appendix :ref:`tut-interacting` for an -introduction to the keys. If nothing appears to happen, or if ``^P`` is -echoed, command line editing isn't available; you'll only be able to use -backspace to remove characters from the current line. - -The interpreter operates somewhat like the Unix shell: when called with standard -input connected to a tty device, it reads and executes commands interactively; -when called with a file name argument or with a file as standard input, it reads -and executes a *script* from that file. - -A second way of starting the interpreter is ``python -c command [arg] ...``, -which executes the statement(s) in *command*, analogous to the shell's -:option:`-c` option. Since Python statements often contain spaces or other -characters that are special to the shell, it is usually advised to quote -*command* in its entirety. - -Some Python modules are also useful as scripts. These can be invoked using -``python -m module [arg] ...``, which executes the source file for *module* as -if you had spelled out its full name on the command line. - -When a script file is used, it is sometimes useful to be able to run the script -and enter interactive mode afterwards. This can be done by passing :option:`-i` -before the script. - -All command line options are described in :ref:`using-on-general`. - - -.. _tut-argpassing: - -Argument Passing ----------------- - -When known to the interpreter, the script name and additional arguments -thereafter are turned into a list of strings and assigned to the ``argv`` -variable in the ``sys`` module. You can access this list by executing ``import -sys``. The length of the list is at least one; when no script and no arguments -are given, ``sys.argv[0]`` is an empty string. When the script name is given as -``'-'`` (meaning standard input), ``sys.argv[0]`` is set to ``'-'``. When -:option:`-c` *command* is used, ``sys.argv[0]`` is set to ``'-c'``. When -:option:`-m` *module* is used, ``sys.argv[0]`` is set to the full name of the -located module. Options found after :option:`-c` *command* or :option:`-m` -*module* are not consumed by the Python interpreter's option processing but -left in ``sys.argv`` for the command or module to handle. - - -.. _tut-interactive: - -Interactive Mode ----------------- - -When commands are read from a tty, the interpreter is said to be in *interactive -mode*. In this mode it prompts for the next command with the *primary prompt*, -usually three greater-than signs (``>>>``); for continuation lines it prompts -with the *secondary prompt*, by default three dots (``...``). The interpreter -prints a welcome message stating its version number and a copyright notice -before printing the first prompt: - -.. code-block:: shell-session - - $ python3.11 - Python 3.11 (default, April 4 2021, 09:25:04) - [GCC 10.2.0] on linux - Type "help", "copyright", "credits" or "license" for more information. - >>> - -.. XXX update for new releases - -Continuation lines are needed when entering a multi-line construct. As an -example, take a look at this :keyword:`if` statement:: - - >>> the_world_is_flat = True - >>> if the_world_is_flat: - ... print("Be careful not to fall off!") - ... - Be careful not to fall off! - - -For more on interactive mode, see :ref:`tut-interac`. - - -.. _tut-interp: - -The Interpreter and Its Environment -=================================== - - -.. _tut-source-encoding: - -Source Code Encoding --------------------- - -By default, Python source files are treated as encoded in UTF-8. In that -encoding, characters of most languages in the world can be used simultaneously -in string literals, identifiers and comments --- although the standard library -only uses ASCII characters for identifiers, a convention that any portable code -should follow. To display all these characters properly, your editor must -recognize that the file is UTF-8, and it must use a font that supports all the -characters in the file. - -To declare an encoding other than the default one, a special comment line -should be added as the *first* line of the file. The syntax is as follows:: - - # -*- coding: encoding -*- - -where *encoding* is one of the valid :mod:`codecs` supported by Python. - -For example, to declare that Windows-1252 encoding is to be used, the first -line of your source code file should be:: - - # -*- coding: cp1252 -*- - -One exception to the *first line* rule is when the source code starts with a -:ref:`UNIX "shebang" line `. In this case, the encoding -declaration should be added as the second line of the file. For example:: - - #!/usr/bin/env python3 - # -*- coding: cp1252 -*- - -.. rubric:: Footnotes - -.. [#] On Unix, the Python 3.x interpreter is by default not installed with the - executable named ``python``, so that it does not conflict with a - simultaneously installed Python 2.x executable. diff --git a/Doc/tutorial/introduction.rst b/Doc/tutorial/introduction.rst deleted file mode 100644 index 731655a5885d14..00000000000000 --- a/Doc/tutorial/introduction.rst +++ /dev/null @@ -1,558 +0,0 @@ -.. _tut-informal: - -********************************** -An Informal Introduction to Python -********************************** - -In the following examples, input and output are distinguished by the presence or -absence of prompts (:term:`>>>` and :term:`...`): to repeat the example, you must type -everything after the prompt, when the prompt appears; lines that do not begin -with a prompt are output from the interpreter. Note that a secondary prompt on a -line by itself in an example means you must type a blank line; this is used to -end a multi-line command. - -.. only:: html - - You can toggle the display of prompts and output by clicking on ``>>>`` - in the upper-right corner of an example box. If you hide the prompts - and output for an example, then you can easily copy and paste the input - lines into your interpreter. - -.. index:: single: # (hash); comment - -Many of the examples in this manual, even those entered at the interactive -prompt, include comments. Comments in Python start with the hash character, -``#``, and extend to the end of the physical line. A comment may appear at the -start of a line or following whitespace or code, but not within a string -literal. A hash character within a string literal is just a hash character. -Since comments are to clarify code and are not interpreted by Python, they may -be omitted when typing in examples. - -Some examples:: - - # this is the first comment - spam = 1 # and this is the second comment - # ... and now a third! - text = "# This is not a comment because it's inside quotes." - - -.. _tut-calculator: - -Using Python as a Calculator -============================ - -Let's try some simple Python commands. Start the interpreter and wait for the -primary prompt, ``>>>``. (It shouldn't take long.) - - -.. _tut-numbers: - -Numbers -------- - -The interpreter acts as a simple calculator: you can type an expression at it -and it will write the value. Expression syntax is straightforward: the -operators ``+``, ``-``, ``*`` and ``/`` can be used to perform -arithmetic; parentheses (``()``) can be used for grouping. -For example:: - - >>> 2 + 2 - 4 - >>> 50 - 5*6 - 20 - >>> (50 - 5*6) / 4 - 5.0 - >>> 8 / 5 # division always returns a floating point number - 1.6 - -The integer numbers (e.g. ``2``, ``4``, ``20``) have type :class:`int`, -the ones with a fractional part (e.g. ``5.0``, ``1.6``) have type -:class:`float`. We will see more about numeric types later in the tutorial. - -Division (``/``) always returns a float. To do :term:`floor division` and -get an integer result you can use the ``//`` operator; to calculate -the remainder you can use ``%``:: - - >>> 17 / 3 # classic division returns a float - 5.666666666666667 - >>> - >>> 17 // 3 # floor division discards the fractional part - 5 - >>> 17 % 3 # the % operator returns the remainder of the division - 2 - >>> 5 * 3 + 2 # floored quotient * divisor + remainder - 17 - -With Python, it is possible to use the ``**`` operator to calculate powers [#]_:: - - >>> 5 ** 2 # 5 squared - 25 - >>> 2 ** 7 # 2 to the power of 7 - 128 - -The equal sign (``=``) is used to assign a value to a variable. Afterwards, no -result is displayed before the next interactive prompt:: - - >>> width = 20 - >>> height = 5 * 9 - >>> width * height - 900 - -If a variable is not "defined" (assigned a value), trying to use it will -give you an error:: - - >>> n # try to access an undefined variable - Traceback (most recent call last): - File "", line 1, in - NameError: name 'n' is not defined - -There is full support for floating point; operators with mixed type operands -convert the integer operand to floating point:: - - >>> 4 * 3.75 - 1 - 14.0 - -In interactive mode, the last printed expression is assigned to the variable -``_``. This means that when you are using Python as a desk calculator, it is -somewhat easier to continue calculations, for example:: - - >>> tax = 12.5 / 100 - >>> price = 100.50 - >>> price * tax - 12.5625 - >>> price + _ - 113.0625 - >>> round(_, 2) - 113.06 - -This variable should be treated as read-only by the user. Don't explicitly -assign a value to it --- you would create an independent local variable with the -same name masking the built-in variable with its magic behavior. - -In addition to :class:`int` and :class:`float`, Python supports other types of -numbers, such as :class:`~decimal.Decimal` and :class:`~fractions.Fraction`. -Python also has built-in support for :ref:`complex numbers `, -and uses the ``j`` or ``J`` suffix to indicate the imaginary part -(e.g. ``3+5j``). - - -.. _tut-strings: - -Text ----- - -Python can manipulate text (represented by type :class:`str`, so-called -"strings") as well as numbers. This includes characters "``!``", words -"``rabbit``", names "``Paris``", sentences "``Got your back.``", etc. -"``Yay! :)``". They can be enclosed in single quotes (``'...'``) or double -quotes (``"..."``) with the same result [#]_. - - >>> 'spam eggs' # single quotes - 'spam eggs' - >>> "Paris rabbit got your back :)! Yay!" # double quotes - 'Paris rabbit got your back :)! Yay!' - >>> '1975' # digits and numerals enclosed in quotes are also strings - '1975' - -To quote a quote, we need to "escape" it, by preceding it with ``\``. -Alternatively, we can use the other type of quotation marks:: - - >>> 'doesn\'t' # use \' to escape the single quote... - "doesn't" - >>> "doesn't" # ...or use double quotes instead - "doesn't" - >>> '"Yes," they said.' - '"Yes," they said.' - >>> "\"Yes,\" they said." - '"Yes," they said.' - >>> '"Isn\'t," they said.' - '"Isn\'t," they said.' - -In the Python shell, the string definition and output string can look -different. The :func:`print` function produces a more readable output, by -omitting the enclosing quotes and by printing escaped and special characters:: - - >>> s = 'First line.\nSecond line.' # \n means newline - >>> s # without print(), special characters are included in the string - 'First line.\nSecond line.' - >>> print(s) # with print(), special characters are interpreted, so \n produces new line - First line. - Second line. - -If you don't want characters prefaced by ``\`` to be interpreted as -special characters, you can use *raw strings* by adding an ``r`` before -the first quote:: - - >>> print('C:\some\name') # here \n means newline! - C:\some - ame - >>> print(r'C:\some\name') # note the r before the quote - C:\some\name - -There is one subtle aspect to raw strings: a raw string may not end in -an odd number of ``\`` characters; see -:ref:`the FAQ entry ` for more information -and workarounds. - -String literals can span multiple lines. One way is using triple-quotes: -``"""..."""`` or ``'''...'''``. End of lines are automatically -included in the string, but it's possible to prevent this by adding a ``\`` at -the end of the line. The following example:: - - print("""\ - Usage: thingy [OPTIONS] - -h Display this usage message - -H hostname Hostname to connect to - """) - -produces the following output (note that the initial newline is not included): - -.. code-block:: text - - Usage: thingy [OPTIONS] - -h Display this usage message - -H hostname Hostname to connect to - -Strings can be concatenated (glued together) with the ``+`` operator, and -repeated with ``*``:: - - >>> # 3 times 'un', followed by 'ium' - >>> 3 * 'un' + 'ium' - 'unununium' - -Two or more *string literals* (i.e. the ones enclosed between quotes) next -to each other are automatically concatenated. :: - - >>> 'Py' 'thon' - 'Python' - -This feature is particularly useful when you want to break long strings:: - - >>> text = ('Put several strings within parentheses ' - ... 'to have them joined together.') - >>> text - 'Put several strings within parentheses to have them joined together.' - -This only works with two literals though, not with variables or expressions:: - - >>> prefix = 'Py' - >>> prefix 'thon' # can't concatenate a variable and a string literal - File "", line 1 - prefix 'thon' - ^^^^^^ - SyntaxError: invalid syntax - >>> ('un' * 3) 'ium' - File "", line 1 - ('un' * 3) 'ium' - ^^^^^ - SyntaxError: invalid syntax - -If you want to concatenate variables or a variable and a literal, use ``+``:: - - >>> prefix + 'thon' - 'Python' - -Strings can be *indexed* (subscripted), with the first character having index 0. -There is no separate character type; a character is simply a string of size -one:: - - >>> word = 'Python' - >>> word[0] # character in position 0 - 'P' - >>> word[5] # character in position 5 - 'n' - -Indices may also be negative numbers, to start counting from the right:: - - >>> word[-1] # last character - 'n' - >>> word[-2] # second-last character - 'o' - >>> word[-6] - 'P' - -Note that since -0 is the same as 0, negative indices start from -1. - -In addition to indexing, *slicing* is also supported. While indexing is used -to obtain individual characters, *slicing* allows you to obtain substring:: - - >>> word[0:2] # characters from position 0 (included) to 2 (excluded) - 'Py' - >>> word[2:5] # characters from position 2 (included) to 5 (excluded) - 'tho' - -Slice indices have useful defaults; an omitted first index defaults to zero, an -omitted second index defaults to the size of the string being sliced. :: - - >>> word[:2] # character from the beginning to position 2 (excluded) - 'Py' - >>> word[4:] # characters from position 4 (included) to the end - 'on' - >>> word[-2:] # characters from the second-last (included) to the end - 'on' - -Note how the start is always included, and the end always excluded. This -makes sure that ``s[:i] + s[i:]`` is always equal to ``s``:: - - >>> word[:2] + word[2:] - 'Python' - >>> word[:4] + word[4:] - 'Python' - -One way to remember how slices work is to think of the indices as pointing -*between* characters, with the left edge of the first character numbered 0. -Then the right edge of the last character of a string of *n* characters has -index *n*, for example:: - - +---+---+---+---+---+---+ - | P | y | t | h | o | n | - +---+---+---+---+---+---+ - 0 1 2 3 4 5 6 - -6 -5 -4 -3 -2 -1 - -The first row of numbers gives the position of the indices 0...6 in the string; -the second row gives the corresponding negative indices. The slice from *i* to -*j* consists of all characters between the edges labeled *i* and *j*, -respectively. - -For non-negative indices, the length of a slice is the difference of the -indices, if both are within bounds. For example, the length of ``word[1:3]`` is -2. - -Attempting to use an index that is too large will result in an error:: - - >>> word[42] # the word only has 6 characters - Traceback (most recent call last): - File "", line 1, in - IndexError: string index out of range - -However, out of range slice indexes are handled gracefully when used for -slicing:: - - >>> word[4:42] - 'on' - >>> word[42:] - '' - -Python strings cannot be changed --- they are :term:`immutable`. -Therefore, assigning to an indexed position in the string results in an error:: - - >>> word[0] = 'J' - Traceback (most recent call last): - File "", line 1, in - TypeError: 'str' object does not support item assignment - >>> word[2:] = 'py' - Traceback (most recent call last): - File "", line 1, in - TypeError: 'str' object does not support item assignment - -If you need a different string, you should create a new one:: - - >>> 'J' + word[1:] - 'Jython' - >>> word[:2] + 'py' - 'Pypy' - -The built-in function :func:`len` returns the length of a string:: - - >>> s = 'supercalifragilisticexpialidocious' - >>> len(s) - 34 - - -.. seealso:: - - :ref:`textseq` - Strings are examples of *sequence types*, and support the common - operations supported by such types. - - :ref:`string-methods` - Strings support a large number of methods for - basic transformations and searching. - - :ref:`f-strings` - String literals that have embedded expressions. - - :ref:`formatstrings` - Information about string formatting with :meth:`str.format`. - - :ref:`old-string-formatting` - The old formatting operations invoked when strings are - the left operand of the ``%`` operator are described in more detail here. - - -.. _tut-lists: - -Lists ------ - -Python knows a number of *compound* data types, used to group together other -values. The most versatile is the *list*, which can be written as a list of -comma-separated values (items) between square brackets. Lists might contain -items of different types, but usually the items all have the same type. :: - - >>> squares = [1, 4, 9, 16, 25] - >>> squares - [1, 4, 9, 16, 25] - -Like strings (and all other built-in :term:`sequence` types), lists can be -indexed and sliced:: - - >>> squares[0] # indexing returns the item - 1 - >>> squares[-1] - 25 - >>> squares[-3:] # slicing returns a new list - [9, 16, 25] - -All slice operations return a new list containing the requested elements. This -means that the following slice returns a -:ref:`shallow copy ` of the list:: - - >>> squares[:] - [1, 4, 9, 16, 25] - -Lists also support operations like concatenation:: - - >>> squares + [36, 49, 64, 81, 100] - [1, 4, 9, 16, 25, 36, 49, 64, 81, 100] - -Unlike strings, which are :term:`immutable`, lists are a :term:`mutable` -type, i.e. it is possible to change their content:: - - >>> cubes = [1, 8, 27, 65, 125] # something's wrong here - >>> 4 ** 3 # the cube of 4 is 64, not 65! - 64 - >>> cubes[3] = 64 # replace the wrong value - >>> cubes - [1, 8, 27, 64, 125] - -You can also add new items at the end of the list, by using -the :meth:`!list.append` *method* (we will see more about methods later):: - - >>> cubes.append(216) # add the cube of 6 - >>> cubes.append(7 ** 3) # and the cube of 7 - >>> cubes - [1, 8, 27, 64, 125, 216, 343] - -Assignment to slices is also possible, and this can even change the size of the -list or clear it entirely:: - - >>> letters = ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - >>> letters - ['a', 'b', 'c', 'd', 'e', 'f', 'g'] - >>> # replace some values - >>> letters[2:5] = ['C', 'D', 'E'] - >>> letters - ['a', 'b', 'C', 'D', 'E', 'f', 'g'] - >>> # now remove them - >>> letters[2:5] = [] - >>> letters - ['a', 'b', 'f', 'g'] - >>> # clear the list by replacing all the elements with an empty list - >>> letters[:] = [] - >>> letters - [] - -The built-in function :func:`len` also applies to lists:: - - >>> letters = ['a', 'b', 'c', 'd'] - >>> len(letters) - 4 - -It is possible to nest lists (create lists containing other lists), for -example:: - - >>> a = ['a', 'b', 'c'] - >>> n = [1, 2, 3] - >>> x = [a, n] - >>> x - [['a', 'b', 'c'], [1, 2, 3]] - >>> x[0] - ['a', 'b', 'c'] - >>> x[0][1] - 'b' - -.. _tut-firststeps: - -First Steps Towards Programming -=============================== - -Of course, we can use Python for more complicated tasks than adding two and two -together. For instance, we can write an initial sub-sequence of the -`Fibonacci series `_ -as follows:: - - >>> # Fibonacci series: - ... # the sum of two elements defines the next - ... a, b = 0, 1 - >>> while a < 10: - ... print(a) - ... a, b = b, a+b - ... - 0 - 1 - 1 - 2 - 3 - 5 - 8 - -This example introduces several new features. - -* The first line contains a *multiple assignment*: the variables ``a`` and ``b`` - simultaneously get the new values 0 and 1. On the last line this is used again, - demonstrating that the expressions on the right-hand side are all evaluated - first before any of the assignments take place. The right-hand side expressions - are evaluated from the left to the right. - -* The :keyword:`while` loop executes as long as the condition (here: ``a < 10``) - remains true. In Python, like in C, any non-zero integer value is true; zero is - false. The condition may also be a string or list value, in fact any sequence; - anything with a non-zero length is true, empty sequences are false. The test - used in the example is a simple comparison. The standard comparison operators - are written the same as in C: ``<`` (less than), ``>`` (greater than), ``==`` - (equal to), ``<=`` (less than or equal to), ``>=`` (greater than or equal to) - and ``!=`` (not equal to). - -* The *body* of the loop is *indented*: indentation is Python's way of grouping - statements. At the interactive prompt, you have to type a tab or space(s) for - each indented line. In practice you will prepare more complicated input - for Python with a text editor; all decent text editors have an auto-indent - facility. When a compound statement is entered interactively, it must be - followed by a blank line to indicate completion (since the parser cannot - guess when you have typed the last line). Note that each line within a basic - block must be indented by the same amount. - -* The :func:`print` function writes the value of the argument(s) it is given. - It differs from just writing the expression you want to write (as we did - earlier in the calculator examples) in the way it handles multiple arguments, - floating point quantities, and strings. Strings are printed without quotes, - and a space is inserted between items, so you can format things nicely, like - this:: - - >>> i = 256*256 - >>> print('The value of i is', i) - The value of i is 65536 - - The keyword argument *end* can be used to avoid the newline after the output, - or end the output with a different string:: - - >>> a, b = 0, 1 - >>> while a < 1000: - ... print(a, end=',') - ... a, b = b, a+b - ... - 0,1,1,2,3,5,8,13,21,34,55,89,144,233,377,610,987, - - -.. rubric:: Footnotes - -.. [#] Since ``**`` has higher precedence than ``-``, ``-3**2`` will be - interpreted as ``-(3**2)`` and thus result in ``-9``. To avoid this - and get ``9``, you can use ``(-3)**2``. - -.. [#] Unlike other languages, special characters such as ``\n`` have the - same meaning with both single (``'...'``) and double (``"..."``) quotes. - The only difference between the two is that within single quotes you don't - need to escape ``"`` (but you have to escape ``\'``) and vice versa. diff --git a/Doc/tutorial/modules.rst b/Doc/tutorial/modules.rst deleted file mode 100644 index bf9e8e0b7b8066..00000000000000 --- a/Doc/tutorial/modules.rst +++ /dev/null @@ -1,601 +0,0 @@ -.. _tut-modules: - -******* -Modules -******* - -If you quit from the Python interpreter and enter it again, the definitions you -have made (functions and variables) are lost. Therefore, if you want to write a -somewhat longer program, you are better off using a text editor to prepare the -input for the interpreter and running it with that file as input instead. This -is known as creating a *script*. As your program gets longer, you may want to -split it into several files for easier maintenance. You may also want to use a -handy function that you've written in several programs without copying its -definition into each program. - -To support this, Python has a way to put definitions in a file and use them in a -script or in an interactive instance of the interpreter. Such a file is called a -*module*; definitions from a module can be *imported* into other modules or into -the *main* module (the collection of variables that you have access to in a -script executed at the top level and in calculator mode). - -A module is a file containing Python definitions and statements. The file name -is the module name with the suffix :file:`.py` appended. Within a module, the -module's name (as a string) is available as the value of the global variable -``__name__``. For instance, use your favorite text editor to create a file -called :file:`fibo.py` in the current directory with the following contents:: - - # Fibonacci numbers module - - def fib(n): # write Fibonacci series up to n - a, b = 0, 1 - while a < n: - print(a, end=' ') - a, b = b, a+b - print() - - def fib2(n): # return Fibonacci series up to n - result = [] - a, b = 0, 1 - while a < n: - result.append(a) - a, b = b, a+b - return result - -Now enter the Python interpreter and import this module with the following -command:: - - >>> import fibo - -This does not add the names of the functions defined in ``fibo`` directly to -the current :term:`namespace` (see :ref:`tut-scopes` for more details); -it only adds the module name ``fibo`` there. Using -the module name you can access the functions:: - - >>> fibo.fib(1000) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 610 987 - >>> fibo.fib2(100) - [0, 1, 1, 2, 3, 5, 8, 13, 21, 34, 55, 89] - >>> fibo.__name__ - 'fibo' - -If you intend to use a function often you can assign it to a local name:: - - >>> fib = fibo.fib - >>> fib(500) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - - -.. _tut-moremodules: - -More on Modules -=============== - -A module can contain executable statements as well as function definitions. -These statements are intended to initialize the module. They are executed only -the *first* time the module name is encountered in an import statement. [#]_ -(They are also run if the file is executed as a script.) - -Each module has its own private namespace, which is used as the global namespace -by all functions defined in the module. Thus, the author of a module can -use global variables in the module without worrying about accidental clashes -with a user's global variables. On the other hand, if you know what you are -doing you can touch a module's global variables with the same notation used to -refer to its functions, ``modname.itemname``. - -Modules can import other modules. It is customary but not required to place all -:keyword:`import` statements at the beginning of a module (or script, for that -matter). The imported module names, if placed at the top level of a module -(outside any functions or classes), are added to the module's global namespace. - -There is a variant of the :keyword:`import` statement that imports names from a -module directly into the importing module's namespace. For example:: - - >>> from fibo import fib, fib2 - >>> fib(500) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - -This does not introduce the module name from which the imports are taken in the -local namespace (so in the example, ``fibo`` is not defined). - -There is even a variant to import all names that a module defines:: - - >>> from fibo import * - >>> fib(500) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - -This imports all names except those beginning with an underscore (``_``). -In most cases Python programmers do not use this facility since it introduces -an unknown set of names into the interpreter, possibly hiding some things -you have already defined. - -Note that in general the practice of importing ``*`` from a module or package is -frowned upon, since it often causes poorly readable code. However, it is okay to -use it to save typing in interactive sessions. - -If the module name is followed by :keyword:`!as`, then the name -following :keyword:`!as` is bound directly to the imported module. - -:: - - >>> import fibo as fib - >>> fib.fib(500) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - -This is effectively importing the module in the same way that ``import fibo`` -will do, with the only difference of it being available as ``fib``. - -It can also be used when utilising :keyword:`from` with similar effects:: - - >>> from fibo import fib as fibonacci - >>> fibonacci(500) - 0 1 1 2 3 5 8 13 21 34 55 89 144 233 377 - - -.. note:: - - For efficiency reasons, each module is only imported once per interpreter - session. Therefore, if you change your modules, you must restart the - interpreter -- or, if it's just one module you want to test interactively, - use :func:`importlib.reload`, e.g. ``import importlib; - importlib.reload(modulename)``. - - -.. _tut-modulesasscripts: - -Executing modules as scripts ----------------------------- - -When you run a Python module with :: - - python fibo.py - -the code in the module will be executed, just as if you imported it, but with -the ``__name__`` set to ``"__main__"``. That means that by adding this code at -the end of your module:: - - if __name__ == "__main__": - import sys - fib(int(sys.argv[1])) - -you can make the file usable as a script as well as an importable module, -because the code that parses the command line only runs if the module is -executed as the "main" file: - -.. code-block:: shell-session - - $ python fibo.py 50 - 0 1 1 2 3 5 8 13 21 34 - -If the module is imported, the code is not run:: - - >>> import fibo - >>> - -This is often used either to provide a convenient user interface to a module, or -for testing purposes (running the module as a script executes a test suite). - - -.. _tut-searchpath: - -The Module Search Path ----------------------- - -.. index:: triple: module; search; path - -When a module named :mod:`!spam` is imported, the interpreter first searches for -a built-in module with that name. These module names are listed in -:data:`sys.builtin_module_names`. If not found, it then searches for a file -named :file:`spam.py` in a list of directories given by the variable -:data:`sys.path`. :data:`sys.path` is initialized from these locations: - -* The directory containing the input script (or the current directory when no - file is specified). -* :envvar:`PYTHONPATH` (a list of directory names, with the same syntax as the - shell variable :envvar:`PATH`). -* The installation-dependent default (by convention including a - ``site-packages`` directory, handled by the :mod:`site` module). - -More details are at :ref:`sys-path-init`. - -.. note:: - On file systems which support symlinks, the directory containing the input - script is calculated after the symlink is followed. In other words the - directory containing the symlink is **not** added to the module search path. - -After initialization, Python programs can modify :data:`sys.path`. The -directory containing the script being run is placed at the beginning of the -search path, ahead of the standard library path. This means that scripts in that -directory will be loaded instead of modules of the same name in the library -directory. This is an error unless the replacement is intended. See section -:ref:`tut-standardmodules` for more information. - -.. % - Do we need stuff on zip files etc. ? DUBOIS - -.. _tut-pycache: - -"Compiled" Python files ------------------------ - -To speed up loading modules, Python caches the compiled version of each module -in the ``__pycache__`` directory under the name :file:`module.{version}.pyc`, -where the version encodes the format of the compiled file; it generally contains -the Python version number. For example, in CPython release 3.3 the compiled -version of spam.py would be cached as ``__pycache__/spam.cpython-33.pyc``. This -naming convention allows compiled modules from different releases and different -versions of Python to coexist. - -Python checks the modification date of the source against the compiled version -to see if it's out of date and needs to be recompiled. This is a completely -automatic process. Also, the compiled modules are platform-independent, so the -same library can be shared among systems with different architectures. - -Python does not check the cache in two circumstances. First, it always -recompiles and does not store the result for the module that's loaded directly -from the command line. Second, it does not check the cache if there is no -source module. To support a non-source (compiled only) distribution, the -compiled module must be in the source directory, and there must not be a source -module. - -Some tips for experts: - -* You can use the :option:`-O` or :option:`-OO` switches on the Python command - to reduce the size of a compiled module. The ``-O`` switch removes assert - statements, the ``-OO`` switch removes both assert statements and __doc__ - strings. Since some programs may rely on having these available, you should - only use this option if you know what you're doing. "Optimized" modules have - an ``opt-`` tag and are usually smaller. Future releases may - change the effects of optimization. - -* A program doesn't run any faster when it is read from a ``.pyc`` - file than when it is read from a ``.py`` file; the only thing that's faster - about ``.pyc`` files is the speed with which they are loaded. - -* The module :mod:`compileall` can create .pyc files for all modules in a - directory. - -* There is more detail on this process, including a flow chart of the - decisions, in :pep:`3147`. - - -.. _tut-standardmodules: - -Standard Modules -================ - -.. index:: pair: module; sys - -Python comes with a library of standard modules, described in a separate -document, the Python Library Reference ("Library Reference" hereafter). Some -modules are built into the interpreter; these provide access to operations that -are not part of the core of the language but are nevertheless built in, either -for efficiency or to provide access to operating system primitives such as -system calls. The set of such modules is a configuration option which also -depends on the underlying platform. For example, the :mod:`winreg` module is only -provided on Windows systems. One particular module deserves some attention: -:mod:`sys`, which is built into every Python interpreter. The variables -``sys.ps1`` and ``sys.ps2`` define the strings used as primary and secondary -prompts:: - - >>> import sys - >>> sys.ps1 - '>>> ' - >>> sys.ps2 - '... ' - >>> sys.ps1 = 'C> ' - C> print('Yuck!') - Yuck! - C> - - -These two variables are only defined if the interpreter is in interactive mode. - -The variable ``sys.path`` is a list of strings that determines the interpreter's -search path for modules. It is initialized to a default path taken from the -environment variable :envvar:`PYTHONPATH`, or from a built-in default if -:envvar:`PYTHONPATH` is not set. You can modify it using standard list -operations:: - - >>> import sys - >>> sys.path.append('/ufs/guido/lib/python') - - -.. _tut-dir: - -The :func:`dir` Function -======================== - -The built-in function :func:`dir` is used to find out which names a module -defines. It returns a sorted list of strings:: - - >>> import fibo, sys - >>> dir(fibo) - ['__name__', 'fib', 'fib2'] - >>> dir(sys) # doctest: +NORMALIZE_WHITESPACE - ['__breakpointhook__', '__displayhook__', '__doc__', '__excepthook__', - '__interactivehook__', '__loader__', '__name__', '__package__', '__spec__', - '__stderr__', '__stdin__', '__stdout__', '__unraisablehook__', - '_clear_type_cache', '_current_frames', '_debugmallocstats', '_framework', - '_getframe', '_git', '_home', '_xoptions', 'abiflags', 'addaudithook', - 'api_version', 'argv', 'audit', 'base_exec_prefix', 'base_prefix', - 'breakpointhook', 'builtin_module_names', 'byteorder', 'call_tracing', - 'callstats', 'copyright', 'displayhook', 'dont_write_bytecode', 'exc_info', - 'excepthook', 'exec_prefix', 'executable', 'exit', 'flags', 'float_info', - 'float_repr_style', 'get_asyncgen_hooks', 'get_coroutine_origin_tracking_depth', - 'getallocatedblocks', 'getdefaultencoding', 'getdlopenflags', - 'getfilesystemencodeerrors', 'getfilesystemencoding', 'getprofile', - 'getrecursionlimit', 'getrefcount', 'getsizeof', 'getswitchinterval', - 'gettrace', 'hash_info', 'hexversion', 'implementation', 'int_info', - 'intern', 'is_finalizing', 'last_traceback', 'last_type', 'last_value', - 'maxsize', 'maxunicode', 'meta_path', 'modules', 'path', 'path_hooks', - 'path_importer_cache', 'platform', 'prefix', 'ps1', 'ps2', 'pycache_prefix', - 'set_asyncgen_hooks', 'set_coroutine_origin_tracking_depth', 'setdlopenflags', - 'setprofile', 'setrecursionlimit', 'setswitchinterval', 'settrace', 'stderr', - 'stdin', 'stdout', 'thread_info', 'unraisablehook', 'version', 'version_info', - 'warnoptions'] - -Without arguments, :func:`dir` lists the names you have defined currently:: - - >>> a = [1, 2, 3, 4, 5] - >>> import fibo - >>> fib = fibo.fib - >>> dir() - ['__builtins__', '__name__', 'a', 'fib', 'fibo', 'sys'] - -Note that it lists all types of names: variables, modules, functions, etc. - -.. index:: pair: module; builtins - -:func:`dir` does not list the names of built-in functions and variables. If you -want a list of those, they are defined in the standard module -:mod:`builtins`:: - - >>> import builtins - >>> dir(builtins) # doctest: +NORMALIZE_WHITESPACE - ['ArithmeticError', 'AssertionError', 'AttributeError', 'BaseException', - 'BlockingIOError', 'BrokenPipeError', 'BufferError', 'BytesWarning', - 'ChildProcessError', 'ConnectionAbortedError', 'ConnectionError', - 'ConnectionRefusedError', 'ConnectionResetError', 'DeprecationWarning', - 'EOFError', 'Ellipsis', 'EnvironmentError', 'Exception', 'False', - 'FileExistsError', 'FileNotFoundError', 'FloatingPointError', - 'FutureWarning', 'GeneratorExit', 'IOError', 'ImportError', - 'ImportWarning', 'IndentationError', 'IndexError', 'InterruptedError', - 'IsADirectoryError', 'KeyError', 'KeyboardInterrupt', 'LookupError', - 'MemoryError', 'NameError', 'None', 'NotADirectoryError', 'NotImplemented', - 'NotImplementedError', 'OSError', 'OverflowError', - 'PendingDeprecationWarning', 'PermissionError', 'ProcessLookupError', - 'ReferenceError', 'ResourceWarning', 'RuntimeError', 'RuntimeWarning', - 'StopIteration', 'SyntaxError', 'SyntaxWarning', 'SystemError', - 'SystemExit', 'TabError', 'TimeoutError', 'True', 'TypeError', - 'UnboundLocalError', 'UnicodeDecodeError', 'UnicodeEncodeError', - 'UnicodeError', 'UnicodeTranslateError', 'UnicodeWarning', 'UserWarning', - 'ValueError', 'Warning', 'ZeroDivisionError', '_', '__build_class__', - '__debug__', '__doc__', '__import__', '__name__', '__package__', 'abs', - 'all', 'any', 'ascii', 'bin', 'bool', 'bytearray', 'bytes', 'callable', - 'chr', 'classmethod', 'compile', 'complex', 'copyright', 'credits', - 'delattr', 'dict', 'dir', 'divmod', 'enumerate', 'eval', 'exec', 'exit', - 'filter', 'float', 'format', 'frozenset', 'getattr', 'globals', 'hasattr', - 'hash', 'help', 'hex', 'id', 'input', 'int', 'isinstance', 'issubclass', - 'iter', 'len', 'license', 'list', 'locals', 'map', 'max', 'memoryview', - 'min', 'next', 'object', 'oct', 'open', 'ord', 'pow', 'print', 'property', - 'quit', 'range', 'repr', 'reversed', 'round', 'set', 'setattr', 'slice', - 'sorted', 'staticmethod', 'str', 'sum', 'super', 'tuple', 'type', 'vars', - 'zip'] - -.. _tut-packages: - -Packages -======== - -Packages are a way of structuring Python's module namespace by using "dotted -module names". For example, the module name :mod:`!A.B` designates a submodule -named ``B`` in a package named ``A``. Just like the use of modules saves the -authors of different modules from having to worry about each other's global -variable names, the use of dotted module names saves the authors of multi-module -packages like NumPy or Pillow from having to worry about -each other's module names. - -Suppose you want to design a collection of modules (a "package") for the uniform -handling of sound files and sound data. There are many different sound file -formats (usually recognized by their extension, for example: :file:`.wav`, -:file:`.aiff`, :file:`.au`), so you may need to create and maintain a growing -collection of modules for the conversion between the various file formats. -There are also many different operations you might want to perform on sound data -(such as mixing, adding echo, applying an equalizer function, creating an -artificial stereo effect), so in addition you will be writing a never-ending -stream of modules to perform these operations. Here's a possible structure for -your package (expressed in terms of a hierarchical filesystem): - -.. code-block:: text - - sound/ Top-level package - __init__.py Initialize the sound package - formats/ Subpackage for file format conversions - __init__.py - wavread.py - wavwrite.py - aiffread.py - aiffwrite.py - auread.py - auwrite.py - ... - effects/ Subpackage for sound effects - __init__.py - echo.py - surround.py - reverse.py - ... - filters/ Subpackage for filters - __init__.py - equalizer.py - vocoder.py - karaoke.py - ... - -When importing the package, Python searches through the directories on -``sys.path`` looking for the package subdirectory. - -The :file:`__init__.py` files are required to make Python treat directories -containing the file as packages. This prevents directories with a common name, -such as ``string``, from unintentionally hiding valid modules that occur later -on the module search path. In the simplest case, :file:`__init__.py` can just be -an empty file, but it can also execute initialization code for the package or -set the ``__all__`` variable, described later. - -Users of the package can import individual modules from the package, for -example:: - - import sound.effects.echo - -This loads the submodule :mod:`!sound.effects.echo`. It must be referenced with -its full name. :: - - sound.effects.echo.echofilter(input, output, delay=0.7, atten=4) - -An alternative way of importing the submodule is:: - - from sound.effects import echo - -This also loads the submodule :mod:`!echo`, and makes it available without its -package prefix, so it can be used as follows:: - - echo.echofilter(input, output, delay=0.7, atten=4) - -Yet another variation is to import the desired function or variable directly:: - - from sound.effects.echo import echofilter - -Again, this loads the submodule :mod:`!echo`, but this makes its function -:func:`!echofilter` directly available:: - - echofilter(input, output, delay=0.7, atten=4) - -Note that when using ``from package import item``, the item can be either a -submodule (or subpackage) of the package, or some other name defined in the -package, like a function, class or variable. The ``import`` statement first -tests whether the item is defined in the package; if not, it assumes it is a -module and attempts to load it. If it fails to find it, an :exc:`ImportError` -exception is raised. - -Contrarily, when using syntax like ``import item.subitem.subsubitem``, each item -except for the last must be a package; the last item can be a module or a -package but can't be a class or function or variable defined in the previous -item. - - -.. _tut-pkg-import-star: - -Importing \* From a Package ---------------------------- - -.. index:: single: __all__ - -Now what happens when the user writes ``from sound.effects import *``? Ideally, -one would hope that this somehow goes out to the filesystem, finds which -submodules are present in the package, and imports them all. This could take a -long time and importing sub-modules might have unwanted side-effects that should -only happen when the sub-module is explicitly imported. - -The only solution is for the package author to provide an explicit index of the -package. The :keyword:`import` statement uses the following convention: if a package's -:file:`__init__.py` code defines a list named ``__all__``, it is taken to be the -list of module names that should be imported when ``from package import *`` is -encountered. It is up to the package author to keep this list up-to-date when a -new version of the package is released. Package authors may also decide not to -support it, if they don't see a use for importing \* from their package. For -example, the file :file:`sound/effects/__init__.py` could contain the following -code:: - - __all__ = ["echo", "surround", "reverse"] - -This would mean that ``from sound.effects import *`` would import the three -named submodules of the :mod:`!sound.effects` package. - -Be aware that submodules might become shadowed by locally defined names. For -example, if you added a ``reverse`` function to the -:file:`sound/effects/__init__.py` file, the ``from sound.effects import *`` -would only import the two submodules ``echo`` and ``surround``, but *not* the -``reverse`` submodule, because it is shadowed by the locally defined -``reverse`` function:: - - __all__ = [ - "echo", # refers to the 'echo.py' file - "surround", # refers to the 'surround.py' file - "reverse", # !!! refers to the 'reverse' function now !!! - ] - - def reverse(msg: str): # <-- this name shadows the 'reverse.py' submodule - return msg[::-1] # in the case of a 'from sound.effects import *' - -If ``__all__`` is not defined, the statement ``from sound.effects import *`` -does *not* import all submodules from the package :mod:`!sound.effects` into the -current namespace; it only ensures that the package :mod:`!sound.effects` has -been imported (possibly running any initialization code in :file:`__init__.py`) -and then imports whatever names are defined in the package. This includes any -names defined (and submodules explicitly loaded) by :file:`__init__.py`. It -also includes any submodules of the package that were explicitly loaded by -previous :keyword:`import` statements. Consider this code:: - - import sound.effects.echo - import sound.effects.surround - from sound.effects import * - -In this example, the :mod:`!echo` and :mod:`!surround` modules are imported in the -current namespace because they are defined in the :mod:`!sound.effects` package -when the ``from...import`` statement is executed. (This also works when -``__all__`` is defined.) - -Although certain modules are designed to export only names that follow certain -patterns when you use ``import *``, it is still considered bad practice in -production code. - -Remember, there is nothing wrong with using ``from package import -specific_submodule``! In fact, this is the recommended notation unless the -importing module needs to use submodules with the same name from different -packages. - - -.. _intra-package-references: - -Intra-package References ------------------------- - -When packages are structured into subpackages (as with the :mod:`!sound` package -in the example), you can use absolute imports to refer to submodules of siblings -packages. For example, if the module :mod:`!sound.filters.vocoder` needs to use -the :mod:`!echo` module in the :mod:`!sound.effects` package, it can use ``from -sound.effects import echo``. - -You can also write relative imports, with the ``from module import name`` form -of import statement. These imports use leading dots to indicate the current and -parent packages involved in the relative import. From the :mod:`!surround` -module for example, you might use:: - - from . import echo - from .. import formats - from ..filters import equalizer - -Note that relative imports are based on the name of the current module. Since -the name of the main module is always ``"__main__"``, modules intended for use -as the main module of a Python application must always use absolute imports. - - -Packages in Multiple Directories --------------------------------- - -Packages support one more special attribute, :attr:`__path__`. This is -initialized to be a list containing the name of the directory holding the -package's :file:`__init__.py` before the code in that file is executed. This -variable can be modified; doing so affects future searches for modules and -subpackages contained in the package. - -While this feature is not often needed, it can be used to extend the set of -modules found in a package. - - -.. rubric:: Footnotes - -.. [#] In fact function definitions are also 'statements' that are 'executed'; the - execution of a module-level function definition adds the function name to - the module's global namespace. diff --git a/Doc/tutorial/stdlib.rst b/Doc/tutorial/stdlib.rst deleted file mode 100644 index 084b09a9e907f3..00000000000000 --- a/Doc/tutorial/stdlib.rst +++ /dev/null @@ -1,354 +0,0 @@ -.. _tut-brieftour: - -********************************** -Brief Tour of the Standard Library -********************************** - - -.. _tut-os-interface: - -Operating System Interface -========================== - -The :mod:`os` module provides dozens of functions for interacting with the -operating system:: - - >>> import os - >>> os.getcwd() # Return the current working directory - 'C:\\Python311' - >>> os.chdir('/server/accesslogs') # Change current working directory - >>> os.system('mkdir today') # Run the command mkdir in the system shell - 0 - -Be sure to use the ``import os`` style instead of ``from os import *``. This -will keep :func:`os.open` from shadowing the built-in :func:`open` function which -operates much differently. - -.. index:: pair: built-in function; help - -The built-in :func:`dir` and :func:`help` functions are useful as interactive -aids for working with large modules like :mod:`os`:: - - >>> import os - >>> dir(os) - - >>> help(os) - - -For daily file and directory management tasks, the :mod:`shutil` module provides -a higher level interface that is easier to use:: - - >>> import shutil - >>> shutil.copyfile('data.db', 'archive.db') - 'archive.db' - >>> shutil.move('/build/executables', 'installdir') - 'installdir' - - -.. _tut-file-wildcards: - -File Wildcards -============== - -The :mod:`glob` module provides a function for making file lists from directory -wildcard searches:: - - >>> import glob - >>> glob.glob('*.py') - ['primes.py', 'random.py', 'quote.py'] - - -.. _tut-command-line-arguments: - -Command Line Arguments -====================== - -Common utility scripts often need to process command line arguments. These -arguments are stored in the :mod:`sys` module's *argv* attribute as a list. For -instance the following output results from running ``python demo.py one two -three`` at the command line:: - - >>> import sys - >>> print(sys.argv) - ['demo.py', 'one', 'two', 'three'] - -The :mod:`argparse` module provides a more sophisticated mechanism to process -command line arguments. The following script extracts one or more filenames -and an optional number of lines to be displayed:: - - import argparse - - parser = argparse.ArgumentParser( - prog='top', - description='Show top lines from each file') - parser.add_argument('filenames', nargs='+') - parser.add_argument('-l', '--lines', type=int, default=10) - args = parser.parse_args() - print(args) - -When run at the command line with ``python top.py --lines=5 alpha.txt -beta.txt``, the script sets ``args.lines`` to ``5`` and ``args.filenames`` -to ``['alpha.txt', 'beta.txt']``. - - -.. _tut-stderr: - -Error Output Redirection and Program Termination -================================================ - -The :mod:`sys` module also has attributes for *stdin*, *stdout*, and *stderr*. -The latter is useful for emitting warnings and error messages to make them -visible even when *stdout* has been redirected:: - - >>> sys.stderr.write('Warning, log file not found starting a new one\n') - Warning, log file not found starting a new one - -The most direct way to terminate a script is to use ``sys.exit()``. - - -.. _tut-string-pattern-matching: - -String Pattern Matching -======================= - -The :mod:`re` module provides regular expression tools for advanced string -processing. For complex matching and manipulation, regular expressions offer -succinct, optimized solutions:: - - >>> import re - >>> re.findall(r'\bf[a-z]*', 'which foot or hand fell fastest') - ['foot', 'fell', 'fastest'] - >>> re.sub(r'(\b[a-z]+) \1', r'\1', 'cat in the the hat') - 'cat in the hat' - -When only simple capabilities are needed, string methods are preferred because -they are easier to read and debug:: - - >>> 'tea for too'.replace('too', 'two') - 'tea for two' - - -.. _tut-mathematics: - -Mathematics -=========== - -The :mod:`math` module gives access to the underlying C library functions for -floating point math:: - - >>> import math - >>> math.cos(math.pi / 4) - 0.70710678118654757 - >>> math.log(1024, 2) - 10.0 - -The :mod:`random` module provides tools for making random selections:: - - >>> import random - >>> random.choice(['apple', 'pear', 'banana']) - 'apple' - >>> random.sample(range(100), 10) # sampling without replacement - [30, 83, 16, 4, 8, 81, 41, 50, 18, 33] - >>> random.random() # random float - 0.17970987693706186 - >>> random.randrange(6) # random integer chosen from range(6) - 4 - -The :mod:`statistics` module calculates basic statistical properties -(the mean, median, variance, etc.) of numeric data:: - - >>> import statistics - >>> data = [2.75, 1.75, 1.25, 0.25, 0.5, 1.25, 3.5] - >>> statistics.mean(data) - 1.6071428571428572 - >>> statistics.median(data) - 1.25 - >>> statistics.variance(data) - 1.3720238095238095 - -The SciPy project has many other modules for numerical -computations. - -.. _tut-internet-access: - -Internet Access -=============== - -There are a number of modules for accessing the internet and processing internet -protocols. Two of the simplest are :mod:`urllib.request` for retrieving data -from URLs and :mod:`smtplib` for sending mail:: - - >>> from urllib.request import urlopen - >>> with urlopen('http://worldtimeapi.org/api/timezone/etc/UTC.txt') as response: - ... for line in response: - ... line = line.decode() # Convert bytes to a str - ... if line.startswith('datetime'): - ... print(line.rstrip()) # Remove trailing newline - ... - datetime: 2022-01-01T01:36:47.689215+00:00 - - >>> import smtplib - >>> server = smtplib.SMTP('localhost') - >>> server.sendmail('soothsayer@example.org', 'jcaesar@example.org', - ... """To: jcaesar@example.org - ... From: soothsayer@example.org - ... - ... Beware the Ides of March. - ... """) - >>> server.quit() - -(Note that the second example needs a mailserver running on localhost.) - - -.. _tut-dates-and-times: - -Dates and Times -=============== - -The :mod:`datetime` module supplies classes for manipulating dates and times in -both simple and complex ways. While date and time arithmetic is supported, the -focus of the implementation is on efficient member extraction for output -formatting and manipulation. The module also supports objects that are timezone -aware. :: - - >>> # dates are easily constructed and formatted - >>> from datetime import date - >>> now = date.today() - >>> now - datetime.date(2003, 12, 2) - >>> now.strftime("%m-%d-%y. %d %b %Y is a %A on the %d day of %B.") - '12-02-03. 02 Dec 2003 is a Tuesday on the 02 day of December.' - - >>> # dates support calendar arithmetic - >>> birthday = date(1964, 7, 31) - >>> age = now - birthday - >>> age.days - 14368 - - -.. _tut-data-compression: - -Data Compression -================ - -Common data archiving and compression formats are directly supported by modules -including: :mod:`zlib`, :mod:`gzip`, :mod:`bz2`, :mod:`lzma`, :mod:`zipfile` and -:mod:`tarfile`. :: - - >>> import zlib - >>> s = b'witch which has which witches wrist watch' - >>> len(s) - 41 - >>> t = zlib.compress(s) - >>> len(t) - 37 - >>> zlib.decompress(t) - b'witch which has which witches wrist watch' - >>> zlib.crc32(s) - 226805979 - - -.. _tut-performance-measurement: - -Performance Measurement -======================= - -Some Python users develop a deep interest in knowing the relative performance of -different approaches to the same problem. Python provides a measurement tool -that answers those questions immediately. - -For example, it may be tempting to use the tuple packing and unpacking feature -instead of the traditional approach to swapping arguments. The :mod:`timeit` -module quickly demonstrates a modest performance advantage:: - - >>> from timeit import Timer - >>> Timer('t=a; a=b; b=t', 'a=1; b=2').timeit() - 0.57535828626024577 - >>> Timer('a,b = b,a', 'a=1; b=2').timeit() - 0.54962537085770791 - -In contrast to :mod:`timeit`'s fine level of granularity, the :mod:`profile` and -:mod:`pstats` modules provide tools for identifying time critical sections in -larger blocks of code. - - -.. _tut-quality-control: - -Quality Control -=============== - -One approach for developing high quality software is to write tests for each -function as it is developed and to run those tests frequently during the -development process. - -The :mod:`doctest` module provides a tool for scanning a module and validating -tests embedded in a program's docstrings. Test construction is as simple as -cutting-and-pasting a typical call along with its results into the docstring. -This improves the documentation by providing the user with an example and it -allows the doctest module to make sure the code remains true to the -documentation:: - - def average(values): - """Computes the arithmetic mean of a list of numbers. - - >>> print(average([20, 30, 70])) - 40.0 - """ - return sum(values) / len(values) - - import doctest - doctest.testmod() # automatically validate the embedded tests - -The :mod:`unittest` module is not as effortless as the :mod:`doctest` module, -but it allows a more comprehensive set of tests to be maintained in a separate -file:: - - import unittest - - class TestStatisticalFunctions(unittest.TestCase): - - def test_average(self): - self.assertEqual(average([20, 30, 70]), 40.0) - self.assertEqual(round(average([1, 5, 7]), 1), 4.3) - with self.assertRaises(ZeroDivisionError): - average([]) - with self.assertRaises(TypeError): - average(20, 30, 70) - - unittest.main() # Calling from the command line invokes all tests - - -.. _tut-batteries-included: - -Batteries Included -================== - -Python has a "batteries included" philosophy. This is best seen through the -sophisticated and robust capabilities of its larger packages. For example: - -* The :mod:`xmlrpc.client` and :mod:`xmlrpc.server` modules make implementing - remote procedure calls into an almost trivial task. Despite the modules' - names, no direct knowledge or handling of XML is needed. - -* The :mod:`email` package is a library for managing email messages, including - MIME and other :rfc:`2822`-based message documents. Unlike :mod:`smtplib` and - :mod:`poplib` which actually send and receive messages, the email package has - a complete toolset for building or decoding complex message structures - (including attachments) and for implementing internet encoding and header - protocols. - -* The :mod:`json` package provides robust support for parsing this - popular data interchange format. The :mod:`csv` module supports - direct reading and writing of files in Comma-Separated Value format, - commonly supported by databases and spreadsheets. XML processing is - supported by the :mod:`xml.etree.ElementTree`, :mod:`xml.dom` and - :mod:`xml.sax` packages. Together, these modules and packages - greatly simplify data interchange between Python applications and - other tools. - -* The :mod:`sqlite3` module is a wrapper for the SQLite database - library, providing a persistent database that can be updated and - accessed using slightly nonstandard SQL syntax. - -* Internationalization is supported by a number of modules including - :mod:`gettext`, :mod:`locale`, and the :mod:`codecs` package. diff --git a/Doc/tutorial/stdlib2.rst b/Doc/tutorial/stdlib2.rst deleted file mode 100644 index 24cf0970e4832d..00000000000000 --- a/Doc/tutorial/stdlib2.rst +++ /dev/null @@ -1,406 +0,0 @@ -.. _tut-brieftourtwo: - -********************************************** -Brief Tour of the Standard Library --- Part II -********************************************** - -This second tour covers more advanced modules that support professional -programming needs. These modules rarely occur in small scripts. - - -.. _tut-output-formatting: - -Output Formatting -================= - -The :mod:`reprlib` module provides a version of :func:`repr` customized for -abbreviated displays of large or deeply nested containers:: - - >>> import reprlib - >>> reprlib.repr(set('supercalifragilisticexpialidocious')) - "{'a', 'c', 'd', 'e', 'f', 'g', ...}" - -The :mod:`pprint` module offers more sophisticated control over printing both -built-in and user defined objects in a way that is readable by the interpreter. -When the result is longer than one line, the "pretty printer" adds line breaks -and indentation to more clearly reveal data structure:: - - >>> import pprint - >>> t = [[[['black', 'cyan'], 'white', ['green', 'red']], [['magenta', - ... 'yellow'], 'blue']]] - ... - >>> pprint.pprint(t, width=30) - [[[['black', 'cyan'], - 'white', - ['green', 'red']], - [['magenta', 'yellow'], - 'blue']]] - -The :mod:`textwrap` module formats paragraphs of text to fit a given screen -width:: - - >>> import textwrap - >>> doc = """The wrap() method is just like fill() except that it returns - ... a list of strings instead of one big string with newlines to separate - ... the wrapped lines.""" - ... - >>> print(textwrap.fill(doc, width=40)) - The wrap() method is just like fill() - except that it returns a list of strings - instead of one big string with newlines - to separate the wrapped lines. - -The :mod:`locale` module accesses a database of culture specific data formats. -The grouping attribute of locale's format function provides a direct way of -formatting numbers with group separators:: - - >>> import locale - >>> locale.setlocale(locale.LC_ALL, 'English_United States.1252') - 'English_United States.1252' - >>> conv = locale.localeconv() # get a mapping of conventions - >>> x = 1234567.8 - >>> locale.format("%d", x, grouping=True) - '1,234,567' - >>> locale.format_string("%s%.*f", (conv['currency_symbol'], - ... conv['frac_digits'], x), grouping=True) - '$1,234,567.80' - - -.. _tut-templating: - -Templating -========== - -The :mod:`string` module includes a versatile :class:`~string.Template` class -with a simplified syntax suitable for editing by end-users. This allows users -to customize their applications without having to alter the application. - -The format uses placeholder names formed by ``$`` with valid Python identifiers -(alphanumeric characters and underscores). Surrounding the placeholder with -braces allows it to be followed by more alphanumeric letters with no intervening -spaces. Writing ``$$`` creates a single escaped ``$``:: - - >>> from string import Template - >>> t = Template('${village}folk send $$10 to $cause.') - >>> t.substitute(village='Nottingham', cause='the ditch fund') - 'Nottinghamfolk send $10 to the ditch fund.' - -The :meth:`~string.Template.substitute` method raises a :exc:`KeyError` when a -placeholder is not supplied in a dictionary or a keyword argument. For -mail-merge style applications, user supplied data may be incomplete and the -:meth:`~string.Template.safe_substitute` method may be more appropriate --- -it will leave placeholders unchanged if data is missing:: - - >>> t = Template('Return the $item to $owner.') - >>> d = dict(item='unladen swallow') - >>> t.substitute(d) - Traceback (most recent call last): - ... - KeyError: 'owner' - >>> t.safe_substitute(d) - 'Return the unladen swallow to $owner.' - -Template subclasses can specify a custom delimiter. For example, a batch -renaming utility for a photo browser may elect to use percent signs for -placeholders such as the current date, image sequence number, or file format:: - - >>> import time, os.path - >>> photofiles = ['img_1074.jpg', 'img_1076.jpg', 'img_1077.jpg'] - >>> class BatchRename(Template): - ... delimiter = '%' - ... - >>> fmt = input('Enter rename style (%d-date %n-seqnum %f-format): ') - Enter rename style (%d-date %n-seqnum %f-format): Ashley_%n%f - - >>> t = BatchRename(fmt) - >>> date = time.strftime('%d%b%y') - >>> for i, filename in enumerate(photofiles): - ... base, ext = os.path.splitext(filename) - ... newname = t.substitute(d=date, n=i, f=ext) - ... print('{0} --> {1}'.format(filename, newname)) - - img_1074.jpg --> Ashley_0.jpg - img_1076.jpg --> Ashley_1.jpg - img_1077.jpg --> Ashley_2.jpg - -Another application for templating is separating program logic from the details -of multiple output formats. This makes it possible to substitute custom -templates for XML files, plain text reports, and HTML web reports. - - -.. _tut-binary-formats: - -Working with Binary Data Record Layouts -======================================= - -The :mod:`struct` module provides :func:`~struct.pack` and -:func:`~struct.unpack` functions for working with variable length binary -record formats. The following example shows -how to loop through header information in a ZIP file without using the -:mod:`zipfile` module. Pack codes ``"H"`` and ``"I"`` represent two and four -byte unsigned numbers respectively. The ``"<"`` indicates that they are -standard size and in little-endian byte order:: - - import struct - - with open('myfile.zip', 'rb') as f: - data = f.read() - - start = 0 - for i in range(3): # show the first 3 file headers - start += 14 - fields = struct.unpack('>> import weakref, gc - >>> class A: - ... def __init__(self, value): - ... self.value = value - ... def __repr__(self): - ... return str(self.value) - ... - >>> a = A(10) # create a reference - >>> d = weakref.WeakValueDictionary() - >>> d['primary'] = a # does not create a reference - >>> d['primary'] # fetch the object if it is still alive - 10 - >>> del a # remove the one reference - >>> gc.collect() # run garbage collection right away - 0 - >>> d['primary'] # entry was automatically removed - Traceback (most recent call last): - File "", line 1, in - d['primary'] # entry was automatically removed - File "C:/python311/lib/weakref.py", line 46, in __getitem__ - o = self.data[key]() - KeyError: 'primary' - - -.. _tut-list-tools: - -Tools for Working with Lists -============================ - -Many data structure needs can be met with the built-in list type. However, -sometimes there is a need for alternative implementations with different -performance trade-offs. - -The :mod:`array` module provides an :class:`~array.array()` object that is like -a list that stores only homogeneous data and stores it more compactly. The -following example shows an array of numbers stored as two byte unsigned binary -numbers (typecode ``"H"``) rather than the usual 16 bytes per entry for regular -lists of Python int objects:: - - >>> from array import array - >>> a = array('H', [4000, 10, 700, 22222]) - >>> sum(a) - 26932 - >>> a[1:3] - array('H', [10, 700]) - -The :mod:`collections` module provides a :class:`~collections.deque()` object -that is like a list with faster appends and pops from the left side but slower -lookups in the middle. These objects are well suited for implementing queues -and breadth first tree searches:: - - >>> from collections import deque - >>> d = deque(["task1", "task2", "task3"]) - >>> d.append("task4") - >>> print("Handling", d.popleft()) - Handling task1 - -:: - - unsearched = deque([starting_node]) - def breadth_first_search(unsearched): - node = unsearched.popleft() - for m in gen_moves(node): - if is_goal(m): - return m - unsearched.append(m) - -In addition to alternative list implementations, the library also offers other -tools such as the :mod:`bisect` module with functions for manipulating sorted -lists:: - - >>> import bisect - >>> scores = [(100, 'perl'), (200, 'tcl'), (400, 'lua'), (500, 'python')] - >>> bisect.insort(scores, (300, 'ruby')) - >>> scores - [(100, 'perl'), (200, 'tcl'), (300, 'ruby'), (400, 'lua'), (500, 'python')] - -The :mod:`heapq` module provides functions for implementing heaps based on -regular lists. The lowest valued entry is always kept at position zero. This -is useful for applications which repeatedly access the smallest element but do -not want to run a full list sort:: - - >>> from heapq import heapify, heappop, heappush - >>> data = [1, 3, 5, 7, 9, 2, 4, 6, 8, 0] - >>> heapify(data) # rearrange the list into heap order - >>> heappush(data, -5) # add a new entry - >>> [heappop(data) for i in range(3)] # fetch the three smallest entries - [-5, 0, 1] - - -.. _tut-decimal-fp: - -Decimal Floating Point Arithmetic -================================= - -The :mod:`decimal` module offers a :class:`~decimal.Decimal` datatype for -decimal floating point arithmetic. Compared to the built-in :class:`float` -implementation of binary floating point, the class is especially helpful for - -* financial applications and other uses which require exact decimal - representation, -* control over precision, -* control over rounding to meet legal or regulatory requirements, -* tracking of significant decimal places, or -* applications where the user expects the results to match calculations done by - hand. - -For example, calculating a 5% tax on a 70 cent phone charge gives different -results in decimal floating point and binary floating point. The difference -becomes significant if the results are rounded to the nearest cent:: - - >>> from decimal import * - >>> round(Decimal('0.70') * Decimal('1.05'), 2) - Decimal('0.74') - >>> round(.70 * 1.05, 2) - 0.73 - -The :class:`~decimal.Decimal` result keeps a trailing zero, automatically -inferring four place significance from multiplicands with two place -significance. Decimal reproduces mathematics as done by hand and avoids -issues that can arise when binary floating point cannot exactly represent -decimal quantities. - -Exact representation enables the :class:`~decimal.Decimal` class to perform -modulo calculations and equality tests that are unsuitable for binary floating -point:: - - >>> Decimal('1.00') % Decimal('.10') - Decimal('0.00') - >>> 1.00 % 0.10 - 0.09999999999999995 - - >>> sum([Decimal('0.1')]*10) == Decimal('1.0') - True - >>> sum([0.1]*10) == 1.0 - False - -The :mod:`decimal` module provides arithmetic with as much precision as needed:: - - >>> getcontext().prec = 36 - >>> Decimal(1) / Decimal(7) - Decimal('0.142857142857142857142857142857142857') - - diff --git a/Doc/tutorial/venv.rst b/Doc/tutorial/venv.rst deleted file mode 100644 index d91fc3de4f92d8..00000000000000 --- a/Doc/tutorial/venv.rst +++ /dev/null @@ -1,212 +0,0 @@ - -.. _tut-venv: - -********************************* -Virtual Environments and Packages -********************************* - -Introduction -============ - -Python applications will often use packages and modules that don't -come as part of the standard library. Applications will sometimes -need a specific version of a library, because the application may -require that a particular bug has been fixed or the application may be -written using an obsolete version of the library's interface. - -This means it may not be possible for one Python installation to meet -the requirements of every application. If application A needs version -1.0 of a particular module but application B needs version 2.0, then -the requirements are in conflict and installing either version 1.0 or 2.0 -will leave one application unable to run. - -The solution for this problem is to create a :term:`virtual environment`, a -self-contained directory tree that contains a Python installation for a -particular version of Python, plus a number of additional packages. - -Different applications can then use different virtual environments. -To resolve the earlier example of conflicting requirements, -application A can have its own virtual environment with version 1.0 -installed while application B has another virtual environment with version 2.0. -If application B requires a library be upgraded to version 3.0, this will -not affect application A's environment. - - -Creating Virtual Environments -============================= - -The module used to create and manage virtual environments is called -:mod:`venv`. :mod:`venv` will usually install the most recent version of -Python that you have available. If you have multiple versions of Python on your -system, you can select a specific Python version by running ``python3`` or -whichever version you want. - -To create a virtual environment, decide upon a directory where you want to -place it, and run the :mod:`venv` module as a script with the directory path:: - - python -m venv tutorial-env - -This will create the ``tutorial-env`` directory if it doesn't exist, -and also create directories inside it containing a copy of the Python -interpreter and various supporting files. - -A common directory location for a virtual environment is ``.venv``. -This name keeps the directory typically hidden in your shell and thus -out of the way while giving it a name that explains why the directory -exists. It also prevents clashing with ``.env`` environment variable -definition files that some tooling supports. - -Once you've created a virtual environment, you may activate it. - -On Windows, run:: - - tutorial-env\Scripts\activate.bat - -On Unix or MacOS, run:: - - source tutorial-env/bin/activate - -(This script is written for the bash shell. If you use the -:program:`csh` or :program:`fish` shells, there are alternate -``activate.csh`` and ``activate.fish`` scripts you should use -instead.) - -Activating the virtual environment will change your shell's prompt to show what -virtual environment you're using, and modify the environment so that running -``python`` will get you that particular version and installation of Python. -For example: - -.. code-block:: bash - - $ source ~/envs/tutorial-env/bin/activate - (tutorial-env) $ python - Python 3.5.1 (default, May 6 2016, 10:59:36) - ... - >>> import sys - >>> sys.path - ['', '/usr/local/lib/python35.zip', ..., - '~/envs/tutorial-env/lib/python3.5/site-packages'] - >>> - -To deactivate a virtual environment, type:: - - deactivate - -into the terminal. - -Managing Packages with pip -========================== - -You can install, upgrade, and remove packages using a program called -:program:`pip`. By default ``pip`` will install packages from the `Python -Package Index `_. You can browse the Python -Package Index by going to it in your web browser. - -``pip`` has a number of subcommands: "install", "uninstall", -"freeze", etc. (Consult the :ref:`installing-index` guide for -complete documentation for ``pip``.) - -You can install the latest version of a package by specifying a package's name: - -.. code-block:: bash - - (tutorial-env) $ python -m pip install novas - Collecting novas - Downloading novas-3.1.1.3.tar.gz (136kB) - Installing collected packages: novas - Running setup.py install for novas - Successfully installed novas-3.1.1.3 - -You can also install a specific version of a package by giving the -package name followed by ``==`` and the version number: - -.. code-block:: bash - - (tutorial-env) $ python -m pip install requests==2.6.0 - Collecting requests==2.6.0 - Using cached requests-2.6.0-py2.py3-none-any.whl - Installing collected packages: requests - Successfully installed requests-2.6.0 - -If you re-run this command, ``pip`` will notice that the requested -version is already installed and do nothing. You can supply a -different version number to get that version, or you can run ``python --m pip install --upgrade`` to upgrade the package to the latest version: - -.. code-block:: bash - - (tutorial-env) $ python -m pip install --upgrade requests - Collecting requests - Installing collected packages: requests - Found existing installation: requests 2.6.0 - Uninstalling requests-2.6.0: - Successfully uninstalled requests-2.6.0 - Successfully installed requests-2.7.0 - -``python -m pip uninstall`` followed by one or more package names will -remove the packages from the virtual environment. - -``python -m pip show`` will display information about a particular package: - -.. code-block:: bash - - (tutorial-env) $ python -m pip show requests - --- - Metadata-Version: 2.0 - Name: requests - Version: 2.7.0 - Summary: Python HTTP for Humans. - Home-page: http://python-requests.org - Author: Kenneth Reitz - Author-email: me@kennethreitz.com - License: Apache 2.0 - Location: /Users/akuchling/envs/tutorial-env/lib/python3.4/site-packages - Requires: - -``python -m pip list`` will display all of the packages installed in -the virtual environment: - -.. code-block:: bash - - (tutorial-env) $ python -m pip list - novas (3.1.1.3) - numpy (1.9.2) - pip (7.0.3) - requests (2.7.0) - setuptools (16.0) - -``python -m pip freeze`` will produce a similar list of the installed packages, -but the output uses the format that ``python -m pip install`` expects. -A common convention is to put this list in a ``requirements.txt`` file: - -.. code-block:: bash - - (tutorial-env) $ python -m pip freeze > requirements.txt - (tutorial-env) $ cat requirements.txt - novas==3.1.1.3 - numpy==1.9.2 - requests==2.7.0 - -The ``requirements.txt`` can then be committed to version control and -shipped as part of an application. Users can then install all the -necessary packages with ``install -r``: - -.. code-block:: bash - - (tutorial-env) $ python -m pip install -r requirements.txt - Collecting novas==3.1.1.3 (from -r requirements.txt (line 1)) - ... - Collecting numpy==1.9.2 (from -r requirements.txt (line 2)) - ... - Collecting requests==2.7.0 (from -r requirements.txt (line 3)) - ... - Installing collected packages: novas, numpy, requests - Running setup.py install for novas - Successfully installed novas-3.1.1.3 numpy-1.9.2 requests-2.7.0 - -``pip`` has many more options. Consult the :ref:`installing-index` -guide for complete documentation for ``pip``. When you've written -a package and want to make it available on the Python Package Index, -consult the `Python packaging user guide`_. - -.. _Python Packaging User Guide: https://packaging.python.org/en/latest/tutorials/packaging-projects/ diff --git a/Doc/tutorial/whatnow.rst b/Doc/tutorial/whatnow.rst deleted file mode 100644 index dbe2d7fc09927e..00000000000000 --- a/Doc/tutorial/whatnow.rst +++ /dev/null @@ -1,73 +0,0 @@ -.. _tut-whatnow: - -********* -What Now? -********* - -Reading this tutorial has probably reinforced your interest in using Python --- -you should be eager to apply Python to solving your real-world problems. Where -should you go to learn more? - -This tutorial is part of Python's documentation set. Some other documents in -the set are: - -* :ref:`library-index`: - - You should browse through this manual, which gives complete (though terse) - reference material about types, functions, and the modules in the standard - library. The standard Python distribution includes a *lot* of additional code. - There are modules to read Unix mailboxes, retrieve documents via HTTP, generate - random numbers, parse command-line options, compress data, - and many other tasks. Skimming through the Library Reference will give you an - idea of what's available. - -* :ref:`installing-index` explains how to install additional modules written - by other Python users. - -* :ref:`reference-index`: A detailed explanation of Python's syntax and - semantics. It's heavy reading, but is useful as a complete guide to the - language itself. - -More Python resources: - -* https://www.python.org: The major Python web site. It contains code, - documentation, and pointers to Python-related pages around the web. - -* https://docs.python.org: Fast access to Python's documentation. - -* https://pypi.org: The Python Package Index, previously also nicknamed - the Cheese Shop [#]_, is an index of user-created Python modules that are available - for download. Once you begin releasing code, you can register it here so that - others can find it. - -* https://code.activestate.com/recipes/langs/python/: The Python Cookbook is a - sizable collection of code examples, larger modules, and useful scripts. - Particularly notable contributions are collected in a book also titled Python - Cookbook (O'Reilly & Associates, ISBN 0-596-00797-3.) - -* https://pyvideo.org collects links to Python-related videos from - conferences and user-group meetings. - -* https://scipy.org: The Scientific Python project includes modules for fast - array computations and manipulations plus a host of packages for such - things as linear algebra, Fourier transforms, non-linear solvers, - random number distributions, statistical analysis and the like. - -For Python-related questions and problem reports, you can post to the newsgroup -:newsgroup:`comp.lang.python`, or send them to the mailing list at -python-list@python.org. The newsgroup and mailing list are gatewayed, so -messages posted to one will automatically be forwarded to the other. There are -hundreds of postings a day, asking (and -answering) questions, suggesting new features, and announcing new modules. -Mailing list archives are available at https://mail.python.org/pipermail/. - -Before posting, be sure to check the list of -:ref:`Frequently Asked Questions ` (also called the FAQ). The -FAQ answers many of the questions that come up again and again, and may -already contain the solution for your problem. - -.. rubric:: Footnotes - -.. [#] "Cheese Shop" is a Monty Python's sketch: a customer enters a cheese shop, - but whatever cheese he asks for, the clerk says it's missing. - diff --git a/Doc/using/cmdline.rst b/Doc/using/cmdline.rst deleted file mode 100644 index c1a6f50ca34979..00000000000000 --- a/Doc/using/cmdline.rst +++ /dev/null @@ -1,1069 +0,0 @@ -.. highlight:: sh - -.. ATTENTION: You probably should update Misc/python.man, too, if you modify - this file. - -.. _using-on-general: - -Command line and environment -============================ - -The CPython interpreter scans the command line and the environment for various -settings. - -.. impl-detail:: - - Other implementations' command line schemes may differ. See - :ref:`implementations` for further resources. - - -.. _using-on-cmdline: - -Command line ------------- - -When invoking Python, you may specify any of these options:: - - python [-bBdEhiIOqsSuvVWx?] [-c command | -m module-name | script | - ] [args] - -The most common use case is, of course, a simple invocation of a script:: - - python myscript.py - - -.. _using-on-interface-options: - -Interface options -~~~~~~~~~~~~~~~~~ - -The interpreter interface resembles that of the UNIX shell, but provides some -additional methods of invocation: - -* When called with standard input connected to a tty device, it prompts for - commands and executes them until an EOF (an end-of-file character, you can - produce that with :kbd:`Ctrl-D` on UNIX or :kbd:`Ctrl-Z, Enter` on Windows) is read. -* When called with a file name argument or with a file as standard input, it - reads and executes a script from that file. -* When called with a directory name argument, it reads and executes an - appropriately named script from that directory. -* When called with ``-c command``, it executes the Python statement(s) given as - *command*. Here *command* may contain multiple statements separated by - newlines. Leading whitespace is significant in Python statements! -* When called with ``-m module-name``, the given module is located on the - Python module path and executed as a script. - -In non-interactive mode, the entire input is parsed before it is executed. - -An interface option terminates the list of options consumed by the interpreter, -all consecutive arguments will end up in :data:`sys.argv` -- note that the first -element, subscript zero (``sys.argv[0]``), is a string reflecting the program's -source. - -.. option:: -c - - Execute the Python code in *command*. *command* can be one or more - statements separated by newlines, with significant leading whitespace as in - normal module code. - - If this option is given, the first element of :data:`sys.argv` will be - ``"-c"`` and the current directory will be added to the start of - :data:`sys.path` (allowing modules in that directory to be imported as top - level modules). - - .. audit-event:: cpython.run_command command cmdoption-c - -.. option:: -m - - Search :data:`sys.path` for the named module and execute its contents as - the :mod:`__main__` module. - - Since the argument is a *module* name, you must not give a file extension - (``.py``). The module name should be a valid absolute Python module name, but - the implementation may not always enforce this (e.g. it may allow you to - use a name that includes a hyphen). - - Package names (including namespace packages) are also permitted. When a - package name is supplied instead - of a normal module, the interpreter will execute ``.__main__`` as - the main module. This behaviour is deliberately similar to the handling - of directories and zipfiles that are passed to the interpreter as the - script argument. - - .. note:: - - This option cannot be used with built-in modules and extension modules - written in C, since they do not have Python module files. However, it - can still be used for precompiled modules, even if the original source - file is not available. - - If this option is given, the first element of :data:`sys.argv` will be the - full path to the module file (while the module file is being located, the - first element will be set to ``"-m"``). As with the :option:`-c` option, - the current directory will be added to the start of :data:`sys.path`. - - :option:`-I` option can be used to run the script in isolated mode where - :data:`sys.path` contains neither the current directory nor the user's - site-packages directory. All ``PYTHON*`` environment variables are - ignored, too. - - Many standard library modules contain code that is invoked on their execution - as a script. An example is the :mod:`timeit` module:: - - python -m timeit -s 'setup here' 'benchmarked code here' - python -m timeit -h # for details - - .. audit-event:: cpython.run_module module-name cmdoption-m - - .. seealso:: - :func:`runpy.run_module` - Equivalent functionality directly available to Python code - - :pep:`338` -- Executing modules as scripts - - .. versionchanged:: 3.1 - Supply the package name to run a ``__main__`` submodule. - - .. versionchanged:: 3.4 - namespace packages are also supported - -.. _cmdarg-dash: - -.. describe:: - - - Read commands from standard input (:data:`sys.stdin`). If standard input is - a terminal, :option:`-i` is implied. - - If this option is given, the first element of :data:`sys.argv` will be - ``"-"`` and the current directory will be added to the start of - :data:`sys.path`. - - .. audit-event:: cpython.run_stdin "" "" - -.. _cmdarg-script: - -.. describe::